/// <summary> /// Create or add/update or remove Syms Entities. Throws exception on failure. /// </summary> internal static async Task CreateOrUpdateSymsEntities(SymsManifestContent symsManifestContent, StorageAdapterBase adapter) { string errorMesg = string.Empty; if (symsManifestContent.InitialSync == true) { await CreateOrUpdateDatabase(symsManifestContent.Database, adapter); } errorMesg = await InvokeMethod(RemoveTableEntity, symsManifestContent.RemovedEntities, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(RemoveRelationshipEntity, symsManifestContent.RemovedRelationships, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(CreateOrUpdateTableEntity, symsManifestContent.Entities, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(CreateOrUpdateRelationshipEntity, symsManifestContent.Relationships, symsManifestContent.Database.Name, adapter); if (!string.IsNullOrEmpty(errorMesg)) { throw new Exception(errorMesg); } }
/// <summary> /// Loads a document from the folder path. /// </summary> /// <param name="folder">The folder that contains the document we want to load.</param> /// <param name="docName">The document name.</param> /// <param name="docContainer">The loaded document, if it was previously loaded.</param> /// <param name="resOpt">Optional parameter. The resolve options.</param> /// <returns>The loaded document.</returns> internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null) { // This makes sure date values are consistently parsed exactly as they appear. // Default behavior auto formats date values. JsonConvert.DefaultSettings = () => new JsonSerializerSettings { DateParseHandling = DateParseHandling.None, }; CdmDocumentDefinition docContent = null; string jsonData = null; DateTimeOffset? fsModifiedTime = null; string docPath = folder.FolderPath + docName; StorageAdapterBase adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace); try { if (adapter.CanRead()) { // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"request file: {docPath}"); jsonData = await adapter.ReadAsync(docPath); // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"received file: {docPath}"); } else { throw new Exception("Storage Adapter is not enabled to read."); } } catch (Exception e) { // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"fail file: {docPath}"); // When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if (resOpt != null && resOpt.ShallowValidation) { Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileReadFailure, docPath, folder.Namespace, e.Message); } else { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistFileReadFailure, docPath, folder.Namespace, e.Message); } return(null); } try { fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath); } catch (Exception e) { Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileModTimeFailure, e.Message); } if (string.IsNullOrWhiteSpace(docName)) { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistNullDocName); return(null); } // If loading an model.json file, check that it is named correctly. if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension)) { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocNameLoadFailure, docName, ModelJsonExtension); return(null); } try { if (Persistence.Syms.Utils.CheckIfSymsAdapter(adapter)) { if (docName.EqualsWithIgnoreCase(SymsDatabases)) { SymsDatabasesResponse databases = JsonConvert.DeserializeObject <SymsDatabasesResponse>(jsonData); docContent = Persistence.Syms.ManifestDatabasesPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, databases) as CdmDocumentDefinition; } else if (docName.Contains(ManifestExtension)) { SymsManifestContent manifestContent = await Persistence.Syms.Utils.GetSymsModel(adapter, jsonData, docPath); docContent = Persistence.Syms.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, manifestContent) as CdmDocumentDefinition; } else if (docName.Contains(CdmExtension)) { // specific table TableEntity table = JsonConvert.DeserializeObject <TableEntity>(jsonData); docContent = Persistence.Syms.DocumentPersistence.FromObject(this.Ctx, folder.Namespace, folder.FolderPath, table); } else { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistSymsUnsupportedCdmConversion, docName); return(null); } } // Check file extensions, which performs a case-insensitive ordinal string comparison else if (docName.EndWithOrdinalIgnoreCase(ManifestExtension) || docName.EndWithOrdinalIgnoreCase(FolioExtension)) { docContent = Persistence.CdmFolder.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition; } else if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension)) { docContent = await Persistence.ModelJson.ManifestPersistence.FromObject(this.Ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder); } else if (docName.EndWithOrdinalIgnoreCase(CdmExtension)) { docContent = Persistence.CdmFolder.DocumentPersistence.FromObject(this.Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <Microsoft.CommonDataModel.ObjectModel.Persistence.CdmFolder.Types.DocumentContent>(jsonData)); } else { // Could not find a registered persistence class to handle this document type. Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistClassMissing, docName); return(null); } } catch (Exception e) { var errorMsg = e.Message; if (e.InnerException != null) { errorMsg += $" InnerException: {e.InnerException.Message}"; } Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocConversionFailure, docName, errorMsg); return(null); } // Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if (docContent != null) { if (docContainer != null) { // there are situations where a previously loaded document must be re-loaded. // the end of that chain of work is here where the old version of the document has been removed from // the corpus and we have created a new document and loaded it from storage and after this call we will probably // add it to the corpus and index it, etc. // it would be really rude to just kill that old object and replace it with this replicant, especially because // the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that // was just killed off but make it contain everything the new document loaded. docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition; } if (folder.Documents.AllItems.Find(x => x.Id == docContent.Id) == null) { folder.Documents.Add(docContent, docName); } docContent._fileSystemModifiedTime = fsModifiedTime; docContent.IsDirty = false; } return(docContent); }
public static CdmManifestDefinition FromObject(CdmCorpusContext ctx, string docName, string nameSpace, string path, SymsManifestContent dataObj) { DatabaseEntity database = dataObj.Database; if (database == null || database.Type != SASEntityType.DATABASE) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbObject); return(null); } DatabaseProperties databaseProperties = ((JToken)database.Properties).ToObject <DatabaseProperties>(); if (databaseProperties == null || databaseProperties.Source == null) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbPropObject); return(null); } var properties = databaseProperties.Properties; var manifest = ctx.Corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef); manifest.ManifestName = dataObj.Database.Name; manifest.Name = docName; manifest.FolderPath = path; manifest.Namespace = nameSpace; manifest.Explanation = databaseProperties.Description; if (properties != null) { if (properties.ContainsKey("cdm:schema")) { manifest.Schema = properties["cdm:schema"].ToObject <string>(); } if (properties.ContainsKey("cdm:jsonSchemaSemanticVersion")) { manifest.JsonSchemaSemanticVersion = properties["cdm:jsonSchemaSemanticVersion"].ToObject <string>(); } if (properties.ContainsKey("cdm:documentVersion")) { manifest.DocumentVersion = properties["cdm:documentVersion"].ToObject <string>(); } else if (properties.ContainsKey("cdm:traits")) { Utils.AddListToCdmCollection(manifest.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"])); } if (properties.ContainsKey("cdm:imports")) { foreach (var importObj in properties["cdm:imports"].ToObject <List <Import> >()) { manifest.Imports.Add(ImportPersistence.FromData(ctx, importObj)); } } if (properties.ContainsKey("cdm:lastFileStatusCheckTime")) { manifest.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastFileModifiedTime")) { manifest.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastChildFileModifiedTime")) { manifest.LastChildFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastChildFileModifiedTime"].ToObject <string>()); } } var t2pm = new TraitToPropertyMap(manifest); var sourceTrait = t2pm.FetchTraitReference(dbLocationTrait); if (sourceTrait == null) { sourceTrait = Utils.CreateSourceTrait(ctx, dbLocationTrait, dbLocationTraitArgName); manifest.ExhibitsTraits.Add(sourceTrait); } var adlsPath = Utils.SymsPathToAdlsAdapterPath(databaseProperties.Source.Location); var adlsCorpusPath = ctx.Corpus.Storage.AdapterPathToCorpusPath(adlsPath); if (adlsCorpusPath == null) { Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(sourceTrait.Arguments[0].Value); if (null == Utils.CreateAndMountAdlsAdapterFromAdlsPath(ctx.Corpus.Storage, adlsPath, pathTuple.Item1)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsAdlsAdapterNotMounted, adlsPath); return(null); } } if (dataObj.Entities != null) { foreach (var entityObj in dataObj.Entities) { if (entityObj.Type == SASEntityType.TABLE) { var entity = LocalEntityDeclarationPersistence.FromData(ctx, entityObj, manifest, databaseProperties.Source.Location); if (entity != null) { manifest.Entities.Add(entity); } else { Logger.Warning((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.WarnPersistSymsEntitySkipped, entityObj.Name); } } } } if (!manifest.Imports.Any((CdmImport importPresent) => importPresent.CorpusPath == Constants.FoundationsCorpusPath)) { manifest.Imports.Add(Constants.FoundationsCorpusPath); } if (dataObj.Relationships != null) { foreach (var relationshipEntity in dataObj.Relationships) { manifest.Relationships.AddRange(E2ERelationshipPersistence.FromData(ctx, relationshipEntity)); } } // TODO : Submanifest return(manifest); }