/// <summary> /// Loads a folder or document given its obejct path. /// </summary> /// <param name="objectPath"></param> /// <param name="forceReload"></param> /// <param name="resOpt"></param> /// <returns></returns> private async Task <CdmContainerDefinition> _LoadFolderOrDocument(string objectPath, bool forceReload = false, ResolveOptions resOpt = null) { if (string.IsNullOrWhiteSpace(objectPath)) { return(null); } // first check for namespace Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(objectPath); if (pathTuple == null) { Logger.Error(this.corpus.Ctx, TAG, nameof(LoadFolderOrDocument), objectPath, CdmLogCode.ErrPathNullObjectPath); return(null); } string nameSpace = !string.IsNullOrWhiteSpace(pathTuple.Item1) ? pathTuple.Item1 : this.corpus.Storage.DefaultNamespace; objectPath = pathTuple.Item2; if (!objectPath.StartsWith("/")) { return(null); } var namespaceFolder = this.corpus.Storage.FetchRootFolder(nameSpace); StorageAdapterBase namespaceAdapter = this.corpus.Storage.FetchAdapter(nameSpace); if (namespaceFolder == null || namespaceAdapter == null) { Logger.Error(this.corpus.Ctx, TAG, nameof(LoadFolderOrDocument), objectPath, CdmLogCode.ErrStorageNamespaceNotRegistered, nameSpace); return(null); } CdmFolderDefinition lastFolder = namespaceFolder.FetchChildFolderFromPath(objectPath, false); // don't create new folders, just go as far as possible if (lastFolder == null) { return(null); } // maybe the search is for a folder? string lastPath = lastFolder.FolderPath; if (lastPath == objectPath) { return(lastFolder); } // remove path to folder and then look in the folder objectPath = StringUtils.Slice(objectPath, lastPath.Length); this.concurrentReadLock.Acquire(); // During this step the document will be added to the pathLookup when it is added to a folder. var doc = await lastFolder.FetchDocumentFromFolderPathAsync(objectPath, forceReload, resOpt); this.concurrentReadLock.Release(); return(doc); }
/// <summary> /// check if adapter is syms. /// </summary> internal static bool CheckIfSymsAdapter(StorageAdapterBase adapter) { if (adapter.GetType().ToString().Equals("Microsoft.CommonDataModel.ObjectModel.Storage.SymsAdapter") || adapter.GetType().ToString().Equals("Microsoft.CommonDataModel.ObjectModel.Adapter.Syms")) { return(true); } return(false); }
/// <summary> /// Convert to SyMs object to CDM object. /// </summary> internal static async Task <SymsManifestContent> GetSymsModel(StorageAdapterBase adapter, string databaseResponse, string docPath) { var database = JsonConvert.DeserializeObject <DatabaseEntity>(databaseResponse); var entities = await adapter.ReadAsync($"/{database.Name}/{database.Name}.manifest.cdm.json/entitydefinition"); var relationships = await adapter.ReadAsync($"{docPath}/relationships"); return(new SymsManifestContent { Database = database, Entities = JsonConvert.DeserializeObject <SymsTableResponse>(entities).Tables, Relationships = JsonConvert.DeserializeObject <SymsRelationshipResponse>(relationships).Relationships, InitialSync = false, RemovedEntities = null, RemovedRelationships = null }); }
/// <summary> /// Remove Relationship Entity. Throws exception on failure. /// </summary> private static async Task RemoveRelationshipEntity(string relationship, string databaseName, StorageAdapterBase adapter) { await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationship}", null); }
/// <summary> /// Remove Table Entity. Throws exception on failure. /// </summary> private static async Task RemoveTableEntity(string tableName, string databaseName, StorageAdapterBase adapter) { await adapter.WriteAsync($"{databaseName}/{tableName}.cdm.json", null); }
/// <summary> /// Create or update Syms RelationshipEntity. Throws exception on failure. /// </summary> private static async Task CreateOrUpdateRelationshipEntity(RelationshipEntity relationshipEntity, string databaseName, StorageAdapterBase adapter) { await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationshipEntity.Name}", JsonConvertSerializeObject(relationshipEntity)); }
/// <summary> /// Create or update Syms Table Entity. Throws exception on failure. /// </summary> private static async Task CreateOrUpdateTableEntity(TableEntity tableEntity, string databaseName, StorageAdapterBase adapter) { await adapter.WriteAsync($"{databaseName}/{tableEntity.Name}.cdm.json", JsonConvertSerializeObject(tableEntity)); }
/// <summary> /// Create or update Syms RelationshipEntity. Throws exception on failure. /// </summary> internal static async Task CreateOrUpdateRelationshipEntity(RelationshipEntity relationshipEntity, StorageAdapterBase adapter) { string databaseName = ((RelationshipProperties)relationshipEntity.Properties).NamespaceProperty.DatabaseName; await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationshipEntity.Name}", JsonConvertSerializeObject(relationshipEntity)); }
/// <summary> /// Create or update Syms Table Entity. Throws exception on failure. /// </summary> internal static async Task CreateOrUpdateTableEntity(TableEntity tableEntity, StorageAdapterBase adapter) { await adapter.WriteAsync($"{((TableProperties)tableEntity.Properties).NamespaceProperty.DatabaseName}/{tableEntity.Name}.cdm.json", JsonConvertSerializeObject(tableEntity)); }
/// <summary> /// Create or update Database. Throws exception on failure. /// </summary> internal static async Task CreateOrUpdateDatabase(DatabaseEntity databaseEntity, StorageAdapterBase adapter) { await adapter.WriteAsync($"{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json", JsonConvertSerializeObject(databaseEntity)); }
/// <summary> /// Invoke specific functions to update Syms database through syms adapter. /// </summary> private static async Task <string> InvokeMethod <T>(Func <T, string, StorageAdapterBase, Task> func, List <T> entities, string dBName, StorageAdapterBase adapter) { ConcurrentDictionary <string, string> errorMesgsHash = new ConcurrentDictionary <string, string>(); string errorMesgs = string.Empty; if (entities != null) { if (entities != null && entities.Count > 0) { var tasks = entities.Select(async entity => { try { await func(entity, dBName, adapter); } catch (Exception e) { string key = string.Empty; if (entity is string) { key = (string)(object)entity; } else if (entity is TableEntity) { key = ((TableEntity)(object)entity).Name; } else if (entity is RelationshipEntity) { key = ((RelationshipEntity)(object)entity).Name; } errorMesgsHash[key] = e.Message; } }); await Task.WhenAll(tasks); foreach (var index in errorMesgsHash) { errorMesgs += $"Failed to update table {index.Key} with exception {index.Value} \n"; } } } return(errorMesgs); }
/// <summary> /// Create or add/update or remove Syms Entities. Throws exception on failure. /// </summary> internal static async Task CreateOrUpdateSymsEntities(SymsManifestContent symsManifestContent, StorageAdapterBase adapter) { string errorMesg = string.Empty; if (symsManifestContent.InitialSync == true) { await CreateOrUpdateDatabase(symsManifestContent.Database, adapter); } errorMesg = await InvokeMethod(RemoveTableEntity, symsManifestContent.RemovedEntities, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(RemoveRelationshipEntity, symsManifestContent.RemovedRelationships, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(CreateOrUpdateTableEntity, symsManifestContent.Entities, symsManifestContent.Database.Name, adapter); errorMesg += await InvokeMethod(CreateOrUpdateRelationshipEntity, symsManifestContent.Relationships, symsManifestContent.Database.Name, adapter); if (!string.IsNullOrEmpty(errorMesg)) { throw new Exception(errorMesg); } }
/// <summary> /// Convert to SyMs table object from CDM object. /// </summary> internal static async Task <TableEntity> ConvertDocToSymsTable(CdmCorpusContext ctx, CdmDocumentDefinition doc, StorageAdapterBase adapter, string name, ResolveOptions resOpt, CopyOptions options) { TableEntity existingTableEntity = JsonConvert.DeserializeObject <TableEntity>(await adapter.ReadAsync(name)); return(Persistence.Syms.DocumentPersistence.ToData(ctx, doc, ((JToken)existingTableEntity.Properties).ToObject <TableProperties>(), resOpt, options)); }
/// <summary> /// Convert to SyMs object from Manifest object. /// </summary> internal static async Task <SymsManifestContent> ConvertManifestToSyms(CdmManifestDefinition doc, StorageAdapterBase adapter, string path, ResolveOptions resOpt, CopyOptions options) { DatabaseEntity databaseEntity = null; bool isDeltaSync = true; IList <TableEntity> existingSymsTables = null; IList <RelationshipEntity> existingSymsRelationshipEntities = null; try { databaseEntity = JsonConvert.DeserializeObject <DatabaseEntity>(await adapter.ReadAsync(path)); } catch (Exception e) { if (e.Message.Contains("NotFound")) { isDeltaSync = false; } else { throw; } } if (isDeltaSync) { var entities = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/entitydefinition"); existingSymsTables = JsonConvert.DeserializeObject <SymsTableResponse>(entities).Tables; var realtionships = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/relationships"); existingSymsRelationshipEntities = JsonConvert.DeserializeObject <SymsRelationshipResponse>(realtionships).Relationships; } return(await Persistence.Syms.ManifestPersistence.ToDataAsync(doc, resOpt, options, isDeltaSync, existingSymsTables, existingSymsRelationshipEntities)); }
/// <summary> /// Loads a document from the folder path. /// </summary> /// <param name="folder">The folder that contains the document we want to load.</param> /// <param name="docName">The document name.</param> /// <param name="docContainer">The loaded document, if it was previously loaded.</param> /// <param name="resOpt">Optional parameter. The resolve options.</param> /// <returns>The loaded document.</returns> internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null) { // This makes sure date values are consistently parsed exactly as they appear. // Default behavior auto formats date values. JsonConvert.DefaultSettings = () => new JsonSerializerSettings { DateParseHandling = DateParseHandling.None, }; CdmDocumentDefinition docContent = null; string jsonData = null; DateTimeOffset? fsModifiedTime = null; string docPath = folder.FolderPath + docName; StorageAdapterBase adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace); try { if (adapter.CanRead()) { // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"request file: {docPath}"); jsonData = await adapter.ReadAsync(docPath); // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"received file: {docPath}"); } else { throw new Exception("Storage Adapter is not enabled to read."); } } catch (Exception e) { // log message used by navigator, do not change or remove Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"fail file: {docPath}"); // When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if (resOpt != null && resOpt.ShallowValidation) { Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileReadFailure, docPath, folder.Namespace, e.Message); } else { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistFileReadFailure, docPath, folder.Namespace, e.Message); } return(null); } try { fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath); } catch (Exception e) { Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileModTimeFailure, e.Message); } if (string.IsNullOrWhiteSpace(docName)) { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistNullDocName); return(null); } // If loading an model.json file, check that it is named correctly. if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension)) { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocNameLoadFailure, docName, ModelJsonExtension); return(null); } try { if (Persistence.Syms.Utils.CheckIfSymsAdapter(adapter)) { if (docName.EqualsWithIgnoreCase(SymsDatabases)) { SymsDatabasesResponse databases = JsonConvert.DeserializeObject <SymsDatabasesResponse>(jsonData); docContent = Persistence.Syms.ManifestDatabasesPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, databases) as CdmDocumentDefinition; } else if (docName.Contains(ManifestExtension)) { SymsManifestContent manifestContent = await Persistence.Syms.Utils.GetSymsModel(adapter, jsonData, docPath); docContent = Persistence.Syms.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, manifestContent) as CdmDocumentDefinition; } else if (docName.Contains(CdmExtension)) { // specific table TableEntity table = JsonConvert.DeserializeObject <TableEntity>(jsonData); docContent = Persistence.Syms.DocumentPersistence.FromObject(this.Ctx, folder.Namespace, folder.FolderPath, table); } else { Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistSymsUnsupportedCdmConversion, docName); return(null); } } // Check file extensions, which performs a case-insensitive ordinal string comparison else if (docName.EndWithOrdinalIgnoreCase(ManifestExtension) || docName.EndWithOrdinalIgnoreCase(FolioExtension)) { docContent = Persistence.CdmFolder.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition; } else if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension)) { docContent = await Persistence.ModelJson.ManifestPersistence.FromObject(this.Ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder); } else if (docName.EndWithOrdinalIgnoreCase(CdmExtension)) { docContent = Persistence.CdmFolder.DocumentPersistence.FromObject(this.Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <Microsoft.CommonDataModel.ObjectModel.Persistence.CdmFolder.Types.DocumentContent>(jsonData)); } else { // Could not find a registered persistence class to handle this document type. Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistClassMissing, docName); return(null); } } catch (Exception e) { var errorMsg = e.Message; if (e.InnerException != null) { errorMsg += $" InnerException: {e.InnerException.Message}"; } Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocConversionFailure, docName, errorMsg); return(null); } // Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if (docContent != null) { if (docContainer != null) { // there are situations where a previously loaded document must be re-loaded. // the end of that chain of work is here where the old version of the document has been removed from // the corpus and we have created a new document and loaded it from storage and after this call we will probably // add it to the corpus and index it, etc. // it would be really rude to just kill that old object and replace it with this replicant, especially because // the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that // was just killed off but make it contain everything the new document loaded. docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition; } if (folder.Documents.AllItems.Find(x => x.Id == docContent.Id) == null) { folder.Documents.Add(docContent, docName); } docContent._fileSystemModifiedTime = fsModifiedTime; docContent.IsDirty = false; } return(docContent); }