/// <summary>
/// Loads a folder or document given its obejct path.
/// </summary>
/// <param name="objectPath"></param>
/// <param name="forceReload"></param>
/// <param name="resOpt"></param>
/// <returns></returns>
private async Task <CdmContainerDefinition> _LoadFolderOrDocument(string objectPath, bool forceReload = false, ResolveOptions resOpt = null)
{
if (string.IsNullOrWhiteSpace(objectPath))
{
return(null);
}
// first check for namespace
Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(objectPath);
if (pathTuple == null)
{
Logger.Error(this.corpus.Ctx, TAG, nameof(LoadFolderOrDocument), objectPath, CdmLogCode.ErrPathNullObjectPath);
return(null);
}
string nameSpace = !string.IsNullOrWhiteSpace(pathTuple.Item1) ? pathTuple.Item1 : this.corpus.Storage.DefaultNamespace;
objectPath = pathTuple.Item2;
if (!objectPath.StartsWith("/"))
{
return(null);
}
var namespaceFolder = this.corpus.Storage.FetchRootFolder(nameSpace);
StorageAdapterBase namespaceAdapter = this.corpus.Storage.FetchAdapter(nameSpace);
if (namespaceFolder == null || namespaceAdapter == null)
{
Logger.Error(this.corpus.Ctx, TAG, nameof(LoadFolderOrDocument), objectPath, CdmLogCode.ErrStorageNamespaceNotRegistered, nameSpace);
return(null);
}
CdmFolderDefinition lastFolder = namespaceFolder.FetchChildFolderFromPath(objectPath, false);
// don't create new folders, just go as far as possible
if (lastFolder == null)
{
return(null);
}
// maybe the search is for a folder?
string lastPath = lastFolder.FolderPath;
if (lastPath == objectPath)
{
return(lastFolder);
}
// remove path to folder and then look in the folder
objectPath = StringUtils.Slice(objectPath, lastPath.Length);
this.concurrentReadLock.Acquire();
// During this step the document will be added to the pathLookup when it is added to a folder.
var doc = await lastFolder.FetchDocumentFromFolderPathAsync(objectPath, forceReload, resOpt);
this.concurrentReadLock.Release();
return(doc);
}
/// <summary>
/// check if adapter is syms.
/// </summary>
internal static bool CheckIfSymsAdapter(StorageAdapterBase adapter)
{
if (adapter.GetType().ToString().Equals("Microsoft.CommonDataModel.ObjectModel.Storage.SymsAdapter") ||
adapter.GetType().ToString().Equals("Microsoft.CommonDataModel.ObjectModel.Adapter.Syms"))
{
return(true);
}
return(false);
}
/// <summary>
/// Convert to SyMs object to CDM object.
/// </summary>
internal static async Task <SymsManifestContent> GetSymsModel(StorageAdapterBase adapter, string databaseResponse, string docPath)
{
var database = JsonConvert.DeserializeObject <DatabaseEntity>(databaseResponse);
var entities = await adapter.ReadAsync($"/{database.Name}/{database.Name}.manifest.cdm.json/entitydefinition");
var relationships = await adapter.ReadAsync($"{docPath}/relationships");
return(new SymsManifestContent
{
Database = database,
Entities = JsonConvert.DeserializeObject <SymsTableResponse>(entities).Tables,
Relationships = JsonConvert.DeserializeObject <SymsRelationshipResponse>(relationships).Relationships,
InitialSync = false,
RemovedEntities = null,
RemovedRelationships = null
});
}
/// <summary>
/// Remove Relationship Entity. Throws exception on failure.
/// </summary>
private static async Task RemoveRelationshipEntity(string relationship, string databaseName, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationship}", null);
}
/// <summary>
/// Remove Table Entity. Throws exception on failure.
/// </summary>
private static async Task RemoveTableEntity(string tableName, string databaseName, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{databaseName}/{tableName}.cdm.json", null);
}
/// <summary>
/// Create or update Syms RelationshipEntity. Throws exception on failure.
/// </summary>
private static async Task CreateOrUpdateRelationshipEntity(RelationshipEntity relationshipEntity, string databaseName, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationshipEntity.Name}",
JsonConvertSerializeObject(relationshipEntity));
}
/// <summary>
/// Create or update Syms Table Entity. Throws exception on failure.
/// </summary>
private static async Task CreateOrUpdateTableEntity(TableEntity tableEntity, string databaseName, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{databaseName}/{tableEntity.Name}.cdm.json",
JsonConvertSerializeObject(tableEntity));
}
/// <summary>
/// Create or update Syms RelationshipEntity. Throws exception on failure.
/// </summary>
internal static async Task CreateOrUpdateRelationshipEntity(RelationshipEntity relationshipEntity, StorageAdapterBase adapter)
{
string databaseName = ((RelationshipProperties)relationshipEntity.Properties).NamespaceProperty.DatabaseName;
await adapter.WriteAsync($"{databaseName}/{databaseName}.manifest.cdm.json/relationships/{relationshipEntity.Name}",
JsonConvertSerializeObject(relationshipEntity));
}
/// <summary>
/// Create or update Syms Table Entity. Throws exception on failure.
/// </summary>
internal static async Task CreateOrUpdateTableEntity(TableEntity tableEntity, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{((TableProperties)tableEntity.Properties).NamespaceProperty.DatabaseName}/{tableEntity.Name}.cdm.json",
JsonConvertSerializeObject(tableEntity));
}
/// <summary>
/// Create or update Database. Throws exception on failure.
/// </summary>
internal static async Task CreateOrUpdateDatabase(DatabaseEntity databaseEntity, StorageAdapterBase adapter)
{
await adapter.WriteAsync($"{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json", JsonConvertSerializeObject(databaseEntity));
}
/// <summary>
/// Invoke specific functions to update Syms database through syms adapter.
/// </summary>
private static async Task <string> InvokeMethod <T>(Func <T, string, StorageAdapterBase, Task> func, List <T> entities, string dBName, StorageAdapterBase adapter)
{
ConcurrentDictionary <string, string> errorMesgsHash = new ConcurrentDictionary <string, string>();
string errorMesgs = string.Empty;
if (entities != null)
{
if (entities != null && entities.Count > 0)
{
var tasks = entities.Select(async entity =>
{
try
{
await func(entity, dBName, adapter);
}
catch (Exception e)
{
string key = string.Empty;
if (entity is string)
{
key = (string)(object)entity;
}
else if (entity is TableEntity)
{
key = ((TableEntity)(object)entity).Name;
}
else if (entity is RelationshipEntity)
{
key = ((RelationshipEntity)(object)entity).Name;
}
errorMesgsHash[key] = e.Message;
}
});
await Task.WhenAll(tasks);
foreach (var index in errorMesgsHash)
{
errorMesgs += $"Failed to update table {index.Key} with exception {index.Value} \n";
}
}
}
return(errorMesgs);
}
/// <summary>
/// Create or add/update or remove Syms Entities. Throws exception on failure.
/// </summary>
internal static async Task CreateOrUpdateSymsEntities(SymsManifestContent symsManifestContent, StorageAdapterBase adapter)
{
string errorMesg = string.Empty;
if (symsManifestContent.InitialSync == true)
{
await CreateOrUpdateDatabase(symsManifestContent.Database, adapter);
}
errorMesg = await InvokeMethod(RemoveTableEntity, symsManifestContent.RemovedEntities, symsManifestContent.Database.Name, adapter);
errorMesg += await InvokeMethod(RemoveRelationshipEntity, symsManifestContent.RemovedRelationships, symsManifestContent.Database.Name, adapter);
errorMesg += await InvokeMethod(CreateOrUpdateTableEntity, symsManifestContent.Entities, symsManifestContent.Database.Name, adapter);
errorMesg += await InvokeMethod(CreateOrUpdateRelationshipEntity, symsManifestContent.Relationships, symsManifestContent.Database.Name, adapter);
if (!string.IsNullOrEmpty(errorMesg))
{
throw new Exception(errorMesg);
}
}
/// <summary>
/// Convert to SyMs table object from CDM object.
/// </summary>
internal static async Task <TableEntity> ConvertDocToSymsTable(CdmCorpusContext ctx, CdmDocumentDefinition doc, StorageAdapterBase adapter, string name,
ResolveOptions resOpt, CopyOptions options)
{
TableEntity existingTableEntity = JsonConvert.DeserializeObject <TableEntity>(await adapter.ReadAsync(name));
return(Persistence.Syms.DocumentPersistence.ToData(ctx, doc, ((JToken)existingTableEntity.Properties).ToObject <TableProperties>(), resOpt, options));
}
/// <summary>
/// Convert to SyMs object from Manifest object.
/// </summary>
internal static async Task <SymsManifestContent> ConvertManifestToSyms(CdmManifestDefinition doc, StorageAdapterBase adapter, string path,
ResolveOptions resOpt, CopyOptions options)
{
DatabaseEntity databaseEntity = null;
bool isDeltaSync = true;
IList <TableEntity> existingSymsTables = null;
IList <RelationshipEntity> existingSymsRelationshipEntities = null;
try
{
databaseEntity = JsonConvert.DeserializeObject <DatabaseEntity>(await adapter.ReadAsync(path));
}
catch (Exception e)
{
if (e.Message.Contains("NotFound"))
{
isDeltaSync = false;
}
else
{
throw;
}
}
if (isDeltaSync)
{
var entities = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/entitydefinition");
existingSymsTables = JsonConvert.DeserializeObject <SymsTableResponse>(entities).Tables;
var realtionships = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/relationships");
existingSymsRelationshipEntities = JsonConvert.DeserializeObject <SymsRelationshipResponse>(realtionships).Relationships;
}
return(await Persistence.Syms.ManifestPersistence.ToDataAsync(doc, resOpt, options, isDeltaSync, existingSymsTables, existingSymsRelationshipEntities));
}
/// <summary>
/// Loads a document from the folder path.
/// </summary>
/// <param name="folder">The folder that contains the document we want to load.</param>
/// <param name="docName">The document name.</param>
/// <param name="docContainer">The loaded document, if it was previously loaded.</param>
/// <param name="resOpt">Optional parameter. The resolve options.</param>
/// <returns>The loaded document.</returns>
internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null)
{
// This makes sure date values are consistently parsed exactly as they appear.
// Default behavior auto formats date values.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None,
};
CdmDocumentDefinition docContent = null;
string jsonData = null;
DateTimeOffset? fsModifiedTime = null;
string docPath = folder.FolderPath + docName;
StorageAdapterBase adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace);
try
{
if (adapter.CanRead())
{
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"request file: {docPath}");
jsonData = await adapter.ReadAsync(docPath);
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"received file: {docPath}");
}
else
{
throw new Exception("Storage Adapter is not enabled to read.");
}
}
catch (Exception e)
{
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"fail file: {docPath}");
// When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if (resOpt != null && resOpt.ShallowValidation)
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileReadFailure, docPath, folder.Namespace, e.Message);
}
else
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistFileReadFailure, docPath, folder.Namespace, e.Message);
}
return(null);
}
try
{
fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath);
}
catch (Exception e)
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileModTimeFailure, e.Message);
}
if (string.IsNullOrWhiteSpace(docName))
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistNullDocName);
return(null);
}
// If loading an model.json file, check that it is named correctly.
if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension))
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocNameLoadFailure, docName, ModelJsonExtension);
return(null);
}
try
{
if (Persistence.Syms.Utils.CheckIfSymsAdapter(adapter))
{
if (docName.EqualsWithIgnoreCase(SymsDatabases))
{
SymsDatabasesResponse databases = JsonConvert.DeserializeObject <SymsDatabasesResponse>(jsonData);
docContent = Persistence.Syms.ManifestDatabasesPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, databases) as CdmDocumentDefinition;
}
else if (docName.Contains(ManifestExtension))
{
SymsManifestContent manifestContent = await Persistence.Syms.Utils.GetSymsModel(adapter, jsonData, docPath);
docContent = Persistence.Syms.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, manifestContent) as CdmDocumentDefinition;
}
else if (docName.Contains(CdmExtension))
{
// specific table
TableEntity table = JsonConvert.DeserializeObject <TableEntity>(jsonData);
docContent = Persistence.Syms.DocumentPersistence.FromObject(this.Ctx, folder.Namespace, folder.FolderPath, table);
}
else
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistSymsUnsupportedCdmConversion, docName);
return(null);
}
}
// Check file extensions, which performs a case-insensitive ordinal string comparison
else if (docName.EndWithOrdinalIgnoreCase(ManifestExtension) || docName.EndWithOrdinalIgnoreCase(FolioExtension))
{
docContent = Persistence.CdmFolder.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition;
}
else if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension))
{
docContent = await Persistence.ModelJson.ManifestPersistence.FromObject(this.Ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder);
}
else if (docName.EndWithOrdinalIgnoreCase(CdmExtension))
{
docContent = Persistence.CdmFolder.DocumentPersistence.FromObject(this.Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <Microsoft.CommonDataModel.ObjectModel.Persistence.CdmFolder.Types.DocumentContent>(jsonData));
}
else
{
// Could not find a registered persistence class to handle this document type.
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistClassMissing, docName);
return(null);
}
}
catch (Exception e)
{
var errorMsg = e.Message;
if (e.InnerException != null)
{
errorMsg += $" InnerException: {e.InnerException.Message}";
}
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocConversionFailure, docName, errorMsg);
return(null);
}
// Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if (docContent != null)
{
if (docContainer != null)
{
// there are situations where a previously loaded document must be re-loaded.
// the end of that chain of work is here where the old version of the document has been removed from
// the corpus and we have created a new document and loaded it from storage and after this call we will probably
// add it to the corpus and index it, etc.
// it would be really rude to just kill that old object and replace it with this replicant, especially because
// the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
// was just killed off but make it contain everything the new document loaded.
docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition;
}
if (folder.Documents.AllItems.Find(x => x.Id == docContent.Id) == null)
{
folder.Documents.Add(docContent, docName);
}
docContent._fileSystemModifiedTime = fsModifiedTime;
docContent.IsDirty = false;
}
return(docContent);
}
