/// <summary>
/// Loads a document from the folder path.
/// </summary>
/// <param name="folder">The folder that contains the document we want to load.</param>
/// <param name="docName">The document name.</param>
/// <param name="docContainer">The loaded document, if it was previously loaded.</param>
/// <param name="resOpt">Optional parameter. The resolve options.</param>
/// <returns>The loaded document.</returns>
internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null)
{
// This makes sure date values are consistently parsed exactly as they appear.
// Default behavior auto formats date values.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None,
};
CdmDocumentDefinition docContent = null;
string jsonData = null;
DateTimeOffset? fsModifiedTime = null;
string docPath = folder.FolderPath + docName;
StorageAdapterBase adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace);
try
{
if (adapter.CanRead())
{
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"request file: {docPath}");
jsonData = await adapter.ReadAsync(docPath);
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"received file: {docPath}");
}
else
{
throw new Exception("Storage Adapter is not enabled to read.");
}
}
catch (Exception e)
{
// log message used by navigator, do not change or remove
Logger.Debug(this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, $"fail file: {docPath}");
// When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if (resOpt != null && resOpt.ShallowValidation)
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileReadFailure, docPath, folder.Namespace, e.Message);
}
else
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistFileReadFailure, docPath, folder.Namespace, e.Message);
}
return(null);
}
try
{
fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath);
}
catch (Exception e)
{
Logger.Warning((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.WarnPersistFileModTimeFailure, e.Message);
}
if (string.IsNullOrWhiteSpace(docName))
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistNullDocName);
return(null);
}
// If loading an model.json file, check that it is named correctly.
if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension))
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocNameLoadFailure, docName, ModelJsonExtension);
return(null);
}
try
{
if (Persistence.Syms.Utils.CheckIfSymsAdapter(adapter))
{
if (docName.EqualsWithIgnoreCase(SymsDatabases))
{
SymsDatabasesResponse databases = JsonConvert.DeserializeObject <SymsDatabasesResponse>(jsonData);
docContent = Persistence.Syms.ManifestDatabasesPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, databases) as CdmDocumentDefinition;
}
else if (docName.Contains(ManifestExtension))
{
SymsManifestContent manifestContent = await Persistence.Syms.Utils.GetSymsModel(adapter, jsonData, docPath);
docContent = Persistence.Syms.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, manifestContent) as CdmDocumentDefinition;
}
else if (docName.Contains(CdmExtension))
{
// specific table
TableEntity table = JsonConvert.DeserializeObject <TableEntity>(jsonData);
docContent = Persistence.Syms.DocumentPersistence.FromObject(this.Ctx, folder.Namespace, folder.FolderPath, table);
}
else
{
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistSymsUnsupportedCdmConversion, docName);
return(null);
}
}
// Check file extensions, which performs a case-insensitive ordinal string comparison
else if (docName.EndWithOrdinalIgnoreCase(ManifestExtension) || docName.EndWithOrdinalIgnoreCase(FolioExtension))
{
docContent = Persistence.CdmFolder.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition;
}
else if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension))
{
docContent = await Persistence.ModelJson.ManifestPersistence.FromObject(this.Ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder);
}
else if (docName.EndWithOrdinalIgnoreCase(CdmExtension))
{
docContent = Persistence.CdmFolder.DocumentPersistence.FromObject(this.Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <Microsoft.CommonDataModel.ObjectModel.Persistence.CdmFolder.Types.DocumentContent>(jsonData));
}
else
{
// Could not find a registered persistence class to handle this document type.
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistClassMissing, docName);
return(null);
}
}
catch (Exception e)
{
var errorMsg = e.Message;
if (e.InnerException != null)
{
errorMsg += $" InnerException: {e.InnerException.Message}";
}
Logger.Error((ResolveContext)this.Ctx, Tag, nameof(LoadDocumentFromPathAsync), docPath, CdmLogCode.ErrPersistDocConversionFailure, docName, errorMsg);
return(null);
}
// Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if (docContent != null)
{
if (docContainer != null)
{
// there are situations where a previously loaded document must be re-loaded.
// the end of that chain of work is here where the old version of the document has been removed from
// the corpus and we have created a new document and loaded it from storage and after this call we will probably
// add it to the corpus and index it, etc.
// it would be really rude to just kill that old object and replace it with this replicant, especially because
// the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
// was just killed off but make it contain everything the new document loaded.
docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition;
}
if (folder.Documents.AllItems.Find(x => x.Id == docContent.Id) == null)
{
folder.Documents.Add(docContent, docName);
}
docContent._fileSystemModifiedTime = fsModifiedTime;
docContent.IsDirty = false;
}
return(docContent);
}
public static async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer)
{
// This makes sure date values are consistently parsed exactly as they appear.
// Default behavior auto formats date values.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
CdmDocumentDefinition docContent = null;
string jsonData = null;
DateTimeOffset? fsModifiedTime = null;
CdmCorpusContext ctx = folder.Ctx;
string docPath = folder.FolderPath + docName;
StorageAdapter adapter = ctx.Corpus.Storage.FetchAdapter(folder.Namespace);
try
{
if (adapter.CanRead())
{
jsonData = await adapter.ReadAsync(docPath);
fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath);
Logger.Info(nameof(CdmFolderDefinition), ctx, $"read file: {docPath}", "LoadDocumentFromPathAsync");
}
}
catch (Exception e)
{
Logger.Error(nameof(CdmFolderDefinition), (ResolveContext)ctx, $"Could not read '{docPath}' from the '{folder.Namespace}' namespace. Reason '{e.Message}'", "LoadDocumentFromPathAsync");
return(null);
}
try
{
// Check file extensions, which performs a case-insensitive ordinal string comparison
if (docPath.EndWithOrdinalIgnoreCase(CdmCorpusDefinition.FetchManifestExtension()) || docPath.EndWithOrdinalIgnoreCase(CdmCorpusDefinition.FetchFolioExtension()))
{
docContent = ManifestPersistence.FromData(ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition;
}
else if (docPath.EndWithOrdinalIgnoreCase(CdmCorpusDefinition.FetchModelJsonExtension()))
{
if (!docName.EqualsWithOrdinalIgnoreCase(CdmCorpusDefinition.FetchModelJsonExtension()))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be model.json.", "LoadDocumentFromPathAsync");
return(null);
}
docContent = await ModelJson.ManifestPersistence.FromData(ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder);
}
else
{
docContent = DocumentPersistence.FromData(ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <DocumentContent>(jsonData));
}
}
catch (Exception e)
{
Logger.Error(nameof(CdmFolderDefinition), (ResolveContext)ctx, $"Could not convert '{docPath}'. Reason '{e.Message}'", "LoadDocumentFromPathAsync");
return(null);
}
// Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if (docContent != null)
{
if (docContainer != null)
{
// there are situations where a previously loaded document must be re-loaded.
// the end of that chain of work is here where the old version of the document has been removed from
// the corpus and we have created a new document and loaded it from storage and after this call we will probably
// add it to the corpus and index it, etc.
// it would be really rude to just kill that old object and replace it with this replicant, especially because
// the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
// was just killed off but make it contain everything the new document loaded.
docContent = docContent.Copy(new ResolveOptions(docContainer), docContainer) as CdmDocumentDefinition;
}
folder.Documents.Add(docContent, docName);
docContent._fileSystemModifiedTime = fsModifiedTime;
docContent.IsDirty = false;
}
return(docContent);
}
/// <summary>
/// Loads a document from the folder path.
/// </summary>
/// <param name="folder">The folder that contains the document we want to load.</param>
/// <param name="docName">The document name.</param>
/// <param name="docContainer">The loaded document, if it was previously loaded.</param>
/// <param name="resOpt">Optional parameter. The resolve options.</param>
/// <returns>The loaded document.</returns>
internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null)
{
// This makes sure date values are consistently parsed exactly as they appear.
// Default behavior auto formats date values.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
CdmDocumentDefinition docContent = null;
string jsonData = null;
DateTimeOffset?fsModifiedTime = null;
string docPath = folder.FolderPath + docName;
StorageAdapter adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace);
try
{
if (adapter.CanRead())
{
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"request file: {docPath}", nameof(LoadDocumentFromPathAsync));
jsonData = await adapter.ReadAsync(docPath);
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"received file: {docPath}", nameof(LoadDocumentFromPathAsync));
}
else
{
throw new Exception("Storage Adapter is not enabled to read.");
}
}
catch (Exception e)
{
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"fail file: {docPath}", nameof(LoadDocumentFromPathAsync));
string message = $"Could not read '{docPath}' from the '{folder.Namespace}' namespace. Reason '{e.Message}'";
// When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if (resOpt != null && resOpt.ShallowValidation)
{
Logger.Warning(nameof(PersistenceLayer), (ResolveContext)this.Ctx, message, nameof(LoadDocumentFromPathAsync));
}
else
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, message, nameof(LoadDocumentFromPathAsync));
}
return(null);
}
try
{
fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath);
}
catch (Exception e)
{
Logger.Warning(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to compute file last modified time. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync));
}
if (string.IsNullOrWhiteSpace(docName))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Document name cannot be null or empty.", nameof(LoadDocumentFromPathAsync));
return(null);
}
// If loading an odi.json/model.json file, check that it is named correctly.
if (docName.EndWithOrdinalIgnoreCase(OdiExtension) && !docName.EqualsWithOrdinalIgnoreCase(OdiExtension))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be {OdiExtension}.", nameof(LoadDocumentFromPathAsync));
return(null);
}
if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be {ModelJsonExtension}.", nameof(LoadDocumentFromPathAsync));
return(null);
}
// Fetch the correct persistence class to use.
Type persistenceClass = FetchRegisteredPersistenceFormat(docName);
if (persistenceClass != null)
{
try
{
MethodInfo method = persistenceClass.GetMethod(nameof(FromData));
object[] parameters = new object[] { this.Ctx, docName, jsonData, folder };
// Check if FromData() is asynchronous for this persistence class.
if (!isRegisteredPersistenceAsync.ContainsKey(persistenceClass))
{
// Cache whether this persistence class has async methods.
isRegisteredPersistenceAsync.TryAdd(persistenceClass, (bool)persistenceClass.GetField("IsPersistenceAsync").GetValue(null));
}
if (isRegisteredPersistenceAsync[persistenceClass])
{
var task = (Task)method.Invoke(null, parameters);
await task;
docContent = task.GetType().GetProperty("Result").GetValue(task) as CdmDocumentDefinition;
}
else
{
docContent = method.Invoke(null, parameters) as CdmDocumentDefinition;
}
}
catch (Exception e)
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not convert '{docName}'. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync));
return(null);
}
}
else
{
// Could not find a registered persistence class to handle this document type.
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not find a persistence class to handle the file '{docName}'", nameof(LoadDocumentFromPathAsync));
return(null);
}
// Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if (docContent != null)
{
if (docContainer != null)
{
// there are situations where a previously loaded document must be re-loaded.
// the end of that chain of work is here where the old version of the document has been removed from
// the corpus and we have created a new document and loaded it from storage and after this call we will probably
// add it to the corpus and index it, etc.
// it would be really rude to just kill that old object and replace it with this replicant, especially because
// the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
// was just killed off but make it contain everything the new document loaded.
docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition;
}
folder.Documents.Add(docContent, docName);
docContent._fileSystemModifiedTime = fsModifiedTime;
docContent.IsDirty = false;
}
return(docContent);
}
/// <summary>
/// Loads a document from the folder path.
/// </summary>
/// <param name="folder">The folder that contains the document we want to load.</param>
/// <param name="docName">The document name.</param>
/// <param name="docContainer">The loaded document, if it was previously loaded.</param>
/// <param name="resOpt">Optional parameter. The resolve options.</param>
/// <returns>The loaded document.</returns>
internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer, ResolveOptions resOpt = null)
{
// This makes sure date values are consistently parsed exactly as they appear.
// Default behavior auto formats date values.
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
CdmDocumentDefinition docContent = null;
string jsonData = null;
DateTimeOffset?fsModifiedTime = null;
string docPath = folder.FolderPath + docName;
StorageAdapter adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace);
try
{
if (adapter.CanRead())
{
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"request file: {docPath}", nameof(LoadDocumentFromPathAsync));
jsonData = await adapter.ReadAsync(docPath);
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"received file: {docPath}", nameof(LoadDocumentFromPathAsync));
}
else
{
throw new Exception("Storage Adapter is not enabled to read.");
}
}
catch (Exception e)
{
// log message used by navigator, do not change or remove
Logger.Debug(nameof(PersistenceLayer), this.Ctx, $"fail file: {docPath}", nameof(LoadDocumentFromPathAsync));
string message = $"Could not read '{docPath}' from the '{folder.Namespace}' namespace. Reason '{e.Message}'";
// When shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if (resOpt != null && resOpt.ShallowValidation)
{
Logger.Warning(nameof(PersistenceLayer), (ResolveContext)this.Ctx, message, nameof(LoadDocumentFromPathAsync));
}
else
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, message, nameof(LoadDocumentFromPathAsync));
}
return(null);
}
try
{
fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath);
}
catch (Exception e)
{
Logger.Warning(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to compute file last modified time. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync));
}
if (string.IsNullOrWhiteSpace(docName))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Document name cannot be null or empty.", nameof(LoadDocumentFromPathAsync));
return(null);
}
// If loading an model.json file, check that it is named correctly.
if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension) && !docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be {ModelJsonExtension}.", nameof(LoadDocumentFromPathAsync));
return(null);
}
try
{
// Check file extensions, which performs a case-insensitive ordinal string comparison
if (docName.EndWithOrdinalIgnoreCase(ManifestExtension) || docName.EndWithOrdinalIgnoreCase(FolioExtension))
{
docContent = Persistence.CdmFolder.ManifestPersistence.FromObject(Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition;
}
else if (docName.EndWithOrdinalIgnoreCase(ModelJsonExtension))
{
if (!docName.EqualsWithOrdinalIgnoreCase(ModelJsonExtension))
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be model.json.", "LoadDocumentFromPathAsync");
return(null);
}
docContent = await Persistence.ModelJson.ManifestPersistence.FromObject(this.Ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder);
}
else if (docName.EndWithOrdinalIgnoreCase(CdmExtension))
{
docContent = Persistence.CdmFolder.DocumentPersistence.FromObject(this.Ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <DocumentContent>(jsonData));
}
else
{
// Could not find a registered persistence class to handle this document type.
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not find a persistence class to handle the file '{docName}'", nameof(LoadDocumentFromPathAsync));
return(null);
}
}
catch (Exception e)
{
Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not convert '{docName}'. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync));
return(null);
}
// Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if (docContent != null)
{
if (docContainer != null)
{
// there are situations where a previously loaded document must be re-loaded.
// the end of that chain of work is here where the old version of the document has been removed from
// the corpus and we have created a new document and loaded it from storage and after this call we will probably
// add it to the corpus and index it, etc.
// it would be really rude to just kill that old object and replace it with this replicant, especially because
// the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
// was just killed off but make it contain everything the new document loaded.
docContent = docContent.Copy(new ResolveOptions(docContainer, this.Ctx.Corpus.DefaultResolutionDirectives), docContainer) as CdmDocumentDefinition;
}
folder.Documents.Add(docContent, docName);
docContent._fileSystemModifiedTime = fsModifiedTime;
docContent.IsDirty = false;
}
return(docContent);
}