public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Partition
{
Name = instance.Name,
Description = instance.Description,
Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
instance.Location, instance.InDocument)),
RefreshTime = instance.RefreshTime,
FileFormatSettings = null,
LastFileModifiedTime = instance.LastFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime
};
if (result.Name == null)
{
Logger.Warning(instance.Ctx, Tag, instance.AtCorpusPath, nameof(ToData), CdmLogCode.WarnPersistPartitionNameNull);
result.Name = "";
}
if (string.IsNullOrEmpty(result.Location))
{
Logger.Warning(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.WarnPersistPartitionLocMissing, result.Name);
}
Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);
if (csvFormatSettings != null)
{
result.FileFormatSettings = csvFormatSettings;
result.FileFormatSettings.Type = "CsvFormatSettings";
}
else
{
Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistCsvProcessingError);
return(null);
}
}
return(result);
}
public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Partition
{
Name = instance.Name,
Description = instance.Description,
Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
instance.Location, instance.InDocument)),
RefreshTime = instance.RefreshTime,
FileFormatSettings = null,
LastFileModifiedTime = instance.LastFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime
};
if (string.IsNullOrEmpty(result.Location))
{
Logger.Warning(nameof(DataPartitionPersistence), instance.Ctx, $"Couldn't find data partition's location for partition {result.Name}.", nameof(ToData));
}
await Utils.ProcessAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);
if (csvFormatSettings != null)
{
result.FileFormatSettings = csvFormatSettings;
result.FileFormatSettings.Type = "CsvFormatSettings";
}
else
{
Logger.Error(nameof(DataPartitionPersistence), instance.Ctx,
"There was a problem while processing csv format trait inside data partition.");
return(null);
}
}
return(result);
}
private static IDictionary <string, JToken> CreateTablePropertyBags(CdmLocalEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options, IDictionary <string, JToken> properties)
{
if (properties == null)
{
properties = new Dictionary <string, JToken>();
}
if (instance.EntityPath != null)
{
Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(instance.EntityPath);
if (pathTuple == null)
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsEntityPathNull, instance.EntityName);
return(null);
}
properties["cdm:entityPath"] = JToken.FromObject(pathTuple.Item2);
}
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (!properties.ContainsKey("cdm:description"))
{
properties["cdm:description"] = instance.Explanation;
}
if (instance.LastChildFileModifiedTime != null)
{
properties["cdm:lastChildFileModifiedTime"] = instance.LastChildFileModifiedTime;
}
if (instance.LastFileModifiedTime != null)
{
properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
}
if (instance.LastFileStatusCheckTime != null)
{
properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
}
if (isHiddenTrait != null)
{
properties["cdm:isHidden"] = true;
}
if (instance.ExhibitsTraits != null && instance.ExhibitsTraits.Count > 0)
{
properties["cdm:entityDecTraits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
});
}
return(properties);
}
public static async Task <ReferenceEntity> ToData(CdmReferencedEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var sourceIndex = instance.EntityPath.LastIndexOf("/");
if (sourceIndex == -1)
{
Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityRefConversionError, instance.EntityName);
return(null);
}
var referenceEntity = new ReferenceEntity
{
Type = "ReferenceEntity",
Name = instance.EntityName,
Source = instance.EntityPath.Slice(sourceIndex + 1),
Description = instance.Explanation,
LastFileModifiedTime = instance.LastFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime
};
Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, referenceEntity, instance.ExhibitsTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
referenceEntity.IsHidden = true;
}
var propertiesTrait = t2pm.FetchTraitReference("is.propertyContent.multiTrait");
if (propertiesTrait != null)
{
referenceEntity.ModelId = propertiesTrait.Arguments.AllItems[0].Value as string;
}
return(referenceEntity);
}
public static async Task <LocalEntity> ToData(CdmLocalEntityDeclarationDefinition instance, CdmManifestDefinition manifest, ResolveOptions resOpt, CopyOptions options)
{
var localEntity = await DocumentPersistence.ToData(instance.EntityPath, manifest, resOpt, options, instance.Ctx);
if (localEntity != null)
{
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (localEntity.Description == null)
{
localEntity.Description = instance.Explanation;
}
localEntity.LastChildFileModifiedTime = instance.LastChildFileModifiedTime;
localEntity.LastFileModifiedTime = instance.LastFileModifiedTime;
localEntity.LastFileStatusCheckTime = instance.LastFileStatusCheckTime;
if (isHiddenTrait != null)
{
localEntity.IsHidden = true;
}
if (t2pm.FetchPropertyValue("cdmSchemas") is List <string> schemas)
{
localEntity.Schemas = schemas;
}
if (instance.DataPartitions != null && instance.DataPartitions.Count > 0)
{
localEntity.Partitions = new List <Partition>();
foreach (var element in instance.DataPartitions)
{
var partition = await DataPartitionPersistence.ToData(element, resOpt, options);
if (partition != null)
{
localEntity.Partitions.Add(partition);
}
else
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityPartitionConversionError);
return(null);
}
}
}
}
return(localEntity);
}
public static async Task <LocalEntity> ToData(CdmLocalEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var localEntity = await DocumentPersistence.ToData(instance.EntityPath, resOpt, options, instance.Ctx);
if (localEntity != null)
{
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
localEntity.Description = instance.Explanation;
localEntity.LastChildFileModifiedTime = instance.LastChildFileModifiedTime;
localEntity.LastFileModifiedTime = instance.LastFileModifiedTime;
localEntity.LastFileStatusCheckTime = instance.LastFileStatusCheckTime;
if (isHiddenTrait != null)
{
localEntity.IsHidden = true;
}
if (t2pm.FetchPropertyValue("cdmSchemas") is List <string> schemas)
{
localEntity.Schemas = schemas;
}
if (instance.DataPartitions != null && instance.DataPartitions.Count > 0)
{
localEntity.Partitions = new List <Partition>();
foreach (var element in instance.DataPartitions)
{
var partition = await DataPartitionPersistence.ToData(element, resOpt, options);
if (partition != null)
{
localEntity.Partitions.Add(partition);
}
else
{
Logger.Error(nameof(LocalEntityDeclarationPersistence), instance.Ctx, "There was an error while trying to convert cdm data partition to model.json partition.");
return(null);
}
}
}
}
return(localEntity);
}
public static DataColumn ToData(CdmTypeAttributeDefinition instance, CdmCorpusContext ctx, ResolveOptions resOpt, CopyOptions options)
{
var properties = CreateProperties(instance, resOpt, options);
var originDataTypeName = new TypeInfo
{
Properties = properties,
IsComplexType = false,
IsNullable = instance.GetProperty("isNullable"),
TypeFamily = "cdm"
};
var t2pm = new TraitToPropertyMap(instance);;
var numericTraits = t2pm.FetchTraitReference("is.dataFormat.numeric.shaped");
if (numericTraits != null)
{
foreach (var numericTraitsArg in numericTraits.Arguments)
{
if (numericTraitsArg.Name == "precision")
{
originDataTypeName.Precision = (int)numericTraitsArg.Value;
}
if (numericTraitsArg.Name == "scale")
{
originDataTypeName.Scale = (int)numericTraitsArg.Value;
}
}
}
var dataFormat = instance.GetProperty("dataFormat");
originDataTypeName = Utils.CdmDataFormatToSymsDataType(dataFormat, originDataTypeName);
if (originDataTypeName.TypeName == null)
{
Logger.Error(ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsUnknownDataFormat, instance.DisplayName);
return(null);
}
return(new DataColumn
{
OriginDataTypeName = originDataTypeName,
Name = instance.Name
});
}
private static DataSource CreateDataSource(CdmManifestDefinition instance)
{
var source = new DataSource();
var t2pm = new TraitToPropertyMap(instance);
var sourceTraits = t2pm.FetchTraitReference(dbLocationTrait);
if (sourceTraits != null && sourceTraits.Arguments != null && sourceTraits.Arguments.Count == 1 &&
sourceTraits.Arguments[0].Name == dbLocationTraitArgName)
{
source.Location = Utils.CorpusPathToSymsPath(sourceTraits.Arguments[0].Value, instance.Ctx.Corpus.Storage);
}
if (source.Location == null)
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath,
CdmLogCode.ErrPersistSymsStorageSourceTraitError, dbLocationTrait, dbLocationTraitArgName);
return(null);
}
return(source);
}
public static async Task <Attribute> ToData(CdmTypeAttributeDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var attribute = new Attribute
{
Name = instance.Name,
DataType = DataTypeToData(instance.DataFormat),
Description = instance.Description
};
await Utils.ProcessAnnotationsToData(instance.Ctx, attribute, instance.AppliedTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
attribute.IsHidden = true;
}
return(attribute);
}
private static IDictionary <string, JToken> FillPropertyBagFromCsvTrait(CdmDataPartitionDefinition instance, IDictionary <string, JToken> properties = null)
{
TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
if (properties == null)
{
properties = new Dictionary <string, JToken>();
}
foreach (var csvTraitArg in csvTrait.Arguments)
{
string key;
// map to syms define properties
switch (csvTraitArg.Name)
{
case "columnHeaders":
key = "header";
break;
case "delimiter":
key = "field.delim";
break;
default:
key = csvTraitArg.Value;
break;
}
properties[key] = csvTraitArg.Value;
}
}
return(properties);
}
public async static Task <Model> ToData(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Model()
{
Name = instance.ManifestName,
Description = instance.Explanation,
ModifiedTime = instance.LastFileModifiedTime,
LastChildFileModifiedTime = instance.LastChildFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime,
DocumentVersion = instance.DocumentVersion
};
TraitToPropertyMap t2pm = new TraitToPropertyMap(instance);
CdmTraitReference isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
CdmTraitReference applicationTrait = t2pm.FetchTraitReference("is.managedBy");
if (applicationTrait != null)
{
result.Application = applicationTrait.Arguments.AllItems[0].Value as string;
}
CdmTraitReference versionTrait = t2pm.FetchTraitReference("is.modelConversion.modelVersion");
if (versionTrait != null)
{
result.Version = versionTrait.Arguments.AllItems[0].Value;
}
else
{
// version property is required. If it doesn't exist set default.
result.Version = "1.0";
}
CdmTraitReference cultureTrait = t2pm.FetchTraitReference("is.partition.culture");
if (cultureTrait != null)
{
result.Culture = cultureTrait.Arguments.AllItems[0].Value as string;
}
Dictionary <string, string> referenceEntityLocations = new Dictionary <string, string>();
Dictionary <string, string> referenceModels = new Dictionary <string, string>();
CdmTraitReference referenceModelsTrait = t2pm.FetchTraitReference("is.modelConversion.referenceModelMap");
if (referenceModelsTrait != null)
{
JArray refModels = referenceModelsTrait.Arguments[0].Value as JArray;
foreach (JObject referenceModel in refModels)
{
var referenceModelId = referenceModel["id"];
var referenceModelIdAsString = referenceModelId.ToString();
var referenceModelLocation = referenceModel["location"];
var referenceModelLocationAsString = referenceModelLocation.ToString();
referenceModels.Add(referenceModelIdAsString, referenceModelLocationAsString);
referenceEntityLocations.Add(referenceModelLocationAsString, referenceModelIdAsString);
}
}
Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
if (instance.Entities != null && instance.Entities.Count > 0)
{
List <Task> promises = new List <Task>();
var obtainedEntities = new ConcurrentBag <JToken>();
foreach (var entity in instance.Entities)
{
Task createdPromise = Task.Run(async() =>
{
dynamic element = null;
if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
element = await LocalEntityDeclarationPersistence.ToData(
entity as CdmLocalEntityDeclarationDefinition,
instance,
resOpt,
options
);
}
else if (entity.ObjectType == CdmObjectType.ReferencedEntityDeclarationDef)
{
element = await ReferencedEntityDeclarationPersistence.ToData(
entity as CdmReferencedEntityDeclarationDefinition,
resOpt,
options
);
var location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(entity.EntityPath);
if (string.IsNullOrEmpty(location))
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Invalid entity path set in entity {entity.EntityName}");
element = null;
}
if (element is ReferenceEntity referenceEntity)
{
location = location.Slice(0, location.LastIndexOf("/"));
if (referenceEntity.ModelId != null)
{
if (referenceModels.TryGetValue(referenceEntity.ModelId, out var savedLocation) && savedLocation != location)
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Same ModelId pointing to different locations");
element = null;
}
else if (savedLocation == null)
{
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
else if (referenceEntity.ModelId == null && referenceEntityLocations.ContainsKey(location))
{
referenceEntity.ModelId = referenceEntityLocations[location];
}
else
{
referenceEntity.ModelId = Guid.NewGuid().ToString();
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
}
if (element != null)
{
obtainedEntities.Add(JToken.FromObject(element));
}
else
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format.");
}
});
try
{
// TODO: Currently function is synchronous. Remove next line to turn it asynchronous.
// Currently some functions called are not thread safe.
await createdPromise;
promises.Add(createdPromise);
}
catch (Exception ex)
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format for reason {ex.Message}.");
}
}
await Task.WhenAll(promises);
result.Entities = obtainedEntities.ToList();
}
if (referenceModels.Count > 0)
{
result.ReferenceModels = new List <ReferenceModel>();
foreach (var referenceModel in referenceModels)
{
result.ReferenceModels.Add(new ReferenceModel()
{
Id = referenceModel.Key,
Location = referenceModel.Value
});
}
}
if (instance.Relationships != null && instance.Relationships.Count > 0)
{
result.Relationships = new List <SingleKeyRelationship>();
foreach (var cdmRelationship in instance.Relationships)
{
var relationship = await RelationshipPersistence.ToData(cdmRelationship, resOpt, options);
if (relationship != null)
{
result.Relationships.Add(relationship);
}
else
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, "There was an error while trying to convert cdm relationship to model.json relationship.");
}
}
}
if (instance.Imports != null && instance.Imports.Count > 0)
{
result.Imports = new List <Import>();
foreach (var element in instance.Imports)
{
result.Imports.Add(CdmFolder.ImportPersistence.ToData(element, resOpt, options));
}
}
return(result);
}
public static CdmManifestDefinition FromObject(CdmCorpusContext ctx, string docName, string nameSpace, string path, SymsManifestContent dataObj)
{
DatabaseEntity database = dataObj.Database;
if (database == null || database.Type != SASEntityType.DATABASE)
{
Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbObject);
return(null);
}
DatabaseProperties databaseProperties = ((JToken)database.Properties).ToObject <DatabaseProperties>();
if (databaseProperties == null || databaseProperties.Source == null)
{
Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbPropObject);
return(null);
}
var properties = databaseProperties.Properties;
var manifest = ctx.Corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef);
manifest.ManifestName = dataObj.Database.Name;
manifest.Name = docName;
manifest.FolderPath = path;
manifest.Namespace = nameSpace;
manifest.Explanation = databaseProperties.Description;
if (properties != null)
{
if (properties.ContainsKey("cdm:schema"))
{
manifest.Schema = properties["cdm:schema"].ToObject <string>();
}
if (properties.ContainsKey("cdm:jsonSchemaSemanticVersion"))
{
manifest.JsonSchemaSemanticVersion = properties["cdm:jsonSchemaSemanticVersion"].ToObject <string>();
}
if (properties.ContainsKey("cdm:documentVersion"))
{
manifest.DocumentVersion = properties["cdm:documentVersion"].ToObject <string>();
}
else if (properties.ContainsKey("cdm:traits"))
{
Utils.AddListToCdmCollection(manifest.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"]));
}
if (properties.ContainsKey("cdm:imports"))
{
foreach (var importObj in properties["cdm:imports"].ToObject <List <Import> >())
{
manifest.Imports.Add(ImportPersistence.FromData(ctx, importObj));
}
}
if (properties.ContainsKey("cdm:lastFileStatusCheckTime"))
{
manifest.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToObject <string>());
}
if (properties.ContainsKey("cdm:lastFileModifiedTime"))
{
manifest.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToObject <string>());
}
if (properties.ContainsKey("cdm:lastChildFileModifiedTime"))
{
manifest.LastChildFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastChildFileModifiedTime"].ToObject <string>());
}
}
var t2pm = new TraitToPropertyMap(manifest);
var sourceTrait = t2pm.FetchTraitReference(dbLocationTrait);
if (sourceTrait == null)
{
sourceTrait = Utils.CreateSourceTrait(ctx, dbLocationTrait, dbLocationTraitArgName);
manifest.ExhibitsTraits.Add(sourceTrait);
}
var adlsPath = Utils.SymsPathToAdlsAdapterPath(databaseProperties.Source.Location);
var adlsCorpusPath = ctx.Corpus.Storage.AdapterPathToCorpusPath(adlsPath);
if (adlsCorpusPath == null)
{
Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(sourceTrait.Arguments[0].Value);
if (null == Utils.CreateAndMountAdlsAdapterFromAdlsPath(ctx.Corpus.Storage, adlsPath, pathTuple.Item1))
{
Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsAdlsAdapterNotMounted, adlsPath);
return(null);
}
}
if (dataObj.Entities != null)
{
foreach (var entityObj in dataObj.Entities)
{
if (entityObj.Type == SASEntityType.TABLE)
{
var entity = LocalEntityDeclarationPersistence.FromData(ctx, entityObj, manifest, databaseProperties.Source.Location);
if (entity != null)
{
manifest.Entities.Add(entity);
}
else
{
Logger.Warning((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.WarnPersistSymsEntitySkipped, entityObj.Name);
}
}
}
}
if (!manifest.Imports.Any((CdmImport importPresent) => importPresent.CorpusPath == Constants.FoundationsCorpusPath))
{
manifest.Imports.Add(Constants.FoundationsCorpusPath);
}
if (dataObj.Relationships != null)
{
foreach (var relationshipEntity in dataObj.Relationships)
{
manifest.Relationships.AddRange(E2ERelationshipPersistence.FromData(ctx, relationshipEntity));
}
}
// TODO : Submanifest
return(manifest);
}
public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options)
{
obj.Properties = new Dictionary <string, JToken>();
if (instance.Name != null)
{
obj.Properties["cdm:name"] = instance.Name;
}
if (instance.LastFileStatusCheckTime != null)
{
obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
}
if (instance.LastFileModifiedTime != null)
{
obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
}
if (instance.ExhibitsTraits != null)
{
TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
instance.ExhibitsTraits.Remove("is.partition.format.CSV");
}
if (instance.ExhibitsTraits.Count > 0)
{
obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
});
}
if (csvTrait != null)
{
instance.ExhibitsTraits.Add(csvTrait);
}
}
var properties = FillPropertyBagFromCsvTrait(instance);
if (properties != null)
{
obj.Format = new FormatInfo
{
InputFormat = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat,
OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat,
SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe,
FormatType = FormatType.Csv,
Properties = properties
};
}
else
{
// error
return(null);
}
return(obj);
}
public async static Task <Model> ToData(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Model()
{
Name = instance.ManifestName,
Description = instance.Explanation,
ModifiedTime = instance.LastFileModifiedTime,
LastChildFileModifiedTime = instance.LastChildFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime,
DocumentVersion = instance.DocumentVersion
};
TraitToPropertyMap t2pm = new TraitToPropertyMap(instance);
CdmTraitReference isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
CdmTraitReference applicationTrait = t2pm.FetchTraitReference("is.managedBy");
if (applicationTrait != null)
{
result.Application = applicationTrait.Arguments.AllItems[0].Value as string;
}
CdmTraitReference versionTrait = t2pm.FetchTraitReference("is.modelConversion.modelVersion");
if (versionTrait != null)
{
result.Version = versionTrait.Arguments.AllItems[0].Value;
}
else
{
// version property is required. If it doesn't exist set default.
result.Version = "1.0";
}
CdmTraitReference cultureTrait = t2pm.FetchTraitReference("is.partition.culture");
if (cultureTrait != null)
{
result.Culture = cultureTrait.Arguments.AllItems[0].Value as string;
}
Dictionary <string, string> referenceEntityLocations = new Dictionary <string, string>();
Dictionary <string, string> referenceModels = new Dictionary <string, string>();
CdmTraitReference referenceModelsTrait = t2pm.FetchTraitReference("is.modelConversion.referenceModelMap");
if (referenceModelsTrait != null)
{
JArray refModels = referenceModelsTrait.Arguments[0].Value as JArray;
foreach (JObject referenceModel in refModels)
{
var referenceModelId = referenceModel["id"];
var referenceModelIdAsString = referenceModelId.ToString();
var referenceModelLocation = referenceModel["location"];
var referenceModelLocationAsString = referenceModelLocation.ToString();
referenceModels.Add(referenceModelIdAsString, referenceModelLocationAsString);
referenceEntityLocations.Add(referenceModelLocationAsString, referenceModelIdAsString);
}
}
await Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
if (instance.Entities != null && instance.Entities.Count > 0)
{
List <Task> promises = new List <Task>();
var obtainedEntities = new ConcurrentBag <JToken>();
foreach (var entity in instance.Entities)
{
Task createdPromise = Task.Run(async() =>
{
dynamic element = null;
if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
element = await LocalEntityDeclarationPersistence.ToData(
entity as CdmLocalEntityDeclarationDefinition,
instance,
resOpt,
options
);
}
else if (entity.ObjectType == CdmObjectType.ReferencedEntityDeclarationDef)
{
element = await ReferencedEntityDeclarationPersistence.ToData(
entity as CdmReferencedEntityDeclarationDefinition,
resOpt,
options
);
var location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(entity.EntityPath);
if (StringUtils.IsBlankByCdmStandard(location))
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonInvalidEntityPath);
element = null;
}
if (element is ReferenceEntity referenceEntity)
{
// path separator can differ depending on the adapter, cover the case where path uses '/' or '\'
int lastSlashLocation = location.LastIndexOf("/") > location.LastIndexOf("\\") ? location.LastIndexOf("/") : location.LastIndexOf("\\");
if (lastSlashLocation > 0)
{
location = location.Slice(0, lastSlashLocation);
}
if (referenceEntity.ModelId != null)
{
if (referenceModels.TryGetValue(referenceEntity.ModelId, out var savedLocation) && savedLocation != location)
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonModelIdDuplication);
element = null;
}
else if (savedLocation == null)
{
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
else if (referenceEntity.ModelId == null && referenceEntityLocations.ContainsKey(location))
{
referenceEntity.ModelId = referenceEntityLocations[location];
}
else
{
referenceEntity.ModelId = Guid.NewGuid().ToString();
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
}
if (element != null)
{
obtainedEntities.Add(JToken.FromObject(element));
}
else
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityDeclarationConversionError, entity.EntityName);
}
});
try
{
// TODO: Currently function is synchronous. Remove next line to turn it asynchronous.
// Currently some functions called are not thread safe.
await createdPromise;
promises.Add(createdPromise);
}
catch (Exception ex)
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityDeclarationConversionFailure, entity.EntityName, ex.Message);
}
}
await Task.WhenAll(promises);
result.Entities = obtainedEntities.ToList();
}
if (referenceModels.Count > 0)
{
result.ReferenceModels = new List <ReferenceModel>();
foreach (var referenceModel in referenceModels)
{
result.ReferenceModels.Add(new ReferenceModel()
{
Id = referenceModel.Key,
Location = referenceModel.Value
});
}
}
if (instance.Relationships != null && instance.Relationships.Count > 0)
{
result.Relationships = new List <SingleKeyRelationship>();
foreach (var cdmRelationship in instance.Relationships)
{
var relationship = await RelationshipPersistence.ToData(cdmRelationship, resOpt, options);
if (relationship != null)
{
result.Relationships.Add(relationship);
}
}
}
result.Imports = new List <Import>();
if (instance.Imports != null && instance.Imports.Count > 0)
{
foreach (var element in instance.Imports)
{
result.Imports.Add(CdmFolder.ImportPersistence.ToData(element, resOpt, options));
}
}
// Importing foundations.cdm.json to resolve trait properly on manifest
if (instance.Imports == null || instance.Imports.Item(Constants.FoundationsCorpusPath, checkMoniker: false) == null)
{
result.Imports.Add(new Import {
CorpusPath = Constants.FoundationsCorpusPath
});
}
return(result);
}