public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options) { var result = new Partition { Name = instance.Name, Description = instance.Description, Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath( instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath( instance.Location, instance.InDocument)), RefreshTime = instance.RefreshTime, FileFormatSettings = null, LastFileModifiedTime = instance.LastFileModifiedTime, LastFileStatusCheckTime = instance.LastFileStatusCheckTime }; if (result.Name == null) { Logger.Warning(instance.Ctx, Tag, instance.AtCorpusPath, nameof(ToData), CdmLogCode.WarnPersistPartitionNameNull); result.Name = ""; } if (string.IsNullOrEmpty(result.Location)) { Logger.Warning(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.WarnPersistPartitionLocMissing, result.Name); } Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits); var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { result.IsHidden = true; } var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV"); if (csvTrait != null) { var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait); if (csvFormatSettings != null) { result.FileFormatSettings = csvFormatSettings; result.FileFormatSettings.Type = "CsvFormatSettings"; } else { Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistCsvProcessingError); return(null); } } return(result); }
public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options) { var result = new Partition { Name = instance.Name, Description = instance.Description, Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath( instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath( instance.Location, instance.InDocument)), RefreshTime = instance.RefreshTime, FileFormatSettings = null, LastFileModifiedTime = instance.LastFileModifiedTime, LastFileStatusCheckTime = instance.LastFileStatusCheckTime }; if (string.IsNullOrEmpty(result.Location)) { Logger.Warning(nameof(DataPartitionPersistence), instance.Ctx, $"Couldn't find data partition's location for partition {result.Name}.", nameof(ToData)); } await Utils.ProcessAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits); var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { result.IsHidden = true; } var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV"); if (csvTrait != null) { var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait); if (csvFormatSettings != null) { result.FileFormatSettings = csvFormatSettings; result.FileFormatSettings.Type = "CsvFormatSettings"; } else { Logger.Error(nameof(DataPartitionPersistence), instance.Ctx, "There was a problem while processing csv format trait inside data partition."); return(null); } } return(result); }
private static IDictionary <string, JToken> CreateTablePropertyBags(CdmLocalEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options, IDictionary <string, JToken> properties) { if (properties == null) { properties = new Dictionary <string, JToken>(); } if (instance.EntityPath != null) { Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(instance.EntityPath); if (pathTuple == null) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsEntityPathNull, instance.EntityName); return(null); } properties["cdm:entityPath"] = JToken.FromObject(pathTuple.Item2); } var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (!properties.ContainsKey("cdm:description")) { properties["cdm:description"] = instance.Explanation; } if (instance.LastChildFileModifiedTime != null) { properties["cdm:lastChildFileModifiedTime"] = instance.LastChildFileModifiedTime; } if (instance.LastFileModifiedTime != null) { properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime; } if (instance.LastFileStatusCheckTime != null) { properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime; } if (isHiddenTrait != null) { properties["cdm:isHidden"] = true; } if (instance.ExhibitsTraits != null && instance.ExhibitsTraits.Count > 0) { properties["cdm:entityDecTraits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options), new JsonSerializer { NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() }); } return(properties); }
public static async Task <ReferenceEntity> ToData(CdmReferencedEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options) { var sourceIndex = instance.EntityPath.LastIndexOf("/"); if (sourceIndex == -1) { Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityRefConversionError, instance.EntityName); return(null); } var referenceEntity = new ReferenceEntity { Type = "ReferenceEntity", Name = instance.EntityName, Source = instance.EntityPath.Slice(sourceIndex + 1), Description = instance.Explanation, LastFileModifiedTime = instance.LastFileModifiedTime, LastFileStatusCheckTime = instance.LastFileStatusCheckTime }; Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, referenceEntity, instance.ExhibitsTraits); var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { referenceEntity.IsHidden = true; } var propertiesTrait = t2pm.FetchTraitReference("is.propertyContent.multiTrait"); if (propertiesTrait != null) { referenceEntity.ModelId = propertiesTrait.Arguments.AllItems[0].Value as string; } return(referenceEntity); }
public static async Task <LocalEntity> ToData(CdmLocalEntityDeclarationDefinition instance, CdmManifestDefinition manifest, ResolveOptions resOpt, CopyOptions options) { var localEntity = await DocumentPersistence.ToData(instance.EntityPath, manifest, resOpt, options, instance.Ctx); if (localEntity != null) { var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (localEntity.Description == null) { localEntity.Description = instance.Explanation; } localEntity.LastChildFileModifiedTime = instance.LastChildFileModifiedTime; localEntity.LastFileModifiedTime = instance.LastFileModifiedTime; localEntity.LastFileStatusCheckTime = instance.LastFileStatusCheckTime; if (isHiddenTrait != null) { localEntity.IsHidden = true; } if (t2pm.FetchPropertyValue("cdmSchemas") is List <string> schemas) { localEntity.Schemas = schemas; } if (instance.DataPartitions != null && instance.DataPartitions.Count > 0) { localEntity.Partitions = new List <Partition>(); foreach (var element in instance.DataPartitions) { var partition = await DataPartitionPersistence.ToData(element, resOpt, options); if (partition != null) { localEntity.Partitions.Add(partition); } else { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityPartitionConversionError); return(null); } } } } return(localEntity); }
public static async Task <LocalEntity> ToData(CdmLocalEntityDeclarationDefinition instance, ResolveOptions resOpt, CopyOptions options) { var localEntity = await DocumentPersistence.ToData(instance.EntityPath, resOpt, options, instance.Ctx); if (localEntity != null) { var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); localEntity.Description = instance.Explanation; localEntity.LastChildFileModifiedTime = instance.LastChildFileModifiedTime; localEntity.LastFileModifiedTime = instance.LastFileModifiedTime; localEntity.LastFileStatusCheckTime = instance.LastFileStatusCheckTime; if (isHiddenTrait != null) { localEntity.IsHidden = true; } if (t2pm.FetchPropertyValue("cdmSchemas") is List <string> schemas) { localEntity.Schemas = schemas; } if (instance.DataPartitions != null && instance.DataPartitions.Count > 0) { localEntity.Partitions = new List <Partition>(); foreach (var element in instance.DataPartitions) { var partition = await DataPartitionPersistence.ToData(element, resOpt, options); if (partition != null) { localEntity.Partitions.Add(partition); } else { Logger.Error(nameof(LocalEntityDeclarationPersistence), instance.Ctx, "There was an error while trying to convert cdm data partition to model.json partition."); return(null); } } } } return(localEntity); }
public static DataColumn ToData(CdmTypeAttributeDefinition instance, CdmCorpusContext ctx, ResolveOptions resOpt, CopyOptions options) { var properties = CreateProperties(instance, resOpt, options); var originDataTypeName = new TypeInfo { Properties = properties, IsComplexType = false, IsNullable = instance.GetProperty("isNullable"), TypeFamily = "cdm" }; var t2pm = new TraitToPropertyMap(instance);; var numericTraits = t2pm.FetchTraitReference("is.dataFormat.numeric.shaped"); if (numericTraits != null) { foreach (var numericTraitsArg in numericTraits.Arguments) { if (numericTraitsArg.Name == "precision") { originDataTypeName.Precision = (int)numericTraitsArg.Value; } if (numericTraitsArg.Name == "scale") { originDataTypeName.Scale = (int)numericTraitsArg.Value; } } } var dataFormat = instance.GetProperty("dataFormat"); originDataTypeName = Utils.CdmDataFormatToSymsDataType(dataFormat, originDataTypeName); if (originDataTypeName.TypeName == null) { Logger.Error(ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsUnknownDataFormat, instance.DisplayName); return(null); } return(new DataColumn { OriginDataTypeName = originDataTypeName, Name = instance.Name }); }
private static DataSource CreateDataSource(CdmManifestDefinition instance) { var source = new DataSource(); var t2pm = new TraitToPropertyMap(instance); var sourceTraits = t2pm.FetchTraitReference(dbLocationTrait); if (sourceTraits != null && sourceTraits.Arguments != null && sourceTraits.Arguments.Count == 1 && sourceTraits.Arguments[0].Name == dbLocationTraitArgName) { source.Location = Utils.CorpusPathToSymsPath(sourceTraits.Arguments[0].Value, instance.Ctx.Corpus.Storage); } if (source.Location == null) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsStorageSourceTraitError, dbLocationTrait, dbLocationTraitArgName); return(null); } return(source); }
public static async Task <Attribute> ToData(CdmTypeAttributeDefinition instance, ResolveOptions resOpt, CopyOptions options) { var attribute = new Attribute { Name = instance.Name, DataType = DataTypeToData(instance.DataFormat), Description = instance.Description }; await Utils.ProcessAnnotationsToData(instance.Ctx, attribute, instance.AppliedTraits); var t2pm = new TraitToPropertyMap(instance); var isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { attribute.IsHidden = true; } return(attribute); }
private static IDictionary <string, JToken> FillPropertyBagFromCsvTrait(CdmDataPartitionDefinition instance, IDictionary <string, JToken> properties = null) { TraitToPropertyMap tpm = new TraitToPropertyMap(instance); var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV"); if (csvTrait != null) { if (properties == null) { properties = new Dictionary <string, JToken>(); } foreach (var csvTraitArg in csvTrait.Arguments) { string key; // map to syms define properties switch (csvTraitArg.Name) { case "columnHeaders": key = "header"; break; case "delimiter": key = "field.delim"; break; default: key = csvTraitArg.Value; break; } properties[key] = csvTraitArg.Value; } } return(properties); }
public async static Task <Model> ToData(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options) { var result = new Model() { Name = instance.ManifestName, Description = instance.Explanation, ModifiedTime = instance.LastFileModifiedTime, LastChildFileModifiedTime = instance.LastChildFileModifiedTime, LastFileStatusCheckTime = instance.LastFileStatusCheckTime, DocumentVersion = instance.DocumentVersion }; TraitToPropertyMap t2pm = new TraitToPropertyMap(instance); CdmTraitReference isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { result.IsHidden = true; } CdmTraitReference applicationTrait = t2pm.FetchTraitReference("is.managedBy"); if (applicationTrait != null) { result.Application = applicationTrait.Arguments.AllItems[0].Value as string; } CdmTraitReference versionTrait = t2pm.FetchTraitReference("is.modelConversion.modelVersion"); if (versionTrait != null) { result.Version = versionTrait.Arguments.AllItems[0].Value; } else { // version property is required. If it doesn't exist set default. result.Version = "1.0"; } CdmTraitReference cultureTrait = t2pm.FetchTraitReference("is.partition.culture"); if (cultureTrait != null) { result.Culture = cultureTrait.Arguments.AllItems[0].Value as string; } Dictionary <string, string> referenceEntityLocations = new Dictionary <string, string>(); Dictionary <string, string> referenceModels = new Dictionary <string, string>(); CdmTraitReference referenceModelsTrait = t2pm.FetchTraitReference("is.modelConversion.referenceModelMap"); if (referenceModelsTrait != null) { JArray refModels = referenceModelsTrait.Arguments[0].Value as JArray; foreach (JObject referenceModel in refModels) { var referenceModelId = referenceModel["id"]; var referenceModelIdAsString = referenceModelId.ToString(); var referenceModelLocation = referenceModel["location"]; var referenceModelLocationAsString = referenceModelLocation.ToString(); referenceModels.Add(referenceModelIdAsString, referenceModelLocationAsString); referenceEntityLocations.Add(referenceModelLocationAsString, referenceModelIdAsString); } } Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits); if (instance.Entities != null && instance.Entities.Count > 0) { List <Task> promises = new List <Task>(); var obtainedEntities = new ConcurrentBag <JToken>(); foreach (var entity in instance.Entities) { Task createdPromise = Task.Run(async() => { dynamic element = null; if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef) { element = await LocalEntityDeclarationPersistence.ToData( entity as CdmLocalEntityDeclarationDefinition, instance, resOpt, options ); } else if (entity.ObjectType == CdmObjectType.ReferencedEntityDeclarationDef) { element = await ReferencedEntityDeclarationPersistence.ToData( entity as CdmReferencedEntityDeclarationDefinition, resOpt, options ); var location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(entity.EntityPath); if (string.IsNullOrEmpty(location)) { Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Invalid entity path set in entity {entity.EntityName}"); element = null; } if (element is ReferenceEntity referenceEntity) { location = location.Slice(0, location.LastIndexOf("/")); if (referenceEntity.ModelId != null) { if (referenceModels.TryGetValue(referenceEntity.ModelId, out var savedLocation) && savedLocation != location) { Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Same ModelId pointing to different locations"); element = null; } else if (savedLocation == null) { referenceModels[referenceEntity.ModelId] = location; referenceEntityLocations[location] = referenceEntity.ModelId; } } else if (referenceEntity.ModelId == null && referenceEntityLocations.ContainsKey(location)) { referenceEntity.ModelId = referenceEntityLocations[location]; } else { referenceEntity.ModelId = Guid.NewGuid().ToString(); referenceModels[referenceEntity.ModelId] = location; referenceEntityLocations[location] = referenceEntity.ModelId; } } } if (element != null) { obtainedEntities.Add(JToken.FromObject(element)); } else { Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format."); } }); try { // TODO: Currently function is synchronous. Remove next line to turn it asynchronous. // Currently some functions called are not thread safe. await createdPromise; promises.Add(createdPromise); } catch (Exception ex) { Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format for reason {ex.Message}."); } } await Task.WhenAll(promises); result.Entities = obtainedEntities.ToList(); } if (referenceModels.Count > 0) { result.ReferenceModels = new List <ReferenceModel>(); foreach (var referenceModel in referenceModels) { result.ReferenceModels.Add(new ReferenceModel() { Id = referenceModel.Key, Location = referenceModel.Value }); } } if (instance.Relationships != null && instance.Relationships.Count > 0) { result.Relationships = new List <SingleKeyRelationship>(); foreach (var cdmRelationship in instance.Relationships) { var relationship = await RelationshipPersistence.ToData(cdmRelationship, resOpt, options); if (relationship != null) { result.Relationships.Add(relationship); } else { Logger.Error(nameof(ManifestPersistence), instance.Ctx, "There was an error while trying to convert cdm relationship to model.json relationship."); } } } if (instance.Imports != null && instance.Imports.Count > 0) { result.Imports = new List <Import>(); foreach (var element in instance.Imports) { result.Imports.Add(CdmFolder.ImportPersistence.ToData(element, resOpt, options)); } } return(result); }
public static CdmManifestDefinition FromObject(CdmCorpusContext ctx, string docName, string nameSpace, string path, SymsManifestContent dataObj) { DatabaseEntity database = dataObj.Database; if (database == null || database.Type != SASEntityType.DATABASE) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbObject); return(null); } DatabaseProperties databaseProperties = ((JToken)database.Properties).ToObject <DatabaseProperties>(); if (databaseProperties == null || databaseProperties.Source == null) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbPropObject); return(null); } var properties = databaseProperties.Properties; var manifest = ctx.Corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef); manifest.ManifestName = dataObj.Database.Name; manifest.Name = docName; manifest.FolderPath = path; manifest.Namespace = nameSpace; manifest.Explanation = databaseProperties.Description; if (properties != null) { if (properties.ContainsKey("cdm:schema")) { manifest.Schema = properties["cdm:schema"].ToObject <string>(); } if (properties.ContainsKey("cdm:jsonSchemaSemanticVersion")) { manifest.JsonSchemaSemanticVersion = properties["cdm:jsonSchemaSemanticVersion"].ToObject <string>(); } if (properties.ContainsKey("cdm:documentVersion")) { manifest.DocumentVersion = properties["cdm:documentVersion"].ToObject <string>(); } else if (properties.ContainsKey("cdm:traits")) { Utils.AddListToCdmCollection(manifest.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"])); } if (properties.ContainsKey("cdm:imports")) { foreach (var importObj in properties["cdm:imports"].ToObject <List <Import> >()) { manifest.Imports.Add(ImportPersistence.FromData(ctx, importObj)); } } if (properties.ContainsKey("cdm:lastFileStatusCheckTime")) { manifest.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastFileModifiedTime")) { manifest.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastChildFileModifiedTime")) { manifest.LastChildFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastChildFileModifiedTime"].ToObject <string>()); } } var t2pm = new TraitToPropertyMap(manifest); var sourceTrait = t2pm.FetchTraitReference(dbLocationTrait); if (sourceTrait == null) { sourceTrait = Utils.CreateSourceTrait(ctx, dbLocationTrait, dbLocationTraitArgName); manifest.ExhibitsTraits.Add(sourceTrait); } var adlsPath = Utils.SymsPathToAdlsAdapterPath(databaseProperties.Source.Location); var adlsCorpusPath = ctx.Corpus.Storage.AdapterPathToCorpusPath(adlsPath); if (adlsCorpusPath == null) { Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(sourceTrait.Arguments[0].Value); if (null == Utils.CreateAndMountAdlsAdapterFromAdlsPath(ctx.Corpus.Storage, adlsPath, pathTuple.Item1)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsAdlsAdapterNotMounted, adlsPath); return(null); } } if (dataObj.Entities != null) { foreach (var entityObj in dataObj.Entities) { if (entityObj.Type == SASEntityType.TABLE) { var entity = LocalEntityDeclarationPersistence.FromData(ctx, entityObj, manifest, databaseProperties.Source.Location); if (entity != null) { manifest.Entities.Add(entity); } else { Logger.Warning((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.WarnPersistSymsEntitySkipped, entityObj.Name); } } } } if (!manifest.Imports.Any((CdmImport importPresent) => importPresent.CorpusPath == Constants.FoundationsCorpusPath)) { manifest.Imports.Add(Constants.FoundationsCorpusPath); } if (dataObj.Relationships != null) { foreach (var relationshipEntity in dataObj.Relationships) { manifest.Relationships.AddRange(E2ERelationshipPersistence.FromData(ctx, relationshipEntity)); } } // TODO : Submanifest return(manifest); }
public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options) { obj.Properties = new Dictionary <string, JToken>(); if (instance.Name != null) { obj.Properties["cdm:name"] = instance.Name; } if (instance.LastFileStatusCheckTime != null) { obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime; } if (instance.LastFileModifiedTime != null) { obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime; } if (instance.ExhibitsTraits != null) { TraitToPropertyMap tpm = new TraitToPropertyMap(instance); var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV"); if (csvTrait != null) { instance.ExhibitsTraits.Remove("is.partition.format.CSV"); } if (instance.ExhibitsTraits.Count > 0) { obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options), new JsonSerializer { NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() }); } if (csvTrait != null) { instance.ExhibitsTraits.Add(csvTrait); } } var properties = FillPropertyBagFromCsvTrait(instance); if (properties != null) { obj.Format = new FormatInfo { InputFormat = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat, OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat, SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe, FormatType = FormatType.Csv, Properties = properties }; } else { // error return(null); } return(obj); }
public async static Task <Model> ToData(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options) { var result = new Model() { Name = instance.ManifestName, Description = instance.Explanation, ModifiedTime = instance.LastFileModifiedTime, LastChildFileModifiedTime = instance.LastChildFileModifiedTime, LastFileStatusCheckTime = instance.LastFileStatusCheckTime, DocumentVersion = instance.DocumentVersion }; TraitToPropertyMap t2pm = new TraitToPropertyMap(instance); CdmTraitReference isHiddenTrait = t2pm.FetchTraitReference("is.hidden"); if (isHiddenTrait != null) { result.IsHidden = true; } CdmTraitReference applicationTrait = t2pm.FetchTraitReference("is.managedBy"); if (applicationTrait != null) { result.Application = applicationTrait.Arguments.AllItems[0].Value as string; } CdmTraitReference versionTrait = t2pm.FetchTraitReference("is.modelConversion.modelVersion"); if (versionTrait != null) { result.Version = versionTrait.Arguments.AllItems[0].Value; } else { // version property is required. If it doesn't exist set default. result.Version = "1.0"; } CdmTraitReference cultureTrait = t2pm.FetchTraitReference("is.partition.culture"); if (cultureTrait != null) { result.Culture = cultureTrait.Arguments.AllItems[0].Value as string; } Dictionary <string, string> referenceEntityLocations = new Dictionary <string, string>(); Dictionary <string, string> referenceModels = new Dictionary <string, string>(); CdmTraitReference referenceModelsTrait = t2pm.FetchTraitReference("is.modelConversion.referenceModelMap"); if (referenceModelsTrait != null) { JArray refModels = referenceModelsTrait.Arguments[0].Value as JArray; foreach (JObject referenceModel in refModels) { var referenceModelId = referenceModel["id"]; var referenceModelIdAsString = referenceModelId.ToString(); var referenceModelLocation = referenceModel["location"]; var referenceModelLocationAsString = referenceModelLocation.ToString(); referenceModels.Add(referenceModelIdAsString, referenceModelLocationAsString); referenceEntityLocations.Add(referenceModelLocationAsString, referenceModelIdAsString); } } await Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits); if (instance.Entities != null && instance.Entities.Count > 0) { List <Task> promises = new List <Task>(); var obtainedEntities = new ConcurrentBag <JToken>(); foreach (var entity in instance.Entities) { Task createdPromise = Task.Run(async() => { dynamic element = null; if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef) { element = await LocalEntityDeclarationPersistence.ToData( entity as CdmLocalEntityDeclarationDefinition, instance, resOpt, options ); } else if (entity.ObjectType == CdmObjectType.ReferencedEntityDeclarationDef) { element = await ReferencedEntityDeclarationPersistence.ToData( entity as CdmReferencedEntityDeclarationDefinition, resOpt, options ); var location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(entity.EntityPath); if (StringUtils.IsBlankByCdmStandard(location)) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonInvalidEntityPath); element = null; } if (element is ReferenceEntity referenceEntity) { // path separator can differ depending on the adapter, cover the case where path uses '/' or '\' int lastSlashLocation = location.LastIndexOf("/") > location.LastIndexOf("\\") ? location.LastIndexOf("/") : location.LastIndexOf("\\"); if (lastSlashLocation > 0) { location = location.Slice(0, lastSlashLocation); } if (referenceEntity.ModelId != null) { if (referenceModels.TryGetValue(referenceEntity.ModelId, out var savedLocation) && savedLocation != location) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonModelIdDuplication); element = null; } else if (savedLocation == null) { referenceModels[referenceEntity.ModelId] = location; referenceEntityLocations[location] = referenceEntity.ModelId; } } else if (referenceEntity.ModelId == null && referenceEntityLocations.ContainsKey(location)) { referenceEntity.ModelId = referenceEntityLocations[location]; } else { referenceEntity.ModelId = Guid.NewGuid().ToString(); referenceModels[referenceEntity.ModelId] = location; referenceEntityLocations[location] = referenceEntity.ModelId; } } } if (element != null) { obtainedEntities.Add(JToken.FromObject(element)); } else { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityDeclarationConversionError, entity.EntityName); } }); try { // TODO: Currently function is synchronous. Remove next line to turn it asynchronous. // Currently some functions called are not thread safe. await createdPromise; promises.Add(createdPromise); } catch (Exception ex) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistModelJsonEntityDeclarationConversionFailure, entity.EntityName, ex.Message); } } await Task.WhenAll(promises); result.Entities = obtainedEntities.ToList(); } if (referenceModels.Count > 0) { result.ReferenceModels = new List <ReferenceModel>(); foreach (var referenceModel in referenceModels) { result.ReferenceModels.Add(new ReferenceModel() { Id = referenceModel.Key, Location = referenceModel.Value }); } } if (instance.Relationships != null && instance.Relationships.Count > 0) { result.Relationships = new List <SingleKeyRelationship>(); foreach (var cdmRelationship in instance.Relationships) { var relationship = await RelationshipPersistence.ToData(cdmRelationship, resOpt, options); if (relationship != null) { result.Relationships.Add(relationship); } } } result.Imports = new List <Import>(); if (instance.Imports != null && instance.Imports.Count > 0) { foreach (var element in instance.Imports) { result.Imports.Add(CdmFolder.ImportPersistence.ToData(element, resOpt, options)); } } // Importing foundations.cdm.json to resolve trait properly on manifest if (instance.Imports == null || instance.Imports.Item(Constants.FoundationsCorpusPath, checkMoniker: false) == null) { result.Imports.Add(new Import { CorpusPath = Constants.FoundationsCorpusPath }); } return(result); }