public static async Task <List <TableEntity> > CreateSymsTablesObjects(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options, string location, IDictionary <string, TableEntity> existingTableEntities = null) { List <TableEntity> symsTables = new List <TableEntity>(); if (instance.Entities != null && instance.Entities.Count > 0) { List <Task> promises = new List <Task>(); var obtainedEntities = new ConcurrentBag <TableEntity>(); foreach (var entity in instance.Entities) { Task createdPromise = Task.Run(async() => { dynamic element = null; if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef && Utils.IsEntityAddedOrModified(entity as CdmLocalEntityDeclarationDefinition, existingTableEntities)) { element = await LocalEntityDeclarationPersistence.ToDataAsync( entity as CdmLocalEntityDeclarationDefinition, instance, location, resOpt, options ); if (element != null) { obtainedEntities.Add(element); } else { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsEntityDeclConversionFailure, entity.EntityName); } } }); try { // TODO: Currently function is synchronous. Remove next line to turn it asynchronous. // Currently some functions called are not thread safe. await createdPromise; promises.Add(createdPromise); } catch (Exception ex) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsEntityDeclConversionException, entity.EntityName, ex.Message); } } await Task.WhenAll(promises); symsTables = obtainedEntities.ToList(); } return(symsTables); }
public static CdmManifestDefinition FromObject(CdmCorpusContext ctx, string docName, string nameSpace, string path, SymsManifestContent dataObj) { DatabaseEntity database = dataObj.Database; if (database == null || database.Type != SASEntityType.DATABASE) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbObject); return(null); } DatabaseProperties databaseProperties = ((JToken)database.Properties).ToObject <DatabaseProperties>(); if (databaseProperties == null || databaseProperties.Source == null) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsInvalidDbPropObject); return(null); } var properties = databaseProperties.Properties; var manifest = ctx.Corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef); manifest.ManifestName = dataObj.Database.Name; manifest.Name = docName; manifest.FolderPath = path; manifest.Namespace = nameSpace; manifest.Explanation = databaseProperties.Description; if (properties != null) { if (properties.ContainsKey("cdm:schema")) { manifest.Schema = properties["cdm:schema"].ToObject <string>(); } if (properties.ContainsKey("cdm:jsonSchemaSemanticVersion")) { manifest.JsonSchemaSemanticVersion = properties["cdm:jsonSchemaSemanticVersion"].ToObject <string>(); } if (properties.ContainsKey("cdm:documentVersion")) { manifest.DocumentVersion = properties["cdm:documentVersion"].ToObject <string>(); } else if (properties.ContainsKey("cdm:traits")) { Utils.AddListToCdmCollection(manifest.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"])); } if (properties.ContainsKey("cdm:imports")) { foreach (var importObj in properties["cdm:imports"].ToObject <List <Import> >()) { manifest.Imports.Add(ImportPersistence.FromData(ctx, importObj)); } } if (properties.ContainsKey("cdm:lastFileStatusCheckTime")) { manifest.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastFileModifiedTime")) { manifest.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastChildFileModifiedTime")) { manifest.LastChildFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastChildFileModifiedTime"].ToObject <string>()); } } var t2pm = new TraitToPropertyMap(manifest); var sourceTrait = t2pm.FetchTraitReference(dbLocationTrait); if (sourceTrait == null) { sourceTrait = Utils.CreateSourceTrait(ctx, dbLocationTrait, dbLocationTraitArgName); manifest.ExhibitsTraits.Add(sourceTrait); } var adlsPath = Utils.SymsPathToAdlsAdapterPath(databaseProperties.Source.Location); var adlsCorpusPath = ctx.Corpus.Storage.AdapterPathToCorpusPath(adlsPath); if (adlsCorpusPath == null) { Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(sourceTrait.Arguments[0].Value); if (null == Utils.CreateAndMountAdlsAdapterFromAdlsPath(ctx.Corpus.Storage, adlsPath, pathTuple.Item1)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.ErrPersistSymsAdlsAdapterNotMounted, adlsPath); return(null); } } if (dataObj.Entities != null) { foreach (var entityObj in dataObj.Entities) { if (entityObj.Type == SASEntityType.TABLE) { var entity = LocalEntityDeclarationPersistence.FromData(ctx, entityObj, manifest, databaseProperties.Source.Location); if (entity != null) { manifest.Entities.Add(entity); } else { Logger.Warning((ResolveContext)ctx, Tag, nameof(FromObject), null, CdmLogCode.WarnPersistSymsEntitySkipped, entityObj.Name); } } } } if (!manifest.Imports.Any((CdmImport importPresent) => importPresent.CorpusPath == Constants.FoundationsCorpusPath)) { manifest.Imports.Add(Constants.FoundationsCorpusPath); } if (dataObj.Relationships != null) { foreach (var relationshipEntity in dataObj.Relationships) { manifest.Relationships.AddRange(E2ERelationshipPersistence.FromData(ctx, relationshipEntity)); } } // TODO : Submanifest return(manifest); }