public static async Task <TableEntity> ToDataAsync(CdmLocalEntityDeclarationDefinition instance, CdmManifestDefinition manifest, string symsRootPath, ResolveOptions resOpt, CopyOptions options) { TableEntity tableEntity = await DocumentPersistence.ToDataAsync(instance.EntityPath, manifest, instance.Ctx, resOpt, options); if (tableEntity != null) { TableProperties teProperties = (TableProperties)tableEntity.Properties; var properties = CreateTablePropertyBags(instance, resOpt, options, teProperties.Properties); if (instance.DataPartitions != null && instance.DataPartitions.Count > 0) { List <string> paths = new List <string>(); foreach (var element in instance.DataPartitions) { if (element.Location != null) { var adlsPath = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(element.Location); string location = element.Location; if (adlsPath == null) { Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath, CdmLogCode.ErrPersistSymsAdlsAdapterMissing, element.Location); return(null); } var symsPath = Utils.AdlsAdapterPathToSymsPath(adlsPath); if (symsPath != null) { location = symsPath; } else { var pathTuple = StorageUtils.SplitNamespacePath(element.Location); location = Utils.CreateSymsAbsolutePath(symsRootPath, pathTuple.Item2); } paths.Add(location); } teProperties.StorageDescriptor = DataPartitionPersistence.ToData(element, teProperties.StorageDescriptor, resOpt, options); } // Logic to find common root folder. IEnumerable <string> samples = paths.ToArray(); teProperties.StorageDescriptor.Source.Location = new string(samples.First().Substring(0, samples.Min(s => s.Length)) .TakeWhile((c, i) => samples.All(s => s[i] == c)).ToArray()); } else { // location and format is mandatory for syms. teProperties.StorageDescriptor.Source.Location = Utils.CreateSymsAbsolutePath(symsRootPath, instance.EntityName); } teProperties.Properties = properties; } return(tableEntity); }
public static CdmLocalEntityDeclarationDefinition FromData(CdmCorpusContext ctx, TableEntity table, CdmManifestDefinition manifest, string symsRootPath) { string tableName = table.Name; var localDec = ctx.Corpus.MakeObject <CdmLocalEntityDeclarationDefinition>( CdmObjectType.LocalEntityDeclarationDef, tableName); localDec.EntityPath = ctx.Corpus.Storage.CreateRelativeCorpusPath($"{tableName}.cdm.json/{tableName}", manifest); TableProperties tableProperties = ((JToken)table.Properties).ToObject <TableProperties>(); var properties = tableProperties.Properties; if (properties != null) { if (properties.ContainsKey("cdm:isHidden")) { var isHiddenTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.hidden", true); isHiddenTrait.IsFromProperty = true; localDec.ExhibitsTraits.Add(isHiddenTrait); } if (properties.ContainsKey("cdm:lastChildFileModifiedTime")) { localDec.LastChildFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastChildFileModifiedTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastFileModifiedTime")) { localDec.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:lastFileStatusCheckTime")) { localDec.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToObject <string>()); } if (properties.ContainsKey("cdm:explanation")) { localDec.Explanation = properties["cdm:explanation"].ToObject <string>(); } if (properties.ContainsKey("cdm:entityDecTraits")) { Utils.AddListToCdmCollection(localDec.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:entityDecTraits"])); } } if (tableProperties.Partitioning != null && tableProperties.Partitioning.Keys != null) { // TODO :This is spark data partitioning. Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsPartitionNotSupported, tableName); return(null); } else { if (tableProperties.StorageDescriptor != null && tableProperties.StorageDescriptor.Format != null) { if (tableProperties.StorageDescriptor.Source.Location == "") { Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsTableMissingDataLocation, tableName); return(null); } string formatType = tableProperties.StorageDescriptor.Format.FormatType; if (!formatType.EqualsWithIgnoreCase(Utils.Csv) && !formatType.EqualsWithIgnoreCase(Utils.Parquet)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsTableFormatTypeNotSupported, tableName); return(null); } // check and get list of wildcards matches in path if any. var matches = Utils.GetWildcardsMatches(tableProperties.StorageDescriptor.Source.Location); if (System.IO.Path.GetExtension(tableProperties.StorageDescriptor.Source.Location) == String.Empty || // check if its a folder matches != null) { var dataPartitionPattern = DataPartitionPatternPersistence.FromData(ctx, tableProperties.StorageDescriptor, $"{table.Name}PartitionPattern", symsRootPath, formatType, matches); localDec.DataPartitionPatterns.Add(dataPartitionPattern); } else if (tableProperties.StorageDescriptor.Source.Location.EndWithOrdinalIgnoreCase(".csv")) { if (formatType.EqualsWithIgnoreCase(Utils.Parquet)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsIncompatibleFileToType, "csv", formatType.ToString()); return(null); } // Location points to file. create data partition. var dataPartition = DataPartitionPersistence.FromData(ctx, tableProperties.StorageDescriptor, symsRootPath, formatType); localDec.DataPartitions.Add(dataPartition); } else if (tableProperties.StorageDescriptor.Source.Location.EndWithOrdinalIgnoreCase(".parquet")) { if (formatType.EqualsWithIgnoreCase(Utils.Csv)) { Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsIncompatibleFileToType, "parquet", formatType.ToString()); return(null); } // Location points to file. create data partition. var dataPartition = DataPartitionPersistence.FromData(ctx, tableProperties.StorageDescriptor, symsRootPath, formatType); localDec.DataPartitions.Add(dataPartition); } else { // restore data partition pattern if exist if (properties != null && properties.ContainsKey("cdm:dataPartitionPatterns")) { var dataPartitionPattern = DataPartitionPatternPersistence.FromData(ctx, properties["cdm:dataPartitionPatterns"], $"{table.Name}PartitionPattern", symsRootPath, formatType); localDec.DataPartitionPatterns.Add(dataPartitionPattern); } else { Logger.Error((ResolveContext)ctx, Tag, nameof(FromData), localDec.AtCorpusPath, CdmLogCode.ErrPersistSymsTableInvalidDataLocation, tableName); return(null); } } } } return(localDec); }