Пример #1
0
        public static DataPartition ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
        {
            var argumentsCopy = new List <Argument>();

            foreach (var argumentList in instance.Arguments)
            {
                foreach (var argumentValue in argumentList.Value)
                {
                    argumentsCopy.Add(new Argument()
                    {
                        Name  = argumentList.Key,
                        Value = argumentValue
                    }
                                      );
                }
            }

            var result = new DataPartition
            {
                Name     = instance.Name,
                Location = instance.Location,
                LastFileStatusCheckTime = TimeUtils.GetFormattedDateString(instance.LastFileStatusCheckTime),
                LastFileModifiedTime    = TimeUtils.GetFormattedDateString(instance.LastFileModifiedTime),
                ExhibitsTraits          = CopyDataUtils.ListCopyData(resOpt, instance.ExhibitsTraits?.Where(trait => !trait.IsFromProperty)?.ToList(), options),
                Arguments         = argumentsCopy,
                SpecializedSchema = instance.SpecializedSchema
            };

            return(result);
        }
Пример #2
0
        public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
        {
            var result = new Partition
            {
                Name        = instance.Name,
                Description = instance.Description,
                Location    = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
                    instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
                        instance.Location, instance.InDocument)),
                RefreshTime             = instance.RefreshTime,
                FileFormatSettings      = null,
                LastFileModifiedTime    = instance.LastFileModifiedTime,
                LastFileStatusCheckTime = instance.LastFileStatusCheckTime
            };

            if (result.Name == null)
            {
                Logger.Warning(instance.Ctx, Tag, instance.AtCorpusPath, nameof(ToData), CdmLogCode.WarnPersistPartitionNameNull);
                result.Name = "";
            }

            if (string.IsNullOrEmpty(result.Location))
            {
                Logger.Warning(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.WarnPersistPartitionLocMissing, result.Name);
            }

            Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);

            var t2pm = new TraitToPropertyMap(instance);

            var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");

            if (isHiddenTrait != null)
            {
                result.IsHidden = true;
            }

            var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");

            if (csvTrait != null)
            {
                var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);

                if (csvFormatSettings != null)
                {
                    result.FileFormatSettings      = csvFormatSettings;
                    result.FileFormatSettings.Type = "CsvFormatSettings";
                }
                else
                {
                    Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistCsvProcessingError);
                    return(null);
                }
            }

            return(result);
        }
Пример #3
0
        public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
        {
            var result = new Partition
            {
                Name        = instance.Name,
                Description = instance.Description,
                Location    = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
                    instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
                        instance.Location, instance.InDocument)),
                RefreshTime             = instance.RefreshTime,
                FileFormatSettings      = null,
                LastFileModifiedTime    = instance.LastFileModifiedTime,
                LastFileStatusCheckTime = instance.LastFileStatusCheckTime
            };

            if (string.IsNullOrEmpty(result.Location))
            {
                Logger.Warning(nameof(DataPartitionPersistence), instance.Ctx, $"Couldn't find data partition's location for partition {result.Name}.", nameof(ToData));
            }

            await Utils.ProcessAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);

            var t2pm = new TraitToPropertyMap(instance);

            var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");

            if (isHiddenTrait != null)
            {
                result.IsHidden = true;
            }

            var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");

            if (csvTrait != null)
            {
                var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);

                if (csvFormatSettings != null)
                {
                    result.FileFormatSettings      = csvFormatSettings;
                    result.FileFormatSettings.Type = "CsvFormatSettings";
                }
                else
                {
                    Logger.Error(nameof(DataPartitionPersistence), instance.Ctx,
                                 "There was a problem while processing csv format trait inside data partition.");

                    return(null);
                }
            }

            return(result);
        }
Пример #4
0
        private static IDictionary <string, JToken> FillPropertyBagFromCsvTrait(CdmDataPartitionDefinition instance, IDictionary <string, JToken> properties = null)
        {
            TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
            var csvTrait           = tpm.FetchTraitReference("is.partition.format.CSV");

            if (csvTrait != null)
            {
                if (properties == null)
                {
                    properties = new Dictionary <string, JToken>();
                }

                foreach (var csvTraitArg in csvTrait.Arguments)
                {
                    string key;
                    // map to syms define properties
                    switch (csvTraitArg.Name)
                    {
                    case "columnHeaders":
                        key = "header";
                        break;

                    case "delimiter":
                        key = "field.delim";
                        break;

                    default:
                        key = csvTraitArg.Value;
                        break;
                    }

                    properties[key] = csvTraitArg.Value;
                }
            }

            return(properties);
        }
 private static void ErrorMessage(CdmCorpusContext ctx, string methodName, string corpusPath, CdmDataPartitionDefinition obj, bool isIncremental)
 {
     if (isIncremental)
     {
         Logger.Error(ctx, Tag, methodName, corpusPath, CdmLogCode.ErrPersistIncrementalConversionError, obj.Name, Constants.IncrementalTraitName, nameof(CdmLocalEntityDeclarationDefinition.DataPartitions));
     }
     else
     {
         Logger.Error(ctx, Tag, methodName, corpusPath, CdmLogCode.ErrPersistNonIncrementalConversionError, obj.Name, Constants.IncrementalTraitName, nameof(CdmLocalEntityDeclarationDefinition.IncrementalPartitions));
     }
 }
Пример #6
0
        public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options)
        {
            obj.Properties = new Dictionary <string, JToken>();

            if (instance.Name != null)
            {
                obj.Properties["cdm:name"] = instance.Name;
            }

            if (instance.LastFileStatusCheckTime != null)
            {
                obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
            }

            if (instance.LastFileModifiedTime != null)
            {
                obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
            }

            if (instance.ExhibitsTraits != null)
            {
                TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
                var csvTrait           = tpm.FetchTraitReference("is.partition.format.CSV");
                if (csvTrait != null)
                {
                    instance.ExhibitsTraits.Remove("is.partition.format.CSV");
                }

                if (instance.ExhibitsTraits.Count > 0)
                {
                    obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
                                                                     new JsonSerializer {
                        NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
                    });
                }
                if (csvTrait != null)
                {
                    instance.ExhibitsTraits.Add(csvTrait);
                }
            }

            var properties = FillPropertyBagFromCsvTrait(instance);

            if (properties != null)
            {
                obj.Format = new FormatInfo
                {
                    InputFormat  = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat,
                    OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat,
                    SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe,
                    FormatType   = FormatType.Csv,
                    Properties   = properties
                };
            }
            else
            {
                // error
                return(null);
            }

            return(obj);
        }