public static DataPartition ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var argumentsCopy = new List <Argument>();
foreach (var argumentList in instance.Arguments)
{
foreach (var argumentValue in argumentList.Value)
{
argumentsCopy.Add(new Argument()
{
Name = argumentList.Key,
Value = argumentValue
}
);
}
}
var result = new DataPartition
{
Name = instance.Name,
Location = instance.Location,
LastFileStatusCheckTime = TimeUtils.GetFormattedDateString(instance.LastFileStatusCheckTime),
LastFileModifiedTime = TimeUtils.GetFormattedDateString(instance.LastFileModifiedTime),
ExhibitsTraits = CopyDataUtils.ListCopyData(resOpt, instance.ExhibitsTraits?.Where(trait => !trait.IsFromProperty)?.ToList(), options),
Arguments = argumentsCopy,
SpecializedSchema = instance.SpecializedSchema
};
return(result);
}
public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Partition
{
Name = instance.Name,
Description = instance.Description,
Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
instance.Location, instance.InDocument)),
RefreshTime = instance.RefreshTime,
FileFormatSettings = null,
LastFileModifiedTime = instance.LastFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime
};
if (result.Name == null)
{
Logger.Warning(instance.Ctx, Tag, instance.AtCorpusPath, nameof(ToData), CdmLogCode.WarnPersistPartitionNameNull);
result.Name = "";
}
if (string.IsNullOrEmpty(result.Location))
{
Logger.Warning(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.WarnPersistPartitionLocMissing, result.Name);
}
Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);
if (csvFormatSettings != null)
{
result.FileFormatSettings = csvFormatSettings;
result.FileFormatSettings.Type = "CsvFormatSettings";
}
else
{
Logger.Error(instance.Ctx, Tag, nameof(ToData), instance.AtCorpusPath, CdmLogCode.ErrPersistCsvProcessingError);
return(null);
}
}
return(result);
}
public static async Task <Partition> ToData(CdmDataPartitionDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Partition
{
Name = instance.Name,
Description = instance.Description,
Location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(
instance.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(
instance.Location, instance.InDocument)),
RefreshTime = instance.RefreshTime,
FileFormatSettings = null,
LastFileModifiedTime = instance.LastFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime
};
if (string.IsNullOrEmpty(result.Location))
{
Logger.Warning(nameof(DataPartitionPersistence), instance.Ctx, $"Couldn't find data partition's location for partition {result.Name}.", nameof(ToData));
}
await Utils.ProcessAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
var csvTrait = t2pm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
var csvFormatSettings = Utils.CreateCsvFormatSettings(csvTrait);
if (csvFormatSettings != null)
{
result.FileFormatSettings = csvFormatSettings;
result.FileFormatSettings.Type = "CsvFormatSettings";
}
else
{
Logger.Error(nameof(DataPartitionPersistence), instance.Ctx,
"There was a problem while processing csv format trait inside data partition.");
return(null);
}
}
return(result);
}
private static IDictionary <string, JToken> FillPropertyBagFromCsvTrait(CdmDataPartitionDefinition instance, IDictionary <string, JToken> properties = null)
{
TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
if (properties == null)
{
properties = new Dictionary <string, JToken>();
}
foreach (var csvTraitArg in csvTrait.Arguments)
{
string key;
// map to syms define properties
switch (csvTraitArg.Name)
{
case "columnHeaders":
key = "header";
break;
case "delimiter":
key = "field.delim";
break;
default:
key = csvTraitArg.Value;
break;
}
properties[key] = csvTraitArg.Value;
}
}
return(properties);
}
private static void ErrorMessage(CdmCorpusContext ctx, string methodName, string corpusPath, CdmDataPartitionDefinition obj, bool isIncremental)
{
if (isIncremental)
{
Logger.Error(ctx, Tag, methodName, corpusPath, CdmLogCode.ErrPersistIncrementalConversionError, obj.Name, Constants.IncrementalTraitName, nameof(CdmLocalEntityDeclarationDefinition.DataPartitions));
}
else
{
Logger.Error(ctx, Tag, methodName, corpusPath, CdmLogCode.ErrPersistNonIncrementalConversionError, obj.Name, Constants.IncrementalTraitName, nameof(CdmLocalEntityDeclarationDefinition.IncrementalPartitions));
}
}
public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options)
{
obj.Properties = new Dictionary <string, JToken>();
if (instance.Name != null)
{
obj.Properties["cdm:name"] = instance.Name;
}
if (instance.LastFileStatusCheckTime != null)
{
obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
}
if (instance.LastFileModifiedTime != null)
{
obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
}
if (instance.ExhibitsTraits != null)
{
TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
instance.ExhibitsTraits.Remove("is.partition.format.CSV");
}
if (instance.ExhibitsTraits.Count > 0)
{
obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
});
}
if (csvTrait != null)
{
instance.ExhibitsTraits.Add(csvTrait);
}
}
var properties = FillPropertyBagFromCsvTrait(instance);
if (properties != null)
{
obj.Format = new FormatInfo
{
InputFormat = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat,
OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat,
SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe,
FormatType = FormatType.Csv,
Properties = properties
};
}
else
{
// error
return(null);
}
return(obj);
}