/// <summary>
/// Determine if calling FileStatusCheckAsync on the given pattern or the given partition is needed.
/// </summary>
/// <param name="incrementalType">The incremental type.</param>
/// <param name="isPattern">Whether the object is a pattern object or a partition object.</param>
/// <param name="patternOrPartitionObj">The pattern object if isPattern is true, otherwise the partition object.</param>
private bool ShouldCallFileStatusCheck(CdmIncrementalPartitionType incrementalType, bool isPattern, CdmObjectDefinitionBase patternOrPartitionObj)
{
var update = true;
CdmTraitReference traitRef = patternOrPartitionObj.ExhibitsTraits.Item(Constants.IncrementalTraitName) as CdmTraitReference;
if (traitRef == null)
{
Logger.Error(patternOrPartitionObj.Ctx, Tag, nameof(ShouldCallFileStatusCheck), patternOrPartitionObj.AtCorpusPath, CdmLogCode.ErrMissingIncrementalPartitionTrait,
isPattern ? nameof(CdmDataPartitionPatternDefinition) : nameof(CdmDataPartitionDefinition), patternOrPartitionObj.FetchObjectDefinitionName(),
Constants.IncrementalTraitName, isPattern ? nameof(IncrementalPartitionPatterns) : nameof(IncrementalPartitions));
}
else
{
// None means update by default
if (incrementalType == CdmIncrementalPartitionType.None)
{
return(update);
}
var traitRefIncrementalTypeValue = traitRef.Arguments?.FetchValue("type");
if (traitRefIncrementalTypeValue == null)
{
update = false;
Logger.Error(patternOrPartitionObj.Ctx, Tag, nameof(ShouldCallFileStatusCheck), patternOrPartitionObj.AtCorpusPath, CdmLogCode.ErrTraitArgumentMissing,
"type", Constants.IncrementalTraitName, patternOrPartitionObj.FetchObjectDefinitionName());
}
else if (traitRefIncrementalTypeValue is string == false)
{
update = false;
Logger.Error(patternOrPartitionObj.Ctx, Tag, nameof(ShouldCallFileStatusCheck), patternOrPartitionObj.AtCorpusPath, CdmLogCode.ErrTraitInvalidArgumentValueType,
"type", Constants.IncrementalTraitName, patternOrPartitionObj.FetchObjectDefinitionName());
}
else
{
bool success = Enum.TryParse(traitRefIncrementalTypeValue.ToString(), out CdmIncrementalPartitionType traitRefIncrementalType);
if (success)
{
update = traitRefIncrementalType == incrementalType;
}
else
{
update = false;
Logger.Error(patternOrPartitionObj.Ctx, Tag, nameof(ShouldCallFileStatusCheck), patternOrPartitionObj.AtCorpusPath, CdmLogCode.ErrEnumConversionFailure,
traitRefIncrementalTypeValue, nameof(CdmIncrementalPartitionType),
$"parameter 'type' of trait '{Constants.IncrementalTraitName}' from '{patternOrPartitionObj.FetchObjectDefinitionName()}'");
}
}
}
return(update);
}
/// <summary>
/// Add incremental partition trait "is.partition.incremental" and supplied arguments to the given data partition pattern.
/// </summary>
/// <param name="patternDef"> The data partition pattern. </param>
/// <param name="type"> The CdmIncrementalPartitionType, this is a required argument for the incremental trait. </param>
/// <param name="fullDataPartitionPatternName"> The name of the full data partition pattern name, this is optional. </param>
private static void AddIncrementalPartitionTrait(CdmDataPartitionPatternDefinition patternDef, CdmIncrementalPartitionType type, string fullDataPartitionPatternName = null)
{
var typeTuple = new Tuple <string, dynamic>("type", type.ToString());
var arguments = new List <Tuple <string, dynamic> >()
{
typeTuple
};
if (!string.IsNullOrEmpty(fullDataPartitionPatternName))
{
arguments.Add(new Tuple <string, dynamic>("fullDataPartitionPatternName", fullDataPartitionPatternName));
}
patternDef.ExhibitsTraits.Add("is.partition.incremental", arguments);
}
/// <inheritdoc />
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None)
{
using ((this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace) as StorageAdapterBase)?.CreateFileQueryCacheContext())
{
string fullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.EntityPath, this.InDocument);
DateTimeOffset?modifiedTime = await this.Ctx.Corpus.ComputeLastModifiedTimeAsync(fullPath, this);
// check patterns first as this is a more performant way of querying file modification times
// from ADLS and we can cache the times for reuse in the individual partition checks below
if (partitionFileStatusCheckType == PartitionFileStatusCheckType.Full || partitionFileStatusCheckType == PartitionFileStatusCheckType.FullAndIncremental)
{
foreach (var pattern in this.DataPartitionPatterns)
{
if (pattern.IsIncremental)
{
Logger.Error(pattern.Ctx, Tag, nameof(FileStatusCheckAsync), pattern.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
nameof(CdmDataPartitionPatternDefinition), pattern.FetchObjectDefinitionName(), Constants.IncrementalTraitName, nameof(DataPartitionPatterns));
}
else
{
await pattern.FileStatusCheckAsync();
}
}
foreach (var partition in this.DataPartitions)
{
if (partition.IsIncremental)
{
Logger.Error(partition.Ctx, Tag, nameof(FileStatusCheckAsync), partition.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
nameof(CdmDataPartitionDefinition), partition.FetchObjectDefinitionName(), Constants.IncrementalTraitName, nameof(DataPartitions));
}
else
{
await partition.FileStatusCheckAsync();
}
}
}
if (partitionFileStatusCheckType == PartitionFileStatusCheckType.Incremental || partitionFileStatusCheckType == PartitionFileStatusCheckType.FullAndIncremental)
{
foreach (var pattern in this.IncrementalPartitionPatterns)
{
if (this.ShouldCallFileStatusCheck(incrementalType, true, pattern))
{
await pattern.FileStatusCheckAsync();
}
}
foreach (var partition in this.IncrementalPartitions)
{
if (this.ShouldCallFileStatusCheck(incrementalType, false, partition))
{
await partition.FileStatusCheckAsync();
}
}
}
// update modified times
this.LastFileStatusCheckTime = DateTimeOffset.UtcNow;
this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
await this.ReportMostRecentTimeAsync(this.LastFileModifiedTime);
}
}
/// <inheritdoc />
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None)
{
using (Logger.EnterScope(nameof(CdmManifestDefinition), Ctx, nameof(FileStatusCheckAsync)))
{
using (this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace)?.CreateFileQueryCacheContext())
{
DateTimeOffset?modifiedTime = await this.Ctx.Corpus.GetLastModifiedTimeFromObjectAsync(this);
this.LastFileStatusCheckTime = DateTimeOffset.UtcNow;
if (this.LastFileModifiedTime == null)
{
this.LastFileModifiedTime = this._fileSystemModifiedTime;
}
// reload the manifest if it has been updated in the file system
if (modifiedTime != this._fileSystemModifiedTime)
{
await this.Reload();
this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
this._fileSystemModifiedTime = this.LastFileModifiedTime;
}
foreach (var entity in this.Entities)
{
if (entity is CdmReferencedEntityDeclarationDefinition)
{
await entity.FileStatusCheckAsync();
}
else if (entity is CdmLocalEntityDeclarationDefinition)
{
await(entity as CdmLocalEntityDeclarationDefinition).FileStatusCheckAsync(partitionFileStatusCheckType, incrementalType);
}
}
foreach (var subManifest in this.SubManifests)
{
await subManifest.FileStatusCheckAsync();
}
}
}
}
