/// <inheritdoc />
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None)
{
using (Logger.EnterScope(nameof(CdmManifestDefinition), Ctx, nameof(FileStatusCheckAsync)))
{
using (this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace)?.CreateFileQueryCacheContext())
{
DateTimeOffset?modifiedTime = await this.Ctx.Corpus.GetLastModifiedTimeFromObjectAsync(this);
this.LastFileStatusCheckTime = DateTimeOffset.UtcNow;
if (this.LastFileModifiedTime == null)
{
this.LastFileModifiedTime = this._fileSystemModifiedTime;
}
// reload the manifest if it has been updated in the file system
if (modifiedTime != this._fileSystemModifiedTime)
{
await this.Reload();
this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
this._fileSystemModifiedTime = this.LastFileModifiedTime;
}
foreach (var entity in this.Entities)
{
if (entity is CdmReferencedEntityDeclarationDefinition)
{
await entity.FileStatusCheckAsync();
}
else if (entity is CdmLocalEntityDeclarationDefinition)
{
await(entity as CdmLocalEntityDeclarationDefinition).FileStatusCheckAsync(partitionFileStatusCheckType, incrementalType);
}
}
foreach (var subManifest in this.SubManifests)
{
await subManifest.FileStatusCheckAsync();
}
}
}
}
/// <inheritdoc />
public async Task FileStatusCheckAsync(PartitionFileStatusCheckType partitionFileStatusCheckType = PartitionFileStatusCheckType.Full, CdmIncrementalPartitionType incrementalType = CdmIncrementalPartitionType.None)
{
using ((this.Ctx.Corpus.Storage.FetchAdapter(this.InDocument.Namespace) as StorageAdapterBase)?.CreateFileQueryCacheContext())
{
string fullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.EntityPath, this.InDocument);
DateTimeOffset?modifiedTime = await this.Ctx.Corpus.ComputeLastModifiedTimeAsync(fullPath, this);
// check patterns first as this is a more performant way of querying file modification times
// from ADLS and we can cache the times for reuse in the individual partition checks below
if (partitionFileStatusCheckType == PartitionFileStatusCheckType.Full || partitionFileStatusCheckType == PartitionFileStatusCheckType.FullAndIncremental)
{
foreach (var pattern in this.DataPartitionPatterns)
{
if (pattern.IsIncremental)
{
Logger.Error(pattern.Ctx, Tag, nameof(FileStatusCheckAsync), pattern.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
nameof(CdmDataPartitionPatternDefinition), pattern.FetchObjectDefinitionName(), Constants.IncrementalTraitName, nameof(DataPartitionPatterns));
}
else
{
await pattern.FileStatusCheckAsync();
}
}
foreach (var partition in this.DataPartitions)
{
if (partition.IsIncremental)
{
Logger.Error(partition.Ctx, Tag, nameof(FileStatusCheckAsync), partition.AtCorpusPath, CdmLogCode.ErrUnexpectedIncrementalPartitionTrait,
nameof(CdmDataPartitionDefinition), partition.FetchObjectDefinitionName(), Constants.IncrementalTraitName, nameof(DataPartitions));
}
else
{
await partition.FileStatusCheckAsync();
}
}
}
if (partitionFileStatusCheckType == PartitionFileStatusCheckType.Incremental || partitionFileStatusCheckType == PartitionFileStatusCheckType.FullAndIncremental)
{
foreach (var pattern in this.IncrementalPartitionPatterns)
{
if (this.ShouldCallFileStatusCheck(incrementalType, true, pattern))
{
await pattern.FileStatusCheckAsync();
}
}
foreach (var partition in this.IncrementalPartitions)
{
if (this.ShouldCallFileStatusCheck(incrementalType, false, partition))
{
await partition.FileStatusCheckAsync();
}
}
}
// update modified times
this.LastFileStatusCheckTime = DateTimeOffset.UtcNow;
this.LastFileModifiedTime = TimeUtils.MaxTime(modifiedTime, this.LastFileModifiedTime);
await this.ReportMostRecentTimeAsync(this.LastFileModifiedTime);
}
}
