public IXTable ReadSource(string tableName, XDatabaseContext context)
{
List <IXTable> sources = new List <IXTable>();
// Find the latest source of this type
ItemVersions sourceVersions = context.StreamProvider.ItemVersions(LocationType.Source, tableName);
ItemVersion latestFullSource = sourceVersions.LatestBeforeCutoff(CrawlType.Full, context.RequestedAsOfDateTime);
// If there are no sources, there's nothing to rebuild from
if (sourceVersions.Versions == null || sourceVersions.Versions.Count == 0)
{
return(null);
}
// Find the latest already converted table
ItemVersions tableVersions = context.StreamProvider.ItemVersions(LocationType.Table, tableName);
ItemVersion latestBuiltTable = tableVersions.LatestBeforeCutoff(CrawlType.Full, context.RequestedAsOfDateTime);
// If no source or table was found, throw
if (latestFullSource == null && latestBuiltTable == null)
{
throw new UsageException(tableName, "[Table]", context.StreamProvider.SourceNames());
}
// Read the Table or the Full Crawl Source, whichever is newer
DateTime incrementalNeededAfterCutoff;
if (latestBuiltTable != null &&
(latestFullSource == null || !IsOutOfDate(latestBuiltTable.AsOfDate, latestFullSource.AsOfDate)) &&
TableMetadataSerializer.UncachedExists(context.StreamProvider, latestBuiltTable.Path))
{
// If the table is current, reuse it
sources.Add(BinaryTableReader.Build(context.StreamProvider, latestBuiltTable.Path));
incrementalNeededAfterCutoff = latestBuiltTable.AsOfDate;
}
else
{
// Otherwise, build a new table from the latest source full crawl
sources.AddRange(context.StreamProvider.Enumerate(latestFullSource.Path, EnumerateTypes.File, true).Select((sa) => new TabularFileReader(context.StreamProvider, sa.Path)));
incrementalNeededAfterCutoff = latestFullSource.AsOfDate;
}
// Add incremental crawls between the full source and the reporting date
DateTime latestComponent = incrementalNeededAfterCutoff;
foreach (ItemVersion incrementalCrawl in sourceVersions.VersionsInRange(CrawlType.Inc, incrementalNeededAfterCutoff, context.RequestedAsOfDateTime))
{
sources.AddRange(context.StreamProvider.Enumerate(incrementalCrawl.Path, EnumerateTypes.File, true).Select((sa) => new TabularFileReader(context.StreamProvider, sa.Path)));
latestComponent = latestComponent.BiggestOf(incrementalCrawl.AsOfDate);
}
// Report the latest incorporated source back
context.NewestDependency = latestComponent;
// Return the source (if a single) or concatenated group (if multiple parts)
return(ConcatenatedTable.Build(sources));
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
if (source != null)
{
throw new ArgumentException($"'read' must be the first stage in a pipeline.");
}
List <IXTable> sources = new List <IXTable>();
// Identify the interval and table name requested
TimeSpan interval = context.Parser.NextTimeSpan();
if (interval < TimeSpan.Zero)
{
throw new ArgumentException($"'interval' must be positive. (For last 7 days, use '7d')");
}
string tableName = (string)context.Parser.NextLiteralValue();
// Determine the range of versions to include (from the as of date or now if not provided)
DateTime rangeEnd = (context.RequestedAsOfDateTime == DateTime.MaxValue ? DateTime.UtcNow : context.RequestedAsOfDateTime);
DateTime rangeStart = rangeEnd.Subtract(interval);
// Find versions available
ItemVersions versions = context.StreamProvider.ItemVersions(LocationType.Source, tableName);
if (versions.Versions.Count == 0)
{
versions = context.StreamProvider.ItemVersions(LocationType.Table, tableName);
}
if (versions.Versions.Count == 0)
{
throw new ArgumentException($"'{tableName}' was not found as a Source or Table.");
}
// Find the first version to include (if any) - the last full version before the start was 'current' at the start moment
ItemVersion previous = versions.LatestBeforeCutoff(CrawlType.Full, rangeStart);
XDatabaseContext historicalContext;
foreach (ItemVersion version in versions.VersionsInRange(CrawlType.Full, rangeStart, rangeEnd))
{
// Add the version before this one, if any (including any incremental pieces)
if (previous != null)
{
historicalContext = new XDatabaseContext(context);
historicalContext.RequestedAsOfDateTime = version.AsOfDate.AddSeconds(-1);
sources.Add(context.Runner.Build(tableName, historicalContext));
}
previous = version;
}
// Add 'last' up to the requested moment
historicalContext = new XDatabaseContext(context);
historicalContext.RequestedAsOfDateTime = rangeEnd;
sources.Add(context.Runner.Build(tableName, historicalContext));
// Communicate the latest component as of date back to the builder
historicalContext.Pop(context);
// Return the source(s) found
return(ConcatenatedTable.Build(sources));
}
