public static void XForm(string xformCommand, int expectedExitCode = 0, XDatabaseContext context = null)
{
if (context == null)
{
context = SampleDatabase.XDatabaseContext;
// Ensure the as-of DateTime is reset for each operation
context.RequestedAsOfDateTime = DateTime.MaxValue;
}
List <string> args = new List <string>();
XqlScanner scanner = new XqlScanner(xformCommand);
while (scanner.Current.Type != TokenType.End)
{
if (scanner.Current.Type == TokenType.Newline)
{
break;
}
args.Add(scanner.Current.Value);
scanner.Next();
}
int result = Program.Run(args.ToArray(), context);
Assert.AreEqual(expectedExitCode, result, $"Unexpected Exit Code for XForm {xformCommand}");
}
public void Database_RunAll()
{
SampleDatabase.EnsureBuilt();
// XForm build each source
foreach (string sourceName in SampleDatabase.XDatabaseContext.Runner.SourceNames)
{
XForm($"build {XqlScanner.Escape(sourceName, TokenType.Value)}", ExpectedResult(sourceName));
}
// When one fails, put it by itself in the test below to debug
}
private static string TestSplitAndJoin(string xqlLine)
{
XqlScanner scanner = new XqlScanner(xqlLine);
List <string> parts = new List <string>();
while (scanner.Current.Type != TokenType.End)
{
parts.Add(scanner.Current.Value);
scanner.Next();
}
return(string.Join("|", parts));
}
public override string ToString()
{
if (_defaultValue != null && _changeToDefaultKinds != ValueKinds.InvalidOrNull)
{
return($"Cast({_column}, {_targetType.Name}, {_errorOnKinds}, {XqlScanner.Escape(_defaultValue, TokenType.Value)}, {_changeToDefaultKinds})");
}
else if (_errorOnKinds != ValueKinds.None)
{
return($"Cast({_column}, {_targetType.Name}, {_errorOnKinds})");
}
else
{
return($"Cast({_column}, {_targetType.Name})");
}
}
public void Current()
{
ColumnCache.IsEnabled = true;
string query;
// Still want to tune schema requests. Current bottleneck: LatestBeforeCutoff
//query = @"
// read WebRequest
// schema
//";
//query = @"
// read Asset.Extended.Release.Typed
// where [Asset_SourceID] = 29 AND [IsBaseline] = 0";
//query = @"
// read Identity.StandingAdmins.AssetAdminSummary.V0
// join [Name] Asset.Extended.Release [Name] Asset. ";
//query = @"
// read WebRequestHuge
// peek [DataCenter]";
//query = @"
// read WebRequestHuge.Typed
// where [EventTime] = null
// count";
query = @"
read WebRequest
where [ClientBrowser]: ""Edge""
choose Max [ResponseBytes] [ClientOS]
limit 10
";
string singleLineQuery = XqlScanner.QueryToSingleLineStyle(query);
using (Benchmarker b = new Benchmarker(singleLineQuery, 10000))
{
b.Measure($"XForm", 1, () =>
{
return(Context.Query(query).Count());
});
}
}
public override string ToString()
{
return(XqlScanner.Escape(Value.ToString(), TokenType.Value, (IsNull || ColumnDetails.Type != typeof(String8))));
}
public override string ToString()
{
return(XqlScanner.Escape((_array.GetValue(0) ?? "null").ToString(), TokenType.Value, (_isNull || ColumnDetails.Type != typeof(String8))));
}
public IXTable Build(string tableName, XDatabaseContext outerContext, bool deferred)
{
// Validate the source name is recognized
if (!Sources.Contains(tableName))
{
// If it wasn't in cache, check individually for it live
if (!XDatabaseContext.StreamProvider.ContainsTable(tableName))
{
throw new UsageException(tableName, "Table", Sources);
}
// If found, update the cache
UpdateSources();
}
// If only a Date was passed for AsOfDate, look for the last version as of that day
if (outerContext.RequestedAsOfDateTime.TimeOfDay == TimeSpan.Zero)
{
outerContext.RequestedAsOfDateTime = outerContext.RequestedAsOfDateTime.AddDays(1).AddSeconds(-1);
}
// If we previously found the latest for this table, just return it again
LatestTableForCutoff previousLatest;
if (_currentTableVersions.TryGet(tableName, out previousLatest) &&
previousLatest.Cutoff >= outerContext.RequestedAsOfDateTime &&
previousLatest.TableVersion.AsOfDate <= outerContext.RequestedAsOfDateTime &&
TableMetadataSerializer.UncachedExists(outerContext.StreamProvider, previousLatest.TableVersion.Path))
{
outerContext.NewestDependency = previousLatest.TableVersion.AsOfDate;
return(BinaryTableReader.Build(outerContext.StreamProvider, previousLatest.TableVersion.Path));
}
// Create a context to track what we're building now
XDatabaseContext innerContext = XDatabaseContext.Push(outerContext);
innerContext.CurrentTable = tableName;
// If this is a query, there won't be a cached table - just build a pipeline to make it
StreamAttributes queryAttributes = innerContext.StreamProvider.Attributes(innerContext.StreamProvider.Path(LocationType.Query, tableName, ".xql"));
if (queryAttributes.Exists)
{
IXTable queryPipeline = innerContext.Query(innerContext.StreamProvider.ReadAllText(queryAttributes.Path));
innerContext.Pop(outerContext);
return(queryPipeline);
}
// Find the latest already built result, and associated query
ItemVersions tableVersions = innerContext.StreamProvider.ItemVersions(LocationType.Table, tableName);
ItemVersion latestTable = tableVersions.LatestBeforeCutoff(CrawlType.Full, outerContext.RequestedAsOfDateTime);
string latestTableQuery = null;
if (latestTable != null)
{
latestTableQuery = TableMetadataSerializer.Read(outerContext.StreamProvider, latestTable.Path).Query;
}
// Set the dependency date to the latest table we've already built (if any)
innerContext.NewestDependency = (latestTable == null ? DateTime.MinValue : latestTable.AsOfDate);
// Determine the XQL to build the table and construct a builder which can do so
string xql;
IXTable builder;
// Find the config to build the table and scan dependency versions to determine whether table is out-of-date
StreamAttributes configAttributes = innerContext.StreamProvider.Attributes(innerContext.StreamProvider.Path(LocationType.Config, tableName, ".xql"));
if (!configAttributes.Exists)
{
// If this is a simple source, just reading it is how to build it
xql = $"read {XqlScanner.Escape(tableName, TokenType.Value)}";
// Build a reader concatenating all needed pieces
builder = ReadSource(tableName, innerContext);
}
else
{
// If there is a config, the config is how to build it
xql = innerContext.StreamProvider.ReadAllText(configAttributes.Path);
// Build a pipeline for the query, recursively creating dependencies
builder = innerContext.Query(xql);
}
// If we don't have the table or the source, we have to throw
if (latestTable == null && builder == null)
{
throw new UsageException(tableName, "Table", innerContext.StreamProvider.Tables());
}
// Get the path we're either reading or building
string tablePath = innerContext.StreamProvider.Path(LocationType.Table, tableName, CrawlType.Full, innerContext.NewestDependency);
// If we can rebuild this table and we need to (sources rebuilt, query changed, out-of-date, deleted), rebuild it
if (builder != null)
{
if (latestTable == null ||
innerContext.RebuiltSomething ||
(latestTableQuery != null && xql != latestTableQuery) ||
IsOutOfDate(latestTable.AsOfDate, innerContext.NewestDependency) ||
!TableMetadataSerializer.UncachedExists(outerContext.StreamProvider, latestTable.Path))
{
// If we're not running now, just return how to build it
if (deferred)
{
return(builder);
}
// Otherwise, build it now; we'll return the query to read the output
innerContext.CurrentQuery = xql;
Trace.WriteLine($"COMPUTE: [{innerContext.NewestDependency.ToString(StreamProviderExtensions.DateTimeFolderFormat)}] {tableName}");
BinaryTableWriter.Build(builder, innerContext, tablePath).RunAndDispose();
innerContext.RebuiltSomething = true;
}
}
// Report the newest dependency in this chain to the components above
innerContext.Pop(outerContext);
_currentTableVersions.Add(tableName, new LatestTableForCutoff(outerContext.RequestedAsOfDateTime, new ItemVersion(LocationType.Table, tableName, CrawlType.Full, innerContext.NewestDependency)));
return(BinaryTableReader.Build(innerContext.StreamProvider, tablePath));
}
public override string ToString()
{
return(XqlScanner.Escape(this.ColumnDetails.Name, TokenType.ColumnName));
}
public override string ToString()
{
return($"{_column} AS {XqlScanner.Escape(ColumnDetails.Name, TokenType.ColumnName)}");
}