public void Database_TryOne()
{
SampleDatabase.EnsureBuilt();
// To debug Main() error handling or argument parsing, run like this:
//XForm("build WebRequest");
// To debug engine execution, run like this:
XqlParser.Parse("read webrequest", null, SampleDatabase.XDatabaseContext).Count();
}
public void Database_Function()
{
SampleDatabase.EnsureBuilt();
SampleDatabase.XDatabaseContext.Query(@"
read WebRequest
set [ClientOsUpper] Trim(ToUpper([ClientOs]))
set [Sample] ToUpper(Trim("" Sample ""))
assert none
where [Sample] != ""SAMPLE""").Count();
}
public void Database_RunAll()
{
SampleDatabase.EnsureBuilt();
// XForm build each source
foreach (string sourceName in SampleDatabase.XDatabaseContext.Runner.SourceNames)
{
XForm($"build {XqlScanner.Escape(sourceName, TokenType.Value)}", ExpectedResult(sourceName));
}
// When one fails, put it by itself in the test below to debug
}
public void Database_Sources()
{
SampleDatabase.EnsureBuilt();
// Validate Database source list as returned by IWorkflowRunner.SourceNames. Don't validate the full list so that as test data is added this test isn't constantly failing.
List <string> sources = new List <string>(SampleDatabase.XDatabaseContext.Runner.SourceNames);
Trace.Write(string.Join("\r\n", sources));
Assert.IsTrue(sources.Contains("WebRequest"), "WebRequest table should exist");
Assert.IsTrue(sources.Contains("WebRequest.Authenticated"), "WebRequest.Authenticated config should exist");
Assert.IsTrue(sources.Contains("WebRequest.Typed"), "WebRequest.Typed config should exist");
Assert.IsTrue(sources.Contains("WebRequest.BigServers"), "WebRequest.BigServers query should exist");
Assert.IsTrue(sources.Contains("WebServer"), "WebServer table should exist");
}
public void Database_ReadRange()
{
SampleDatabase.EnsureBuilt();
// Asking for 2d from 2017-12-04 should get 2017-12-03 and 2017-12-02 crawls
XDatabaseContext historicalContext = new XDatabaseContext(SampleDatabase.XDatabaseContext)
{
RequestedAsOfDateTime = new DateTime(2017, 12, 04, 00, 00, 00, DateTimeKind.Utc)
};
Assert.AreEqual(2000, historicalContext.Query("readRange 2d WebRequest").RunAndDispose());
// Asking for 3d should get all three crawls
Assert.AreEqual(3000, historicalContext.Query("readRange 3d WebRequest").RunAndDispose());
// Asking for 4d should error (no version for the range start)
Verify.Exception <UsageException>(() => historicalContext.Query("readRange 4d WebRequest").RunAndDispose());
}
public void Database_BranchedScenario()
{
SampleDatabase.EnsureBuilt();
// Make a branch of the database in "Database.Branched"
string branchedFolder = Path.Combine(s_RootPath, @"..\Database.Branched");
IStreamProvider branchedStreamProvider = new LocalFileStreamProvider(branchedFolder);
branchedStreamProvider.Delete(".");
IStreamProvider mainStreamProvider = SampleDatabase.XDatabaseContext.StreamProvider;
XDatabaseContext branchedContext = new XDatabaseContext(SampleDatabase.XDatabaseContext);
branchedContext.StreamProvider = new MultipleSourceStreamProvider(branchedStreamProvider, mainStreamProvider, MultipleSourceStreamConfiguration.LocalBranch);
branchedContext.Runner = new WorkflowRunner(branchedContext);
// Ask for WebRequest in the main database; verify built
XForm("build WebRequest", 0);
Assert.IsTrue(mainStreamProvider.Attributes("Table\\WebRequest").Exists);
// Ask for WebRequest in the branch. Verify the main one is loaded where it is
XForm("build WebRequest", 0, branchedContext);
Assert.IsFalse(branchedStreamProvider.Attributes("Table\\WebRequest").Exists);
// Ask for WebRequest.Authenticated in the main database; verify built
XForm("build WebRequest.Authenticated", 0);
Assert.IsTrue(mainStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
Assert.IsFalse(branchedStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
// Make a custom query in the branch. Verify the branched source has a copy with the new query, but it isn't published back
string webRequestAuthenticatedConfigNew = @"
read WebRequest
# Slightly different query
where [UserName] != ""
where [UserName] != null";
branchedStreamProvider.WriteAllText("Config\\WebRequest.Authenticated.xql", webRequestAuthenticatedConfigNew);
XForm("build WebRequest.Authenticated", 0, branchedContext);
Assert.IsTrue(branchedStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
Assert.AreEqual(webRequestAuthenticatedConfigNew, ((BinaryTableReader)branchedContext.Runner.Build("WebRequest.Authenticated", branchedContext)).Query);
Assert.AreNotEqual(webRequestAuthenticatedConfigNew, ((BinaryTableReader)SampleDatabase.XDatabaseContext.Runner.Build("WebRequest.Authenticated", SampleDatabase.XDatabaseContext)).Query);
}
public void Database_Report()
{
SampleDatabase.EnsureBuilt();
ItemVersion latestReport;
// Build WebRequest.tsv. Verify it's a 2017-12-03 version. Verify the TSV is found
XForm($"build WebRequest tsv");
latestReport = SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Report, "WebRequest").LatestBeforeCutoff(CrawlType.Full, DateTime.UtcNow);
Assert.AreEqual(new DateTime(2017, 12, 03, 00, 00, 00, DateTimeKind.Utc), latestReport.AsOfDate);
Assert.IsTrue(SampleDatabase.XDatabaseContext.StreamProvider.Attributes(Path.Combine(latestReport.Path, "Report.tsv")).Exists);
// Ask for a 2017-12-02 report. Verify 2017-12-02 version is created
DateTime cutoff = new DateTime(2017, 12, 02, 11, 50, 00, DateTimeKind.Utc);
XForm($"build WebRequest tsv \"{cutoff:yyyy-MM-dd hh:mm:ssZ}");
latestReport = SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Report, "WebRequest").LatestBeforeCutoff(CrawlType.Full, cutoff);
Assert.AreEqual(new DateTime(2017, 12, 02, 00, 00, 00, DateTimeKind.Utc), latestReport.AsOfDate);
Assert.IsTrue(SampleDatabase.XDatabaseContext.StreamProvider.Attributes(Path.Combine(latestReport.Path, "Report.tsv")).Exists);
}
public void Database_RequestHistorical()
{
SampleDatabase.EnsureBuilt();
// Ask for WebRequest as of 2017-12-02 just before noon. The 2017-12-02 version should be latest
DateTime cutoff = new DateTime(2017, 12, 02, 11, 50, 00, DateTimeKind.Utc);
XForm($"build WebRequest xform \"{cutoff:yyyy-MM-dd hh:mm:ssZ}");
// Verify it has been created
DateTime versionFound = SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Table, "WebRequest").LatestBeforeCutoff(CrawlType.Full, cutoff).AsOfDate;
Assert.AreEqual(new DateTime(2017, 12, 02, 00, 00, 00, DateTimeKind.Utc), versionFound);
// Ask for WebRequest.Authenticated. Verify a 2017-12-02 version is also built for it
XForm($"build WebRequest.Authenticated xform \"{cutoff:yyyy-MM-dd hh:mm:ssZ}");
// Verify it has been created
versionFound = SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Table, "WebRequest.Authenticated").LatestBeforeCutoff(CrawlType.Full, cutoff).AsOfDate;
Assert.AreEqual(new DateTime(2017, 12, 02, 00, 00, 00, DateTimeKind.Utc), versionFound);
}
public void Database_IncrementalSources()
{
SampleDatabase.EnsureBuilt();
XDatabaseContext reportContext = new XDatabaseContext(SampleDatabase.XDatabaseContext);
// Build WebServer as of 2017-11-25; should have 86 original rows, verify built copy cached
reportContext.NewestDependency = DateTime.MinValue;
reportContext.RequestedAsOfDateTime = new DateTime(2017, 11, 25, 00, 00, 00, DateTimeKind.Utc);
Assert.AreEqual(86, SampleDatabase.XDatabaseContext.Runner.Build("WebServer", reportContext).Count());
Assert.AreEqual(new DateTime(2017, 11, 25, 00, 00, 00, DateTimeKind.Utc), SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Table, "WebServer").LatestBeforeCutoff(CrawlType.Full, DateTime.UtcNow).AsOfDate);
// Build WebServer as of 2017-11-27; should have 91 rows (one incremental added)
reportContext.NewestDependency = DateTime.MinValue;
reportContext.RequestedAsOfDateTime = new DateTime(2017, 11, 27, 00, 00, 00, DateTimeKind.Utc);
Assert.AreEqual(91, SampleDatabase.XDatabaseContext.Runner.Build("WebServer", reportContext).Count());
Assert.AreEqual(new DateTime(2017, 11, 27, 00, 00, 00, DateTimeKind.Utc), SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Table, "WebServer").LatestBeforeCutoff(CrawlType.Full, DateTime.UtcNow).AsOfDate);
// Build WebServer as of 2017-11-29; should have 96 rows (two incrementals added)
reportContext.NewestDependency = DateTime.MinValue;
reportContext.RequestedAsOfDateTime = new DateTime(2017, 11, 29, 00, 00, 00, DateTimeKind.Utc);
Assert.AreEqual(96, SampleDatabase.XDatabaseContext.Runner.Build("WebServer", reportContext).Count());
Assert.AreEqual(new DateTime(2017, 11, 29, 00, 00, 00, DateTimeKind.Utc), SampleDatabase.XDatabaseContext.StreamProvider.ItemVersions(LocationType.Table, "WebServer").LatestBeforeCutoff(CrawlType.Full, DateTime.UtcNow).AsOfDate);
}
public void Database_ReadRange()
{
SampleDatabase.EnsureBuilt();
// Asking for 2d from 2017-12-04 should get 2017-12-03 and 2017-12-02 crawls
XDatabaseContext historicalContext = new XDatabaseContext(SampleDatabase.XDatabaseContext)
{
RequestedAsOfDateTime = new DateTime(2017, 12, 04, 00, 00, 00, DateTimeKind.Utc)
};
Assert.AreEqual(2000, historicalContext.Query("readRange 2d WebRequest").Count());
// Asking for 3d should get all three crawls
Assert.AreEqual(3000, historicalContext.Query("readRange 3d WebRequest").Count());
// Asking for 4d should get all three crawls (allowed to ask for beyond first version)
Assert.AreEqual(3000, historicalContext.Query("readRange 4d WebRequest").Count());
// Asking for unknown table should error
Verify.Exception <UsageException>(() => historicalContext.Query("readRange 4d WebRequester").Count());
// Asking for negative TimeSpan should error
Verify.Exception <UsageException>(() => historicalContext.Query("readRange -4d WebRequest").Count());
}