public void Dispose()
{
if (_source != null)
{
_source.Dispose();
_source = null;
}
if (_writer != null)
{
try
{
_writer.Dispose();
// On Dispose, tell the StreamProvider to publish the table
if (_streamProvider != null)
{
_streamProvider.Publish(_outputFilePath);
}
}
finally
{
_writer = null;
}
}
}
public static string Build(string tableName, XDatabaseContext context, string outputFormat)
{
IXTable builder = null;
try
{
Stopwatch w = Stopwatch.StartNew();
// Reset the dependency DateTime check
context.NewestDependency = DateTime.MinValue;
// Recursively build dependencies and return a reader for the result table
builder = context.Runner.Build(tableName, context);
string outputPath = null;
if ((String.IsNullOrEmpty(outputFormat) || outputFormat.Equals("xform", StringComparison.OrdinalIgnoreCase)) && builder is BinaryTableReader)
{
// If the binary format was requested and we already built it, return the path written
outputPath = ((BinaryTableReader)builder).TablePath;
}
else
{
ItemVersion latestReportVersion = context.StreamProvider.ItemVersions(LocationType.Report, tableName).LatestBeforeCutoff(CrawlType.Full, context.RequestedAsOfDateTime);
if (latestReportVersion != null)
{
outputPath = Path.Combine(context.StreamProvider.Path(LocationType.Report, tableName, CrawlType.Full, latestReportVersion.AsOfDate), $"Report.{outputFormat}");
}
if (latestReportVersion == null || latestReportVersion.AsOfDate < context.NewestDependency || context.RebuiltSomething || !context.StreamProvider.Attributes(outputPath).Exists)
{
// If the report needs to be rebuilt, make it and return the path
outputPath = Path.Combine(context.StreamProvider.Path(LocationType.Report, tableName, CrawlType.Full, context.NewestDependency), $"Report.{outputFormat}");
if (outputFormat.Equals("xform", StringComparison.OrdinalIgnoreCase))
{
BinaryTableWriter.Build(builder, context, outputPath).RunAndDispose();
}
else
{
new TabularFileWriter(builder, context.StreamProvider, outputPath).RunAndDispose();
}
context.RebuiltSomething = true;
}
}
w.Stop();
Trace.WriteLine($"Done. \"{outputPath}\" {(context.RebuiltSomething ? "written" : "up-to-date")} in {w.Elapsed.ToFriendlyString()}.");
return(outputPath);
}
finally
{
if (builder != null)
{
builder.Dispose();
builder = null;
}
}
}
public void Dispose()
{
if (_source != null)
{
_source.Dispose();
_source = null;
}
}
public void Dispose()
{
DisposeCalled = true;
if (_inner != null)
{
_inner.Dispose();
_inner = null;
}
}
private void CountWithinTimeout(string query, TimeSpan timeout, DateTime asOfDate, IHttpResponse response)
{
IXTable pipeline = null;
try
{
XDatabaseContext context = _xDatabaseContext;
// Build for another moment in time if requested
if (asOfDate != _xDatabaseContext.RequestedAsOfDateTime)
{
context = new XDatabaseContext(_xDatabaseContext)
{
RequestedAsOfDateTime = asOfDate
};
}
// Build a Pipeline for the Query
pipeline = context.Query(query);
// If there was no query, return an empty result
if (pipeline == null)
{
return;
}
// Try to get the count up to the timeout
if (Debugger.IsAttached)
{
timeout = TimeSpan.MaxValue;
}
RunResult result = pipeline.RunUntilTimeout(timeout);
using (ITabularWriter writer = WriterForFormat("json", response))
{
writer.SetColumns(new string[] { "Count", "IsComplete", "RuntimeMs" });
writer.Write(result.RowCount);
writer.Write(result.IsComplete);
writer.Write((int)result.Elapsed.TotalMilliseconds);
writer.NextRow();
}
}
finally
{
if (pipeline != null)
{
pipeline.Dispose();
pipeline = null;
}
}
}
public void Dispose()
{
if (_source != null)
{
_source.Dispose();
_source = null;
}
if (_joinToSource != null)
{
_joinToSource.Dispose();
_joinToSource = null;
}
}
public static void XTable_All(string configurationLine, int expectedRowCount, string[] requiredColumns = null)
{
SampleDatabase.EnsureBuilt();
int requiredColumnCount = (requiredColumns == null ? 0 : requiredColumns.Length);
long actualRowCount;
using (CancellationTokenSource source = new CancellationTokenSource())
{
IXTable pipeline = null;
ValidatingTable innerValidator = null;
CancellationToken token = source.Token;
try
{
pipeline = SampleDatabase.XDatabaseContext.Load("WebRequest");
innerValidator = new ValidatingTable(pipeline);
pipeline = SampleDatabase.XDatabaseContext.Query(configurationLine, innerValidator);
// Run without requesting any columns. Validate.
actualRowCount = pipeline.RunWithoutDispose(token).RowCount;
Assert.AreEqual(expectedRowCount, actualRowCount, "XTable should return correct count with no requested columns.");
// Reset; Request all columns. Validate.
pipeline.Reset();
pipeline = SampleDatabase.XDatabaseContext.Query("write \"Sample.output.csv\"", pipeline);
actualRowCount = pipeline.RunWithoutDispose(token).RowCount;
}
finally
{
if (pipeline != null)
{
pipeline.Dispose();
pipeline = null;
if (innerValidator != null)
{
Assert.IsTrue(innerValidator.DisposeCalled, "Source must call Dispose on nested sources.");
}
}
}
}
}
public long Run()
{
long lastCount = 0;
try
{
while (true)
{
Console.Write("> ");
// Read the next query line
string nextLine = Console.ReadLine();
Stopwatch w = Stopwatch.StartNew();
try
{
if (String.IsNullOrEmpty(nextLine))
{
return(lastCount);
}
string[] parts = nextLine.Split(' ');
string command = parts[0].ToLowerInvariant();
switch (command)
{
case "quit":
case "exit":
// Stop on empty, "quit", or "exit"
return(lastCount);
case "back":
case "undo":
// Unwrap on "back" or "undo"
IXTable last = _stages.LastOrDefault();
if (last != null)
{
_pipeline = last;
_stages.RemoveAt(_stages.Count - 1);
_commands.RemoveAt(_commands.Count - 1);
}
break;
case "save":
string tableName = parts[1];
string queryPath = _xDatabaseContext.StreamProvider.Path(LocationType.Query, tableName, ".xql");
_xDatabaseContext.StreamProvider.WriteAllText(queryPath, String.Join(Environment.NewLine, _commands));
Console.WriteLine($"Query saved to \"{tableName}\".");
_commands.Clear();
_commands.Add($"read \"{tableName}\"");
_pipeline = null;
_pipeline = AddStage(_commands[0]);
break;
case "run":
LoadScript(parts[1]);
break;
case "rerun":
LoadScript(s_commandCachePath);
break;
default:
try
{
_pipeline = AddStage(nextLine);
break;
}
catch (Exception ex) when(!Debugger.IsAttached)
{
Console.WriteLine($"Error: {ex.Message}");
continue;
}
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
continue;
}
SaveScript(s_commandCachePath);
// Get the first 10 results and 10 columns
IXTable firstTenWrapper = _pipeline;
firstTenWrapper = _xDatabaseContext.Query("limit 10 10", firstTenWrapper);
firstTenWrapper = _xDatabaseContext.Query("write cout", firstTenWrapper);
lastCount = firstTenWrapper.RunWithoutDispose();
// Get the count
RunResult result = _pipeline.RunUntilTimeout(TimeSpan.FromSeconds(3));
lastCount += result.RowCount;
firstTenWrapper.Reset();
Console.WriteLine();
Console.WriteLine($"{lastCount:n0} rows in {w.Elapsed.ToFriendlyString()}. {(result.IsComplete ? "" : "[incomplete]")}");
Console.WriteLine();
}
}
finally
{
if (_pipeline != null)
{
_pipeline.Dispose();
}
}
}