public int Next(int desiredCount, CancellationToken cancellationToken)
{
// If this is the first call, walk all rows once to find best rows
if (!_isDictionaryBuilt)
{
_isDictionaryBuilt = true;
BuildChooseDictionary(cancellationToken);
_source.Reset();
}
int outerCount;
while ((outerCount = _source.Next(desiredCount, cancellationToken)) > 0)
{
// Ask for the indices of rows which were chosen in this page
XArray chosenRows = _dictionary.GetChosenRows(_totalRowsRead, _totalRowsRead + outerCount, _totalRowsRead);
// Track the total row count (so we know which rows to ask for chosens each time)
_totalRowsRead += outerCount;
// Tell the remapper to filter to chosen rows
_chosenRowsFilter.SetMatches((int[])chosenRows.Array, chosenRows.Count);
if (chosenRows.Count > 0)
{
CurrentRowCount = chosenRows.Count;
return(CurrentRowCount);
}
}
return(0);
}
public static void RunAndCompare(XArray input, string inputColumnName, XArray expected, string outputColumnName, string queryText)
{
XDatabaseContext context = new XDatabaseContext
{
RequestedAsOfDateTime = TestAsOfDateTime
};
IXTable query = TableTestHarness.DatabaseContext.FromArrays(input.Count)
.WithColumn(new ColumnDetails(inputColumnName, input.Array.GetType().GetElementType()), input)
.Query(queryText, context);
Func <XArray> resultGetter = query.Columns.Find(outputColumnName).CurrentGetter();
int pageCount;
// Get one row only and verify
pageCount = query.Next(1);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 0, 1), resultGetter(), pageCount);
// Get another (ensure values set again)
pageCount = query.Next(1);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 1, 2), resultGetter(), pageCount);
// Get the rest (ensure arrays expanded when needed)
pageCount = query.Next(expected.Count - 2);
Assert.AreEqual(expected.Count - 2, pageCount);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 2, expected.Count), resultGetter(), pageCount);
// Reset and ask for all of them at once
query.Reset();
pageCount = query.Next(expected.Count + 1);
Assert.AreEqual(expected.Count, pageCount);
TableTestHarness.AssertAreEqual(expected, resultGetter(), pageCount);
}
public void Reset()
{
_source.Reset();
// If this is a reset, ensure the old writer is Disposed (and flushes output)
if (_writer != null)
{
_writer.Dispose();
_writer = null;
// On Dispose, tell the StreamProvider to publish the table
_streamProvider.Publish(_outputFilePath);
}
}
public static void XTable_All(string configurationLine, int expectedRowCount, string[] requiredColumns = null)
{
SampleDatabase.EnsureBuilt();
int requiredColumnCount = (requiredColumns == null ? 0 : requiredColumns.Length);
long actualRowCount;
using (CancellationTokenSource source = new CancellationTokenSource())
{
IXTable pipeline = null;
ValidatingTable innerValidator = null;
CancellationToken token = source.Token;
try
{
pipeline = SampleDatabase.XDatabaseContext.Load("WebRequest");
innerValidator = new ValidatingTable(pipeline);
pipeline = SampleDatabase.XDatabaseContext.Query(configurationLine, innerValidator);
// Run without requesting any columns. Validate.
actualRowCount = pipeline.RunWithoutDispose(token).RowCount;
Assert.AreEqual(expectedRowCount, actualRowCount, "XTable should return correct count with no requested columns.");
// Reset; Request all columns. Validate.
pipeline.Reset();
pipeline = SampleDatabase.XDatabaseContext.Query("write \"Sample.output.csv\"", pipeline);
actualRowCount = pipeline.RunWithoutDispose(token).RowCount;
}
finally
{
if (pipeline != null)
{
pipeline.Dispose();
pipeline = null;
if (innerValidator != null)
{
Assert.IsTrue(innerValidator.DisposeCalled, "Source must call Dispose on nested sources.");
}
}
}
}
}
/// <summary>
/// Write a table to the Tracing system for debugging.
/// </summary>
public static void TraceWrite(IXTable table, int rowCount = XTableExtensions.DefaultBatchSize)
{
Func <XArray>[] columnGetters = new Func <XArray> [table.Columns.Count];
XArray[] columns = new XArray[table.Columns.Count];
for (int i = 0; i < columns.Length; ++i)
{
columnGetters[i] = table.Columns[i].CurrentGetter();
}
table.Next(rowCount);
for (int i = 0; i < columns.Length; ++i)
{
columns[i] = columnGetters[i]();
}
TraceWrite(columns, table.Columns.Select((col) => col.ColumnDetails).ToArray());
table.Reset();
}
public void Reset()
{
_source.Reset();
}
public void Reset()
{
NextCalled = false;
_inner.Reset();
}
public long Run()
{
long lastCount = 0;
try
{
while (true)
{
Console.Write("> ");
// Read the next query line
string nextLine = Console.ReadLine();
Stopwatch w = Stopwatch.StartNew();
try
{
if (String.IsNullOrEmpty(nextLine))
{
return(lastCount);
}
string[] parts = nextLine.Split(' ');
string command = parts[0].ToLowerInvariant();
switch (command)
{
case "quit":
case "exit":
// Stop on empty, "quit", or "exit"
return(lastCount);
case "back":
case "undo":
// Unwrap on "back" or "undo"
IXTable last = _stages.LastOrDefault();
if (last != null)
{
_pipeline = last;
_stages.RemoveAt(_stages.Count - 1);
_commands.RemoveAt(_commands.Count - 1);
}
break;
case "save":
string tableName = parts[1];
string queryPath = _xDatabaseContext.StreamProvider.Path(LocationType.Query, tableName, ".xql");
_xDatabaseContext.StreamProvider.WriteAllText(queryPath, String.Join(Environment.NewLine, _commands));
Console.WriteLine($"Query saved to \"{tableName}\".");
_commands.Clear();
_commands.Add($"read \"{tableName}\"");
_pipeline = null;
_pipeline = AddStage(_commands[0]);
break;
case "run":
LoadScript(parts[1]);
break;
case "rerun":
LoadScript(s_commandCachePath);
break;
default:
try
{
_pipeline = AddStage(nextLine);
break;
}
catch (Exception ex) when(!Debugger.IsAttached)
{
Console.WriteLine($"Error: {ex.Message}");
continue;
}
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
continue;
}
SaveScript(s_commandCachePath);
// Get the first 10 results and 10 columns
IXTable firstTenWrapper = _pipeline;
firstTenWrapper = _xDatabaseContext.Query("limit 10 10", firstTenWrapper);
firstTenWrapper = _xDatabaseContext.Query("write cout", firstTenWrapper);
lastCount = firstTenWrapper.RunWithoutDispose();
// Get the count
RunResult result = _pipeline.RunUntilTimeout(TimeSpan.FromSeconds(3));
lastCount += result.RowCount;
firstTenWrapper.Reset();
Console.WriteLine();
Console.WriteLine($"{lastCount:n0} rows in {w.Elapsed.ToFriendlyString()}. {(result.IsComplete ? "" : "[incomplete]")}");
Console.WriteLine();
}
}
finally
{
if (_pipeline != null)
{
_pipeline.Dispose();
}
}
}
private static int RunAndCount(string query, IXTable source, XDatabaseContext context)
{
source.Reset();
return((int)context.Query(query, source).RunWithoutDispose());
}
public static void AssertAreEqual(IXTable expected, IXTable actual, int pageSize)
{
// Reset both tables (so they can be used for repeated scenarios)
expected.Reset();
actual.Reset();
// Get the column getters for every expected column and the columns of the same names in actual
Func <XArray>[] expectedGetters = new Func <XArray> [expected.Columns.Count];
Func <XArray>[] actualGetters = new Func <XArray> [actual.Columns.Count];
for (int i = 0; i < expected.Columns.Count; ++i)
{
expectedGetters[i] = expected.Columns[i].CurrentGetter();
actualGetters[i] = actual.Columns.Find(expected.Columns[i].ColumnDetails.Name).CurrentGetter();
}
// Loop over rows, comparing as many rows as available each time
int totalRowCount = 0;
int expectedCurrentCount = 0, expectedNextIndex = 0;
int actualCurrentCount = 0, actualNextIndex = 0;
XArray[] expectedArrays = new XArray[expected.Columns.Count];
XArray[] actualArrays = new XArray[expected.Columns.Count];
while (true)
{
// Get new expected rows if we've compared all of the current ones already
if (expectedNextIndex >= expectedCurrentCount)
{
expectedNextIndex = 0;
expectedCurrentCount = expected.Next(pageSize);
for (int i = 0; i < expected.Columns.Count; ++i)
{
expectedArrays[i] = expectedGetters[i]();
}
}
// Get new actual rows if we've compared all of the current ones already
if (actualNextIndex >= actualCurrentCount)
{
actualNextIndex = 0;
actualCurrentCount = actual.Next(pageSize);
for (int i = 0; i < expected.Columns.Count; ++i)
{
actualArrays[i] = actualGetters[i]();
}
}
// If we're out of rows from both sides, stop
if (expectedCurrentCount == 0 && actualCurrentCount == 0)
{
break;
}
// Figure out how many rows we can compare this time (the minimum available from both sides)
int countToCompare = Math.Min(expectedCurrentCount - expectedNextIndex, actualCurrentCount - actualNextIndex);
string errorMessage = "";
int firstMismatchedRow = -1;
// If we ran out of rows on one side before the other, fail
if (countToCompare == 0)
{
errorMessage = $"Ran out of rows after {totalRowCount + expectedCurrentCount - expectedNextIndex:n0} Expected rows but {totalRowCount + actualCurrentCount - actualNextIndex:n0} Actual rows.";
firstMismatchedRow = Math.Max(expectedCurrentCount - expectedNextIndex, actualCurrentCount - actualNextIndex);
}
else
{
// Get the current xarray for each column, slice to the set of rows to compare, and compare them
for (int i = 0; i < expected.Columns.Count; ++i)
{
XArray expectedArray = expectedArrays[i].Slice(expectedNextIndex, expectedNextIndex + countToCompare);
XArray actualArray = actualArrays[i].Slice(actualNextIndex, actualNextIndex + countToCompare);
firstMismatchedRow = FirstMismatchedRow(
expectedArray,
actualArray,
countToCompare,
expected.Columns[i].ColumnDetails.Name,
out errorMessage);
if (!String.IsNullOrEmpty(errorMessage))
{
break;
}
}
}
// If the table spans weren't equal, show the rows and error message
if (!String.IsNullOrEmpty(errorMessage))
{
Trace.WriteLine("Expected:");
TraceWrite(expectedArrays, expected.Columns.Select((col) => col.ColumnDetails).ToArray(), expectedNextIndex + firstMismatchedRow, expectedCurrentCount - (expectedNextIndex + firstMismatchedRow));
Trace.WriteLine("Actual:");
TraceWrite(actualArrays, expected.Columns.Select((col) => col.ColumnDetails).ToArray(), actualNextIndex + firstMismatchedRow, actualCurrentCount - (actualNextIndex + firstMismatchedRow));
Assert.Fail(errorMessage);
}
expectedNextIndex += countToCompare;
actualNextIndex += countToCompare;
totalRowCount += countToCompare;
}
}
public void Reset()
{
_count = -1;
_source.Reset();
}
public virtual void Reset()
{
_source.Reset();
}
