public int Next(int desiredCount, CancellationToken cancellationToken) { // If this is the first call, walk all rows once to find best rows if (!_isDictionaryBuilt) { _isDictionaryBuilt = true; BuildChooseDictionary(cancellationToken); _source.Reset(); } int outerCount; while ((outerCount = _source.Next(desiredCount, cancellationToken)) > 0) { // Ask for the indices of rows which were chosen in this page XArray chosenRows = _dictionary.GetChosenRows(_totalRowsRead, _totalRowsRead + outerCount, _totalRowsRead); // Track the total row count (so we know which rows to ask for chosens each time) _totalRowsRead += outerCount; // Tell the remapper to filter to chosen rows _chosenRowsFilter.SetMatches((int[])chosenRows.Array, chosenRows.Count); if (chosenRows.Count > 0) { CurrentRowCount = chosenRows.Count; return(CurrentRowCount); } } return(0); }
public static void RunAndCompare(XArray input, string inputColumnName, XArray expected, string outputColumnName, string queryText) { XDatabaseContext context = new XDatabaseContext { RequestedAsOfDateTime = TestAsOfDateTime }; IXTable query = TableTestHarness.DatabaseContext.FromArrays(input.Count) .WithColumn(new ColumnDetails(inputColumnName, input.Array.GetType().GetElementType()), input) .Query(queryText, context); Func <XArray> resultGetter = query.Columns.Find(outputColumnName).CurrentGetter(); int pageCount; // Get one row only and verify pageCount = query.Next(1); TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 0, 1), resultGetter(), pageCount); // Get another (ensure values set again) pageCount = query.Next(1); TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 1, 2), resultGetter(), pageCount); // Get the rest (ensure arrays expanded when needed) pageCount = query.Next(expected.Count - 2); Assert.AreEqual(expected.Count - 2, pageCount); TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 2, expected.Count), resultGetter(), pageCount); // Reset and ask for all of them at once query.Reset(); pageCount = query.Next(expected.Count + 1); Assert.AreEqual(expected.Count, pageCount); TableTestHarness.AssertAreEqual(expected, resultGetter(), pageCount); }
public void Reset() { _source.Reset(); // If this is a reset, ensure the old writer is Disposed (and flushes output) if (_writer != null) { _writer.Dispose(); _writer = null; // On Dispose, tell the StreamProvider to publish the table _streamProvider.Publish(_outputFilePath); } }
public static void XTable_All(string configurationLine, int expectedRowCount, string[] requiredColumns = null) { SampleDatabase.EnsureBuilt(); int requiredColumnCount = (requiredColumns == null ? 0 : requiredColumns.Length); long actualRowCount; using (CancellationTokenSource source = new CancellationTokenSource()) { IXTable pipeline = null; ValidatingTable innerValidator = null; CancellationToken token = source.Token; try { pipeline = SampleDatabase.XDatabaseContext.Load("WebRequest"); innerValidator = new ValidatingTable(pipeline); pipeline = SampleDatabase.XDatabaseContext.Query(configurationLine, innerValidator); // Run without requesting any columns. Validate. actualRowCount = pipeline.RunWithoutDispose(token).RowCount; Assert.AreEqual(expectedRowCount, actualRowCount, "XTable should return correct count with no requested columns."); // Reset; Request all columns. Validate. pipeline.Reset(); pipeline = SampleDatabase.XDatabaseContext.Query("write \"Sample.output.csv\"", pipeline); actualRowCount = pipeline.RunWithoutDispose(token).RowCount; } finally { if (pipeline != null) { pipeline.Dispose(); pipeline = null; if (innerValidator != null) { Assert.IsTrue(innerValidator.DisposeCalled, "Source must call Dispose on nested sources."); } } } } }
/// <summary> /// Write a table to the Tracing system for debugging. /// </summary> public static void TraceWrite(IXTable table, int rowCount = XTableExtensions.DefaultBatchSize) { Func <XArray>[] columnGetters = new Func <XArray> [table.Columns.Count]; XArray[] columns = new XArray[table.Columns.Count]; for (int i = 0; i < columns.Length; ++i) { columnGetters[i] = table.Columns[i].CurrentGetter(); } table.Next(rowCount); for (int i = 0; i < columns.Length; ++i) { columns[i] = columnGetters[i](); } TraceWrite(columns, table.Columns.Select((col) => col.ColumnDetails).ToArray()); table.Reset(); }
public void Reset() { _source.Reset(); }
public void Reset() { NextCalled = false; _inner.Reset(); }
public long Run() { long lastCount = 0; try { while (true) { Console.Write("> "); // Read the next query line string nextLine = Console.ReadLine(); Stopwatch w = Stopwatch.StartNew(); try { if (String.IsNullOrEmpty(nextLine)) { return(lastCount); } string[] parts = nextLine.Split(' '); string command = parts[0].ToLowerInvariant(); switch (command) { case "quit": case "exit": // Stop on empty, "quit", or "exit" return(lastCount); case "back": case "undo": // Unwrap on "back" or "undo" IXTable last = _stages.LastOrDefault(); if (last != null) { _pipeline = last; _stages.RemoveAt(_stages.Count - 1); _commands.RemoveAt(_commands.Count - 1); } break; case "save": string tableName = parts[1]; string queryPath = _xDatabaseContext.StreamProvider.Path(LocationType.Query, tableName, ".xql"); _xDatabaseContext.StreamProvider.WriteAllText(queryPath, String.Join(Environment.NewLine, _commands)); Console.WriteLine($"Query saved to \"{tableName}\"."); _commands.Clear(); _commands.Add($"read \"{tableName}\""); _pipeline = null; _pipeline = AddStage(_commands[0]); break; case "run": LoadScript(parts[1]); break; case "rerun": LoadScript(s_commandCachePath); break; default: try { _pipeline = AddStage(nextLine); break; } catch (Exception ex) when(!Debugger.IsAttached) { Console.WriteLine($"Error: {ex.Message}"); continue; } } } catch (ArgumentException ex) { Console.WriteLine(ex.Message); continue; } SaveScript(s_commandCachePath); // Get the first 10 results and 10 columns IXTable firstTenWrapper = _pipeline; firstTenWrapper = _xDatabaseContext.Query("limit 10 10", firstTenWrapper); firstTenWrapper = _xDatabaseContext.Query("write cout", firstTenWrapper); lastCount = firstTenWrapper.RunWithoutDispose(); // Get the count RunResult result = _pipeline.RunUntilTimeout(TimeSpan.FromSeconds(3)); lastCount += result.RowCount; firstTenWrapper.Reset(); Console.WriteLine(); Console.WriteLine($"{lastCount:n0} rows in {w.Elapsed.ToFriendlyString()}. {(result.IsComplete ? "" : "[incomplete]")}"); Console.WriteLine(); } } finally { if (_pipeline != null) { _pipeline.Dispose(); } } }
private static int RunAndCount(string query, IXTable source, XDatabaseContext context) { source.Reset(); return((int)context.Query(query, source).RunWithoutDispose()); }
public static void AssertAreEqual(IXTable expected, IXTable actual, int pageSize) { // Reset both tables (so they can be used for repeated scenarios) expected.Reset(); actual.Reset(); // Get the column getters for every expected column and the columns of the same names in actual Func <XArray>[] expectedGetters = new Func <XArray> [expected.Columns.Count]; Func <XArray>[] actualGetters = new Func <XArray> [actual.Columns.Count]; for (int i = 0; i < expected.Columns.Count; ++i) { expectedGetters[i] = expected.Columns[i].CurrentGetter(); actualGetters[i] = actual.Columns.Find(expected.Columns[i].ColumnDetails.Name).CurrentGetter(); } // Loop over rows, comparing as many rows as available each time int totalRowCount = 0; int expectedCurrentCount = 0, expectedNextIndex = 0; int actualCurrentCount = 0, actualNextIndex = 0; XArray[] expectedArrays = new XArray[expected.Columns.Count]; XArray[] actualArrays = new XArray[expected.Columns.Count]; while (true) { // Get new expected rows if we've compared all of the current ones already if (expectedNextIndex >= expectedCurrentCount) { expectedNextIndex = 0; expectedCurrentCount = expected.Next(pageSize); for (int i = 0; i < expected.Columns.Count; ++i) { expectedArrays[i] = expectedGetters[i](); } } // Get new actual rows if we've compared all of the current ones already if (actualNextIndex >= actualCurrentCount) { actualNextIndex = 0; actualCurrentCount = actual.Next(pageSize); for (int i = 0; i < expected.Columns.Count; ++i) { actualArrays[i] = actualGetters[i](); } } // If we're out of rows from both sides, stop if (expectedCurrentCount == 0 && actualCurrentCount == 0) { break; } // Figure out how many rows we can compare this time (the minimum available from both sides) int countToCompare = Math.Min(expectedCurrentCount - expectedNextIndex, actualCurrentCount - actualNextIndex); string errorMessage = ""; int firstMismatchedRow = -1; // If we ran out of rows on one side before the other, fail if (countToCompare == 0) { errorMessage = $"Ran out of rows after {totalRowCount + expectedCurrentCount - expectedNextIndex:n0} Expected rows but {totalRowCount + actualCurrentCount - actualNextIndex:n0} Actual rows."; firstMismatchedRow = Math.Max(expectedCurrentCount - expectedNextIndex, actualCurrentCount - actualNextIndex); } else { // Get the current xarray for each column, slice to the set of rows to compare, and compare them for (int i = 0; i < expected.Columns.Count; ++i) { XArray expectedArray = expectedArrays[i].Slice(expectedNextIndex, expectedNextIndex + countToCompare); XArray actualArray = actualArrays[i].Slice(actualNextIndex, actualNextIndex + countToCompare); firstMismatchedRow = FirstMismatchedRow( expectedArray, actualArray, countToCompare, expected.Columns[i].ColumnDetails.Name, out errorMessage); if (!String.IsNullOrEmpty(errorMessage)) { break; } } } // If the table spans weren't equal, show the rows and error message if (!String.IsNullOrEmpty(errorMessage)) { Trace.WriteLine("Expected:"); TraceWrite(expectedArrays, expected.Columns.Select((col) => col.ColumnDetails).ToArray(), expectedNextIndex + firstMismatchedRow, expectedCurrentCount - (expectedNextIndex + firstMismatchedRow)); Trace.WriteLine("Actual:"); TraceWrite(actualArrays, expected.Columns.Select((col) => col.ColumnDetails).ToArray(), actualNextIndex + firstMismatchedRow, actualCurrentCount - (actualNextIndex + firstMismatchedRow)); Assert.Fail(errorMessage); } expectedNextIndex += countToCompare; actualNextIndex += countToCompare; totalRowCount += countToCompare; } }
public void Reset() { _count = -1; _source.Reset(); }
public virtual void Reset() { _source.Reset(); }