public IXTable Build(IXTable source, XDatabaseContext context)
{
return(new AssertCommand(source,
context.Parser.NextEnum <AssertType>(),
context));
}
public AssertCommand(IXTable source, AssertType type, XDatabaseContext context) : base(source)
{
_type = type;
_singlePageSource = new SinglePageEnumerator(source);
_assertPipeline = context.Parser.NextQuery(_singlePageSource);
}
public TermExpression(IXTable source, IXColumn left, CompareOperator op, IXColumn right)
{
_evaluate = EvaluateNormal;
// Save arguments as-is for ToString()
_left = left;
_cOp = op;
_right = right;
// Disallow constant <op> constant [likely error not wrapping column name]
if (_left is ConstantColumn && _right is ConstantColumn)
{
throw new ArgumentException($"({left} {op.ToQueryForm()} {right}) is comparing two constants. Wrap [ColumnNames] in braces.");
}
// If the left side is a constant and the operator can be swapped, move it to the right side.
// Comparers can check if the right side is constant and run a faster loop when that's the case.
if (_left.IsConstantColumn() && !(_right.IsConstantColumn()))
{
if (op.TryInvertCompareOperator(out op))
{
_left = right;
_right = left;
}
}
// Disallow unquoted constants used as strings
if (_right.IsConstantColumn() && _left.ColumnDetails.Type == typeof(String8) && _right.ColumnDetails.Type == typeof(String8))
{
ConstantColumn cRight = _right as ConstantColumn;
if (cRight != null && !cRight.IsNull && cRight.WasUnwrappedLiteral)
{
throw new ArgumentException($"{right} is compared to a string, but is unquoted. Strings must be quoted.");
}
}
// Convert the right side to the left side type if required
// This means constants will always be casted to the other side type.
if (_left.ColumnDetails.Type != _right.ColumnDetails.Type)
{
_right = CastedColumn.Build(source, _right, _left.ColumnDetails.Type, ValueKinds.Invalid);
}
// Get the left and right getters
_leftGetter = _left.CurrentGetter();
_rightGetter = _right.CurrentGetter();
// Null comparison is generic
if (_right.IsNullConstant())
{
if (op == CompareOperator.Equal)
{
_comparer = WhereIsNull;
}
else if (op == CompareOperator.NotEqual)
{
_comparer = WhereIsNotNull;
}
else
{
throw new ArgumentException($"Only equals and not equals operators are supported against null.");
}
}
else if (_left.IsNullConstant())
{
_left = _right;
if (op == CompareOperator.Equal)
{
_comparer = WhereIsNull;
}
else if (op == CompareOperator.NotEqual)
{
_comparer = WhereIsNotNull;
}
else
{
throw new ArgumentException($"Only equals and not equals operators are supported against null.");
}
}
else
{
// Get a comparer which can compare the values
_comparer = TypeProviderFactory.Get(left.ColumnDetails.Type).TryGetComparer(op);
if (_comparer == null)
{
throw new ArgumentException($"No comparer found for type {left.ColumnDetails.Type.Name}.");
}
}
// Optimize Enum to Constant comparisons to use the underlying indices
if (_left.IsEnumColumn() && _right.IsConstantColumn())
{
// Get an optimized comparer against the indices rather than values
IXColumn replacedRight = _right;
_comparer = SetComparer.ConvertToEnumIndexComparer(_left, _comparer, ref replacedRight, source);
// Get the indices on the left side
_leftGetter = _left.IndicesCurrentGetter();
// Use the updated value for the right side
_rightGetter = replacedRight.CurrentGetter();
}
// Allow String8 to constant Contains queries to compare on the raw byte[] and int[]
if (op == CompareOperator.Contains && _right.IsConstantColumn() && _left.ColumnDetails.Type == typeof(String8) && !_left.IsEnumColumn())
{
Func <object> rawGetter = _left.ComponentGetter("String8Raw");
if (rawGetter != null)
{
String8 rightValue = (String8)_right.ValuesGetter()().Array.GetValue(0);
String8Comparer string8Comparer = new String8Comparer();
_evaluate = (vector) =>
{
String8Raw raw = (String8Raw)rawGetter();
string8Comparer.WhereContains(raw, rightValue, vector);
};
}
}
}
public IAggregator Build(IXTable source, XDatabaseContext context)
{
return(new CountAggregator());
}
/// <summary>
/// ConvertToEnumIndexComparer takes a comparer on the values of the enum and translates it to a comparer
/// which can operate on the byte[] indices instead.
/// </summary>
/// <param name="leftColumn">EnumColumn being compared</param>
/// <param name="currentComparer">Current Comparison function requested by TermExpression</param>
/// <param name="rightColumn">Constant being compared against</param>
/// <param name="source">IXTable containing comparison</param>
/// <returns>Comparer to compare the (updated) right Constant to the EnumColumn.Indices (rather than Values)</returns>
public static ComparerExtensions.Comparer ConvertToEnumIndexComparer(IXColumn leftColumn, ComparerExtensions.Comparer currentComparer, ref IXColumn rightColumn, IXTable source)
{
Func <XArray> valuesGetter = leftColumn.ValuesGetter();
if (valuesGetter == null)
{
throw new ArgumentException("ConvertToEnumIndexComparer is only valid for columns implementing Values.");
}
// Get all distinct values from the left side
XArray left = valuesGetter();
// If there are no values, return none
if (left.Count == 0)
{
return(None);
}
// Get right side and compare
XArray right = rightColumn.ValuesGetter()();
BitVector set = new BitVector(left.Count);
currentComparer(left, right, set);
// NOTE: When EnumColumn values are sorted, can convert comparisons to non-equality native accelerated compare.
if (set.Count == 0)
{
// If there were no matches, always return none
return(None);
}
else if (set.Count == left.Count)
{
// If everything matched, always return everything
return(All);
}
else if (set.Count == 1)
{
// Convert the constant to the one matching index and make the comparison for index equals that
rightColumn = new ConstantColumn(source, (byte)set.GetSingle(), typeof(byte));
return(TypeProviderFactory.Get(typeof(byte)).TryGetComparer(CompareOperator.Equal));
}
else if (set.Count == left.Count - 1)
{
set.Not();
// Convert the constant to the one non-matching index and make the comparison for index doesn't equal that
rightColumn = new ConstantColumn(source, (byte)set.GetSingle(), typeof(byte));
return(TypeProviderFactory.Get(typeof(byte)).TryGetComparer(CompareOperator.NotEqual));
}
else
{
// Otherwise, build a matcher for values in the set
return(new SetComparer(set).Evaluate);
}
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn column = context.Parser.NextColumn(source, context);
return(new NullTransformFunction(column));
}
/// <summary>
/// Get a specific desired count of rows from the table.
/// </summary>
/// <param name="table">IXTable to enumerate</param>
/// <param name="desiredCount">Maximum number of rows to return</param>
/// <returns>Count of rows returned; zero means no more rows</returns>
public static int Next(this IXTable table, int desiredCount)
{
return(table.Next(desiredCount, default(CancellationToken)));
}
/// <summary>
/// Run an XQL Query on this source and return the query result.
/// </summary>
/// <param name="source">IXTable to query</param>
/// <param name="xqlQuery">XQL query to run</param>
/// <param name="context">XDatabaseContext for loading location, as-of-date, and other context</param>
/// <returns>IXTable of result</returns>
public static IXTable Query(this IXTable source, string xqlQuery, XDatabaseContext context)
{
return(XqlParser.Parse(xqlQuery, source, context));
}
/// <summary>
/// Get the next batch of rows from the table.
/// </summary>
/// <param name="table">IXTable to enumerate</param>
/// <returns>Count of rows returned; zero means no more rows</returns>
public static int Next(this IXTable table)
{
return(table.Next(DefaultBatchSize, default(CancellationToken)));
}
/// <summary>
/// Save the source or query as a binary format table with the given name.
/// </summary>
/// <param name="source">IXTable to save</param>
/// <param name="tableName">Table Name to save table as</param>
/// <param name="context">XDatabaseContext for location to save to, as-of-date of result, and other context</param>
/// <param name="xarraySize">Number of rows to process in each iteration</param>
/// <returns>Row Count Written</returns>
public static long Save(this IXTable source, string tableName, XDatabaseContext context, int xarraySize = DefaultBatchSize)
{
string tableRootPath = context.StreamProvider.Path(LocationType.Table, tableName, CrawlType.Full, context.RequestedAsOfDateTime);
return(new BinaryTableWriter(source, context, tableRootPath).RunAndDispose());
}
/// <summary>
/// Get the count of rows from the source.
/// </summary>
/// <param name="pipeline">IXTable to count</param>
/// <param name="batchSize">Number of rows to process on each iteration</param>
/// <returns>Count of rows in this source or query.</returns>
public static long Count(this IXTable pipeline, CancellationToken cancellationToken = default(CancellationToken), int batchSize = DefaultBatchSize)
{
return(RunAndDispose(pipeline, cancellationToken, batchSize));
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
return(new Set(source,
context.Parser.NextOutputColumnName(source),
context.Parser.NextColumn(source, context)));
}
public long Run()
{
long lastCount = 0;
try
{
while (true)
{
Console.Write("> ");
// Read the next query line
string nextLine = Console.ReadLine();
Stopwatch w = Stopwatch.StartNew();
try
{
if (String.IsNullOrEmpty(nextLine))
{
return(lastCount);
}
string[] parts = nextLine.Split(' ');
string command = parts[0].ToLowerInvariant();
switch (command)
{
case "quit":
case "exit":
// Stop on empty, "quit", or "exit"
return(lastCount);
case "back":
case "undo":
// Unwrap on "back" or "undo"
IXTable last = _stages.LastOrDefault();
if (last != null)
{
_pipeline = last;
_stages.RemoveAt(_stages.Count - 1);
_commands.RemoveAt(_commands.Count - 1);
}
break;
case "save":
string tableName = parts[1];
string queryPath = _xDatabaseContext.StreamProvider.Path(LocationType.Query, tableName, ".xql");
_xDatabaseContext.StreamProvider.WriteAllText(queryPath, String.Join(Environment.NewLine, _commands));
Console.WriteLine($"Query saved to \"{tableName}\".");
_commands.Clear();
_commands.Add($"read \"{tableName}\"");
_pipeline = null;
_pipeline = AddStage(_commands[0]);
break;
case "run":
LoadScript(parts[1]);
break;
case "rerun":
LoadScript(s_commandCachePath);
break;
default:
try
{
_pipeline = AddStage(nextLine);
break;
}
catch (Exception ex) when(!Debugger.IsAttached)
{
Console.WriteLine($"Error: {ex.Message}");
continue;
}
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
continue;
}
SaveScript(s_commandCachePath);
// Get the first 10 results and 10 columns
IXTable firstTenWrapper = _pipeline;
firstTenWrapper = _xDatabaseContext.Query("limit 10 10", firstTenWrapper);
firstTenWrapper = _xDatabaseContext.Query("write cout", firstTenWrapper);
lastCount = firstTenWrapper.Count();
// Get the count
RunResult result = _pipeline.RunUntilTimeout(TimeSpan.FromSeconds(3));
lastCount += result.RowCount;
firstTenWrapper.Reset();
Console.WriteLine();
Console.WriteLine($"{lastCount:n0} rows in {w.Elapsed.ToFriendlyString()}. {(result.IsComplete ? "" : "[incomplete]")}");
Console.WriteLine();
}
}
finally
{
if (_pipeline != null)
{
_pipeline.Dispose();
}
}
}
