public static void RunAndCompare(XArray input, string inputColumnName, XArray expected, string outputColumnName, string queryText)
{
XDatabaseContext context = new XDatabaseContext
{
RequestedAsOfDateTime = TestAsOfDateTime
};
IXTable query = TableTestHarness.DatabaseContext.FromArrays(input.Count)
.WithColumn(new ColumnDetails(inputColumnName, input.Array.GetType().GetElementType()), input)
.Query(queryText, context);
Func <XArray> resultGetter = query.Columns.Find(outputColumnName).CurrentGetter();
int pageCount;
// Get one row only and verify
pageCount = query.Next(1);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 0, 1), resultGetter(), pageCount);
// Get another (ensure values set again)
pageCount = query.Next(1);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 1, 2), resultGetter(), pageCount);
// Get the rest (ensure arrays expanded when needed)
pageCount = query.Next(expected.Count - 2);
Assert.AreEqual(expected.Count - 2, pageCount);
TableTestHarness.AssertAreEqual(TableTestHarness.Slice(expected, 2, expected.Count), resultGetter(), pageCount);
// Reset and ask for all of them at once
query.Reset();
pageCount = query.Next(expected.Count + 1);
Assert.AreEqual(expected.Count, pageCount);
TableTestHarness.AssertAreEqual(expected, resultGetter(), pageCount);
}
public static void XForm(string xformCommand, int expectedExitCode = 0, XDatabaseContext context = null)
{
if (context == null)
{
context = SampleDatabase.XDatabaseContext;
// Ensure the as-of DateTime is reset for each operation
context.RequestedAsOfDateTime = DateTime.MaxValue;
}
List <string> args = new List <string>();
XqlScanner scanner = new XqlScanner(xformCommand);
while (scanner.Current.Type != TokenType.End)
{
if (scanner.Current.Type == TokenType.Newline)
{
break;
}
args.Add(scanner.Current.Value);
scanner.Next();
}
int result = Program.Run(args.ToArray(), context);
Assert.AreEqual(expectedExitCode, result, $"Unexpected Exit Code for XForm {xformCommand}");
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
// Column and ToType are required
IXColumn column = context.Parser.NextColumn(source, context);
Type toType = context.Parser.NextType();
// ErrorOn, DefaultValue, and ChangeToDefaultOn are optional
ValueKinds errorOn = ValueKindsDefaults.ErrorOn;
object defaultValue = null;
ValueKinds changeToDefaultOn = ValueKindsDefaults.ChangeToDefault;
if (context.Parser.HasAnotherArgument)
{
// Parse ErrorOn if there's another argument
errorOn = context.Parser.NextEnum <ValueKinds>();
// If there's another argument, both of the last two are required
if (context.Parser.HasAnotherArgument)
{
defaultValue = context.Parser.NextLiteralValue();
changeToDefaultOn = context.Parser.NextEnum <ValueKinds>();
}
}
return(CastedColumn.Build(source, column, toType, errorOn, defaultValue, changeToDefaultOn));
}
private static void RoundTrip(string columnName, int[] array, int batchSize = 128)
{
XDatabaseContext context = new XDatabaseContext();
string columnPath = Path.Combine("VariableIntegerReaderWriterTests", columnName);
string columnPrefix = Path.Combine(columnPath, "Vl");
context.StreamProvider.Delete(columnPath);
Directory.CreateDirectory(columnPath);
XArray values = XArray.All(array, array.Length);
using (IColumnWriter writer = new VariableIntegerWriter(context.StreamProvider, columnPrefix))
{
ArraySelector page = ArraySelector.All(0).NextPage(array.Length, batchSize);
while (page.Count > 0)
{
writer.Append(values.Reselect(page));
page = page.NextPage(array.Length, batchSize);
}
}
XArray returned = default(XArray);
using (IColumnReader reader = new VariableIntegerReader(context.StreamProvider, columnPrefix, CachingOption.AsConfigured))
{
returned = reader.Read(ArraySelector.All(array.Length));
}
TableTestHarness.AssertAreEqual(values, returned, array.Length);
context.StreamProvider.Delete(columnPath);
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
if (source != null)
{
throw new ArgumentException($"'read' must be the first stage in a pipeline.");
}
List <IXTable> sources = new List <IXTable>();
// Identify the interval and table name requested
TimeSpan interval = context.Parser.NextTimeSpan();
DateTime rangeStart = context.RequestedAsOfDateTime.Subtract(interval);
string tableName = (string)context.Parser.NextLiteralValue();
// Add rows *just before* each full source in range (the previous full crawl and all incremental ones)
foreach (ItemVersion fullSource in context.StreamProvider.ItemVersions(LocationType.Source, tableName).VersionsInRange(CrawlType.Full, rangeStart, context.RequestedAsOfDateTime))
{
// Ask for the state just before this source
XDatabaseContext historicalContext = new XDatabaseContext(context);
historicalContext.RequestedAsOfDateTime = fullSource.AsOfDate.AddSeconds(-1);
sources.Add(context.Runner.Build(tableName, historicalContext));
}
// Add the last full source and incremental ones up to the RequestedAsOfDateTime
sources.Add(context.Runner.Build(tableName, context));
// Return the source(s) found
if (sources.Count == 1)
{
return(sources[0]);
}
return(new ConcatenatingReader(sources));
}
private static IExpression ParseExpression(string query, IXTable source, XDatabaseContext context)
{
XqlParser parser = new XqlParser(query, context);
context.Parser = parser;
return(parser.NextExpression(source, context));
}
private void Initialize(HttpContext context)
{
string productionFolderPath = ConfigurationManager.AppSettings["XFormProductionFolder"];
if (String.IsNullOrEmpty(productionFolderPath))
{
throw new InvalidOperationException("XForm.IIS requires web.config to contain appSetting 'XFormProductionFolder'.");
}
if (!Path.IsPathRooted(productionFolderPath))
{
productionFolderPath = context.Server.MapPath(productionFolderPath);
}
// Enable XForm Native Acceleration
NativeAccelerator.Enable();
// Build the Database Context
XDatabaseContext db = new XDatabaseContext();
db.RequestedAsOfDateTime = DateTime.MaxValue;
db.StreamProvider = new StreamProviderCache(new LocalFileStreamProvider(productionFolderPath));
db.Runner = new WorkflowRunner(db);
// Build and save an HttpService instance to run queries
s_httpService = new HttpService(db);
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
int rowLimit = context.Parser.NextInteger();
int colLimit = (context.Parser.HasAnotherArgument ? context.Parser.NextInteger() : -1);
return(new Limit(source, rowLimit, colLimit));
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
List <IXColumn> groupByColumns = new List <IXColumn>();
List <IAggregator> aggregators = new List <IAggregator>();
// Parse GroupBy columns
do
{
IXColumn column = context.Parser.NextColumn(source, context);
if (column.IsConstantColumn())
{
throw new ArgumentException("GroupBy can't aggregate across a constant.");
}
groupByColumns.Add(column);
} while (context.Parser.HasAnotherPart && !context.Parser.NextTokenText.Equals("with", StringComparison.OrdinalIgnoreCase));
// If 'GET', parse Aggregators
if (context.Parser.HasAnotherPart)
{
context.Parser.NextSingleKeyword("with");
do
{
aggregators.Add(context.Parser.NextAggregator(source, context));
} while (context.Parser.HasAnotherPart);
}
return(new GroupBy(source, groupByColumns, aggregators));
}
/// <summary>
/// Build constructs an IXTable to write a (potentially multi-partition) Binary Table.
/// </summary>
/// <param name="source">IXTable to write data from</param>
/// <param name="xDatabaseContext">XDatabaseContext for database</param>
/// <param name="tableRootPath">Table Path (Table\[Name]\Full\[UtcDateTime])</param>
/// <returns>IXTable to enumerate to write the table and pass through rows</returns>
public static IXTable Build(IXTable source, XDatabaseContext xDatabaseContext, string tableRootPath)
{
//return new BinaryTableWriter(source, xDatabaseContext, tableRootPath);
// Enable Partitioning
return(new PartitionedBinaryTableWriter(source, xDatabaseContext, tableRootPath));
}
public IExpression NextExpression(IXTable source, XDatabaseContext context)
{
List <IExpression> terms = new List <IExpression>();
// Parse the first term (and any 'AND'ed terms)
terms.Add(NextAndExpression(source, context));
while (HasAnotherArgument)
{
// If this is not an OR, stop
BooleanOperator bOp;
if (!_scanner.Current.Value.TryParseBooleanOperator(out bOp))
{
break;
}
if (bOp != BooleanOperator.Or)
{
break;
}
_scanner.Next();
// Parse the next term
terms.Add(NextAndExpression(source, context));
}
// Return the full expression
if (terms.Count == 1)
{
return(terms[0]);
}
return(new OrExpression(terms.ToArray()));
}
public static IXTable Parse(string xqlQuery, IXTable source, XDatabaseContext outerContext)
{
if (outerContext == null)
{
throw new ArgumentNullException("outerContext");
}
if (outerContext.StreamProvider == null)
{
throw new ArgumentNullException("outerContext.StreamProvider");
}
if (outerContext.Runner == null)
{
throw new ArgumentNullException("outerContext.Runner");
}
// Build an inner context to hold this copy of the parser
XDatabaseContext innerContext = XDatabaseContext.Push(outerContext);
XqlParser parser = new XqlParser(xqlQuery, innerContext);
innerContext.CurrentQuery = xqlQuery;
innerContext.Parser = parser;
// Build the Pipeline
IXTable result = parser.NextQuery(source);
// Copy inner context results back out to the outer context
innerContext.Pop(outerContext);
return(result);
}
public IAggregator NextAggregator(IXTable source, XDatabaseContext context)
{
IAggregatorBuilder builder = null;
ParseNextOrThrow(
() => Factories.AggregatorFactory.TryGetBuilder(_scanner.Current.Value, out builder),
"Aggregator",
TokenType.FunctionName,
Factories.AggregatorFactory.SupportedNames);
_currentlyBuilding.Push(builder);
// Parse the open paren
ParseNextOrThrow(() => true, "(", TokenType.OpenParen);
try
{
IAggregator result = builder.Build(source, context);
// Ensure we've parsed all arguments, and consume the close paren
ParseNextOrThrow(() => true, ")", TokenType.CloseParen);
_currentlyBuilding.Pop();
return(result);
}
catch (Exception ex)
{
Rethrow(ex);
return(null);
}
}
public XqlParser(string xqlQuery, XDatabaseContext workflow)
{
EnsureLoaded();
_scanner = new XqlScanner(xqlQuery);
_workflow = workflow;
_currentlyBuilding = new Stack <IUsage>();
}
public void XqlParser_QueryEvaluation()
{
XDatabaseContext context = SampleDatabase.XDatabaseContext;
IXTable source = context.Query(@"
read WebRequest
cast [ServerPort], Int32
cast [ResponseBytes], Int32, None, 0, InvalidOrNull
");
// Results from WebRequest.20171202.r5.n1000, counts validated against Excel
// Single Term
Assert.AreEqual(423, RunAndCount("where [ServerPort] = 80", source, context));
// OR
Assert.AreEqual(1000, RunAndCount("where [ServerPort] = 80 OR [ServerPort] = 443", source, context));
// AND
Assert.AreEqual(278, RunAndCount("where [ServerPort] = 80 AND [ResponseBytes] > 1000", source, context));
// Precedence: Counts are correct for precedence; default is 'AND' terms evaluate together first
Assert.AreEqual(55, RunAndCount("where [ServerPort] = 80 AND ([ResponseBytes] > 1200 OR [ResponseBytes] < 900)", source, context));
Assert.AreEqual(22 + 95, RunAndCount("where ([ServerPort] = 80 AND [ResponseBytes] > 1200) OR [ResponseBytes] < 900", source, context));
Assert.AreEqual(22 + 95, RunAndCount("where [ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900", source, context));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
// Strings in XForm are stored in the type 'String8'.
// Make a String8Block (like StringBuilder) to allocate space for the new strings for the resolved IP addresses.
String8Block block = new String8Block();
return(SimpleTransformFunction <String8, String8> .Build(Name, source,
context.Parser.NextColumn(source, context, typeof(String8)),
(name8) =>
{
// Convert the value passed for the name column to a string
string machineName = name8.ToString();
// Run the code to do the lookup
string firstIpAddress = LookupName(machineName);
// Convert back to XForm's String8 type
return block.GetCopy(firstIpAddress);
},
() =>
{
// Before each page, clear the String8Block to reuse the memory
block.Clear();
}
));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn left = CastedColumn.Build(source, context.Parser.NextColumn(source, context), typeof(long));
IXColumn right = CastedColumn.Build(source, context.Parser.NextColumn(source, context), typeof(long));
IXArrayComputer computer = new LongComputer();
return(BlockTwoArgumentFunction.Build("Sum", ReturnType, source, left, right, computer.Add));
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
if (source != null)
{
throw new ArgumentException($"'read' must be the first stage in a pipeline.");
}
return(context.Parser.NextTableSource());
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
return(SimpleTransformFunction <String8, String8> .Build(
Name,
source,
context.Parser.NextColumn(source, context, typeof(String8)),
(string8) => string8.Trim()));
}
public void Verb_Choose()
{
XDatabaseContext context = new XDatabaseContext();
int[] rankPattern = new int[] { 2, 3, 1 };
// Build three arrays
int distinctCount = 100;
int length = 3 * distinctCount;
int[] id = new int[length];
int[] rank = new int[length];
int[] value = new int[length];
for (int i = 0; i < length; ++i)
{
// ID is the same for three rows at a time
id[i] = i / 3;
// Rank is [2, 3, 1] repeating (so the middle is the biggest)
rank[i] = rankPattern[i % 3];
// Value is the index of the real row
value[i] = i;
}
// Build the expected results - one for each distinct ID, each with max rank and from the right row
int[] expectedIds = new int[distinctCount];
int[] expectedRanks = new int[distinctCount];
int[] expectedValues = new int[distinctCount];
for (int i = 0; i < distinctCount; ++i)
{
expectedIds[i] = i;
expectedRanks[i] = 3;
expectedValues[i] = 3 * i + 1;
}
IXTable expected = TableTestHarness.DatabaseContext.FromArrays(distinctCount)
.WithColumn("ID", expectedIds)
.WithColumn("Rank", expectedRanks)
.WithColumn("Value", expectedValues);
IXTable actual = TableTestHarness.DatabaseContext.FromArrays(length)
.WithColumn("ID", id)
.WithColumn("Rank", rank)
.WithColumn("Value", value);
// Run and compare (as integer)
TableTestHarness.AssertAreEqual(expected, actual.Query("choose Max [Rank] [ID]", context), distinctCount / 3);
// Run and compare (as String8)
TableTestHarness.AssertAreEqual(
expected.Query("select Cast([ID], String8), Cast([Rank], String8), Cast([Value], String8)", context),
actual.Query("select Cast([ID], String8), Cast([Rank], String8), Cast([Value], String8)", context).Query("choose Max [Rank] [ID]", context),
distinctCount);
}
public void XqlParser_QueryParsing()
{
XDatabaseContext context = SampleDatabase.XDatabaseContext;
IXTable source = context.Query(@"read WebRequest.Typed");
// Single Term
Assert.AreEqual("[ServerPort] = 80", ParseExpression("[ServerPort] = 80", source, context).ToString());
// Column to Column
Assert.AreEqual("[ServerPort] < [RequestBytes]", ParseExpression("[ServerPort] < [RequestBytes]", source, context).ToString());
// Column to Function(Constant)
Assert.AreEqual("[ServerName] = ToUpper(\"ws-front-4\")", ParseExpression("[ServerName] = ToUpper(\"ws-front-4\")", source, context).ToString());
// Column to Function(Column)
Assert.AreEqual("[ServerName] = ToUpper([ServerName])", ParseExpression("[ServerName] = ToUpper([ServerName])", source, context).ToString());
// Compare to null and empty
Assert.AreEqual("[ServerName] = \"\"", ParseExpression("[ServerName] = \"\"", source, context).ToString());
Assert.AreEqual("[ServerName] = null", ParseExpression("[ServerName] = null", source, context).ToString());
// Multiple Clauses, explicit AND
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 900", ParseExpression("[ServerPort] = 80 AND [ResponseBytes] > 900", source, context).ToString());
// Multiple Clauses, implicit AND
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 900", ParseExpression("[ServerPort] = 80 [ResponseBytes] > 900", source, context).ToString());
// AND and OR with no parens, AND is tighter, parens omitted on ToString
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900", ParseExpression("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900", source, context).ToString());
// AND and OR with AND parens, AND is tighter, parens omitted because same as default precedence
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900", ParseExpression("([ServerPort] = 80 AND [ResponseBytes] > 1200) OR [ResponseBytes] < 900", source, context).ToString());
// AND and OR with OR parens, parens on output to maintain evaluation order
Assert.AreEqual("[ServerPort] = 80 AND ([ResponseBytes] > 1200 OR [ResponseBytes] < 900)", ParseExpression("[ServerPort] = 80 AND ([ResponseBytes] > 1200 OR [ResponseBytes] < 900)", source, context).ToString());
// AND after OR [OrExpression parsing falls out correctly]
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900 AND [ServerPort] != 443", ParseExpression("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR [ResponseBytes] < 900 AND [ServerPort] != 443", source, context).ToString());
// NOT
Assert.AreEqual("NOT([ServerPort] = 80)", ParseExpression("NOT([ServerPort] = 80)", source, context).ToString());
// Operators are case insensitive
Assert.AreEqual("[ServerPort] = 80 AND [ResponseBytes] > 1200 OR NOT([ResponseBytes] < 900)", ParseExpression("[ServerPort] = 80 aNd [ResponseBytes] > 1200 oR nOT [ResponseBytes] < 900", source, context).ToString());
// Unclosed quotes shouldn't parse across lines
Assert.AreEqual("[ServerName] != \"8\"", ParseExpression("[ServerName] != \"8\r\nschema", source, context).ToString());
Assert.AreEqual("[ServerName] != \"\"", ParseExpression("[ServerName] != \"\nschema", source, context).ToString());
Assert.AreEqual("[ServerName] != \" \"", ParseExpression("[ServerName] != \" \nschema", source, context).ToString());
// Constant = Constant rule
Verify.Exception <ArgumentException>(() => ParseExpression("80 = 80", source, context));
// String = Quoted Only Constant rule
Verify.Exception <ArgumentException>(() => ParseExpression("[ServerName] = 80", source, context));
Assert.AreEqual("[ServerName] = \"80\"", ParseExpression("[ServerName] = \"80\"", source, context).ToString());
}
private BinaryTableWriter(IXTable source, XDatabaseContext xDatabaseContext, string tableRootPath) : base(source)
{
_xDatabaseContext = xDatabaseContext;
_tableRootPath = tableRootPath;
Metadata = new TableMetadata();
Metadata.Query = xDatabaseContext.CurrentQuery;
// Defer subscribing to columns; if wrapped in a PartitionedBinaryTableWriter, it will take that over.
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
// Set can be evaluated in parallel, so keep parallel
return(source.WrapParallel(context.Parser, (part) =>
new Set(
part,
context.Parser.NextOutputColumnName(part),
context.Parser.NextColumn(part, context))
));
}
public void Where_Parallel()
{
XDatabaseContext historicalContext = new XDatabaseContext(SampleDatabase.XDatabaseContext)
{
RequestedAsOfDateTime = new DateTime(2017, 12, 04, 00, 00, 00, DateTimeKind.Utc)
};
Assert.AreEqual(3000, historicalContext.Query("readRange \"3d\" WebRequest\r\nwhere [ID] != \"\"").Count());
Assert.AreEqual(732, historicalContext.Query("readRange \"3d\" WebRequest\r\nwhere Cast([IsPremiumUser], Boolean) = \"\"").Count());
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
Dictionary <string, string> columnNameMappings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
while (context.Parser.HasAnotherPart)
{
columnNameMappings[context.Parser.NextColumnName(source)] = context.Parser.NextOutputColumnName(source);
}
return(new Rename(source, columnNameMappings));
}
public IXTable Build(IXTable source, XDatabaseContext context)
{
List <string> columnNames = new List <string>();
while (context.Parser.HasAnotherPart)
{
columnNames.Add(context.Parser.NextColumnName(source));
}
return(new Remove(source, columnNames));
}
public PartitionedBinaryTableWriter(IXTable source, XDatabaseContext xDatabaseContext, string tableRootPath) : base(source)
{
_xDatabaseContext = xDatabaseContext;
_tableRootPath = tableRootPath;
_metadata = new TableMetadata();
_metadata.Query = xDatabaseContext.CurrentQuery;
_currentArrays = new XArray[source.Columns.Count];
_getters = _source.Columns.Select((col) => col.CurrentGetter()).ToArray();
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn value = context.Parser.NextColumn(source, context, typeof(String8));
int limit = context.Parser.NextInteger();
return(SimpleTransformFunction <String8, String8> .Build(
Name,
source,
value,
(string8) => Truncate(string8, limit)));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn baseDateTime = context.Parser.NextColumn(source, context, typeof(DateTime));
TimeSpan offsetSpan = context.Parser.NextTimeSpan();
return(SimpleTransformFunction <DateTime, DateTime> .Build(
Name,
source,
baseDateTime,
(dateTime) => dateTime.Add(offsetSpan)
));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn text = context.Parser.NextColumn(source, context, typeof(String8));
string value = context.Parser.NextString();
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
return(SimpleTransformFunction <String8, int> .Build(
Name,
source,
text,
(string8) => string8.IndexOf(value8)));
}
