public DataBlockResult Compute(Partition p)
{
if (p == null)
{
throw new ArgumentNullException("p");
}
DataBlockResult result = new DataBlockResult(this);
// Find matches for the remaining query
ShortSet baseQueryMatches = new ShortSet(p.Count);
this.Where.TryEvaluate(p, baseQueryMatches, result.Details);
// Find and count matches per column for the term in the outer query
List <Tuple <string, int> > matchCountPerColumn = new List <Tuple <string, int> >();
if (baseQueryMatches.Count() > 0)
{
TermExpression bareTerm = new TermExpression(this.Term);
ShortSet termMatchesForColumn = new ShortSet(p.Count);
bool succeeded = false;
ExecutionDetails perColumnDetails = new ExecutionDetails();
foreach (IColumn <object> column in p.Columns.Values)
{
termMatchesForColumn.Clear();
perColumnDetails.Succeeded = true;
column.TryWhere(Operator.Matches, this.Term, termMatchesForColumn, perColumnDetails);
succeeded |= perColumnDetails.Succeeded;
termMatchesForColumn.And(baseQueryMatches);
ushort matchCount = termMatchesForColumn.Count();
if (matchCount > 0)
{
matchCountPerColumn.Add(new Tuple <string, int>(column.Name, (int)matchCount));
}
}
// Sort results by count of matches descending
matchCountPerColumn.Sort((left, right) => right.Item2.CompareTo(left.Item2));
}
// Copy to a DataBlock and return it
int index = 0;
DataBlock block = new DataBlock(new string[] { "ColumnName", "Count" }, matchCountPerColumn.Count);
foreach (var column in matchCountPerColumn)
{
block[index, 0] = column.Item1;
block[index, 1] = column.Item2;
index++;
}
result.Values = block;
result.Total = baseQueryMatches.Count();
return(result);
}
public void TypedColumn_Boolean_Basic()
{
IColumn <object> c = ColumnFactory.Build(new ColumnDetails("IsDuplicate", "bool", true), 0);
c.SetSize(5);
c[0] = Value.Create(true);
c[1] = Value.Create(false);
c[2] = Value.Create("True");
c[3] = Value.Create("false");
CommitIfRequired(c);
ShortSet set = new ShortSet(c.Count);
// True - set and default
set.Clear();
c.TryWhere(Operator.Equals, "true", set, null);
Assert.AreEqual("0, 2, 4", String.Join(", ", set.Values));
// False - set
set.Clear();
c.TryWhere(Operator.Equals, "false", set, null);
Assert.AreEqual("1, 3", String.Join(", ", set.Values));
// False (Matches)
set.Clear();
c.TryWhere(Operator.Matches, "false", set, null);
Assert.AreEqual("1, 3", String.Join(", ", set.Values));
// Not False
set.Clear();
c.TryWhere(Operator.NotEquals, "false", set, null);
Assert.AreEqual("0, 2, 4", String.Join(", ", set.Values));
// Values works, *including if not asking for all items in order*
bool[] values = (bool[])c.GetValues(new ushort[] { 3, 2, 1, 0 });
Assert.AreEqual("False, True, False, True", String.Join(", ", values));
}
public void ShortSet_Performance_Set()
{
// Goal: Set operations are <10k instructions, so at 2M instructions per millisecond, 200 per millisecond (Release build)
// Set operations are used to combine where clauses and sets for specific words when word searching.
Random r = new Random();
ShortSet s1 = BuildRandom(ushort.MaxValue, 1000, r);
ShortSet s2 = BuildRandom(ushort.MaxValue, 10000, r);
ShortSet s3 = BuildRandom(ushort.MaxValue, 50000, r);
ushort[] s4 = { 1, 126, 950, 1024, 1025, 1670, 19240 };
ShortSet scratch = new ShortSet(ushort.MaxValue);
// 9 Operations x 10k iterations = 90k operations.
// Goal is 100ms.
int iterations = 2500;
Stopwatch w = Stopwatch.StartNew();
for (int i = 0; i < iterations; ++i)
{
// Singleton Operations / Reset
scratch.Not();
scratch.Clear();
scratch.Or(s1);
// Enumerable Operations
scratch.And(s4);
scratch.Or(s4);
scratch.AndNot(s4);
// ShortSet Operations
scratch.Or(s2);
scratch.And(s3);
scratch.AndNot(s2);
}
int operations = (9 * iterations);
double milliseconds = w.ElapsedMilliseconds;
double operationsPerMillisecond = operations / milliseconds;
Trace.Write(String.Format("{0:n0} operations in {1:n0} milliseconds; {2:n0} per millisecond.", operations, milliseconds, operationsPerMillisecond));
Assert.IsTrue(operationsPerMillisecond > 100, "Not within 200% of goal.");
}
public void TryEvaluate(Partition partition, ShortSet result, ExecutionDetails details)
{
if (details == null)
{
throw new ArgumentNullException("details");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (partition == null)
{
throw new ArgumentNullException("partition");
}
// Include all items - clear any and then add everything.
// ShortSet will scope the set to the ID range valid within the data set.
result.Clear();
result.Not();
}
public void ShortSet_MismatchedCapacities()
{
ShortSet s1 = new ShortSet(10);
s1.Add(1);
s1.Add(3);
ShortSet s2 = new ShortSet(20);
s2.Add(2);
s2.Add(4);
s2.Add(10);
// Verify values below common capacity are set, larger values not set.
s1.Or(s2);
Assert.AreEqual("1, 2, 3, 4", String.Join(", ", s1.Values));
// Verify values above common capacity are left alone.
s2.Or(s1);
Assert.AreEqual("1, 2, 3, 4, 10", String.Join(", ", s2.Values));
// Verify values below common capacity are cleared, values above not unexpectedly set.
s1.AndNot(s2);
Assert.AreEqual("", String.Join(", ", s1.Values));
// Verify values above common capacity are left set
s1.Clear();
s1.Add(1);
s1.Add(3);
s2.AndNot(s1);
Assert.AreEqual("2, 4, 10", String.Join(", ", s2.Values));
// Verify values above common capacity are not set
s2.Or(s1);
s1.And(s2);
Assert.AreEqual("1, 3", String.Join(", ", s1.Values));
// Verify values above common capacity *are* cleared)
s2.And(s1);
Assert.AreEqual("1, 3", String.Join(", ", s2.Values));
}
public void TryEvaluate(Partition partition, ShortSet result, ExecutionDetails details)
{
if (partition == null)
{
throw new ArgumentNullException("partition");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
ushort itemCount = partition.Count;
ShortSet expressionResults = null;
ShortSet partResults = new ShortSet(itemCount);
foreach (IExpression part in _set)
{
partResults.Clear();
part.TryEvaluate(partition, partResults, details);
if (expressionResults == null)
{
expressionResults = new ShortSet(itemCount);
expressionResults.Or(partResults);
}
else
{
expressionResults.And(partResults);
}
if (expressionResults.IsEmpty())
{
break;
}
}
result.Or(expressionResults);
}
public AggregationResult Compute(Partition p)
{
if (p == null)
{
throw new ArgumentNullException("p");
}
Stopwatch w = Stopwatch.StartNew();
AggregationResult result = new AggregationResult(this);
result.AggregationContext = this.Aggregator.CreateContext();
// Get any columns passed to the aggregation function
IUntypedColumn[] columns = null;
if (this.AggregationColumns != null)
{
columns = new IUntypedColumn[this.AggregationColumns.Length];
for (int i = 0; i < this.AggregationColumns.Length; ++i)
{
string columnName = this.AggregationColumns[i];
if (!p.Columns.TryGetValue(columnName, out columns[i]))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, columnName);
return(result);
}
}
}
// Find the number of dimensions and number of "cells" for which we'll aggregate
List <string> resultBlockColumns = new List <string>();
int rowCount = 1;
for (int i = 0; i < this.Dimensions.Count; ++i)
{
AggregationDimension dimension = this.Dimensions[i];
if (!String.IsNullOrEmpty(dimension.Name))
{
resultBlockColumns.Add(dimension.Name);
}
else
{
resultBlockColumns.Add(StringExtensions.Format("Dimension {0}", i + 1));
}
rowCount *= (dimension.GroupByWhere.Count + 1);
}
resultBlockColumns.Add("Aggregate");
// Create the DataBlock to hold the final results
result.Values = new DataBlock(resultBlockColumns, rowCount);
// Find the set of items in the base query
ShortSet baseWhereSet = new ShortSet(p.Count);
this.Where.TryEvaluate(p, baseWhereSet, result.Details);
result.Total = baseWhereSet.Count();
// If this is only one dimension, use only one ShortSet and aggregate as we go
if (this.Dimensions.Count == 1)
{
AggregationDimension dimension = this.Dimensions[0];
ShortSet setForDimension = new ShortSet(p.Count);
int nextBlockRow = 0;
foreach (IExpression dimensionValue in dimension.GroupByWhere)
{
// Get the set for this value intersected with the base set
setForDimension.Clear();
dimensionValue.TryEvaluate(p, setForDimension, result.Details);
setForDimension.And(baseWhereSet);
// Compute and store the aggregate value
if (!setForDimension.IsEmpty())
{
result.Values[nextBlockRow, 1] = this.Aggregator.Aggregate(result.AggregationContext, setForDimension, columns);
}
nextBlockRow++;
}
// Add the total
result.Values[nextBlockRow, 1] = this.Aggregator.Aggregate(result.AggregationContext, baseWhereSet, columns);
}
else
{
// Compute the set of items actually matching each dimension-value
List <List <Tuple <IExpression, ShortSet> > > allDimensionValueSets = new List <List <Tuple <IExpression, ShortSet> > >();
foreach (AggregationDimension dimension in this.Dimensions)
{
List <Tuple <IExpression, ShortSet> > dimensionSet = new List <Tuple <IExpression, ShortSet> >();
// Add one item for each value in this dimension
foreach (IExpression dimensionValue in dimension.GroupByWhere)
{
ShortSet setForDimensionValue = new ShortSet(p.Count);
dimensionValue.TryEvaluate(p, setForDimensionValue, result.Details);
dimensionSet.Add(new Tuple <IExpression, ShortSet>(dimensionValue, setForDimensionValue));
}
// Add one 'Total row' item
dimensionSet.Add(new Tuple <IExpression, ShortSet>(new AllExpression(), baseWhereSet));
allDimensionValueSets.Add(dimensionSet);
}
// Run the aggregator over the items
AggregateAllDimensionsFlat(result.AggregationContext, result.Values, p.Count, baseWhereSet, allDimensionValueSets, columns, this.Aggregator);
}
// Add the dimension names to the result if this is the only partition; otherwise, merge will add it
if (p.Mask.Equals(PartitionMask.All))
{
AddDimensionsToBlock(result.Values);
}
// Capture timing and return
result.Runtime = w.Elapsed;
return(result);
}
public void ShortSet_Basic()
{
// Constructor
ShortSet s1 = new ShortSet(100);
// Empty
Assert.AreEqual("", String.Join(", ", s1.Values));
// Set value and enumerate
s1.Add(0);
Assert.AreEqual("0", String.Join(", ", s1.Values));
// Set additional values
s1.Add(15);
s1.Add(64);
Assert.AreEqual("0, 15, 64", String.Join(", ", s1.Values));
// Clear values
s1.Remove(64);
Assert.AreEqual("0, 15", String.Join(", ", s1.Values));
// Or
ShortSet s2 = new ShortSet(120);
s2.Or(new ushort[] { 0, 1, 2 });
s1.Or(s2);
Assert.AreEqual("0, 1, 2, 15", String.Join(", ", s1.Values));
Assert.AreEqual("0, 1, 2", String.Join(", ", s2.Values));
Verify.Exception <ArgumentNullException>(() => s1.Or((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.Or((IEnumerable <ushort>)null));
// OrNot [only 15, 16 not set, so only they should be added]
ShortSet s3 = new ShortSet(100);
s3.Not();
s3.Remove(15);
s3.Remove(16);
s1.OrNot(s3);
Assert.AreEqual("0, 1, 2, 15, 16", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.OrNot((ShortSet)null));
// And
s1.And(s2);
s1.And(new ushort[] { 1, 2 });
Assert.AreEqual("1, 2", String.Join(", ", s1.Values));
s1.And(new ushort[] { 1 });
Assert.AreEqual("1", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.And((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.And((IEnumerable <ushort>)null));
// AndNot
s1.Add(96);
s1.Add(64);
s1.AndNot(s2);
s1.AndNot(new ushort[] { 96 });
Assert.AreEqual("64", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.AndNot((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.AndNot((IEnumerable <ushort>)null));
// Clear
s1.Clear();
Assert.AreEqual("", String.Join(", ", s1.Values));
// From
s1.From(s2);
Assert.AreEqual("0, 1, 2", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.From((ShortSet)null));
// FromAnd
ShortSet s4 = new ShortSet(100);
s4.Or(new ushort[] { 1, 2, 3 });
s1.Clear();
s1.Not();
s1.FromAnd(s2, s4);
Assert.AreEqual("1, 2", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.FromAnd((ShortSet)null, s2));
Verify.Exception <ArgumentNullException>(() => s1.FromAnd(s2, (ShortSet)null));
// ToString
Assert.AreEqual("[1, 2]", s1.ToString());
}
