public SelectContext(SelectQuery query)
{
this.Count = query.Count;
this.Where = query.Where;
this.OrderByColumn = query.OrderByColumn;
this.OrderByDescending = query.OrderByDescending;
this.Query = query;
this.Columns = query.Columns;
this.Pass1Results = null;
}
/// <summary>
/// Identify the partition from which items should come to be sorted overall.
/// </summary>
/// <param name="partitionResults">DataBlock per partition with items to merge, sort column last</param>
public SelectResult Merge(SelectResult[] partitionResults)
{
if (partitionResults == null)
{
throw new ArgumentNullException("partitionResults");
}
SelectResult mergedResult = new SelectResult(this.Query);
// Aggregate the total across partitions
long totalFound = 0;
long totalReturned = 0;
for (int i = 0; i < partitionResults.Length; ++i)
{
totalFound += partitionResults[i].Total;
totalReturned += partitionResults[i].CountReturned;
mergedResult.Details.Merge(partitionResults[i].Details);
}
mergedResult.Total = totalFound;
mergedResult.CountReturned = (ushort)Math.Min(this.Count, totalReturned);
if (mergedResult.Details.Succeeded)
{
DataBlock mergedBlock = new DataBlock(partitionResults[0].Values.Columns, mergedResult.CountReturned);
DataBlock mergedOrderbyBlock = new DataBlock(partitionResults[0].OrderByValues.Columns, mergedResult.CountReturned);
// Find the next value according to the sort order and add it until we have enough
bool orderByDescending = this.OrderByDescending;
int partitionCount = partitionResults.Length;
int itemIndex = 0;
int[] nextIndexPerPartition = new int[partitionCount];
while (itemIndex < mergedResult.CountReturned)
{
int bestPartition = -1;
IComparable bestValue = null;
// Find the column with the next value to merge
for (int partitionIndex = 0; partitionIndex < partitionCount; ++partitionIndex)
{
int potentialIndex = nextIndexPerPartition[partitionIndex];
if (potentialIndex == partitionResults[partitionIndex].Values.RowCount)
{
continue;
}
// TODO: in the future, we can allow multiple columns for order by as well as place the ID column
// at the end (to ensure stable sorting). This will need to become a cascading comparison
IComparable potentialValue = (IComparable)partitionResults[partitionIndex].OrderByValues.GetValue(potentialIndex, 0);
int cmp = 0;
if (bestValue != null)
{
cmp = potentialValue.CompareTo(bestValue);
}
if (bestPartition == -1 || (orderByDescending && cmp > 0) || (!orderByDescending && cmp < 0))
{
bestPartition = partitionIndex;
bestValue = potentialValue;
}
}
// Copy the data to the results set
for (int columnIndex = 0; columnIndex < mergedBlock.ColumnCount; ++columnIndex)
{
mergedBlock.SetValue(itemIndex, columnIndex, partitionResults[bestPartition].Values.GetValue(nextIndexPerPartition[bestPartition], columnIndex));
}
// Copy the order-by fields to the result set
mergedOrderbyBlock.SetValue(itemIndex, 0, partitionResults[bestPartition].OrderByValues.GetValue(nextIndexPerPartition[bestPartition], 0));
itemIndex++;
nextIndexPerPartition[bestPartition]++;
}
mergedResult.Values = mergedBlock;
mergedResult.OrderByValues = mergedOrderbyBlock;
}
return(mergedResult);
}
private static ushort[] GetLIDsToReturnDense(Partition p, SelectContext context, SelectResult result, ShortSet whereSet)
{
// Get the sorted IDs, *if available*
IList <ushort> sortedLIDs;
int sortedLIDsCount;
IColumn <object> orderByColumn = p.Columns[context.OrderByColumn];
if (!orderByColumn.TryGetSortedIndexes(out sortedLIDs, out sortedLIDsCount))
{
return(GetLIDsToReturnSparse(p, context, result, whereSet));
}
// Determine how many to return. Stop if none.
int countToReturn = Math.Min(context.Count, (int)(result.Total));
ushort[] lidsToReturn = new ushort[countToReturn];
if (countToReturn == 0)
{
return(lidsToReturn);
}
// Enumerate matches in OrderBy order and return the requested columns for them
ushort countAdded = 0;
int sortedIndex = (context.OrderByDescending ? orderByColumn.Count - 1 : 0);
int lastIndex = (context.OrderByDescending ? -1 : orderByColumn.Count);
int step = (context.OrderByDescending ? -1 : 1);
// Return the next 'count' matches
for (; sortedIndex != lastIndex; sortedIndex += step)
{
ushort lid = sortedLIDs[sortedIndex];
if (whereSet.Contains(lid))
{
lidsToReturn[countAdded] = lid;
if (++countAdded == countToReturn)
{
break;
}
}
}
return(lidsToReturn);
}
private static ushort[] GetLIDsToReturnSparse(Partition p, SelectContext context, SelectResult result, ShortSet whereSet)
{
// Get the set of matching IDs
ushort[] lids = whereSet.Values;
// Lame - store the total to return here so we don't have to compute again
result.Total = (uint)(lids.Length);
// Compute the count to return - the count or the number left after skipping
int countToReturn = Math.Min(context.Count, (int)(lids.Length));
// Get the values for all matches in the Order By column and IDs by the order by values
Array orderByValues = p.Columns[context.OrderByColumn].GetValues(lids);
Array.Sort(orderByValues, lids);
// Walk in ascending or descending order and return the matches
int count = 0;
ushort[] lidsToReturn = new ushort[countToReturn];
if (countToReturn == 0)
{
return(lidsToReturn);
}
int index, end, step;
if (context.OrderByDescending)
{
index = (int)(lids.Length - 1);
end = index - countToReturn;
step = -1;
}
else
{
index = 0;
end = index + countToReturn;
step = 1;
}
for (; index != end; index += step)
{
lidsToReturn[count] = lids[index];
count++;
}
return(lidsToReturn);
}
private static ushort[] GetLIDsToReturn(Partition p, SelectContext context, SelectResult result, ShortSet whereSet)
{
if (result.Total == 0)
{
return(new ushort[0]);
}
// If no ORDER BY is provided, the default is the ID column descending
if (String.IsNullOrEmpty(context.OrderByColumn))
{
context.OrderByColumn = p.IDColumn.Name;
context.OrderByDescending = true;
}
// Compute the most efficient way to scan.
// Sparse - get and sort the order by values for all matches.
// Dense - walk the order by column in order until we find enough matches.
// Walking in order measures about 20 times faster than Array.Sort() (cache locality; instruction count)
int sparseCompareCount = (int)(result.Total * Math.Log(result.Total, 2));
double densePercentageToScan = Math.Min(1.0d, (double)(context.Count) / (double)(result.Total));
int denseCheckCount = (int)(p.Count * densePercentageToScan);
if (sparseCompareCount * 20 < denseCheckCount)
{
return(GetLIDsToReturnSparse(p, context, result, whereSet));
}
else
{
return(GetLIDsToReturnDense(p, context, result, whereSet));
}
}
public SelectResult Compute(Partition p)
{
if (p == null)
{
throw new ArgumentNullException("p");
}
SelectResult result = new SelectResult(this.Query);
// Find the set of items matching all terms
ShortSet whereSet = new ShortSet(p.Count);
this.Where.TryEvaluate(p, whereSet, result.Details);
// Verify that the ORDER BY column exists
if (!String.IsNullOrEmpty(this.OrderByColumn) && !p.Columns.ContainsKey(this.OrderByColumn))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, this.OrderByColumn);
return(result);
}
if (result.Details.Succeeded)
{
IUntypedColumn column = null;
result.Total = whereSet.Count();
// Find the set of IDs to return for the query (up to 'Count' after 'Skip' in ORDER BY order)
ushort[] lidsToReturn = GetLIDsToReturn(p, this, result, whereSet);
result.CountReturned = (ushort)lidsToReturn.Length;
// Get the order-by column
if (!p.Columns.TryGetValue(this.OrderByColumn, out column))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, this.OrderByColumn);
return(result);
}
Array orderByColumn = column.GetValues(lidsToReturn);
// Get all of the response columns and return them
Array columns = new Array[this.Columns.Count];
for (int i = 0; i < this.Columns.Count; ++i)
{
string columnName = this.Columns[i];
if (columnName == this.OrderByColumn)
{
columns.SetValue(orderByColumn, i);
}
else
{
if (!p.Columns.TryGetValue(columnName, out column))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, columnName);
return(result);
}
Array values = column.GetValues(lidsToReturn);
if (Query.Highlighter != null)
{
Query.Highlighter.Highlight(values, column, Query);
}
columns.SetValue(values, i);
}
}
result.Values = new DataBlock(p.GetDetails(this.Columns), result.CountReturned, columns);
result.OrderByValues = new DataBlock(p.GetDetails(new string[] { this.OrderByColumn }), result.CountReturned, new Array[] { orderByColumn });
}
return(result);
}
