private ushort[] FindOrAssignLIDs(DataBlock.ReadOnlyDataBlock values, int idColumnIndex, AddOrUpdateMode mode)
{
Type idColumnDataType = values.GetTypeForColumn(idColumnIndex);
// If the insert array matches types with the column then we can use the native type to do a direct assignment from the input array
// to the column array. If the types do not match, we need to fallback to object to allow the Value class to handle the type conversion
ITypedAddOrUpdateWorker worker = NativeContainer.CreateTypedInstance <ITypedAddOrUpdateWorker>(typeof(AddOrUpdateWorker <>), idColumnDataType);
return(worker.FindOrAssignLIDs(this, values, idColumnIndex, mode));
}
public virtual DistinctResult Merge(DistinctResult[] partitionResults)
{
if (partitionResults == null)
{
throw new ArgumentNullException("partitionResults");
}
if (partitionResults.Length == 0)
{
throw new ArgumentException("Length==0 not supported", "partitionResults");
}
if (!partitionResults[0].Details.Succeeded)
{
return(partitionResults[0]);
}
DistinctResult mergedResult = new DistinctResult(this);
mergedResult.ColumnType = partitionResults[0].ColumnType;
mergedResult.AllValuesReturned = true;
// Construct a helper object of the correct type to natively work with the column
GetUniqueValuesWorker helper = NativeContainer.CreateTypedInstance <GetUniqueValuesWorker>(typeof(GetUniqueValuesWorker <>), partitionResults[0].ColumnType);
IUniqueValueMerger merger = helper.GetMerger();
for (int i = 0; i < partitionResults.Length; ++i)
{
DistinctResult result = partitionResults[i];
// Merge Details
mergedResult.Details.Merge(result.Details);
// Merge whether values remain in any partition
mergedResult.AllValuesReturned &= result.AllValuesReturned;
// Add the values themselves to merge
if (result.Values != null)
{
merger.Add(result.Values.GetColumn(0));
}
}
Array uniqueValues = merger.GetUniqueValues((int)this.Count);
// Copy the merged values into a block
DataBlock mergedBlock = new DataBlock(new string[] { this.Column }, uniqueValues.GetLength(0));
mergedBlock.SetColumn(0, uniqueValues);
mergedResult.Values = mergedBlock;
// If the merge didn't return everything, we didn't return everything
mergedResult.AllValuesReturned &= (uniqueValues.GetLength(0) == merger.Count);
return(mergedResult);
}
private void FillPartitionColumn(DataBlock.ReadOnlyDataBlock values, int columnIndex, ushort[] itemLIDs)
{
string columnName = values.Columns[columnIndex].Name;
if (columnName.Equals(this.IDColumn.Name, StringComparison.OrdinalIgnoreCase))
{
return;
}
Type dataBlockColumnDataType = values.GetTypeForColumn(columnIndex);
// If the insert array matches types with the column then we can use the native type to do a direct assignment from the input array
// to the column array. If the types do not match, we need to fallback to object to allow the Value class to handle the type conversion
ITypedAddOrUpdateWorker worker = NativeContainer.CreateTypedInstance <ITypedAddOrUpdateWorker>(typeof(AddOrUpdateWorker <>), dataBlockColumnDataType);
worker.FillPartitionColumn(this, values, columnIndex, itemLIDs);
}
public virtual DistinctResult Compute(Partition p)
{
if (p == null)
{
throw new ArgumentNullException("p");
}
ushort countToReturnPerPartition = (this.Count == 0 || this.Count > ushort.MaxValue) ? ushort.MaxValue : (ushort)this.Count;
DistinctResult result = new DistinctResult(this);
// Verify the column exists
if (!p.ContainsColumn(this.Column))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, this.Column);
return(result);
}
// Find the set of items matching the where clause
ShortSet whereSet = new ShortSet(p.Count);
this.Where.TryEvaluate(p, whereSet, result.Details);
if (result.Details.Succeeded)
{
IUntypedColumn column = p.Columns[this.Column];
// Construct a helper object of the correct type to natively work with the column
GetUniqueValuesWorker helper = NativeContainer.CreateTypedInstance <GetUniqueValuesWorker>(typeof(GetUniqueValuesWorker <>), column.ColumnType);
bool allValuesReturned;
Array uniqueValues = helper.GetUniqueValuesFromColumn(column.InnerColumn, whereSet, countToReturnPerPartition, out allValuesReturned);
result.ColumnType = column.ColumnType;
result.AllValuesReturned = allValuesReturned;
// Build a DataBlock with the results and return it
DataBlock resultValues = new DataBlock(new string[] { this.Column }, uniqueValues.GetLength(0));
resultValues.SetColumn(0, uniqueValues);
result.Values = resultValues;
}
return(result);
}
public void OnBeforeQuery(ITable table)
{
// Find approximate 10th and 90th percentile values for this query
PercentilesQuery pq = new PercentilesQuery();
pq.Column = this.Column;
pq.TableName = this.TableName;
pq.Where = this.Where;
pq.Percentiles = new double[] { 0.10, 0.90 };
DataBlockResult result = table.Query(pq);
if (result.Values != null)
{
// Try to choose buckets if the 10th and 90th percentile values were returned [returns null for unsupported types]
Bucketer bucketer = NativeContainer.CreateTypedInstance <Bucketer>(typeof(Bucketer <>), ((Table)table).GetColumnType(this.Column));
this.Buckets = bucketer.GetBuckets(result.Values, this.Inclusive, this.BucketCount);
}
}
public DataBlockResult Compute(Partition p)
{
if (p == null)
{
throw new ArgumentNullException("p");
}
DataBlockResult result = new DataBlockResult(this);
// Verify the column exists
if (!p.ContainsColumn(this.Column))
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotExist, this.Column);
return(result);
}
// Verify we were able to get percentile values
if (this.Buckets == null)
{
result.Details.AddError(ExecutionDetails.ColumnDoesNotSupportOperator, "percentile", this.Column);
return(result);
}
// Find the set of items matching the where clause
ShortSet whereSet = new ShortSet(p.Count);
this.Where.TryEvaluate(p, whereSet, result.Details);
IUntypedColumn column = p.Columns[this.Column];
if (result.Details.Succeeded)
{
Bucketer bucketer = NativeContainer.CreateTypedInstance <Bucketer>(typeof(Bucketer <>), column.ColumnType);
result.Values = bucketer.Bucket(column.InnerColumn, whereSet, this.Buckets, this.Inclusive);
result.Total = whereSet.Count();
}
return(result);
}
/// <summary>
/// Add or Update the given items with the given values. The ID column must be passed
/// and must be the first column. If an ID is not known, the item will be added.
/// For each item, the value for each column is set to the provided values.
/// </summary>
/// <param name="values">Set of Columns and values to add or update</param>
/// <param name="options">Options to adjust behavior of AddOrUpdate</param>
public void AddOrUpdate(DataBlock.ReadOnlyDataBlock values, AddOrUpdateOptions options)
{
_locker.EnterWriteLock();
try
{
// Add columns from data, if this is the first data and columns weren't predefined
if (options.AddMissingColumns)
{
AddColumnsFromBlock(values);
}
ColumnDetails idColumn = _partitions[0].IDColumn;
if (idColumn == null)
{
throw new ArribaException("Items cannot be added to this Table because it does not yet have an ID column defined. Call AddColumn with exactly one column with 'IsPrimaryKey' true and then items may be added.");
}
int idColumnIndex = values.IndexOfColumn(idColumn.Name);
if (idColumnIndex == -1)
{
throw new ArribaException(StringExtensions.Format("AddOrUpdates must be passed the ID column, '{0}', in order to tell which items to update.", idColumn.Name));
}
// Verify all passed columns exist
foreach (ColumnDetails column in values.Columns)
{
ColumnDetails foundColumn;
if (!_partitions[0].DetailsByColumn.TryGetValue(column.Name, out foundColumn))
{
throw new ArribaException(StringExtensions.Format("AddOrUpdate failed because values were passed for column '{0}', which is not in the table. Use AddColumn to add all columns first or ensure the first block added to the Table has all desired columns.", column.Name));
}
}
// Non-Parallel Implementation
if (_partitions.Count == 1)
{
_partitions[0].AddOrUpdate(values, options);
return;
}
// Determine the partition each item should go to
int[] partitionIds;
TargetPartitionInfo[] partitionInfo;
Type idColumnArrayType = values.GetTypeForColumn(idColumnIndex);
if (_splitter == null || _splitter.Item2 == null || _splitter.Item1 != idColumnArrayType)
{
IComputePartition splitter = NativeContainer.CreateTypedInstance <IComputePartition>(typeof(ComputePartitionHelper <>), idColumnArrayType);
_splitter = Tuple.Create(idColumnArrayType, splitter);
}
_splitter.Item2.ComputePartition(this, values, idColumnIndex, out partitionIds, out partitionInfo);
// Sort/group the incoming items by paritition and then by index to ensure they
// are processed in the order they were presented in the input ReadOnlyDataBlock
int[] sortOrder = new int[values.RowCount];
for (int i = 0; i < values.RowCount; ++i)
{
int p = partitionIds[i];
int startIndex = partitionInfo[p].StartIndex + partitionInfo[p].Count;
sortOrder[startIndex] = i;
partitionInfo[p].Count++;
}
Action <Tuple <int, int>, ParallelLoopState> forBody =
delegate(Tuple <int, int> range, ParallelLoopState unused)
{
for (int p = range.Item1; p < range.Item2; ++p)
{
int startIndex = partitionInfo[p].StartIndex;
int length = partitionInfo[p].Count;
DataBlock.ReadOnlyDataBlock partitionValues = values.ProjectChain(sortOrder, startIndex, length);
_partitions[p].AddOrUpdate(partitionValues, options);
}
};
// In parallel, each partition will add items which belong to it
if (this.RunParallel)
{
var rangePartitioner = Partitioner.Create(0, _partitions.Count);
Parallel.ForEach(rangePartitioner, this.ParallelOptions, forBody);
}
else
{
var range = Tuple.Create(0, _partitions.Count);
forBody(range, null);
}
}
finally
{
_locker.ExitWriteLock();
}
}
