private IDriverDataEnumerator CreatePseudoEnumeratorForInsertValues(RequestExecutionContext context)
{
// generate a single logical entry and break
// actual values are computed as part of universal insert/update routine
return(new SourcedEnumerator(
DriverRowData.DeriveRepresentationType(context.ParsedRequest.TargetEntityPkField.DbType)));
}
public DocumentDataContainerEnumerator_IndexScan(
int untrimmedCount,
DriverRowData rowData,
DocumentDataContainer dataContainer,
IReadOnlyList <FieldMetadata> fields,
int countOfMainFields,
SortIndex sortIndex,
bool descending)
: base(untrimmedCount, rowData, dataContainer, fields, countOfMainFields)
{
if (sortIndex == null)
{
throw new ArgumentNullException("sortIndex");
}
// note that we ignore value of sortIndex.IsValid here
// that's because invalidation of index only happens when the data is stale
// we only check state of an index and optionally update it in the beginning of processing pipeline
if (sortIndex.OrderData == null || sortIndex.OrderData.Length > untrimmedCount)
{
throw new ArgumentException("Index on column is in invalid state", "sortIndex");
}
m_sortIndex = sortIndex;
m_descending = descending;
PositionInIndex = descending ? m_sortIndex.ValidDocCount : -1;
ReadStructureAndTakeLocks();
}
private static Expression GetOrAddParameterRefToCompilationContext(ParsedRequest parsedRequest, PqlCompilerState compilerState, int parameterOrdinal)
{
Tuple <ParameterExpression, Expression> refTuple;
if (compilerState.ParamRefs.TryGetValue(parameterOrdinal, out refTuple))
{
return(refTuple.Item1);
}
ParameterExpression paramRef;
Expression paramExtractor;
var localOrdinal = parsedRequest.Params.OrdinalToLocalOrdinal[parameterOrdinal];
var dbType = parsedRequest.Params.DataTypes[parameterOrdinal];
if (BitVector.Get(compilerState.RequestParameters.IsCollectionFlags, parameterOrdinal))
{
var rowData = Expression.Field(compilerState.Context, "InputParametersCollections");
var hashsetType = typeof(HashSet <>).MakeGenericType(DriverRowData.DeriveSystemType(dbType));
paramExtractor = Expression.Convert(Expression.ArrayIndex(rowData, Expression.Constant(localOrdinal, typeof(int))), hashsetType);
paramRef = Expression.Variable(paramExtractor.Type);
}
else
{
var rowData = Expression.Field(compilerState.Context, "InputParametersRow");
paramExtractor = DriverRowData.CreateReadAccessor(rowData, dbType, localOrdinal);
paramRef = Expression.Variable(paramExtractor.Type);
}
compilerState.ParamRefs.Add(
parameterOrdinal, new Tuple <ParameterExpression, Expression>(paramRef, paramExtractor));
return(paramRef);
}
/// <summary>
/// Ctr.
/// </summary>
public DriverChangeBuffer(int targetEntity, int ordinalOfPrimaryKey, FieldMetadata[] fields)
{
Fields = fields;
TargetEntity = targetEntity;
Data = new DriverRowData(fields.Select(x => x.DbType).ToArray());
OrdinalOfPrimaryKey = ordinalOfPrimaryKey;
}
public static Expression CompileFieldValueExtractorClause(PqlCompilerState compilerState, FieldMetadata field, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
{
var ordinal = GetFieldOrdinalInDriverFetchFields(cacheInfo.ParsedRequest, field);
var rowData = Expression.Field(compilerState.Context, "InputRow");
return(s_expressionRuntime.AdjustReturnType(DriverRowData.CreateReadAccessor(rowData, field.DbType, ordinal), returnType));
}
private static ColumnDataBase CreateColumnStore(DbType dbType, IUnmanagedAllocator allocator, ColumnDataBase migrated)
{
var dataType = DriverRowData.DeriveSystemType(dbType);
var columnStoreType = typeof(ColumnData <>).MakeGenericType(dataType);
return(migrated == null
? (ColumnDataBase)Activator.CreateInstance(columnStoreType, dbType, allocator)
: (ColumnDataBase)Activator.CreateInstance(columnStoreType, migrated, allocator));
}
private static void CompileInsertUpdateClauses(
IStorageDriver storageDriver, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, DriverChangeType changeType)
{
var updates = cacheInfo.ParsedRequest.Modify.UpdateAssignments;
var clauses = cacheInfo.ParsedRequest.Modify.InsertUpdateSetClauses;
var fields = cacheInfo.ParsedRequest.Modify.ModifiedFields;
if (clauses.Count != fields.Count)
{
throw new Exception(string.Format("Internal error: insert/update clauses count ({0}) does not match count of modified fields ({1})",
clauses.Count, fields.Count));
}
// compile field assignment clauses (SET clauses or value inserts)
for (var ordinal = 0; ordinal < clauses.Count; ordinal++)
{
var clause = clauses[ordinal];
var field = fields[ordinal];
// for bulk requests, primary key is there but it is only used to lookup the record
// for non-bulk requests, primary key should not be in the list of UPDATE clauses
if (changeType == DriverChangeType.Update && !cacheInfo.ParsedRequest.IsBulk)
{
if (!storageDriver.CanUpdateField(field.FieldId))
{
throw new Exception(string.Format("Cannot update field {0}/{1} on entity {2}", field.FieldId, field.Name, cacheInfo.ParsedRequest.TargetEntity.Name));
}
}
// prepare Action compilation context
var compilerState = QueryParser.PrepareCompilerState(containerDescriptor, cacheInfo, null);
compilerState.CompileToAction = true;
// extractor has signature like Func<ClauseEvaluationContext, T>
var extractor = clause == null
? QueryParser.CompileFieldValueExtractorClause(compilerState, field, containerDescriptor, cacheInfo, MakeNullableType(field.DbType))
: QueryParser.CompileClause(compilerState, clause, containerDescriptor, cacheInfo, MakeNullableType(field.DbType));
// get the value into local variable, to prevent multiple invokations when row writer checks for null
var extractedValue = Expression.Variable(extractor.Type);
// now take the extractor and create another method, that will take the value and then put it into the changebuffer's data
var changeBufferData = Expression.Field(Expression.Field(compilerState.Context, "ChangeBuffer"), "Data");
var blockBody = Expression.Block(
new[] { extractedValue },
Expression.Assign(extractedValue, extractor),
DriverRowData.CreateWriteAccessor(extractedValue, changeBufferData, field.DbType, ordinal));
updates.Add(
new ParsedRequest.FieldAssignment
{
Field = field,
CompiledExpression = (Action <ClauseEvaluationContext>)QueryParser.CompileClause(blockBody, compilerState)
});
}
}
private Action <int, DriverRowData, int> GenerateAssignToDriverRowAction()
{
// NOTE: this method assumes that source value is NOT NULL
// this should be verified by caller
var docIndex = Expression.Parameter(typeof(int), "docIndex");
var rowData = Expression.Parameter(typeof(DriverRowData), "rowData");
var fieldArrayIndex = Expression.Parameter(typeof(int), "indexInArray");
var arrayData = Expression.Field(Expression.Constant(this), "DataArray");
var dataBlock = Expression.Call(arrayData, "GetBlock", null, docIndex);
var localIndex = Expression.Call(arrayData, "GetLocalIndex", null, docIndex);
var dataElement = Expression.ArrayIndex(dataBlock, localIndex);
Expression dest;
Expression assign;
string subPropName;
var storageType = DriverRowData.DeriveRepresentationType(DbType);
switch (storageType)
{
case DriverRowData.DataTypeRepresentation.ByteArray:
dest = Expression.ArrayAccess(Expression.Field(rowData, "BinaryData"), fieldArrayIndex);
var copyFrom = typeof(SizableArrayOfByte).GetMethod("CopyFrom", new[] { typeof(SizableArrayOfByte) });
// we assume that DriverRowData always has destination byte array initialized
assign = Expression.Call(dest, copyFrom, dataElement);
break;
case DriverRowData.DataTypeRepresentation.String:
dest = Expression.ArrayAccess(Expression.Field(rowData, "StringData"), fieldArrayIndex);
assign = Expression.Assign(dest, dataElement);
break;
case DriverRowData.DataTypeRepresentation.Value8Bytes:
subPropName = DriverRowData.DeriveSystemType(DbType).Name;
dest = Expression.Field(Expression.ArrayAccess(Expression.Field(rowData, "ValueData8Bytes"), fieldArrayIndex), "As" + subPropName);
assign = Expression.Assign(dest, dataElement);
break;
case DriverRowData.DataTypeRepresentation.Value16Bytes:
subPropName = DriverRowData.DeriveSystemType(DbType).Name;
dest = Expression.Field(Expression.ArrayAccess(Expression.Field(rowData, "ValueData16Bytes"), fieldArrayIndex), "As" + subPropName);
assign = Expression.Assign(dest, dataElement);
break;
default:
throw new InvalidOperationException("Invalid value for DbType: " + DbType);
}
var lambda = Expression.Lambda(
Expression.GetActionType(new[] { typeof(int), typeof(DriverRowData), typeof(int) }),
assign, docIndex, rowData, fieldArrayIndex);
return((Action <int, DriverRowData, int>)lambda.Compile());
}
public DocumentDataContainerEnumerator_FullScan(
int untrimmedCount,
DriverRowData rowData,
DocumentDataContainer dataContainer,
IReadOnlyList <FieldMetadata> fields,
int countOfMainFields)
: base(untrimmedCount, rowData, dataContainer, fields, countOfMainFields)
{
ReadStructureAndTakeLocks();
}
public ColumnData(DbType dbType, IUnmanagedAllocator allocator)
: base(allocator)
{
m_dbType = dbType;
DataArray = new ExpandableArray <T>(1, typeof(T).IsValueType ? DriverRowData.GetByteCount(dbType) : IntPtr.Size);
AssignFromDriverRow = GenerateAssignFromDriverRowAction();
AssignToDriverRow = GenerateAssignToDriverRowAction();
WriteData = GenerateWriteDataAction();
ReadData = GenerateReadDataAction();
}
public IDriverDataEnumerator GetUnorderedEnumerator(
IReadOnlyList <FieldMetadata> fields, int countOfMainFields, DriverRowData driverRow)
{
var untrimmedCount = m_untrimmedDocumentCount;
if (untrimmedCount == 0)
{
return(null);
}
return(new DocumentDataContainerEnumerator_FullScan(untrimmedCount, driverRow, this, fields, countOfMainFields));
}
public IDriverDataEnumerator GetOrderedEnumerator(
IReadOnlyList <FieldMetadata> fields, int countOfMainFields, DriverRowData driverRow, int orderFieldId, bool descending)
{
var untrimmedCount = m_untrimmedDocumentCount;
if (untrimmedCount == 0)
{
return(null);
}
var index = SortIndexManager.GetIndex(orderFieldId, m_untrimmedDocumentCount);
return(new DocumentDataContainerEnumerator_IndexScan(untrimmedCount, driverRow, this, fields, countOfMainFields, index, descending));
}
public DocumentDataContainerEnumerator_BulkPkScan(
int untrimmedCount, DriverRowData rowData, DocumentDataContainer dataContainer, List <FieldMetadata> fields, IDriverDataEnumerator inputDataEnumerator)
:
base(untrimmedCount, rowData, dataContainer, fields, fields.Count - 1)
{
if (inputDataEnumerator == null)
{
throw new ArgumentNullException("inputDataEnumerator");
}
m_inputEnumerator = inputDataEnumerator;
ReadStructureAndTakeLocks();
}
/// <summary>
/// Ctr.
/// </summary>
/// <param name="countToRead">Number of <see cref="RowData"/> items to read, can be zero</param>
/// <param name="fieldTypes">Expected field types</param>
/// <param name="stream">Incoming data stream</param>
/// <param name="driverRowData">Buffer to put field values into, after each call to <see cref="MoveNext"/></param>
public InputDataStreamEnumerator(int countToRead, DbType[] fieldTypes, Stream stream, DriverRowData driverRowData)
{
if (countToRead < 0)
{
throw new ArgumentOutOfRangeException("countToRead", countToRead, "Count may not be negative");
}
if (fieldTypes == null)
{
throw new ArgumentNullException("fieldTypes");
}
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (driverRowData == null)
{
throw new ArgumentNullException("driverRowData");
}
if (fieldTypes.Length == 0)
{
throw new ArgumentException("Count of fields in client row buffer must be greater than zero, first column must contain primary key value");
}
if (driverRowData.FieldTypes.Length != fieldTypes.Length)
{
throw new ArgumentException(string.Format("Count of fields in client row buffer ({0}) must be equal to count of fields in driver buffer ({1})"
, fieldTypes.Length, driverRowData.FieldTypes.Length));
}
for (var ordinal = 0; ordinal < driverRowData.FieldTypes.Length; ordinal++)
{
if (driverRowData.FieldTypes[ordinal] != fieldTypes[ordinal])
{
throw new ArgumentException(string.Format("Field type mismatch at ordinal {0}. Client type: {1}, driver type: {2}",
ordinal, fieldTypes[ordinal], m_driverRowData.FieldTypes[ordinal]));
}
}
m_countToRead = countToRead;
m_driverRowData = driverRowData;
m_reader = new BinaryReader(stream, Encoding.UTF8, true);
m_readerBuffer = new RowData(fieldTypes);
m_pkFieldType = m_readerBuffer.FieldRepresentationTypes[0];
}
private static void CompileSelectClauses(DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo)
{
var parsedRequest = cacheInfo.ParsedRequest;
if (parsedRequest.IsBulk)
{
foreach (var field in parsedRequest.Select.SelectFields)
{
var exprType = MakeNullableType(field.DbType);
parsedRequest.Select.OutputColumns.Add(
new ParsedRequest.SelectOutputColumn
{
Label = field.Name,
CompiledExpression = QueryParser.CompileFieldValueExtractorClause(field, containerDescriptor, cacheInfo, exprType),
DbType = field.DbType,
IsNullable = exprType.IsNullableType()
});
}
}
else
{
foreach (var clause in parsedRequest.Select.SelectClauses)
{
// under column item, there is a "columnSource" element (with a single child), and optional "Id" element for alias
var columnExpressionNode = clause.RequireChild("columnSource", 0).RequireChild(null, 0);
var compiled = QueryParser.CompileClause(columnExpressionNode, containerDescriptor, cacheInfo, null);
var returnType = compiled.GetType().GetGenericArguments()[1];
var dbType = DriverRowData.DeriveDataType(returnType.GetUnderlyingType());
// get alias
var aliasNode = clause.TryGetChild("Id", 1) ?? columnExpressionNode;
var span = aliasNode.Span;
var label = span.Length > 0
? cacheInfo.CommandText.Substring(span.Location.Position, span.Length)
: aliasNode.FindTokenAndGetText();
parsedRequest.Select.OutputColumns.Add(
new ParsedRequest.SelectOutputColumn
{
Label = label,
CompiledExpression = compiled,
DbType = dbType,
IsNullable = returnType.IsNullableType()
});
}
}
}
protected DocumentDataContainerEnumeratorBase(
int untrimmedCount,
DriverRowData rowData,
DocumentDataContainer dataContainer,
IReadOnlyList <FieldMetadata> fields,
int countOfMainFields)
{
if (untrimmedCount < 0)
{
throw new ArgumentOutOfRangeException("untrimmedCount", untrimmedCount, "Untrimmed count cannot be negative");
}
if (rowData == null)
{
throw new ArgumentNullException("rowData");
}
if (dataContainer == null)
{
throw new ArgumentNullException("dataContainer");
}
if (fields == null)
{
throw new ArgumentNullException("fields");
}
if (CountOfMainFields < 0 || CountOfMainFields > fields.Count)
{
throw new ArgumentOutOfRangeException("countOfMainFields", countOfMainFields, "Invalid number of first-priority fetching fields");
}
Position = -1;
UntrimmedCount = untrimmedCount;
RowData = rowData;
DataContainer = dataContainer;
Fields = fields;
CountOfMainFields = countOfMainFields;
RowDataOrdinalToColumnStoreIndex = new int[RowData.FieldTypes.Length];
// ancestors must also invoke ReadStructureAndTakeLocks in their constructor
}
private static Expression GetOrAddFieldRefToCompilationContext(ParsedRequest parsedRequest, PqlCompilerState compilerState, FieldMetadata field)
{
Tuple <ParameterExpression, Expression> refTuple;
if (compilerState.FieldRefs.TryGetValue(field.FieldId, out refTuple))
{
return(refTuple.Item1);
}
var ordinal = GetFieldOrdinalInDriverFetchFields(parsedRequest, field);
var rowData = Expression.Field(compilerState.Context, "InputRow");
var fieldAccessor = DriverRowData.CreateReadAccessor(rowData, field.DbType, ordinal);
var fieldRef = Expression.Variable(fieldAccessor.Type);
compilerState.FieldRefs.Add(
field.FieldId, new Tuple <ParameterExpression, Expression>(fieldRef, fieldAccessor));
return(fieldRef);
}
private void ReadPrimitiveValue(DriverRowData rowData, int ordinal, BinaryReader reader, StringBuilder stringBuilder)
{
var indexInArray = rowData.FieldArrayIndexes[ordinal];
switch (rowData.FieldTypes[ordinal])
{
//case DbType.VarNumeric:
// break;
case DbType.AnsiString:
case DbType.String:
case DbType.AnsiStringFixedLength:
case DbType.StringFixedLength:
case DbType.Xml:
{
var len = RowData.Read7BitEncodedInt(reader);
if (len > 0)
{
stringBuilder.Clear();
stringBuilder.EnsureCapacity(len);
for (var i = 0; i < len; i++)
{
stringBuilder.Append((char)RowData.Read7BitEncodedInt(reader));
}
rowData.StringData[indexInArray] = stringBuilder.ToString();
}
else
{
rowData.StringData[indexInArray] = string.Empty;
}
}
break;
case DbType.Binary:
case DbType.Object:
{
var data = rowData.BinaryData[indexInArray];
data.SetLength(RowData.Read7BitEncodedInt(reader));
var bytesRead = 0;
while (bytesRead < data.Length)
{
var count = reader.Read(data.Data, bytesRead, data.Length - bytesRead);
if (count == 0)
{
throw new DataException("Unexpected end of stream");
}
bytesRead += count;
}
}
break;
case DbType.SByte:
case DbType.Byte:
rowData.ValueData8Bytes[indexInArray].AsByte = reader.ReadByte();
break;
case DbType.Boolean:
rowData.ValueData8Bytes[indexInArray].AsBoolean = reader.ReadBoolean();
break;
case DbType.Decimal:
case DbType.Currency:
case DbType.Guid:
case DbType.DateTimeOffset:
rowData.ValueData16Bytes[indexInArray].Lo = reader.ReadInt64();
rowData.ValueData16Bytes[indexInArray].Hi = reader.ReadInt64();
break;
case DbType.Int16:
case DbType.UInt16:
rowData.ValueData8Bytes[indexInArray].AsInt16 = reader.ReadInt16();
break;
case DbType.Int32:
case DbType.UInt32:
case DbType.Single:
rowData.ValueData8Bytes[indexInArray].AsInt32 = reader.ReadInt32();
break;
case DbType.Date:
case DbType.DateTime:
case DbType.DateTime2:
case DbType.Time:
case DbType.Int64:
case DbType.UInt64:
case DbType.Double:
rowData.ValueData8Bytes[indexInArray].AsInt64 = reader.ReadInt64();
break;
default:
throw new DataException("Invalid DbType: " + rowData.FieldTypes[ordinal]);
}
}
private Action <int, DriverRowData, int> GenerateAssignFromDriverRowAction()
{
// NOTE: this method assumes that source value is NOT NULL
// this should be verified by caller
var docIndex = Expression.Parameter(typeof(int), "docIndex");
var rowData = Expression.Parameter(typeof(DriverRowData), "rowData");
var fieldArrayIndex = Expression.Parameter(typeof(int), "indexInArray");
var arrayData = Expression.Field(Expression.Constant(this), "DataArray");
var dataBlock = Expression.Call(arrayData, "GetBlock", null, docIndex);
var localIndex = Expression.Call(arrayData, "GetLocalIndex", null, docIndex);
Expression source;
Expression assign;
string subPropName;
var storageType = DriverRowData.DeriveRepresentationType(DbType);
switch (storageType)
{
case DriverRowData.DataTypeRepresentation.ByteArray:
// column data may have this destination element uninitialized yet
// use Interlocked.CompareExchange when setting its value
var target = Expression.Variable(typeof(SizableArrayOfByte), "target");
var initIfNull = Expression.IfThen(
Expression.ReferenceEqual(Expression.Constant(null), target),
Expression.Block(
Expression.Assign(target, Expression.New(typeof(SizableArrayOfByte))),
Expression.Call(
typeof(Interlocked), "CompareExchange", new [] { typeof(SizableArrayOfByte) },
Expression.ArrayIndex(dataBlock, localIndex), target, Expression.Constant(null, typeof(SizableArrayOfByte))))
);
source = Expression.ArrayIndex(Expression.Field(rowData, "BinaryData"), fieldArrayIndex);
var copyFrom = typeof(SizableArrayOfByte).GetMethod("CopyFrom", new [] { typeof(SizableArrayOfByte) });
var setter = Expression.Call(target, copyFrom, source);
assign = Expression.Block(
new [] { target },
Expression.Assign(target, Expression.ArrayIndex(dataBlock, localIndex)),
initIfNull,
setter);
break;
case DriverRowData.DataTypeRepresentation.String:
source = Expression.ArrayIndex(Expression.Field(rowData, "StringData"), fieldArrayIndex);
assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source);
break;
case DriverRowData.DataTypeRepresentation.Value8Bytes:
subPropName = DriverRowData.DeriveSystemType(DbType).Name;
source = Expression.Field(Expression.ArrayIndex(Expression.Field(rowData, "ValueData8Bytes"), fieldArrayIndex), "As" + subPropName);
assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source);
break;
case DriverRowData.DataTypeRepresentation.Value16Bytes:
subPropName = DriverRowData.DeriveSystemType(DbType).Name;
source = Expression.Field(Expression.ArrayIndex(Expression.Field(rowData, "ValueData16Bytes"), fieldArrayIndex), "As" + subPropName);
assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source);
break;
default:
throw new InvalidOperationException("Invalid value for DbType: " + DbType);
}
var lambda = Expression.Lambda(
Expression.GetActionType(new [] { typeof(int), typeof(DriverRowData), typeof(int) }),
assign, docIndex, rowData, fieldArrayIndex);
return((Action <int, DriverRowData, int>)lambda.Compile());
}
public IDriverDataEnumerator GetBulkUpdateEnumerator(List <FieldMetadata> fields, DriverRowData driverRow, IDriverDataEnumerator inputDataEnumerator)
{
var untrimmedCount = m_untrimmedDocumentCount;
if (untrimmedCount == 0)
{
return(null);
}
return(new DocumentDataContainerEnumerator_BulkPkScan(untrimmedCount, driverRow, this, fields, inputDataEnumerator));
}
private static Type MakeNullableType(DbType dbType)
{
var underlyingType = DriverRowData.DeriveSystemType(dbType);
return(underlyingType.IsValueType ? typeof(UnboxableNullable <>).MakeGenericType(underlyingType) : underlyingType);
}
