private IDriverDataEnumerator CreatePseudoEnumeratorForInsertValues(RequestExecutionContext context) { // generate a single logical entry and break // actual values are computed as part of universal insert/update routine return(new SourcedEnumerator( DriverRowData.DeriveRepresentationType(context.ParsedRequest.TargetEntityPkField.DbType))); }
public DocumentDataContainerEnumerator_IndexScan( int untrimmedCount, DriverRowData rowData, DocumentDataContainer dataContainer, IReadOnlyList <FieldMetadata> fields, int countOfMainFields, SortIndex sortIndex, bool descending) : base(untrimmedCount, rowData, dataContainer, fields, countOfMainFields) { if (sortIndex == null) { throw new ArgumentNullException("sortIndex"); } // note that we ignore value of sortIndex.IsValid here // that's because invalidation of index only happens when the data is stale // we only check state of an index and optionally update it in the beginning of processing pipeline if (sortIndex.OrderData == null || sortIndex.OrderData.Length > untrimmedCount) { throw new ArgumentException("Index on column is in invalid state", "sortIndex"); } m_sortIndex = sortIndex; m_descending = descending; PositionInIndex = descending ? m_sortIndex.ValidDocCount : -1; ReadStructureAndTakeLocks(); }
private static Expression GetOrAddParameterRefToCompilationContext(ParsedRequest parsedRequest, PqlCompilerState compilerState, int parameterOrdinal) { Tuple <ParameterExpression, Expression> refTuple; if (compilerState.ParamRefs.TryGetValue(parameterOrdinal, out refTuple)) { return(refTuple.Item1); } ParameterExpression paramRef; Expression paramExtractor; var localOrdinal = parsedRequest.Params.OrdinalToLocalOrdinal[parameterOrdinal]; var dbType = parsedRequest.Params.DataTypes[parameterOrdinal]; if (BitVector.Get(compilerState.RequestParameters.IsCollectionFlags, parameterOrdinal)) { var rowData = Expression.Field(compilerState.Context, "InputParametersCollections"); var hashsetType = typeof(HashSet <>).MakeGenericType(DriverRowData.DeriveSystemType(dbType)); paramExtractor = Expression.Convert(Expression.ArrayIndex(rowData, Expression.Constant(localOrdinal, typeof(int))), hashsetType); paramRef = Expression.Variable(paramExtractor.Type); } else { var rowData = Expression.Field(compilerState.Context, "InputParametersRow"); paramExtractor = DriverRowData.CreateReadAccessor(rowData, dbType, localOrdinal); paramRef = Expression.Variable(paramExtractor.Type); } compilerState.ParamRefs.Add( parameterOrdinal, new Tuple <ParameterExpression, Expression>(paramRef, paramExtractor)); return(paramRef); }
/// <summary> /// Ctr. /// </summary> public DriverChangeBuffer(int targetEntity, int ordinalOfPrimaryKey, FieldMetadata[] fields) { Fields = fields; TargetEntity = targetEntity; Data = new DriverRowData(fields.Select(x => x.DbType).ToArray()); OrdinalOfPrimaryKey = ordinalOfPrimaryKey; }
public static Expression CompileFieldValueExtractorClause(PqlCompilerState compilerState, FieldMetadata field, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType) { var ordinal = GetFieldOrdinalInDriverFetchFields(cacheInfo.ParsedRequest, field); var rowData = Expression.Field(compilerState.Context, "InputRow"); return(s_expressionRuntime.AdjustReturnType(DriverRowData.CreateReadAccessor(rowData, field.DbType, ordinal), returnType)); }
private static ColumnDataBase CreateColumnStore(DbType dbType, IUnmanagedAllocator allocator, ColumnDataBase migrated) { var dataType = DriverRowData.DeriveSystemType(dbType); var columnStoreType = typeof(ColumnData <>).MakeGenericType(dataType); return(migrated == null ? (ColumnDataBase)Activator.CreateInstance(columnStoreType, dbType, allocator) : (ColumnDataBase)Activator.CreateInstance(columnStoreType, migrated, allocator)); }
private static void CompileInsertUpdateClauses( IStorageDriver storageDriver, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, DriverChangeType changeType) { var updates = cacheInfo.ParsedRequest.Modify.UpdateAssignments; var clauses = cacheInfo.ParsedRequest.Modify.InsertUpdateSetClauses; var fields = cacheInfo.ParsedRequest.Modify.ModifiedFields; if (clauses.Count != fields.Count) { throw new Exception(string.Format("Internal error: insert/update clauses count ({0}) does not match count of modified fields ({1})", clauses.Count, fields.Count)); } // compile field assignment clauses (SET clauses or value inserts) for (var ordinal = 0; ordinal < clauses.Count; ordinal++) { var clause = clauses[ordinal]; var field = fields[ordinal]; // for bulk requests, primary key is there but it is only used to lookup the record // for non-bulk requests, primary key should not be in the list of UPDATE clauses if (changeType == DriverChangeType.Update && !cacheInfo.ParsedRequest.IsBulk) { if (!storageDriver.CanUpdateField(field.FieldId)) { throw new Exception(string.Format("Cannot update field {0}/{1} on entity {2}", field.FieldId, field.Name, cacheInfo.ParsedRequest.TargetEntity.Name)); } } // prepare Action compilation context var compilerState = QueryParser.PrepareCompilerState(containerDescriptor, cacheInfo, null); compilerState.CompileToAction = true; // extractor has signature like Func<ClauseEvaluationContext, T> var extractor = clause == null ? QueryParser.CompileFieldValueExtractorClause(compilerState, field, containerDescriptor, cacheInfo, MakeNullableType(field.DbType)) : QueryParser.CompileClause(compilerState, clause, containerDescriptor, cacheInfo, MakeNullableType(field.DbType)); // get the value into local variable, to prevent multiple invokations when row writer checks for null var extractedValue = Expression.Variable(extractor.Type); // now take the extractor and create another method, that will take the value and then put it into the changebuffer's data var changeBufferData = Expression.Field(Expression.Field(compilerState.Context, "ChangeBuffer"), "Data"); var blockBody = Expression.Block( new[] { extractedValue }, Expression.Assign(extractedValue, extractor), DriverRowData.CreateWriteAccessor(extractedValue, changeBufferData, field.DbType, ordinal)); updates.Add( new ParsedRequest.FieldAssignment { Field = field, CompiledExpression = (Action <ClauseEvaluationContext>)QueryParser.CompileClause(blockBody, compilerState) }); } }
private Action <int, DriverRowData, int> GenerateAssignToDriverRowAction() { // NOTE: this method assumes that source value is NOT NULL // this should be verified by caller var docIndex = Expression.Parameter(typeof(int), "docIndex"); var rowData = Expression.Parameter(typeof(DriverRowData), "rowData"); var fieldArrayIndex = Expression.Parameter(typeof(int), "indexInArray"); var arrayData = Expression.Field(Expression.Constant(this), "DataArray"); var dataBlock = Expression.Call(arrayData, "GetBlock", null, docIndex); var localIndex = Expression.Call(arrayData, "GetLocalIndex", null, docIndex); var dataElement = Expression.ArrayIndex(dataBlock, localIndex); Expression dest; Expression assign; string subPropName; var storageType = DriverRowData.DeriveRepresentationType(DbType); switch (storageType) { case DriverRowData.DataTypeRepresentation.ByteArray: dest = Expression.ArrayAccess(Expression.Field(rowData, "BinaryData"), fieldArrayIndex); var copyFrom = typeof(SizableArrayOfByte).GetMethod("CopyFrom", new[] { typeof(SizableArrayOfByte) }); // we assume that DriverRowData always has destination byte array initialized assign = Expression.Call(dest, copyFrom, dataElement); break; case DriverRowData.DataTypeRepresentation.String: dest = Expression.ArrayAccess(Expression.Field(rowData, "StringData"), fieldArrayIndex); assign = Expression.Assign(dest, dataElement); break; case DriverRowData.DataTypeRepresentation.Value8Bytes: subPropName = DriverRowData.DeriveSystemType(DbType).Name; dest = Expression.Field(Expression.ArrayAccess(Expression.Field(rowData, "ValueData8Bytes"), fieldArrayIndex), "As" + subPropName); assign = Expression.Assign(dest, dataElement); break; case DriverRowData.DataTypeRepresentation.Value16Bytes: subPropName = DriverRowData.DeriveSystemType(DbType).Name; dest = Expression.Field(Expression.ArrayAccess(Expression.Field(rowData, "ValueData16Bytes"), fieldArrayIndex), "As" + subPropName); assign = Expression.Assign(dest, dataElement); break; default: throw new InvalidOperationException("Invalid value for DbType: " + DbType); } var lambda = Expression.Lambda( Expression.GetActionType(new[] { typeof(int), typeof(DriverRowData), typeof(int) }), assign, docIndex, rowData, fieldArrayIndex); return((Action <int, DriverRowData, int>)lambda.Compile()); }
public DocumentDataContainerEnumerator_FullScan( int untrimmedCount, DriverRowData rowData, DocumentDataContainer dataContainer, IReadOnlyList <FieldMetadata> fields, int countOfMainFields) : base(untrimmedCount, rowData, dataContainer, fields, countOfMainFields) { ReadStructureAndTakeLocks(); }
public ColumnData(DbType dbType, IUnmanagedAllocator allocator) : base(allocator) { m_dbType = dbType; DataArray = new ExpandableArray <T>(1, typeof(T).IsValueType ? DriverRowData.GetByteCount(dbType) : IntPtr.Size); AssignFromDriverRow = GenerateAssignFromDriverRowAction(); AssignToDriverRow = GenerateAssignToDriverRowAction(); WriteData = GenerateWriteDataAction(); ReadData = GenerateReadDataAction(); }
public IDriverDataEnumerator GetUnorderedEnumerator( IReadOnlyList <FieldMetadata> fields, int countOfMainFields, DriverRowData driverRow) { var untrimmedCount = m_untrimmedDocumentCount; if (untrimmedCount == 0) { return(null); } return(new DocumentDataContainerEnumerator_FullScan(untrimmedCount, driverRow, this, fields, countOfMainFields)); }
public IDriverDataEnumerator GetOrderedEnumerator( IReadOnlyList <FieldMetadata> fields, int countOfMainFields, DriverRowData driverRow, int orderFieldId, bool descending) { var untrimmedCount = m_untrimmedDocumentCount; if (untrimmedCount == 0) { return(null); } var index = SortIndexManager.GetIndex(orderFieldId, m_untrimmedDocumentCount); return(new DocumentDataContainerEnumerator_IndexScan(untrimmedCount, driverRow, this, fields, countOfMainFields, index, descending)); }
public DocumentDataContainerEnumerator_BulkPkScan( int untrimmedCount, DriverRowData rowData, DocumentDataContainer dataContainer, List <FieldMetadata> fields, IDriverDataEnumerator inputDataEnumerator) : base(untrimmedCount, rowData, dataContainer, fields, fields.Count - 1) { if (inputDataEnumerator == null) { throw new ArgumentNullException("inputDataEnumerator"); } m_inputEnumerator = inputDataEnumerator; ReadStructureAndTakeLocks(); }
/// <summary> /// Ctr. /// </summary> /// <param name="countToRead">Number of <see cref="RowData"/> items to read, can be zero</param> /// <param name="fieldTypes">Expected field types</param> /// <param name="stream">Incoming data stream</param> /// <param name="driverRowData">Buffer to put field values into, after each call to <see cref="MoveNext"/></param> public InputDataStreamEnumerator(int countToRead, DbType[] fieldTypes, Stream stream, DriverRowData driverRowData) { if (countToRead < 0) { throw new ArgumentOutOfRangeException("countToRead", countToRead, "Count may not be negative"); } if (fieldTypes == null) { throw new ArgumentNullException("fieldTypes"); } if (stream == null) { throw new ArgumentNullException("stream"); } if (driverRowData == null) { throw new ArgumentNullException("driverRowData"); } if (fieldTypes.Length == 0) { throw new ArgumentException("Count of fields in client row buffer must be greater than zero, first column must contain primary key value"); } if (driverRowData.FieldTypes.Length != fieldTypes.Length) { throw new ArgumentException(string.Format("Count of fields in client row buffer ({0}) must be equal to count of fields in driver buffer ({1})" , fieldTypes.Length, driverRowData.FieldTypes.Length)); } for (var ordinal = 0; ordinal < driverRowData.FieldTypes.Length; ordinal++) { if (driverRowData.FieldTypes[ordinal] != fieldTypes[ordinal]) { throw new ArgumentException(string.Format("Field type mismatch at ordinal {0}. Client type: {1}, driver type: {2}", ordinal, fieldTypes[ordinal], m_driverRowData.FieldTypes[ordinal])); } } m_countToRead = countToRead; m_driverRowData = driverRowData; m_reader = new BinaryReader(stream, Encoding.UTF8, true); m_readerBuffer = new RowData(fieldTypes); m_pkFieldType = m_readerBuffer.FieldRepresentationTypes[0]; }
private static void CompileSelectClauses(DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo) { var parsedRequest = cacheInfo.ParsedRequest; if (parsedRequest.IsBulk) { foreach (var field in parsedRequest.Select.SelectFields) { var exprType = MakeNullableType(field.DbType); parsedRequest.Select.OutputColumns.Add( new ParsedRequest.SelectOutputColumn { Label = field.Name, CompiledExpression = QueryParser.CompileFieldValueExtractorClause(field, containerDescriptor, cacheInfo, exprType), DbType = field.DbType, IsNullable = exprType.IsNullableType() }); } } else { foreach (var clause in parsedRequest.Select.SelectClauses) { // under column item, there is a "columnSource" element (with a single child), and optional "Id" element for alias var columnExpressionNode = clause.RequireChild("columnSource", 0).RequireChild(null, 0); var compiled = QueryParser.CompileClause(columnExpressionNode, containerDescriptor, cacheInfo, null); var returnType = compiled.GetType().GetGenericArguments()[1]; var dbType = DriverRowData.DeriveDataType(returnType.GetUnderlyingType()); // get alias var aliasNode = clause.TryGetChild("Id", 1) ?? columnExpressionNode; var span = aliasNode.Span; var label = span.Length > 0 ? cacheInfo.CommandText.Substring(span.Location.Position, span.Length) : aliasNode.FindTokenAndGetText(); parsedRequest.Select.OutputColumns.Add( new ParsedRequest.SelectOutputColumn { Label = label, CompiledExpression = compiled, DbType = dbType, IsNullable = returnType.IsNullableType() }); } } }
protected DocumentDataContainerEnumeratorBase( int untrimmedCount, DriverRowData rowData, DocumentDataContainer dataContainer, IReadOnlyList <FieldMetadata> fields, int countOfMainFields) { if (untrimmedCount < 0) { throw new ArgumentOutOfRangeException("untrimmedCount", untrimmedCount, "Untrimmed count cannot be negative"); } if (rowData == null) { throw new ArgumentNullException("rowData"); } if (dataContainer == null) { throw new ArgumentNullException("dataContainer"); } if (fields == null) { throw new ArgumentNullException("fields"); } if (CountOfMainFields < 0 || CountOfMainFields > fields.Count) { throw new ArgumentOutOfRangeException("countOfMainFields", countOfMainFields, "Invalid number of first-priority fetching fields"); } Position = -1; UntrimmedCount = untrimmedCount; RowData = rowData; DataContainer = dataContainer; Fields = fields; CountOfMainFields = countOfMainFields; RowDataOrdinalToColumnStoreIndex = new int[RowData.FieldTypes.Length]; // ancestors must also invoke ReadStructureAndTakeLocks in their constructor }
private static Expression GetOrAddFieldRefToCompilationContext(ParsedRequest parsedRequest, PqlCompilerState compilerState, FieldMetadata field) { Tuple <ParameterExpression, Expression> refTuple; if (compilerState.FieldRefs.TryGetValue(field.FieldId, out refTuple)) { return(refTuple.Item1); } var ordinal = GetFieldOrdinalInDriverFetchFields(parsedRequest, field); var rowData = Expression.Field(compilerState.Context, "InputRow"); var fieldAccessor = DriverRowData.CreateReadAccessor(rowData, field.DbType, ordinal); var fieldRef = Expression.Variable(fieldAccessor.Type); compilerState.FieldRefs.Add( field.FieldId, new Tuple <ParameterExpression, Expression>(fieldRef, fieldAccessor)); return(fieldRef); }
private void ReadPrimitiveValue(DriverRowData rowData, int ordinal, BinaryReader reader, StringBuilder stringBuilder) { var indexInArray = rowData.FieldArrayIndexes[ordinal]; switch (rowData.FieldTypes[ordinal]) { //case DbType.VarNumeric: // break; case DbType.AnsiString: case DbType.String: case DbType.AnsiStringFixedLength: case DbType.StringFixedLength: case DbType.Xml: { var len = RowData.Read7BitEncodedInt(reader); if (len > 0) { stringBuilder.Clear(); stringBuilder.EnsureCapacity(len); for (var i = 0; i < len; i++) { stringBuilder.Append((char)RowData.Read7BitEncodedInt(reader)); } rowData.StringData[indexInArray] = stringBuilder.ToString(); } else { rowData.StringData[indexInArray] = string.Empty; } } break; case DbType.Binary: case DbType.Object: { var data = rowData.BinaryData[indexInArray]; data.SetLength(RowData.Read7BitEncodedInt(reader)); var bytesRead = 0; while (bytesRead < data.Length) { var count = reader.Read(data.Data, bytesRead, data.Length - bytesRead); if (count == 0) { throw new DataException("Unexpected end of stream"); } bytesRead += count; } } break; case DbType.SByte: case DbType.Byte: rowData.ValueData8Bytes[indexInArray].AsByte = reader.ReadByte(); break; case DbType.Boolean: rowData.ValueData8Bytes[indexInArray].AsBoolean = reader.ReadBoolean(); break; case DbType.Decimal: case DbType.Currency: case DbType.Guid: case DbType.DateTimeOffset: rowData.ValueData16Bytes[indexInArray].Lo = reader.ReadInt64(); rowData.ValueData16Bytes[indexInArray].Hi = reader.ReadInt64(); break; case DbType.Int16: case DbType.UInt16: rowData.ValueData8Bytes[indexInArray].AsInt16 = reader.ReadInt16(); break; case DbType.Int32: case DbType.UInt32: case DbType.Single: rowData.ValueData8Bytes[indexInArray].AsInt32 = reader.ReadInt32(); break; case DbType.Date: case DbType.DateTime: case DbType.DateTime2: case DbType.Time: case DbType.Int64: case DbType.UInt64: case DbType.Double: rowData.ValueData8Bytes[indexInArray].AsInt64 = reader.ReadInt64(); break; default: throw new DataException("Invalid DbType: " + rowData.FieldTypes[ordinal]); } }
private Action <int, DriverRowData, int> GenerateAssignFromDriverRowAction() { // NOTE: this method assumes that source value is NOT NULL // this should be verified by caller var docIndex = Expression.Parameter(typeof(int), "docIndex"); var rowData = Expression.Parameter(typeof(DriverRowData), "rowData"); var fieldArrayIndex = Expression.Parameter(typeof(int), "indexInArray"); var arrayData = Expression.Field(Expression.Constant(this), "DataArray"); var dataBlock = Expression.Call(arrayData, "GetBlock", null, docIndex); var localIndex = Expression.Call(arrayData, "GetLocalIndex", null, docIndex); Expression source; Expression assign; string subPropName; var storageType = DriverRowData.DeriveRepresentationType(DbType); switch (storageType) { case DriverRowData.DataTypeRepresentation.ByteArray: // column data may have this destination element uninitialized yet // use Interlocked.CompareExchange when setting its value var target = Expression.Variable(typeof(SizableArrayOfByte), "target"); var initIfNull = Expression.IfThen( Expression.ReferenceEqual(Expression.Constant(null), target), Expression.Block( Expression.Assign(target, Expression.New(typeof(SizableArrayOfByte))), Expression.Call( typeof(Interlocked), "CompareExchange", new [] { typeof(SizableArrayOfByte) }, Expression.ArrayIndex(dataBlock, localIndex), target, Expression.Constant(null, typeof(SizableArrayOfByte)))) ); source = Expression.ArrayIndex(Expression.Field(rowData, "BinaryData"), fieldArrayIndex); var copyFrom = typeof(SizableArrayOfByte).GetMethod("CopyFrom", new [] { typeof(SizableArrayOfByte) }); var setter = Expression.Call(target, copyFrom, source); assign = Expression.Block( new [] { target }, Expression.Assign(target, Expression.ArrayIndex(dataBlock, localIndex)), initIfNull, setter); break; case DriverRowData.DataTypeRepresentation.String: source = Expression.ArrayIndex(Expression.Field(rowData, "StringData"), fieldArrayIndex); assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source); break; case DriverRowData.DataTypeRepresentation.Value8Bytes: subPropName = DriverRowData.DeriveSystemType(DbType).Name; source = Expression.Field(Expression.ArrayIndex(Expression.Field(rowData, "ValueData8Bytes"), fieldArrayIndex), "As" + subPropName); assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source); break; case DriverRowData.DataTypeRepresentation.Value16Bytes: subPropName = DriverRowData.DeriveSystemType(DbType).Name; source = Expression.Field(Expression.ArrayIndex(Expression.Field(rowData, "ValueData16Bytes"), fieldArrayIndex), "As" + subPropName); assign = Expression.Assign(Expression.ArrayAccess(dataBlock, localIndex), source); break; default: throw new InvalidOperationException("Invalid value for DbType: " + DbType); } var lambda = Expression.Lambda( Expression.GetActionType(new [] { typeof(int), typeof(DriverRowData), typeof(int) }), assign, docIndex, rowData, fieldArrayIndex); return((Action <int, DriverRowData, int>)lambda.Compile()); }
public IDriverDataEnumerator GetBulkUpdateEnumerator(List <FieldMetadata> fields, DriverRowData driverRow, IDriverDataEnumerator inputDataEnumerator) { var untrimmedCount = m_untrimmedDocumentCount; if (untrimmedCount == 0) { return(null); } return(new DocumentDataContainerEnumerator_BulkPkScan(untrimmedCount, driverRow, this, fields, inputDataEnumerator)); }
private static Type MakeNullableType(DbType dbType) { var underlyingType = DriverRowData.DeriveSystemType(dbType); return(underlyingType.IsValueType ? typeof(UnboxableNullable <>).MakeGenericType(underlyingType) : underlyingType); }