public void GetValue() { var readState = new RecordReadState(); var type = new SqlBit(readState, CompressionContext.NoCompression); // No bytes read - length is one Assert.AreEqual(1, type.FixedLength); // Load byte and check length is 0 readState.LoadBitByte(0xD2); Assert.AreEqual(0, type.FixedLength); Assert.IsFalse((bool)type.GetValue(new byte[0])); Assert.IsTrue((bool)type.GetValue(new byte[0])); Assert.IsFalse((bool)type.GetValue(new byte[0])); Assert.IsFalse((bool)type.GetValue(new byte[0])); Assert.IsTrue((bool)type.GetValue(new byte[0])); Assert.IsFalse((bool)type.GetValue(new byte[0])); Assert.IsTrue((bool)type.GetValue(new byte[0])); // One bit left - length should still be 0 Assert.AreEqual(0, type.FixedLength); Assert.IsTrue((bool)type.GetValue(new byte[0])); // All bits consumed - length should be 1 Assert.AreEqual(1, type.FixedLength); }
public void General() { var state = new RecordReadState(); // No bits available Assert.IsTrue(state.AllBitsConsumed); state.LoadBitByte(0xD2); // 11010010 // Bits available Assert.IsFalse(state.AllBitsConsumed); // Reading bit values Assert.IsFalse(state.GetNextBit()); Assert.IsTrue(state.GetNextBit()); Assert.IsFalse(state.GetNextBit()); Assert.IsFalse(state.GetNextBit()); Assert.IsTrue(state.GetNextBit()); Assert.IsFalse(state.GetNextBit()); Assert.IsTrue(state.GetNextBit()); // One bit left Assert.IsFalse(state.AllBitsConsumed); Assert.IsTrue(state.GetNextBit()); // Bits exhausted, ready for next byte Assert.IsTrue(state.AllBitsConsumed); }
internal override IEnumerable <Row> GetEntities(DataExtractorHelper schema) { foreach (var record in page.Records) { var dataRow = schema.NewRow(); var readState = new RecordReadState(schema.Columns.Count(x => x.UnderlyingType == ColumnType.Bit)); int columnIndex = 0; foreach (DataColumn col in dataRow.Columns) { var sqlType = SqlTypeFactory.Create(col, readState, new CompressionContext(CompressionLevel.Row, true)); IVariableLengthDataProxy dataProxy = record.GetPhysicalColumnBytes(columnIndex); if (dataProxy == null) { dataRow[col] = null; } else { dataRow[col] = sqlType.GetValue(dataProxy.GetBytes().ToArray()); } columnIndex++; } yield return(dataRow); } }
internal IEnumerable <Row> GetEntities(Row schema, CompressionContext compression) { for (int i = 0; i < Records.Length; i++) { var record = Records[i]; short fixedOffset = 0; short variableColumnIndex = 0; int columnIndex = 0; var readState = new RecordReadState(); var dataRow = schema.NewRow(); foreach (DataColumn col in dataRow.Columns) { var sqlType = SqlTypeFactory.Create(col, readState, compression); object columnValue = null; if (sqlType.IsVariableLength) { if (!record.HasNullBitmap || !record.NullBitmap[columnIndex]) { // If a nullable varlength column does not have a value, it may be not even appear in the varlength column array if it's at the tail if (record.VariableLengthColumnData.Count <= variableColumnIndex) { columnValue = sqlType.GetValue(new byte[] { }); } else { columnValue = sqlType.GetValue(record.VariableLengthColumnData[variableColumnIndex].GetBytes().ToArray()); } } variableColumnIndex++; } else { // Must cache type FixedLength as it may change after getting a value (e.g. SqlBit) short fixedLength = sqlType.FixedLength.Value; if (!record.HasNullBitmap || !record.NullBitmap[columnIndex]) { columnValue = sqlType.GetValue(record.FixedLengthData.Skip(fixedOffset).Take(fixedLength).ToArray()); } fixedOffset += fixedLength; } columnIndex++; dataRow[col] = columnValue; } yield return(dataRow); } }
internal override IEnumerable <Row> GetEntities(DataExtractorHelper schema) { foreach (var record in page.Records) { // Don't process forwarded blob fragments as they should only be processed from the referenced record if (_recordsToSkip.Contains(record.Type) || record.IsGhostForwardedRecord) { continue; } short fixedOffset = 0; short variableColumnIndex = 0; var dataRow = schema.NewRow(); var readState = new RecordReadState(schema.BitColumnsCount); var bitColumnBytes = new byte[0]; foreach (var col in schema.Columns) { var sqlType = SqlTypeFactory.Create(col, readState, compression); object columnValue = null; // Sparse columns needs to retrieve their values from the sparse vector, contained in the very last // variable length column in the record. if (col.IsSparse) { // We may encounter records that don't have any sparse vectors, for instance if no sparse columns have values if (record.SparseVector != null) { // Column ID's are stored as ints in general. In the sparse vector though, they're stored as shorts. if (record.SparseVector.ColumnValues.ContainsKey((short)col.ColumnID)) { columnValue = sqlType.GetValue(record.SparseVector.ColumnValues[(short)col.ColumnID]); } } } else { var nonSparseIndex = schema.NonSparseIndexes[col.Name]; // Before we even try to parse the column & make a null bitmap lookup, ensure that it's present in the record. // There may be columns > record.NumberOfColumns caused by nullable columns added to the schema after the record was written. if (nonSparseIndex < record.NumberOfColumns && col.UnderlyingType != ColumnType.Computed) { if (sqlType.IsVariableLength) { // If there's either no null bitmap, or the null bitmap defines the column as non-null. if (!record.HasNullBitmap || !record.NullBitmap[nonSparseIndex]) { // If the current variable length column index exceeds the number of stored // variable length columns, the value is empty by definition (that is, 0 bytes, but not null). if (variableColumnIndex < record.NumberOfVariableLengthColumns) { var data = record.VariableLengthColumnData[variableColumnIndex].GetBytes()?.ToArray(); columnValue = sqlType.GetValue(data ?? new byte[0]); } else { columnValue = sqlType.GetValue(new byte[0]); } } variableColumnIndex++; } else { // Must cache type FixedLength as it may change after getting a value (e.g. SqlBit) var fixedLength = sqlType.FixedLength.Value; if ((!record.HasNullBitmap || !record.NullBitmap[nonSparseIndex]) && col.UnderlyingType != ColumnType.Bit) { var valueBytes = record.FixedLengthData.Skip(fixedOffset).Take(fixedLength).ToArray(); // We may run out of fixed length bytes. In certain conditions a null integer may have been added without // there being a null bitmap. In such a case, we detect the null condition by there not being enough fixed // length bytes to process. if (valueBytes.Length > 0) { columnValue = sqlType.GetValue(valueBytes); } } else if (col.UnderlyingType == ColumnType.Bit && !schema.IsDroppedColumn(col)) { if (readState.IsFirstBit) { bitColumnBytes = record.FixedLengthData.Skip(fixedOffset).Take(fixedLength).ToArray(); } var value = sqlType.GetValue(bitColumnBytes); columnValue = !record.HasNullBitmap || !record.NullBitmap[nonSparseIndex] ? value : null; } fixedOffset += fixedLength; } } else if (col.UnderlyingType == ColumnType.Computed) { columnValue = sqlType.GetValue(null); } else if (!col.IsNullable) { columnValue = schema.GetDefaultValue(col, sqlType); } } if (!schema.IsDroppedColumn(col)) { dataRow[col] = columnValue; } } yield return(dataRow); } }
internal override IEnumerable<Row> GetEntities(Row schema) { foreach (var record in page.Records) { // Don't process forwarded blob fragments as they should only be processed from the referenced record if (record.Type == RecordType.BlobFragment) continue; short fixedOffset = 0; short variableColumnIndex = 0; short nonSparseColumnIndex = 0; var readState = new RecordReadState(); var dataRow = schema.NewRow(); foreach (DataColumn col in dataRow.Columns) { var sqlType = SqlTypeFactory.Create(col, readState, compression); object columnValue = null; // Sparse columns needs to retrieve their values from the sparse vector, contained in the very last // variable length column in the record. if (col.IsSparse) { // We may encounter records that don't have any sparse vectors, for instance if no sparse columns have values if (record.SparseVector != null) { // Column ID's are stored as ints in general. In the sparse vector though, they're stored as shorts. if (record.SparseVector.ColumnValues.ContainsKey((short)col.ColumnID)) columnValue = sqlType.GetValue(record.SparseVector.ColumnValues[(short)col.ColumnID]); } } else { // Before we even try to parse the column & make a null bitmap lookup, ensure that it's present in the record. // There may be columns > record.NumberOfColumns caused by nullable columns added to the schema after the record was written. if (nonSparseColumnIndex < record.NumberOfColumns) { if (sqlType.IsVariableLength) { // If there's either no null bitmap, or the null bitmap defines the column as non-null. if (!record.HasNullBitmap || !record.NullBitmap[nonSparseColumnIndex]) { // If the current variable length column index exceeds the number of stored // variable length columns, the value is empty by definition (that is, 0 bytes, but not null). if (variableColumnIndex < record.NumberOfVariableLengthColumns) columnValue = sqlType.GetValue(record.VariableLengthColumnData[variableColumnIndex].GetBytes().ToArray()); else columnValue = sqlType.GetValue(new byte[0]); } variableColumnIndex++; } else { // Must cache type FixedLength as it may change after getting a value (e.g. SqlBit) short fixedLength = sqlType.FixedLength.Value; if (!record.HasNullBitmap || !record.NullBitmap[nonSparseColumnIndex]) { byte[] valueBytes = record.FixedLengthData.Skip(fixedOffset).Take(fixedLength).ToArray(); // We may run out of fixed length bytes. In certain conditions a null integer may have been added without // there being a null bitmap. In such a case, we detect the null condition by there not being enough fixed // length bytes to process. if (valueBytes.Length > 0) columnValue = sqlType.GetValue(valueBytes); } fixedOffset += fixedLength; } // Sparse columns don't have an entry in the null bitmap, thus we should only increment it if the current // column was not a sparse column. nonSparseColumnIndex++; } } dataRow[col] = columnValue; } yield return dataRow; } }
public SqlBit(RecordReadState readState, CompressionContext compression) : base(compression) { this._readState = readState; }
public static ISqlType Create(DataColumn column, RecordReadState readState, CompressionContext compression) { switch (column.UnderlyingType) { case ColumnType.Binary: return(new SqlBinary((short)column.VariableFixedLength, compression)); case ColumnType.BigInt: return(new SqlBigInt(compression)); case ColumnType.Bit: return(new SqlBit(readState, compression)); case ColumnType.Char: return(new SqlChar((short)column.VariableFixedLength, column.Encoding ?? DefaultEncoding, compression)); case ColumnType.DateTime: return(new SqlDateTime(compression)); case ColumnType.Float: return(new SqlFloat(column.Precision, compression)); case ColumnType.Decimal: return(new SqlDecimal(column.Precision, column.Scale, compression)); case ColumnType.Image: return(new SqlImage(compression)); case ColumnType.Int: return(new SqlInt(compression)); case ColumnType.Money: return(new SqlMoney(compression)); case ColumnType.NChar: return(new SqlNChar((short)column.VariableFixedLength, compression)); case ColumnType.NText: return(new SqlNText(compression)); case ColumnType.NVarchar: return(new SqlNVarchar(compression)); case ColumnType.RID: return(new SqlRID(compression)); case ColumnType.SmallDatetime: return(new SqlSmallDateTime(compression)); case ColumnType.Date: return(new SqlDate(compression)); case ColumnType.DateTimeOffset: return(new SqlDateTimeOffset(column.Scale, compression)); case ColumnType.DateTime2: return(new SqlDateTime2(column.Scale, compression)); case ColumnType.Time: return(new SqlTime(column.Scale, compression)); case ColumnType.SmallInt: return(new SqlSmallInt(compression)); case ColumnType.SmallMoney: return(new SqlSmallMoney(compression)); case ColumnType.Text: return(new SqlText(compression)); case ColumnType.TinyInt: return(new SqlTinyInt(compression)); case ColumnType.UniqueIdentifier: return(new SqlUniqueIdentifier(compression)); case ColumnType.Uniquifier: return(new SqlUniquifier(compression)); case ColumnType.VarBinary: return(new SqlVarBinary(compression)); case ColumnType.Varchar: return(new SqlVarchar(compression, column.Encoding ?? DefaultEncoding)); case ColumnType.Variant: return(new SqlVariant(compression)); case ColumnType.Computed: return(new SqlComputed(compression)); } throw new ArgumentException("Unsupported type: " + column); }
internal override IEnumerable <Row> GetEntities(Row schema) { foreach (var record in page.Records) { // Don't process forwarded blob fragments as they should only be processed from the referenced record if (record.Type == RecordType.BlobFragment) { continue; } short fixedOffset = 0; short variableColumnIndex = 0; short nonSparseColumnIndex = 0; var readState = new RecordReadState(); var dataRow = schema.NewRow(); foreach (DataColumn col in dataRow.Columns) { var sqlType = SqlTypeFactory.Create(col, readState, compression); object columnValue = null; // Sparse columns needs to retrieve their values from the sparse vector, contained in the very last // variable length column in the record. if (col.IsSparse) { // We may encounter records that don't have any sparse vectors, for instance if no sparse columns have values if (record.SparseVector != null) { // Column ID's are stored as ints in general. In the sparse vector though, they're stored as shorts. if (record.SparseVector.ColumnValues.ContainsKey((short)col.ColumnID)) { columnValue = sqlType.GetValue(record.SparseVector.ColumnValues[(short)col.ColumnID]); } } } else { // Before we even try to parse the column & make a null bitmap lookup, ensure that it's present in the record. // There may be columns > record.NumberOfColumns caused by nullable columns added to the schema after the record was written. if (nonSparseColumnIndex < record.NumberOfColumns) { if (sqlType.IsVariableLength) { // If there's either no null bitmap, or the null bitmap defines the column as non-null. if (!record.HasNullBitmap || !record.NullBitmap[nonSparseColumnIndex]) { // If the current variable length column index exceeds the number of stored // variable length columns, the value is empty by definition (that is, 0 bytes, but not null). if (variableColumnIndex < record.NumberOfVariableLengthColumns) { columnValue = sqlType.GetValue(record.VariableLengthColumnData[variableColumnIndex].GetBytes().ToArray()); } else { columnValue = sqlType.GetValue(new byte[0]); } } variableColumnIndex++; } else { // Must cache type FixedLength as it may change after getting a value (e.g. SqlBit) short fixedLength = sqlType.FixedLength.Value; if (!record.HasNullBitmap || !record.NullBitmap[nonSparseColumnIndex]) { byte[] valueBytes = record.FixedLengthData.Skip(fixedOffset).Take(fixedLength).ToArray(); // We may run out of fixed length bytes. In certain conditions a null integer may have been added without // there being a null bitmap. In such a case, we detect the null condition by there not being enough fixed // length bytes to process. if (valueBytes.Length > 0) { columnValue = sqlType.GetValue(valueBytes); } } fixedOffset += fixedLength; } // Sparse columns don't have an entry in the null bitmap, thus we should only increment it if the current // column was not a sparse column. nonSparseColumnIndex++; } } dataRow[col] = columnValue; } yield return(dataRow); } }