/// <summary> /// Change the default value for a column. `newDefault` must not be null or /// else throws. /// </summary> /// <param name="name">Name of the column.</param> /// <param name="newDefault">The new default value.</param> public AlterTableBuilder ChangeDefault(string name, object newDefault) { if (newDefault is null) { ThrowDefaultValueNullException(); } var column = _table.Schema.GetColumn(name); var defaultValue = KuduEncoder.EncodeDefaultValue(column, newDefault); _request.AlterSchemaSteps.Add(new Step { Type = StepType.AlterColumn, AlterColumn = new AlterColumn { Delta = new ColumnSchemaDeltaPB { Name = name, DefaultValue = UnsafeByteOperations.UnsafeWrap(defaultValue) } } }); return(this); }
private static void EncodeColumn( PartialRow row, int columnIndex, bool isLast, Span <byte> destination, out int bytesWritten) { var schema = row.Schema; var column = schema.GetColumn(columnIndex); if (column.IsFixedSize) { var size = column.Size; var slice = destination.Slice(0, size); var data = row.GetRowAllocColumn(columnIndex, size); data.CopyTo(slice); // Row data is little endian, but key encoding is big endian. slice.Reverse(); if (column.IsSigned) { KuduEncoder.XorLeftMostBit(slice); } bytesWritten = size; } else { var data = row.GetVarLengthColumn(columnIndex); EncodeBinary(data, destination, isLast, out bytesWritten); } }
public float GetFloat(int columnIndex) { CheckType(columnIndex, KuduType.Float); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4); return(KuduEncoder.DecodeFloat(data)); }
public int GetInt32(int columnIndex) { CheckType(columnIndex, KuduTypeFlags.Int32 | KuduTypeFlags.Date); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4); return(KuduEncoder.DecodeInt32(data)); }
public double GetDouble(int columnIndex) { CheckType(columnIndex, KuduType.Double); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8); return(KuduEncoder.DecodeDouble(data)); }
public short GetInt16(int columnIndex) { CheckType(columnIndex, KuduType.Int16); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 2); return(KuduEncoder.DecodeInt16(data)); }
public sbyte GetSByte(int columnIndex) { CheckType(columnIndex, KuduType.Int8); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 1); return(KuduEncoder.DecodeInt8(data)); }
public bool GetBool(int columnIndex) { CheckType(columnIndex, KuduType.Bool); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 1); return(KuduEncoder.DecodeBool(data)); }
public string GetString(int columnIndex) { CheckType(columnIndex, KuduTypeFlags.String | KuduTypeFlags.Varchar); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetVarLengthColumn(columnIndex); return(KuduEncoder.DecodeString(data)); }
public long GetInt64(int columnIndex) { CheckType(columnIndex, KuduTypeFlags.Int64 | KuduTypeFlags.UnixtimeMicros); CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8); return(KuduEncoder.DecodeInt64(data)); }
public decimal GetDecimal(int columnIndex) { ColumnSchema column = CheckType(columnIndex, KuduTypeFlags.Decimal32 | KuduTypeFlags.Decimal64 | KuduTypeFlags.Decimal128); int scale = column.TypeAttributes !.Scale.GetValueOrDefault(); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, column.Size); return(KuduEncoder.DecodeDecimal(data, column.Type, scale)); }
public void SetDecimal(int columnIndex, decimal value) { var column = CheckType(columnIndex, KuduTypeFlags.Decimal32 | KuduTypeFlags.Decimal64 | KuduTypeFlags.Decimal128); var typeAttributes = column.TypeAttributes !; int precision = typeAttributes.Precision.GetValueOrDefault(); int scale = typeAttributes.Scale.GetValueOrDefault(); Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, column.Size); KuduEncoder.EncodeDecimal(span, column.Type, value, precision, scale); }
public PartialRow(KuduSchema schema) { Schema = schema; var columnBitmapSize = KuduEncoder.BitsToBytes(schema.Columns.Count); var headerSize = columnBitmapSize; if (schema.HasNullableColumns) { // nullsBitSet is the same size as the columnBitSet. // Bits for non-nullable columns are ignored. headerSize += columnBitmapSize; _nullOffset = columnBitmapSize; } _rowAlloc = new byte[headerSize + schema.RowAllocSize]; _headerSize = headerSize; _varLengthData = new byte[schema.VarLengthColumnCount][]; }
public static ColumnSchema FromProtobuf(ColumnSchemaPB columnSchemaPb) { var type = (KuduType)columnSchemaPb.Type; var typeAttributes = columnSchemaPb.TypeAttributes.ToTypeAttributes(); var defaultValue = columnSchemaPb.HasWriteDefaultValue ? KuduEncoder.DecodeDefaultValue( type, typeAttributes, columnSchemaPb.WriteDefaultValue.Span) : null; return(new ColumnSchema( columnSchemaPb.Name, type, columnSchemaPb.IsKey, columnSchemaPb.IsNullable, defaultValue, columnSchemaPb.CfileBlockSize, (EncodingType)columnSchemaPb.Encoding, (CompressionType)columnSchemaPb.Compression, typeAttributes, columnSchemaPb.Comment)); }
public DateTime GetDateTime(int columnIndex) { var column = Schema.GetColumn(columnIndex); var type = column.Type; if (type == KuduType.UnixtimeMicros) { CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8); return(KuduEncoder.DecodeDateTime(data)); } else if (type == KuduType.Date) { CheckValue(columnIndex); ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4); return(KuduEncoder.DecodeDate(data)); } return(KuduTypeValidation.ThrowException <DateTime>(column, KuduTypeFlags.UnixtimeMicros | KuduTypeFlags.Date)); }
private void WriteDate(int columnIndex, DateTime value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4); KuduEncoder.EncodeDate(span, value); }
// Used to convert the rowwise data to the newer columnar format, // to avoid virtual calls on ResultSet. // This is only used if the Kudu server is 1.11 or older. public static ResultSet Convert( KuduMessage message, KuduSchema schema, RowwiseRowBlockPB rowPb) { var numColumns = schema.Columns.Count; int columnOffsetsSize = numColumns; if (schema.HasNullableColumns) { columnOffsetsSize++; } var columnOffsets = new int[columnOffsetsSize]; int currentOffset = 0; columnOffsets[0] = currentOffset; // Pre-compute the columns offsets in rowData for easier lookups later. // If the schema has nullables, we also add the offset for the null bitmap at the end. for (int i = 1; i < columnOffsetsSize; i++) { ColumnSchema column = schema.GetColumn(i - 1); int previousSize = column.Size; columnOffsets[i] = previousSize + currentOffset; currentOffset += previousSize; } var rowData = GetRowData(message, rowPb); var indirectData = GetIndirectData(message, rowPb); int nonNullBitmapOffset = columnOffsets[columnOffsets.Length - 1]; int rowSize = schema.RowSize; int numRows = rowPb.NumRows; var dataSidecarOffsets = new SidecarOffset[numColumns]; var varlenDataSidecarOffsets = new SidecarOffset[numColumns]; var nonNullBitmapSidecarOffsets = new SidecarOffset[numColumns]; int nonNullBitmapSize = KuduEncoder.BitsToBytes(numRows); int offset = 0; for (int i = 0; i < numColumns; i++) { var column = schema.GetColumn(i); var dataSize = column.IsFixedSize ? column.Size * numRows : (4 * numRows) + 4; dataSidecarOffsets[i] = new SidecarOffset(offset, dataSize); offset += dataSize; if (column.IsNullable) { nonNullBitmapSidecarOffsets[i] = new SidecarOffset(offset, nonNullBitmapSize); offset += nonNullBitmapSize; } else { nonNullBitmapSidecarOffsets[i] = new SidecarOffset(-1, 0); } } var buffer = new ArrayPoolBuffer <byte>(offset + indirectData.Length); var data = buffer.Buffer; data.AsSpan().Clear(); var varlenData = data.AsSpan(offset); int currentDataOffset = 0; int currentVarlenOffset = 0; for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) { var column = schema.GetColumn(columnIndex); var isFixedSize = column.IsFixedSize; var columnarSize = isFixedSize ? column.Size : 4; var rowwiseSize = column.Size; var dataOffset = dataSidecarOffsets[columnIndex]; var nonNullOffset = nonNullBitmapSidecarOffsets[columnIndex].Start; var dataOutput = data.AsSpan(dataOffset.Start, dataOffset.Length); for (int rowIndex = 0; rowIndex < numRows; rowIndex++) { bool isSet = true; var rowSlice = rowData.Slice(rowSize * rowIndex, rowSize); if (nonNullOffset > 0) { isSet = !rowSlice.GetBit(nonNullBitmapOffset, columnIndex); if (isSet) { data.SetBit(nonNullOffset, rowIndex); } } if (isSet) { if (isFixedSize) { var rawData = rowSlice.Slice(currentDataOffset, columnarSize); rawData.CopyTo(dataOutput); } else { var offsetData = rowSlice.Slice(currentDataOffset, 8); var lengthData = rowSlice.Slice(currentDataOffset + 8, 8); int start = (int)KuduEncoder.DecodeInt64(offsetData); int length = (int)KuduEncoder.DecodeInt64(lengthData); var indirectSlice = indirectData.Slice(start, length); indirectSlice.CopyTo(varlenData); varlenData = varlenData.Slice(length); KuduEncoder.EncodeInt32(dataOutput, currentVarlenOffset); currentVarlenOffset += length; } } dataOutput = dataOutput.Slice(columnarSize); } currentDataOffset += rowwiseSize; if (!isFixedSize) { KuduEncoder.EncodeInt32(dataOutput, currentVarlenOffset); varlenDataSidecarOffsets[columnIndex] = new SidecarOffset(offset, currentVarlenOffset); offset += currentVarlenOffset; currentVarlenOffset = 0; } } return(new ResultSet( buffer, schema, numRows, dataSidecarOffsets, varlenDataSidecarOffsets, nonNullBitmapSidecarOffsets)); }
/// <summary> /// Increments the column at the given index, returning false if the /// value is already the maximum. /// </summary> /// <param name="index">The column index to increment.</param> internal bool IncrementColumn(int index) { if (!IsSet(index)) { throw new ArgumentException($"Column index {index} has not been set."); } ColumnSchema column = Schema.GetColumn(index); if (column.IsFixedSize) { KuduType type = column.Type; Span <byte> data = GetRowAllocColumn(index, column.Size); switch (type) { case KuduType.Bool: { bool isFalse = data[0] == 0; data[0] = 1; return(isFalse); } case KuduType.Int8: { sbyte existing = KuduEncoder.DecodeInt8(data); if (existing == sbyte.MaxValue) { return(false); } KuduEncoder.EncodeInt8(data, (sbyte)(existing + 1)); return(true); } case KuduType.Int16: { short existing = KuduEncoder.DecodeInt16(data); if (existing == short.MaxValue) { return(false); } KuduEncoder.EncodeInt16(data, (short)(existing + 1)); return(true); } case KuduType.Int32: { int existing = KuduEncoder.DecodeInt32(data); if (existing == int.MaxValue) { return(false); } KuduEncoder.EncodeInt32(data, existing + 1); return(true); } case KuduType.Date: { int existing = KuduEncoder.DecodeInt32(data); if (existing == EpochTime.MaxDateValue) { return(false); } KuduEncoder.EncodeInt32(data, existing + 1); return(true); } case KuduType.Int64: case KuduType.UnixtimeMicros: { long existing = KuduEncoder.DecodeInt64(data); if (existing == long.MaxValue) { return(false); } KuduEncoder.EncodeInt64(data, existing + 1); return(true); } case KuduType.Float: { float existing = KuduEncoder.DecodeFloat(data); float incremented = existing.NextUp(); if (existing == incremented) { return(false); } KuduEncoder.EncodeFloat(data, incremented); return(true); } case KuduType.Double: { double existing = KuduEncoder.DecodeDouble(data); double incremented = existing.NextUp(); if (existing == incremented) { return(false); } KuduEncoder.EncodeDouble(data, incremented); return(true); } case KuduType.Decimal32: { int existing = KuduEncoder.DecodeInt32(data); int precision = column.TypeAttributes !.Precision.GetValueOrDefault(); if (existing == DecimalUtil.MaxDecimal32(precision)) { return(false); } KuduEncoder.EncodeInt32(data, existing + 1); return(true); } case KuduType.Decimal64: { long existing = KuduEncoder.DecodeInt64(data); int precision = column.TypeAttributes !.Precision.GetValueOrDefault(); if (existing == DecimalUtil.MaxDecimal64(precision)) { return(false); } KuduEncoder.EncodeInt64(data, existing + 1); return(true); } case KuduType.Decimal128: { KuduInt128 existing = KuduEncoder.DecodeInt128(data); int precision = column.TypeAttributes !.Precision.GetValueOrDefault(); if (existing == DecimalUtil.MaxDecimal128(precision)) { return(false); } KuduEncoder.EncodeInt128(data, existing + 1); return(true); } default: throw new Exception($"Unsupported data type {type}"); } } else { // Column is either string, binary, or varchar. ReadOnlySpan <byte> data = GetVarLengthColumn(index); var incremented = new byte[data.Length + 1]; data.CopyTo(incremented); WriteBinary(index, incremented); return(true); } }
/// <summary> /// Sets the column to the minimum possible value for the column's type. /// </summary> /// <param name="index">The index of the column to set to the minimum.</param> internal void SetMin(int index) { ColumnSchema column = Schema.GetColumn(index); KuduType type = column.Type; switch (type) { case KuduType.Bool: WriteBool(index, false); break; case KuduType.Int8: WriteSByte(index, sbyte.MinValue); break; case KuduType.Int16: WriteInt16(index, short.MinValue); break; case KuduType.Int32: WriteInt32(index, int.MinValue); break; case KuduType.Date: WriteInt32(index, EpochTime.MinDateValue); break; case KuduType.Int64: case KuduType.UnixtimeMicros: WriteInt64(index, long.MinValue); break; case KuduType.Float: WriteFloat(index, float.MinValue); break; case KuduType.Double: WriteDouble(index, double.MinValue); break; case KuduType.Decimal32: WriteInt32(index, DecimalUtil.MinDecimal32( column.TypeAttributes !.Precision.GetValueOrDefault())); break; case KuduType.Decimal64: WriteInt64(index, DecimalUtil.MinDecimal64( column.TypeAttributes !.Precision.GetValueOrDefault())); break; case KuduType.Decimal128: { KuduInt128 min = DecimalUtil.MinDecimal128( column.TypeAttributes !.Precision.GetValueOrDefault()); Span <byte> span = GetSpanInRowAllocAndSetBitSet(index, 16); KuduEncoder.EncodeInt128(span, min); break; } case KuduType.String: case KuduType.Varchar: WriteString(index, string.Empty); break; case KuduType.Binary: WriteBinary(index, Array.Empty <byte>()); break; default: throw new Exception($"Unsupported data type {type}"); } }
private static ResultSet CreateResultSet( KuduMessage message, KuduSchema schema, ColumnarRowBlockPB data) { var columns = data.Columns; var numColumns = columns.Count; if (numColumns != schema.Columns.Count) { ThrowColumnCountMismatchException(schema.Columns.Count, numColumns); } if (data.Columns.Count == 0 || data.NumRows == 0) { // Empty projection, usually used for quick row counting. return(CreateEmptyResultSet(schema, data.NumRows)); } var numRows = checked ((int)data.NumRows); var bufferLength = message.Buffer.Length; var nonNullBitmapLength = KuduEncoder.BitsToBytes(numRows); var dataSidecarOffsets = new SidecarOffset[numColumns]; var varlenDataSidecarOffsets = new SidecarOffset[numColumns]; var nonNullBitmapSidecarOffsets = new SidecarOffset[numColumns]; for (int i = 0; i < numColumns; i++) { var column = columns[i]; var columnSchema = schema.GetColumn(i); if (column.HasDataSidecar) { var offset = message.GetSidecarOffset(column.DataSidecar); var length = GetColumnDataSize(columnSchema, numRows); ValidateSidecar(offset, length, bufferLength); dataSidecarOffsets[i] = offset; } else { ThrowMissingDataSidecarException(columnSchema); } if (column.HasVarlenDataSidecar) { var offset = message.GetSidecarOffset(column.VarlenDataSidecar); varlenDataSidecarOffsets[i] = offset; } if (column.HasNonNullBitmapSidecar) { var offset = message.GetSidecarOffset(column.NonNullBitmapSidecar); ValidateSidecar(offset, nonNullBitmapLength, bufferLength); nonNullBitmapSidecarOffsets[i] = offset; } else { nonNullBitmapSidecarOffsets[i] = new SidecarOffset(-1, 0); } } var buffer = message.TakeMemory(); return(new ResultSet( buffer, schema, data.NumRows, dataSidecarOffsets, varlenDataSidecarOffsets, nonNullBitmapSidecarOffsets)); }
private void WriteString(int columnIndex, string value) { var data = KuduEncoder.EncodeString(value); WriteBinary(columnIndex, data); }
private void WriteBool(int columnIndex, bool value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 1); KuduEncoder.EncodeBool(span, value); }
private void WriteSByte(int columnIndex, sbyte value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 1); KuduEncoder.EncodeInt8(span, value); }
private void WriteInt32(int columnIndex, int value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4); KuduEncoder.EncodeInt32(span, value); }
private void WriteDouble(int columnIndex, double value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 8); KuduEncoder.EncodeDouble(span, value); }
private void WriteInt64(int columnIndex, long value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 8); KuduEncoder.EncodeInt64(span, value); }
private void WriteFloat(int columnIndex, float value) { Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4); KuduEncoder.EncodeFloat(span, value); }
public async Task TestNonNullRows(bool includeNullsInSchema) { await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync(); await using var client = miniCluster.CreateClient(); await using var session = client.NewSession(); var builder = ClientTestUtil.CreateAllTypesSchema(includeNullsInSchema) .SetTableName(nameof(TestNonNullRows)) .CreateBasicRangePartition(); var table = await client.CreateTableAsync(builder); int numRows = 5; int currentRow = 0; for (int i = 0; i < numRows; i++) { var insert = ClientTestUtil.CreateAllTypesInsert(table, i); await session.EnqueueAsync(insert); } await session.FlushAsync(); var scanner = client.NewScanBuilder(table).Build(); await foreach (var resultSet in scanner) { foreach (var row in resultSet) { Assert.Equal(currentRow, row.GetInt32("key")); Assert.Equal(currentRow, row.GetNullableInt32("key")); Assert.Equal(KuduEncoder.EncodeInt32(currentRow), row.GetSpan("key").ToArray()); Assert.Equal(42, row.GetByte("int8")); Assert.Equal((byte?)42, row.GetNullableByte("int8")); Assert.Equal(42, row.GetSByte("int8")); Assert.Equal((sbyte?)42, row.GetNullableSByte("int8")); Assert.Equal(43, row.GetInt16("int16")); Assert.Equal((short?)43, row.GetNullableInt16("int16")); Assert.Equal(44, row.GetInt32("int32")); Assert.Equal(44, row.GetNullableInt32("int32")); Assert.Equal(45, row.GetInt64("int64")); Assert.Equal(45, row.GetNullableInt64("int64")); Assert.True(row.GetBool("bool")); Assert.True(row.GetNullableBool("bool")); Assert.Equal(52.35f, row.GetFloat("float")); Assert.Equal(52.35f, row.GetNullableFloat("float")); Assert.Equal(53.35, row.GetDouble("double")); Assert.Equal(53.35, row.GetNullableDouble("double")); Assert.Equal("fun with ütf\0", row.GetString("string")); Assert.Equal("fun with ütf\0", row.GetNullableString("string")); Assert.Equal("árvíztűrő ", row.GetString("varchar")); Assert.Equal("árvíztűrő ", row.GetNullableString("varchar")); Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, row.GetBinary("binary")); Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, row.GetSpan("binary").ToArray()); Assert.Equal(DateTime.Parse("8/19/2020 7:50 PM").ToUniversalTime(), row.GetDateTime("timestamp")); Assert.Equal(DateTime.Parse("8/19/2020 7:50 PM").ToUniversalTime(), row.GetNullableDateTime("timestamp")); Assert.Equal(DateTime.Parse("8/19/2020").ToUniversalTime().Date, row.GetDateTime("date")); Assert.Equal(DateTime.Parse("8/19/2020").ToUniversalTime().Date, row.GetNullableDateTime("date")); Assert.Equal(12.345m, row.GetDecimal("decimal32")); Assert.Equal(12.345m, row.GetNullableDecimal("decimal32")); Assert.Equal(12.346m, row.GetDecimal("decimal64")); Assert.Equal(12.346m, row.GetNullableDecimal("decimal64")); Assert.Equal(12.347m, row.GetDecimal("decimal128")); Assert.Equal(12.347m, row.GetNullableDecimal("decimal128")); currentRow++; } } Assert.Equal(numRows, currentRow); }