/// <summary>
/// Change the default value for a column. `newDefault` must not be null or
/// else throws.
/// </summary>
/// <param name="name">Name of the column.</param>
/// <param name="newDefault">The new default value.</param>
public AlterTableBuilder ChangeDefault(string name, object newDefault)
{
if (newDefault is null)
{
ThrowDefaultValueNullException();
}
var column = _table.Schema.GetColumn(name);
var defaultValue = KuduEncoder.EncodeDefaultValue(column, newDefault);
_request.AlterSchemaSteps.Add(new Step
{
Type = StepType.AlterColumn,
AlterColumn = new AlterColumn
{
Delta = new ColumnSchemaDeltaPB
{
Name = name,
DefaultValue = UnsafeByteOperations.UnsafeWrap(defaultValue)
}
}
});
return(this);
}
private static void EncodeColumn(
PartialRow row,
int columnIndex,
bool isLast,
Span <byte> destination,
out int bytesWritten)
{
var schema = row.Schema;
var column = schema.GetColumn(columnIndex);
if (column.IsFixedSize)
{
var size = column.Size;
var slice = destination.Slice(0, size);
var data = row.GetRowAllocColumn(columnIndex, size);
data.CopyTo(slice);
// Row data is little endian, but key encoding is big endian.
slice.Reverse();
if (column.IsSigned)
{
KuduEncoder.XorLeftMostBit(slice);
}
bytesWritten = size;
}
else
{
var data = row.GetVarLengthColumn(columnIndex);
EncodeBinary(data, destination, isLast, out bytesWritten);
}
}
public float GetFloat(int columnIndex)
{
CheckType(columnIndex, KuduType.Float);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4);
return(KuduEncoder.DecodeFloat(data));
}
public int GetInt32(int columnIndex)
{
CheckType(columnIndex, KuduTypeFlags.Int32 | KuduTypeFlags.Date);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4);
return(KuduEncoder.DecodeInt32(data));
}
public double GetDouble(int columnIndex)
{
CheckType(columnIndex, KuduType.Double);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8);
return(KuduEncoder.DecodeDouble(data));
}
public short GetInt16(int columnIndex)
{
CheckType(columnIndex, KuduType.Int16);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 2);
return(KuduEncoder.DecodeInt16(data));
}
public sbyte GetSByte(int columnIndex)
{
CheckType(columnIndex, KuduType.Int8);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 1);
return(KuduEncoder.DecodeInt8(data));
}
public bool GetBool(int columnIndex)
{
CheckType(columnIndex, KuduType.Bool);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 1);
return(KuduEncoder.DecodeBool(data));
}
public string GetString(int columnIndex)
{
CheckType(columnIndex, KuduTypeFlags.String | KuduTypeFlags.Varchar);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetVarLengthColumn(columnIndex);
return(KuduEncoder.DecodeString(data));
}
public long GetInt64(int columnIndex)
{
CheckType(columnIndex, KuduTypeFlags.Int64 | KuduTypeFlags.UnixtimeMicros);
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8);
return(KuduEncoder.DecodeInt64(data));
}
public decimal GetDecimal(int columnIndex)
{
ColumnSchema column = CheckType(columnIndex,
KuduTypeFlags.Decimal32 |
KuduTypeFlags.Decimal64 |
KuduTypeFlags.Decimal128);
int scale = column.TypeAttributes !.Scale.GetValueOrDefault();
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, column.Size);
return(KuduEncoder.DecodeDecimal(data, column.Type, scale));
}
public void SetDecimal(int columnIndex, decimal value)
{
var column = CheckType(columnIndex,
KuduTypeFlags.Decimal32 |
KuduTypeFlags.Decimal64 |
KuduTypeFlags.Decimal128);
var typeAttributes = column.TypeAttributes !;
int precision = typeAttributes.Precision.GetValueOrDefault();
int scale = typeAttributes.Scale.GetValueOrDefault();
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, column.Size);
KuduEncoder.EncodeDecimal(span, column.Type, value, precision, scale);
}
public PartialRow(KuduSchema schema)
{
Schema = schema;
var columnBitmapSize = KuduEncoder.BitsToBytes(schema.Columns.Count);
var headerSize = columnBitmapSize;
if (schema.HasNullableColumns)
{
// nullsBitSet is the same size as the columnBitSet.
// Bits for non-nullable columns are ignored.
headerSize += columnBitmapSize;
_nullOffset = columnBitmapSize;
}
_rowAlloc = new byte[headerSize + schema.RowAllocSize];
_headerSize = headerSize;
_varLengthData = new byte[schema.VarLengthColumnCount][];
}
public static ColumnSchema FromProtobuf(ColumnSchemaPB columnSchemaPb)
{
var type = (KuduType)columnSchemaPb.Type;
var typeAttributes = columnSchemaPb.TypeAttributes.ToTypeAttributes();
var defaultValue = columnSchemaPb.HasWriteDefaultValue
? KuduEncoder.DecodeDefaultValue(
type, typeAttributes, columnSchemaPb.WriteDefaultValue.Span)
: null;
return(new ColumnSchema(
columnSchemaPb.Name,
type,
columnSchemaPb.IsKey,
columnSchemaPb.IsNullable,
defaultValue,
columnSchemaPb.CfileBlockSize,
(EncodingType)columnSchemaPb.Encoding,
(CompressionType)columnSchemaPb.Compression,
typeAttributes,
columnSchemaPb.Comment));
}
public DateTime GetDateTime(int columnIndex)
{
var column = Schema.GetColumn(columnIndex);
var type = column.Type;
if (type == KuduType.UnixtimeMicros)
{
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 8);
return(KuduEncoder.DecodeDateTime(data));
}
else if (type == KuduType.Date)
{
CheckValue(columnIndex);
ReadOnlySpan <byte> data = GetRowAllocColumn(columnIndex, 4);
return(KuduEncoder.DecodeDate(data));
}
return(KuduTypeValidation.ThrowException <DateTime>(column,
KuduTypeFlags.UnixtimeMicros |
KuduTypeFlags.Date));
}
private void WriteDate(int columnIndex, DateTime value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4);
KuduEncoder.EncodeDate(span, value);
}
// Used to convert the rowwise data to the newer columnar format,
// to avoid virtual calls on ResultSet.
// This is only used if the Kudu server is 1.11 or older.
public static ResultSet Convert(
KuduMessage message,
KuduSchema schema,
RowwiseRowBlockPB rowPb)
{
var numColumns = schema.Columns.Count;
int columnOffsetsSize = numColumns;
if (schema.HasNullableColumns)
{
columnOffsetsSize++;
}
var columnOffsets = new int[columnOffsetsSize];
int currentOffset = 0;
columnOffsets[0] = currentOffset;
// Pre-compute the columns offsets in rowData for easier lookups later.
// If the schema has nullables, we also add the offset for the null bitmap at the end.
for (int i = 1; i < columnOffsetsSize; i++)
{
ColumnSchema column = schema.GetColumn(i - 1);
int previousSize = column.Size;
columnOffsets[i] = previousSize + currentOffset;
currentOffset += previousSize;
}
var rowData = GetRowData(message, rowPb);
var indirectData = GetIndirectData(message, rowPb);
int nonNullBitmapOffset = columnOffsets[columnOffsets.Length - 1];
int rowSize = schema.RowSize;
int numRows = rowPb.NumRows;
var dataSidecarOffsets = new SidecarOffset[numColumns];
var varlenDataSidecarOffsets = new SidecarOffset[numColumns];
var nonNullBitmapSidecarOffsets = new SidecarOffset[numColumns];
int nonNullBitmapSize = KuduEncoder.BitsToBytes(numRows);
int offset = 0;
for (int i = 0; i < numColumns; i++)
{
var column = schema.GetColumn(i);
var dataSize = column.IsFixedSize
? column.Size * numRows
: (4 * numRows) + 4;
dataSidecarOffsets[i] = new SidecarOffset(offset, dataSize);
offset += dataSize;
if (column.IsNullable)
{
nonNullBitmapSidecarOffsets[i] = new SidecarOffset(offset, nonNullBitmapSize);
offset += nonNullBitmapSize;
}
else
{
nonNullBitmapSidecarOffsets[i] = new SidecarOffset(-1, 0);
}
}
var buffer = new ArrayPoolBuffer <byte>(offset + indirectData.Length);
var data = buffer.Buffer;
data.AsSpan().Clear();
var varlenData = data.AsSpan(offset);
int currentDataOffset = 0;
int currentVarlenOffset = 0;
for (int columnIndex = 0; columnIndex < numColumns; columnIndex++)
{
var column = schema.GetColumn(columnIndex);
var isFixedSize = column.IsFixedSize;
var columnarSize = isFixedSize ? column.Size : 4;
var rowwiseSize = column.Size;
var dataOffset = dataSidecarOffsets[columnIndex];
var nonNullOffset = nonNullBitmapSidecarOffsets[columnIndex].Start;
var dataOutput = data.AsSpan(dataOffset.Start, dataOffset.Length);
for (int rowIndex = 0; rowIndex < numRows; rowIndex++)
{
bool isSet = true;
var rowSlice = rowData.Slice(rowSize * rowIndex, rowSize);
if (nonNullOffset > 0)
{
isSet = !rowSlice.GetBit(nonNullBitmapOffset, columnIndex);
if (isSet)
{
data.SetBit(nonNullOffset, rowIndex);
}
}
if (isSet)
{
if (isFixedSize)
{
var rawData = rowSlice.Slice(currentDataOffset, columnarSize);
rawData.CopyTo(dataOutput);
}
else
{
var offsetData = rowSlice.Slice(currentDataOffset, 8);
var lengthData = rowSlice.Slice(currentDataOffset + 8, 8);
int start = (int)KuduEncoder.DecodeInt64(offsetData);
int length = (int)KuduEncoder.DecodeInt64(lengthData);
var indirectSlice = indirectData.Slice(start, length);
indirectSlice.CopyTo(varlenData);
varlenData = varlenData.Slice(length);
KuduEncoder.EncodeInt32(dataOutput, currentVarlenOffset);
currentVarlenOffset += length;
}
}
dataOutput = dataOutput.Slice(columnarSize);
}
currentDataOffset += rowwiseSize;
if (!isFixedSize)
{
KuduEncoder.EncodeInt32(dataOutput, currentVarlenOffset);
varlenDataSidecarOffsets[columnIndex] = new SidecarOffset(offset, currentVarlenOffset);
offset += currentVarlenOffset;
currentVarlenOffset = 0;
}
}
return(new ResultSet(
buffer,
schema,
numRows,
dataSidecarOffsets,
varlenDataSidecarOffsets,
nonNullBitmapSidecarOffsets));
}
/// <summary>
/// Increments the column at the given index, returning false if the
/// value is already the maximum.
/// </summary>
/// <param name="index">The column index to increment.</param>
internal bool IncrementColumn(int index)
{
if (!IsSet(index))
{
throw new ArgumentException($"Column index {index} has not been set.");
}
ColumnSchema column = Schema.GetColumn(index);
if (column.IsFixedSize)
{
KuduType type = column.Type;
Span <byte> data = GetRowAllocColumn(index, column.Size);
switch (type)
{
case KuduType.Bool:
{
bool isFalse = data[0] == 0;
data[0] = 1;
return(isFalse);
}
case KuduType.Int8:
{
sbyte existing = KuduEncoder.DecodeInt8(data);
if (existing == sbyte.MaxValue)
{
return(false);
}
KuduEncoder.EncodeInt8(data, (sbyte)(existing + 1));
return(true);
}
case KuduType.Int16:
{
short existing = KuduEncoder.DecodeInt16(data);
if (existing == short.MaxValue)
{
return(false);
}
KuduEncoder.EncodeInt16(data, (short)(existing + 1));
return(true);
}
case KuduType.Int32:
{
int existing = KuduEncoder.DecodeInt32(data);
if (existing == int.MaxValue)
{
return(false);
}
KuduEncoder.EncodeInt32(data, existing + 1);
return(true);
}
case KuduType.Date:
{
int existing = KuduEncoder.DecodeInt32(data);
if (existing == EpochTime.MaxDateValue)
{
return(false);
}
KuduEncoder.EncodeInt32(data, existing + 1);
return(true);
}
case KuduType.Int64:
case KuduType.UnixtimeMicros:
{
long existing = KuduEncoder.DecodeInt64(data);
if (existing == long.MaxValue)
{
return(false);
}
KuduEncoder.EncodeInt64(data, existing + 1);
return(true);
}
case KuduType.Float:
{
float existing = KuduEncoder.DecodeFloat(data);
float incremented = existing.NextUp();
if (existing == incremented)
{
return(false);
}
KuduEncoder.EncodeFloat(data, incremented);
return(true);
}
case KuduType.Double:
{
double existing = KuduEncoder.DecodeDouble(data);
double incremented = existing.NextUp();
if (existing == incremented)
{
return(false);
}
KuduEncoder.EncodeDouble(data, incremented);
return(true);
}
case KuduType.Decimal32:
{
int existing = KuduEncoder.DecodeInt32(data);
int precision = column.TypeAttributes !.Precision.GetValueOrDefault();
if (existing == DecimalUtil.MaxDecimal32(precision))
{
return(false);
}
KuduEncoder.EncodeInt32(data, existing + 1);
return(true);
}
case KuduType.Decimal64:
{
long existing = KuduEncoder.DecodeInt64(data);
int precision = column.TypeAttributes !.Precision.GetValueOrDefault();
if (existing == DecimalUtil.MaxDecimal64(precision))
{
return(false);
}
KuduEncoder.EncodeInt64(data, existing + 1);
return(true);
}
case KuduType.Decimal128:
{
KuduInt128 existing = KuduEncoder.DecodeInt128(data);
int precision = column.TypeAttributes !.Precision.GetValueOrDefault();
if (existing == DecimalUtil.MaxDecimal128(precision))
{
return(false);
}
KuduEncoder.EncodeInt128(data, existing + 1);
return(true);
}
default:
throw new Exception($"Unsupported data type {type}");
}
}
else
{
// Column is either string, binary, or varchar.
ReadOnlySpan <byte> data = GetVarLengthColumn(index);
var incremented = new byte[data.Length + 1];
data.CopyTo(incremented);
WriteBinary(index, incremented);
return(true);
}
}
/// <summary>
/// Sets the column to the minimum possible value for the column's type.
/// </summary>
/// <param name="index">The index of the column to set to the minimum.</param>
internal void SetMin(int index)
{
ColumnSchema column = Schema.GetColumn(index);
KuduType type = column.Type;
switch (type)
{
case KuduType.Bool:
WriteBool(index, false);
break;
case KuduType.Int8:
WriteSByte(index, sbyte.MinValue);
break;
case KuduType.Int16:
WriteInt16(index, short.MinValue);
break;
case KuduType.Int32:
WriteInt32(index, int.MinValue);
break;
case KuduType.Date:
WriteInt32(index, EpochTime.MinDateValue);
break;
case KuduType.Int64:
case KuduType.UnixtimeMicros:
WriteInt64(index, long.MinValue);
break;
case KuduType.Float:
WriteFloat(index, float.MinValue);
break;
case KuduType.Double:
WriteDouble(index, double.MinValue);
break;
case KuduType.Decimal32:
WriteInt32(index, DecimalUtil.MinDecimal32(
column.TypeAttributes !.Precision.GetValueOrDefault()));
break;
case KuduType.Decimal64:
WriteInt64(index, DecimalUtil.MinDecimal64(
column.TypeAttributes !.Precision.GetValueOrDefault()));
break;
case KuduType.Decimal128:
{
KuduInt128 min = DecimalUtil.MinDecimal128(
column.TypeAttributes !.Precision.GetValueOrDefault());
Span <byte> span = GetSpanInRowAllocAndSetBitSet(index, 16);
KuduEncoder.EncodeInt128(span, min);
break;
}
case KuduType.String:
case KuduType.Varchar:
WriteString(index, string.Empty);
break;
case KuduType.Binary:
WriteBinary(index, Array.Empty <byte>());
break;
default:
throw new Exception($"Unsupported data type {type}");
}
}
private static ResultSet CreateResultSet(
KuduMessage message,
KuduSchema schema,
ColumnarRowBlockPB data)
{
var columns = data.Columns;
var numColumns = columns.Count;
if (numColumns != schema.Columns.Count)
{
ThrowColumnCountMismatchException(schema.Columns.Count, numColumns);
}
if (data.Columns.Count == 0 || data.NumRows == 0)
{
// Empty projection, usually used for quick row counting.
return(CreateEmptyResultSet(schema, data.NumRows));
}
var numRows = checked ((int)data.NumRows);
var bufferLength = message.Buffer.Length;
var nonNullBitmapLength = KuduEncoder.BitsToBytes(numRows);
var dataSidecarOffsets = new SidecarOffset[numColumns];
var varlenDataSidecarOffsets = new SidecarOffset[numColumns];
var nonNullBitmapSidecarOffsets = new SidecarOffset[numColumns];
for (int i = 0; i < numColumns; i++)
{
var column = columns[i];
var columnSchema = schema.GetColumn(i);
if (column.HasDataSidecar)
{
var offset = message.GetSidecarOffset(column.DataSidecar);
var length = GetColumnDataSize(columnSchema, numRows);
ValidateSidecar(offset, length, bufferLength);
dataSidecarOffsets[i] = offset;
}
else
{
ThrowMissingDataSidecarException(columnSchema);
}
if (column.HasVarlenDataSidecar)
{
var offset = message.GetSidecarOffset(column.VarlenDataSidecar);
varlenDataSidecarOffsets[i] = offset;
}
if (column.HasNonNullBitmapSidecar)
{
var offset = message.GetSidecarOffset(column.NonNullBitmapSidecar);
ValidateSidecar(offset, nonNullBitmapLength, bufferLength);
nonNullBitmapSidecarOffsets[i] = offset;
}
else
{
nonNullBitmapSidecarOffsets[i] = new SidecarOffset(-1, 0);
}
}
var buffer = message.TakeMemory();
return(new ResultSet(
buffer,
schema,
data.NumRows,
dataSidecarOffsets,
varlenDataSidecarOffsets,
nonNullBitmapSidecarOffsets));
}
private void WriteString(int columnIndex, string value)
{
var data = KuduEncoder.EncodeString(value);
WriteBinary(columnIndex, data);
}
private void WriteBool(int columnIndex, bool value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 1);
KuduEncoder.EncodeBool(span, value);
}
private void WriteSByte(int columnIndex, sbyte value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 1);
KuduEncoder.EncodeInt8(span, value);
}
private void WriteInt32(int columnIndex, int value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4);
KuduEncoder.EncodeInt32(span, value);
}
private void WriteDouble(int columnIndex, double value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 8);
KuduEncoder.EncodeDouble(span, value);
}
private void WriteInt64(int columnIndex, long value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 8);
KuduEncoder.EncodeInt64(span, value);
}
private void WriteFloat(int columnIndex, float value)
{
Span <byte> span = GetSpanInRowAllocAndSetBitSet(columnIndex, 4);
KuduEncoder.EncodeFloat(span, value);
}
public async Task TestNonNullRows(bool includeNullsInSchema)
{
await using var miniCluster = await new MiniKuduClusterBuilder().BuildAsync();
await using var client = miniCluster.CreateClient();
await using var session = client.NewSession();
var builder = ClientTestUtil.CreateAllTypesSchema(includeNullsInSchema)
.SetTableName(nameof(TestNonNullRows))
.CreateBasicRangePartition();
var table = await client.CreateTableAsync(builder);
int numRows = 5;
int currentRow = 0;
for (int i = 0; i < numRows; i++)
{
var insert = ClientTestUtil.CreateAllTypesInsert(table, i);
await session.EnqueueAsync(insert);
}
await session.FlushAsync();
var scanner = client.NewScanBuilder(table).Build();
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
Assert.Equal(currentRow, row.GetInt32("key"));
Assert.Equal(currentRow, row.GetNullableInt32("key"));
Assert.Equal(KuduEncoder.EncodeInt32(currentRow), row.GetSpan("key").ToArray());
Assert.Equal(42, row.GetByte("int8"));
Assert.Equal((byte?)42, row.GetNullableByte("int8"));
Assert.Equal(42, row.GetSByte("int8"));
Assert.Equal((sbyte?)42, row.GetNullableSByte("int8"));
Assert.Equal(43, row.GetInt16("int16"));
Assert.Equal((short?)43, row.GetNullableInt16("int16"));
Assert.Equal(44, row.GetInt32("int32"));
Assert.Equal(44, row.GetNullableInt32("int32"));
Assert.Equal(45, row.GetInt64("int64"));
Assert.Equal(45, row.GetNullableInt64("int64"));
Assert.True(row.GetBool("bool"));
Assert.True(row.GetNullableBool("bool"));
Assert.Equal(52.35f, row.GetFloat("float"));
Assert.Equal(52.35f, row.GetNullableFloat("float"));
Assert.Equal(53.35, row.GetDouble("double"));
Assert.Equal(53.35, row.GetNullableDouble("double"));
Assert.Equal("fun with ütf\0", row.GetString("string"));
Assert.Equal("fun with ütf\0", row.GetNullableString("string"));
Assert.Equal("árvíztűrő ", row.GetString("varchar"));
Assert.Equal("árvíztűrő ", row.GetNullableString("varchar"));
Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, row.GetBinary("binary"));
Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, row.GetSpan("binary").ToArray());
Assert.Equal(DateTime.Parse("8/19/2020 7:50 PM").ToUniversalTime(), row.GetDateTime("timestamp"));
Assert.Equal(DateTime.Parse("8/19/2020 7:50 PM").ToUniversalTime(), row.GetNullableDateTime("timestamp"));
Assert.Equal(DateTime.Parse("8/19/2020").ToUniversalTime().Date, row.GetDateTime("date"));
Assert.Equal(DateTime.Parse("8/19/2020").ToUniversalTime().Date, row.GetNullableDateTime("date"));
Assert.Equal(12.345m, row.GetDecimal("decimal32"));
Assert.Equal(12.345m, row.GetNullableDecimal("decimal32"));
Assert.Equal(12.346m, row.GetDecimal("decimal64"));
Assert.Equal(12.346m, row.GetNullableDecimal("decimal64"));
Assert.Equal(12.347m, row.GetDecimal("decimal128"));
Assert.Equal(12.347m, row.GetNullableDecimal("decimal128"));
currentRow++;
}
}
Assert.Equal(numRows, currentRow);
}
