public void SetMin()
{
PartialRow row = GetPartialRowWithAllTypes();
for (int i = 0; i < row.Schema.Columns.Count; i++)
{
row.SetMin(i);
}
Assert.False(row.GetBool("bool"));
Assert.Equal(sbyte.MinValue, row.GetSByte("int8"));
Assert.Equal(short.MinValue, row.GetInt16("int16"));
Assert.Equal(int.MinValue, row.GetInt32("int32"));
Assert.Equal(long.MinValue, row.GetInt64("int64"));
Assert.Equal(long.MinValue, row.GetInt64("timestamp"));
Assert.Equal(EpochTime.FromUnixTimeDays(EpochTime.MinDateValue), row.GetDateTime("date"));
Assert.Equal(float.MinValue, row.GetFloat("float"));
Assert.Equal(double.MinValue, row.GetDouble("double"));
Assert.Equal("", row.GetString("string"));
Assert.Equal("", row.GetString("varchar"));
Assert.Equal(new byte[0], row.GetBinary("binary"));
Assert.Equal(-99.999m, row.GetDecimal("decimal32"));
Assert.Equal(-99.999m, row.GetDecimal("decimal64"));
Assert.Equal(-99.999m, row.GetDecimal("decimal128"));
}
private static void EncodeColumn(
PartialRow row,
int columnIndex,
bool isLast,
Span <byte> destination,
out int bytesWritten)
{
var schema = row.Schema;
var column = schema.GetColumn(columnIndex);
if (column.IsFixedSize)
{
var size = column.Size;
var slice = destination.Slice(0, size);
var data = row.GetRowAllocColumn(columnIndex, size);
data.CopyTo(slice);
// Row data is little endian, but key encoding is big endian.
slice.Reverse();
if (column.IsSigned)
{
KuduEncoder.XorLeftMostBit(slice);
}
bytesWritten = size;
}
else
{
var data = row.GetVarLengthColumn(columnIndex);
EncodeBinary(data, destination, isLast, out bytesWritten);
}
}
public void TestPrimaryKeys()
{
// Test that we get the correct memcmp result,
// the bytes are in big-endian order in a key
var schema = GetSchema(new TableBuilder()
.AddColumn("key", KuduType.Int32, opt => opt.Key(true))
.AddColumn("key2", KuduType.String, opt => opt.Key(true)));
var small = new PartialRow(schema);
small.SetInt32("key", 20);
small.SetString("key2", "data");
var smallPK = KeyEncoder.EncodePrimaryKey(small);
Assert.Equal(
new byte[] { 0x80, 0, 0, 20, (byte)'d', (byte)'a', (byte)'t', (byte)'a' },
smallPK);
var big = new PartialRow(schema);
big.SetInt32("key", 10000);
big.SetString("key2", "data");
byte[] bigPK = KeyEncoder.EncodePrimaryKey(big);
Assert.Equal(
new byte[] { 0x80, 0, 0x27, 0x10, (byte)'d', (byte)'a', (byte)'t', (byte)'a' },
bigPK);
Assert.True(smallPK.SequenceCompareTo(bigPK) < 0);
}
/// <summary>
/// Encodes the provided row into a range partition key.
/// </summary>
/// <param name="row">The row to encode.</param>
/// <param name="rangeSchema">The range partition schema.</param>
public static byte[] EncodeRangePartitionKey(
PartialRow row, RangeSchema rangeSchema)
{
int maxSize = CalculateMaxPrimaryKeySize(row);
Span <byte> buffer = stackalloc byte[maxSize];
EncodeColumns(row, rangeSchema.ColumnIds, buffer, out int bytesWritten);
return(buffer.Slice(0, bytesWritten).ToArray());
}
public static int CalculateMaxPartitionKeySize(
PartialRow row, PartitionSchema partitionSchema)
{
var hashSchemaSize = partitionSchema.HashBucketSchemas.Count * 4;
// While the final partition key might not include any range columns,
// we still need temporary space to encode the data to hash it.
return(CalculateMaxPrimaryKeySize(row) + hashSchemaSize);
}
public void TestSimplePrimaryKey()
{
var schema = GetSchema(new TableBuilder()
.AddColumn("key", KuduType.String, opt => opt.Key(true)));
var row = new PartialRow(schema);
row.SetString("key", "foo");
Assert.Equal("foo".ToUtf8ByteArray(), KeyEncoder.EncodePrimaryKey(row));
}
public static int GetHashBucket(PartialRow row, HashBucketSchema hashSchema, int maxSize)
{
Span <byte> buffer = stackalloc byte[maxSize];
EncodeColumns(row, hashSchema.ColumnIds, buffer, out int bytesWritten);
var slice = buffer.Slice(0, bytesWritten);
var hash = Murmur2.Hash64(slice, hashSchema.Seed);
var bucket = hash % (uint)hashSchema.NumBuckets;
return((int)bucket);
}
/// <summary>
/// Creates a new partial row by deep-copying the data-fields of the
/// provided partial row.
/// </summary>
/// <param name="row">The partial row to copy.</param>
internal PartialRow(PartialRow row)
{
Schema = row.Schema;
_rowAlloc = CloneArray(row._rowAlloc);
_headerSize = row._headerSize;
_nullOffset = row._nullOffset;
_varLengthData = new byte[row._varLengthData.Length][];
for (int i = 0; i < _varLengthData.Length; i++)
{
_varLengthData[i] = CloneArray(row._varLengthData[i]);
}
}
/// <summary>
/// Determine the partition index that the given row falls into.
/// </summary>
/// <param name="row">The row to be partitioned.</param>
/// <returns>
/// The resulting partition index.
/// The result will be less than <see cref="NumPartitions"/>.
/// </returns>
public int PartitionRow(PartialRow row)
{
var result = GetResult(row);
if (result.IsNonCoveredRange)
{
throw new NonCoveredRangeException(
result.NonCoveredRangeStart,
result.NonCoveredRangeEnd);
}
return(result.Index);
}
/// <summary>
/// Counts the partitions touched by a scan with optional primary key bounds.
/// The table is assumed to have three INT8 columns as the primary key.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="lowerBoundPrimaryKey">The optional lower bound primary key.</param>
/// <param name="upperBoundPrimaryKey">The optional upper bound primary key.</param>
private async Task CheckPartitionsPrimaryKeyAsync(
int expectedTablets,
KuduTable table,
List <Partition> partitions,
sbyte[] lowerBoundPrimaryKey,
sbyte[] upperBoundPrimaryKey)
{
var scanBuilder = _client.NewScanTokenBuilder(table);
if (lowerBoundPrimaryKey != null)
{
var lower = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
lower.SetSByte(i, lowerBoundPrimaryKey[i]);
}
scanBuilder.LowerBound(lower);
}
if (upperBoundPrimaryKey != null)
{
var upper = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
upper.SetSByte(i, upperBoundPrimaryKey[i]);
}
scanBuilder.ExclusiveUpperBound(upper);
}
var pruner = PartitionPruner.Create(scanBuilder);
int scannedPartitions = 0;
foreach (var partition in partitions)
{
if (!pruner.ShouldPruneForTests(partition))
{
scannedPartitions++;
}
}
// Check that the number of ScanTokens built for the scan matches.
var tokens = await scanBuilder.BuildAsync();
Assert.Equal(expectedTablets, scannedPartitions);
Assert.Equal(scannedPartitions, tokens.Count);
Assert.Equal(expectedTablets == 0 ? 0 : 1, pruner.NumRangesRemaining);
}
private FindTabletResult GetResult(PartialRow row)
{
var partitionSchema = _partitionSchema;
int maxSize = KeyEncoder.CalculateMaxPartitionKeySize(row, partitionSchema);
Span <byte> buffer = stackalloc byte[maxSize];
KeyEncoder.EncodePartitionKey(
row,
partitionSchema,
buffer,
out int bytesWritten);
var partitionKey = buffer.Slice(0, bytesWritten);
return(RemoteTabletExtensions.FindTablet(_tablets, partitionKey));
}
private static void EncodeColumns(
PartialRow row, List <int> columnIds, Span <byte> destination, out int bytesWritten)
{
var numColumns = columnIds.Count;
var written = 0;
for (int i = 0; i < numColumns; i++)
{
bool isLast = i + 1 == numColumns;
var columnIndex = row.Schema.GetColumnIndex(columnIds[i]);
EncodeColumn(row, columnIndex, isLast, destination, out int localBytesWritten);
destination = destination.Slice(localBytesWritten);
written += localBytesWritten;
}
bytesWritten = written;
}
private static void CheckPartitionKey(
PartialRow row,
PartitionSchema partitionSchema,
byte[] expectedPartitionKey)
{
var maxSize = KeyEncoder.CalculateMaxPartitionKeySize(row, partitionSchema);
Span <byte> buffer = stackalloc byte[maxSize];
KeyEncoder.EncodePartitionKey(
row,
partitionSchema,
buffer,
out int bytesWritten);
var partitionKey = buffer.Slice(0, bytesWritten).ToArray();
Assert.Equal(expectedPartitionKey, partitionKey);
}
/// <summary>
/// Encodes the primary key of the row.
/// </summary>
/// <param name="row">The row to encode.</param>
public static byte[] EncodePrimaryKey(PartialRow row)
{
var schema = row.Schema;
int primaryKeyColumnCount = schema.PrimaryKeyColumnCount;
int maxSize = CalculateMaxPrimaryKeySize(row);
Span <byte> buffer = stackalloc byte[maxSize];
var slice = buffer;
int bytesWritten = 0;
for (int columnIdx = 0; columnIdx < primaryKeyColumnCount; columnIdx++)
{
bool isLast = columnIdx + 1 == primaryKeyColumnCount;
EncodeColumn(row, columnIdx, isLast, slice, out int localBytesWritten);
slice = slice.Slice(localBytesWritten);
bytesWritten += localBytesWritten;
}
return(buffer.Slice(0, bytesWritten).ToArray());
}
public void ReadScore(string score)
{
string line = "";
List<PartialRow> rows = new List<PartialRow>();
StringReader sr = new StringReader(score);
int idx = 0;
while ((line = sr.ReadLine()) != null)
{
Debug.Log(line);
string[] cells = line.Split(',');
List<string> cs = new List<string>();
foreach(string c in cells){
if(c.Trim().Length > 0)
cs.Add(c);
}
cells = cs.ToArray();
print(cells.Length);
if(cells.Length <= 0)
continue;
PartialRow row = new PartialRow();
rows.Add(row);
List<Partial> l = new List<Partial>();
for(int i=0; i<cells.Length-2; i+=2){
//print(""+cells[i]+":"+cells[i+1]);
float t = float.Parse(cells[i].Trim());
float pow = float.Parse(cells[i+1].Trim());
float d = float.Parse(cells[i+2].Trim()) - t;
Partial p = CreatePartial(t, d, pow);
l.Add(p);
}
row.parts = l.ToArray();
}
partials = rows.ToArray();
}
public void TestPrimaryKeyWithNull()
{
var schema = GetSchema(new TableBuilder()
.AddColumn("key", KuduType.String, opt => opt.Key(true))
.AddColumn("key2", KuduType.String, opt => opt.Key(true)));
var row1 = new PartialRow(schema);
row1.SetString("key", "foo");
row1.SetString("key2", "bar");
Assert.Equal(
"foo\0\0bar".ToUtf8ByteArray(),
KeyEncoder.EncodePrimaryKey(row1));
var row2 = new PartialRow(schema);
row2.SetString("key", "xxx\0yyy");
row2.SetString("key2", "bar");
Assert.Equal(
"xxx\0\u0001yyy\0\0bar".ToUtf8ByteArray(),
KeyEncoder.EncodePrimaryKey(row2));
}
public static void EncodePartitionKey(
PartialRow row,
PartitionSchema partitionSchema,
Span <byte> destination,
out int bytesWritten)
{
int localBytesWritten = 0;
foreach (var hashSchema in partitionSchema.HashBucketSchemas)
{
var bucket = GetHashBucket(row, hashSchema, destination.Length);
var slice = destination.Slice(0, 4);
BinaryPrimitives.WriteInt32BigEndian(slice, bucket);
destination = destination.Slice(4);
localBytesWritten += 4;
}
var rangeColumns = partitionSchema.RangeSchema.ColumnIds;
EncodeColumns(row, rangeColumns, destination, out int written);
bytesWritten = localBytesWritten + written;
}
public static int CalculateMaxPrimaryKeySize(PartialRow row)
{
var schema = row.Schema;
var primaryKeyColumnCount = schema.PrimaryKeyColumnCount;
int size = 0;
for (int i = 0; i < primaryKeyColumnCount; i++)
{
var column = schema.GetColumn(i);
if (column.IsFixedSize)
{
size += column.Size;
}
else
{
var isLast = i + 1 == primaryKeyColumnCount;
var data = row.GetVarLengthColumn(i);
if (isLast)
{
// The last column is copied directly without any special encoding.
size += data.Length;
}
else
{
// For all other columns, the worst case would be a value that
// contained all zeros, as each '0' would have to be encoded as '01'.
// Additionally, each column is terminated by '00'.
size += data.Length * 2 + 2;
}
}
}
return(size);
}
/// <summary>
/// Randomizes the fields in a given PartialRow.
/// </summary>
/// <param name="row">The PartialRow to randomize.</param>
/// <param name="randomizeKeys">True if the key columns should be randomized.</param>
public void RandomizeRow(PartialRow row, bool randomizeKeys)
{
var schema = row.Schema;
var columns = schema.Columns;
foreach (var col in columns)
{
if (col.IsKey && !randomizeKeys)
{
continue;
}
var type = col.Type;
var name = col.Name;
if (col.IsNullable && _random.NextDouble() <= _nullRate)
{
// Sometimes set nullable columns to null.
row.SetNull(name);
continue;
}
if (col.DefaultValue != null && !col.IsKey && _random.NextDouble() <= _defaultRate)
{
// Sometimes use the column default value.
continue;
}
switch (type)
{
case KuduType.Bool:
row.SetBool(name, _random.NextBool());
break;
case KuduType.Int8:
row.SetByte(name, (byte)_random.Next());
break;
case KuduType.Int16:
row.SetInt16(name, (short)_random.Next());
break;
case KuduType.Int32:
row.SetInt32(name, _random.Next());
break;
case KuduType.Date:
row.SetDateTime(name, RandomDate(_random));
break;
case KuduType.Int64:
case KuduType.UnixtimeMicros:
row.SetInt64(name, _random.NextInt64());
break;
case KuduType.Float:
row.SetFloat(name, (float)_random.NextDouble());
break;
case KuduType.Double:
row.SetDouble(name, _random.NextDouble());
break;
case KuduType.Decimal32:
case KuduType.Decimal64:
case KuduType.Decimal128:
row.SetDecimal(name, RandomDecimal(col.TypeAttributes, _random));
break;
case KuduType.String:
row.SetString(name, RandomString(_stringLength, _random));
break;
case KuduType.Varchar:
row.SetString(name, RandomString(col.TypeAttributes.Length.Value, _random));
break;
case KuduType.Binary:
row.SetBinary(name, RandomBinary(_binaryLength, _random));
break;
default:
throw new NotSupportedException($"Unsupported type {type}");
}
}
}
public void IncrementColumn()
{
PartialRow row = GetPartialRowWithAllTypes();
// Boolean
int boolIndex = row.Schema.GetColumnIndex("bool");
row.SetBool(boolIndex, false);
Assert.True(row.IncrementColumn(boolIndex));
Assert.True(row.GetBool(boolIndex));
Assert.False(row.IncrementColumn(boolIndex));
// Int8
int int8Index = row.Schema.GetColumnIndex("int8");
row.SetSByte(int8Index, sbyte.MaxValue - 1);
Assert.True(row.IncrementColumn(int8Index));
Assert.Equal(sbyte.MaxValue, row.GetSByte(int8Index));
Assert.False(row.IncrementColumn(int8Index));
// Int16
int int16Index = row.Schema.GetColumnIndex("int16");
row.SetInt16(int16Index, short.MaxValue - 1);
Assert.True(row.IncrementColumn(int16Index));
Assert.Equal(short.MaxValue, row.GetInt16(int16Index));
Assert.False(row.IncrementColumn(int16Index));
// Int32
int int32Index = row.Schema.GetColumnIndex("int32");
row.SetInt32(int32Index, int.MaxValue - 1);
Assert.True(row.IncrementColumn(int32Index));
Assert.Equal(int.MaxValue, row.GetInt32(int32Index));
Assert.False(row.IncrementColumn(int32Index));
// Int64
int int64Index = row.Schema.GetColumnIndex("int64");
row.SetInt64(int64Index, long.MaxValue - 1);
Assert.True(row.IncrementColumn(int64Index));
Assert.Equal(long.MaxValue, row.GetInt64(int64Index));
Assert.False(row.IncrementColumn(int64Index));
// Date
int dateIndex = row.Schema.GetColumnIndex("date");
row.SetDateTime(dateIndex, EpochTime.FromUnixTimeDays(EpochTime.MaxDateValue - 1));
Assert.True(row.IncrementColumn(dateIndex));
Assert.Equal(EpochTime.FromUnixTimeDays(EpochTime.MaxDateValue), row.GetDateTime(dateIndex));
Assert.False(row.IncrementColumn(dateIndex));
// Float
int floatIndex = row.Schema.GetColumnIndex("float");
row.SetFloat(floatIndex, float.MaxValue);
Assert.True(row.IncrementColumn(floatIndex));
Assert.Equal(float.PositiveInfinity, row.GetFloat(floatIndex));
Assert.False(row.IncrementColumn(floatIndex));
// Double
int doubleIndex = row.Schema.GetColumnIndex("double");
row.SetDouble(doubleIndex, double.MaxValue);
Assert.True(row.IncrementColumn(doubleIndex));
Assert.Equal(double.PositiveInfinity, row.GetDouble(doubleIndex));
Assert.False(row.IncrementColumn(doubleIndex));
// Decimal32
int decimalIndex32 = row.Schema.GetColumnIndex("decimal32");
// Decimal with precision 5, scale 3 has a max of 99.999
row.SetDecimal(decimalIndex32, 99.998m);
Assert.True(row.IncrementColumn(decimalIndex32));
Assert.Equal(99.999m, row.GetDecimal(decimalIndex32));
Assert.False(row.IncrementColumn(decimalIndex32));
// Decimal64
int decimalIndex64 = row.Schema.GetColumnIndex("decimal64");
// Decimal with precision 5, scale 3 has a max of 99.999
row.SetDecimal(decimalIndex64, 99.998m);
Assert.True(row.IncrementColumn(decimalIndex64));
Assert.Equal(99.999m, row.GetDecimal(decimalIndex64));
Assert.False(row.IncrementColumn(decimalIndex64));
// Decimal128
int decimalIndex128 = row.Schema.GetColumnIndex("decimal128");
// Decimal with precision 5, scale 3 has a max of 99.999
row.SetDecimal(decimalIndex128, 99.998m);
Assert.True(row.IncrementColumn(decimalIndex128));
Assert.Equal(99.999m, row.GetDecimal(decimalIndex128));
Assert.False(row.IncrementColumn(decimalIndex128));
// String
int stringIndex = row.Schema.GetColumnIndex("string");
row.SetString(stringIndex, "hello");
Assert.True(row.IncrementColumn(stringIndex));
Assert.Equal("hello\0", row.GetString(stringIndex));
// Binary
int binaryIndex = row.Schema.GetColumnIndex("binary");
row.SetBinary(binaryIndex, new byte[] { 0, 1, 2, 3, 4 });
Assert.True(row.IncrementColumn(binaryIndex));
Assert.Equal(new byte[] { 0, 1, 2, 3, 4, 0 }, row.GetBinary(binaryIndex));
// Varchar
int varcharIndex = row.Schema.GetColumnIndex("varchar");
row.SetString(varcharIndex, "hello");
Assert.True(row.IncrementColumn(varcharIndex));
Assert.Equal("hello\0", row.GetString(varcharIndex));
}
/// <summary>
/// Add an upper bound (exclusive) primary key for the scan.
/// If any bound is already added, this bound is intersected with that one.
/// </summary>
/// <param name="row">A partial row with specified key columns.</param>
public TBuilder ExclusiveUpperBound(PartialRow row)
{
byte[] endPrimaryKey = KeyEncoder.EncodePrimaryKey(row);
return(ExclusiveUpperBoundRaw(endPrimaryKey));
}
/// <summary>
/// Add a lower bound (inclusive) primary key for the scan.
/// If any bound is already added, this bound is intersected with that one.
/// </summary>
/// <param name="row">A partial row with specified key columns.</param>
public TBuilder LowerBound(PartialRow row)
{
byte[] startPrimaryKey = KeyEncoder.EncodePrimaryKey(row);
return(LowerBoundRaw(startPrimaryKey));
}
public void TestPartitionKeyEncoding()
{
var builder = new TableBuilder()
.AddColumn("a", KuduType.Int32, opt => opt.Key(true))
.AddColumn("b", KuduType.String, opt => opt.Key(true))
.AddColumn("c", KuduType.String, opt => opt.Key(true));
var schema = GetSchema(builder);
var partitionSchema = new PartitionSchema(
new RangeSchema(new List <int> {
0, 1, 2
}),
new List <HashBucketSchema>
{
new HashBucketSchema(new List <int> {
0, 1
}, 32, 0),
new HashBucketSchema(new List <int> {
2
}, 32, 42)
},
schema);
var rowA = new PartialRow(schema);
rowA.SetInt32(0, 0);
rowA.SetString(1, "");
rowA.SetString(2, "");
CheckPartitionKey(rowA, partitionSchema, new byte[]
{
0, 0, 0, 0, // hash(0, "")
0, 0, 0, 0x14, // hash("")
0x80, 0, 0, 0, // a = 0
0, 0 // b = ""; c is elided
});
var rowB = new PartialRow(schema);
rowB.SetInt32(0, 1);
rowB.SetString(1, "");
rowB.SetString(2, "");
CheckPartitionKey(rowB, partitionSchema, new byte[]
{
0, 0, 0, 0x5, // hash(1, "")
0, 0, 0, 0x14, // hash("")
0x80, 0, 0, 1, // a = 1
0, 0 // b = ""; c is elided
});
var rowC = new PartialRow(schema);
rowC.SetInt32(0, 0);
rowC.SetString(1, "b");
rowC.SetString(2, "c");
CheckPartitionKey(rowC, partitionSchema, new byte[]
{
0, 0, 0, 0x1A, // hash(0, "b")
0, 0, 0, 0x1D, // hash("c")
0x80, 0, 0, 0, // a = 0
(byte)'b', 0, 0, // b = "b"
(byte)'c' // c = "c"
});
var rowD = new PartialRow(schema);
rowD.SetInt32(0, 1);
rowD.SetString(1, "b");
rowD.SetString(2, "c");
CheckPartitionKey(rowD, partitionSchema, new byte[]
{
0, 0, 0, 0, // hash(1, "b")
0, 0, 0, 0x1D, // hash("c")
0x80, 0, 0, 1, // a = 1
(byte)'b', 0, 0, // b = "b"
(byte)'c' // c = "c"
});
}
private async Task TestPartitionSchemaAsync(TableBuilder tableBuilder)
{
var table = await _client.CreateTableAsync(tableBuilder);
var schema = table.Schema;
var rows = CreateRows();
await InsertRowsAsync(table, rows);
var tableScanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.Build();
// Full table scan
Assert.Equal(rows, await CollectRowsAsync(tableScanner));
{ // Lower bound
var minRow = new Row("1", "3", "5");
var lowerBound = new PartialRow(schema);
minRow.FillPartialRow(lowerBound);
var expected = rows
.Where(r => r.CompareTo(minRow) >= 0)
.ToHashSet();
var scanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBound(lowerBound)
.Build();
var results = await CollectRowsAsync(scanner);
Assert.Equal(expected, results);
var scanTokenBuilder = _client.NewScanTokenBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBound(lowerBound);
var tokenResults = await CollectRowsAsync(scanTokenBuilder);
Assert.Equal(expected, tokenResults);
}
{ // Upper bound
var maxRow = new Row("1", "3", "5");
var upperBound = new PartialRow(schema);
maxRow.FillPartialRow(upperBound);
var expected = rows
.Where(r => r.CompareTo(maxRow) < 0)
.ToHashSet();
var scanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.ExclusiveUpperBound(upperBound)
.Build();
var results = await CollectRowsAsync(scanner);
Assert.Equal(expected, results);
var scanTokenBuilder = _client.NewScanTokenBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.ExclusiveUpperBound(upperBound);
var tokenResults = await CollectRowsAsync(scanTokenBuilder);
Assert.Equal(expected, tokenResults);
}
{ // Lower & Upper bounds
var minRow = new Row("1", "3", "5");
var maxRow = new Row("2", "4", "");
var lowerBound = new PartialRow(schema);
minRow.FillPartialRow(lowerBound);
var upperBound = new PartialRow(schema);
maxRow.FillPartialRow(upperBound);
var expected = rows
.Where(r => r.CompareTo(minRow) >= 0 && r.CompareTo(maxRow) < 0)
.ToHashSet();
var scanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBound(lowerBound)
.ExclusiveUpperBound(upperBound)
.Build();
var results = await CollectRowsAsync(scanner);
Assert.Equal(expected, results);
var scanTokenBuilder = _client.NewScanTokenBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBound(lowerBound)
.ExclusiveUpperBound(upperBound);
var tokenResults = await CollectRowsAsync(scanTokenBuilder);
Assert.Equal(expected, tokenResults);
}
var tablets = await _client.GetTableLocationsAsync(
table.TableId, null, 100);
{ // Per-tablet scan
var results = new HashSet <Row>();
foreach (var tablet in tablets)
{
var scanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBoundPartitionKeyRaw(tablet.Partition.PartitionKeyStart)
.ExclusiveUpperBoundPartitionKeyRaw(tablet.Partition.PartitionKeyEnd)
.Build();
var tabletResults = await CollectRowsAsync(scanner);
var intersection = results.Intersect(tabletResults);
Assert.Empty(intersection);
foreach (var row in tabletResults)
{
results.Add(row);
}
}
Assert.Equal(rows, results);
}
{ // Per-tablet scan with lower & upper bounds
var minRow = new Row("1", "3", "5");
var maxRow = new Row("2", "4", "");
var lowerBound = new PartialRow(schema);
minRow.FillPartialRow(lowerBound);
var upperBound = new PartialRow(schema);
maxRow.FillPartialRow(upperBound);
var expected = rows
.Where(r => r.CompareTo(minRow) >= 0 && r.CompareTo(maxRow) < 0)
.ToHashSet();
var results = new HashSet <Row>();
foreach (var tablet in tablets)
{
var scanner = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.LowerBound(lowerBound)
.ExclusiveUpperBound(upperBound)
.LowerBoundPartitionKeyRaw(tablet.Partition.PartitionKeyStart)
.ExclusiveUpperBoundPartitionKeyRaw(tablet.Partition.PartitionKeyEnd)
.Build();
var tabletResults = await CollectRowsAsync(scanner);
var intersection = results.Intersect(tabletResults);
Assert.Empty(intersection);
foreach (var row in tabletResults)
{
results.Add(row);
}
}
Assert.Equal(expected, results);
}
}
public void ReadScore(string score)
{
string line = "";
int maxCol = 0;
int rowNum = 0;
StringReader sr1 = new StringReader(score);
while ((line = sr1.ReadLine()) != null)
{
string[] cells = line.Split(',');
if(cells.Length <= 0 || cells[0].Contains("//"))
continue;
maxCol = Mathf.Max(maxCol, cells.Length);
rowNum++;
}
sr1.Close();
partials = new PartialRow[rowNum];
StringReader sr = new StringReader(score);
Dictionary<string, List<Partial>> dic = new Dictionary<string, List<Partial>>();
int idx = 0;
while ((line = sr.ReadLine()) != null)
{
//Debug.Log(line);
string[] cells = line.Split(',');
if(cells.Length <= 0 || cells[0].Contains("//"))
continue;
string id = cells[0];
if(!dic.ContainsKey(id)){
dic.Add(id, new List<Partial>());
}
for(int i=1;i<cells.Length;i++){
if(cells[i].Trim().Length > 0){
if(i == cells.Length-1){
//last cell
Debug.Log(""+id+":"+i+"-"+maxCol);
Debug.Log(""+cells[i]);
string[] cols = cells[i].Split(';');
Partial p = CreatePartial(id, i-1, maxCol-1, cols);
dic[id].Add(p);
}else {
for(int j=i+1;i<cells.Length;j++){
if(cells[j].Trim().Length > 0 || j == cells.Length-1){
Debug.Log(""+id+":"+i+"-"+j);
Debug.Log(""+cells[i]);
string[] cols = cells[i].Split(';');
Partial p = CreatePartial(id, i-1, j-1, cols);
dic[id].Add(p);
i = j-1;
break;
}
}
}
}
}
}
for (int i=0; i<rowNum; i++)
{
partials[i] = new PartialRow();
partials[i].num = i;
partials[i].parts = dic[""+i].ToArray();
}
}
public void TestPrimaryKeyEncoding()
{
var schema = GetSchema(new TableBuilder()
.AddColumn("int8", KuduType.Int8, opt => opt.Key(true))
.AddColumn("int16", KuduType.Int16, opt => opt.Key(true))
.AddColumn("int32", KuduType.Int32, opt => opt.Key(true))
.AddColumn("int64", KuduType.Int64, opt => opt.Key(true))
.AddColumn("decimal32", KuduType.Decimal32, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal32Precision, 0))
.AddColumn("decimal64", KuduType.Decimal64, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal64Precision, 0))
.AddColumn("decimal128", KuduType.Decimal128, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal128Precision, 0))
.AddColumn("varchar", KuduType.Varchar, opt => opt.Key(true)
.VarcharAttributes(10))
.AddColumn("string", KuduType.String, opt => opt.Key(true))
.AddColumn("binary", KuduType.Binary, opt => opt.Key(true)));
var rowA = new PartialRow(schema);
rowA.SetSByte("int8", sbyte.MinValue);
rowA.SetInt16("int16", short.MinValue);
rowA.SetInt32("int32", int.MinValue);
rowA.SetInt64("int64", long.MinValue);
// Note: The decimal value is not the minimum of the underlying int32,
// int64, int128 type so we don't use "minimum" values in the test.
rowA.SetDecimal("decimal32", 5m);
rowA.SetDecimal("decimal64", 6m);
rowA.SetDecimal("decimal128", 7m);
rowA.SetString("varchar", "");
rowA.SetString("string", "");
rowA.SetBinary("binary", Array.Empty <byte>());
var rowAEncoded = KeyEncoder.EncodePrimaryKey(rowA);
Assert.Equal(new byte[]
{
0,
0, 0,
0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0x80, 0, 0, 5,
0x80, 0, 0, 0, 0, 0, 0, 6,
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
0, 0,
0, 0
}, rowAEncoded);
var rowB = new PartialRow(schema);
rowB.SetSByte("int8", sbyte.MaxValue);
rowB.SetInt16("int16", short.MaxValue);
rowB.SetInt32("int32", int.MaxValue);
rowB.SetInt64("int64", long.MaxValue);
// Note: The decimal value is not the maximum of the underlying int32,
// int64, int128 type so we don't use "maximum" values in the test.
rowB.SetDecimal("decimal32", 5m);
rowB.SetDecimal("decimal64", 6m);
rowB.SetDecimal("decimal128", 7m);
rowB.SetString("varchar", "abc\u0001\0defghij");
rowB.SetString("string", "abc\u0001\0def");
rowB.SetBinary("binary", "\0\u0001binary".ToUtf8ByteArray());
var rowBEncoded = KeyEncoder.EncodePrimaryKey(rowB);
Assert.Equal(ToByteArray(new int[]
{
0xff,
0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x80, 0, 0, 5,
0x80, 0, 0, 0, 0, 0, 0, 6,
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 'g', 'h', 0, 0,
'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 0, 0,
0, 1, 'b', 'i', 'n', 'a', 'r', 'y'
}), rowBEncoded);
var rowC = new PartialRow(schema);
rowC.SetSByte("int8", 1);
rowC.SetInt16("int16", 2);
rowC.SetInt32("int32", 3);
rowC.SetInt64("int64", 4);
rowC.SetDecimal("decimal32", 5m);
rowC.SetDecimal("decimal64", 6m);
rowC.SetDecimal("decimal128", 7m);
rowC.SetString("varchar", "abc\n12345678");
rowC.SetString("string", "abc\n123");
rowC.SetBinary("binary", "\0\u0001\u0002\u0003\u0004\u0005".ToUtf8ByteArray());
var rowCEncoded = KeyEncoder.EncodePrimaryKey(rowC);
Assert.Equal(ToByteArray(new int[]
{
0x81,
0x80, 2,
0x80, 0, 0, 3,
0x80, 0, 0, 0, 0, 0, 0, 4,
0x80, 0, 0, 5,
0x80, 0, 0, 0, 0, 0, 0, 6,
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
'a', 'b', 'c', '\n', '1', '2', '3', '4', '5', '6', 0, 0,
'a', 'b', 'c', '\n', '1', '2', '3', 0, 0,
0, 1, 2, 3, 4, 5
}), rowCEncoded);
var rowD = new PartialRow(schema);
rowD.SetSByte("int8", -1);
rowD.SetInt16("int16", -2);
rowD.SetInt32("int32", -3);
rowD.SetInt64("int64", -4);
rowD.SetDecimal("decimal32", -5m);
rowD.SetDecimal("decimal64", -6m);
rowD.SetDecimal("decimal128", -7m);
rowD.SetString("varchar", "\0abc\n\u0001\u0001\0 123\u0001\0");
rowD.SetString("string", "\0abc\n\u0001\u0001\0 123\u0001\0");
rowD.SetBinary("binary", "\0\u0001\u0002\u0003\u0004\u0005\0".ToUtf8ByteArray());
var rowDEncoded = KeyEncoder.EncodePrimaryKey(rowD);
Assert.Equal(ToByteArray(new int[]
{
127,
127, -2,
127, -1, -1, -3,
127, -1, -1, -1, -1, -1, -1, -4,
127, -1, -1, -5,
127, -1, -1, -1, -1, -1, -1, -6,
127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -7,
0, 1, 'a', 'b', 'c', '\n', 1, 1, 0, 1, ' ', '1', 0, 0,
0, 1, 'a', 'b', 'c', '\n', 1, 1, 0, 1, ' ', '1', '2', '3', 1, 0, 1, 0, 0,
0, 1, 2, 3, 4, 5, 0,
}), rowDEncoded);
}
/// <summary>
/// Determine if the given row falls into a valid partition.
/// </summary>
/// <param name="row">The row to check.</param>
public bool IsCovered(PartialRow row)
{
var result = GetResult(row);
return(result.IsCoveredRange);
}
public void FillPartialRow(PartialRow row)
{
row.SetString("a", ValA);
row.SetString("b", ValB);
row.SetString("c", ValC);
}
/// <summary>
/// Randomizes the fields in a given PartialRow.
/// </summary>
/// <param name="row">The PartialRow to randomize.</param>
public void RandomizeRow(PartialRow row)
{
RandomizeRow(row, true);
}
