public void SetMin() { PartialRow row = GetPartialRowWithAllTypes(); for (int i = 0; i < row.Schema.Columns.Count; i++) { row.SetMin(i); } Assert.False(row.GetBool("bool")); Assert.Equal(sbyte.MinValue, row.GetSByte("int8")); Assert.Equal(short.MinValue, row.GetInt16("int16")); Assert.Equal(int.MinValue, row.GetInt32("int32")); Assert.Equal(long.MinValue, row.GetInt64("int64")); Assert.Equal(long.MinValue, row.GetInt64("timestamp")); Assert.Equal(EpochTime.FromUnixTimeDays(EpochTime.MinDateValue), row.GetDateTime("date")); Assert.Equal(float.MinValue, row.GetFloat("float")); Assert.Equal(double.MinValue, row.GetDouble("double")); Assert.Equal("", row.GetString("string")); Assert.Equal("", row.GetString("varchar")); Assert.Equal(new byte[0], row.GetBinary("binary")); Assert.Equal(-99.999m, row.GetDecimal("decimal32")); Assert.Equal(-99.999m, row.GetDecimal("decimal64")); Assert.Equal(-99.999m, row.GetDecimal("decimal128")); }
private static void EncodeColumn( PartialRow row, int columnIndex, bool isLast, Span <byte> destination, out int bytesWritten) { var schema = row.Schema; var column = schema.GetColumn(columnIndex); if (column.IsFixedSize) { var size = column.Size; var slice = destination.Slice(0, size); var data = row.GetRowAllocColumn(columnIndex, size); data.CopyTo(slice); // Row data is little endian, but key encoding is big endian. slice.Reverse(); if (column.IsSigned) { KuduEncoder.XorLeftMostBit(slice); } bytesWritten = size; } else { var data = row.GetVarLengthColumn(columnIndex); EncodeBinary(data, destination, isLast, out bytesWritten); } }
public void TestPrimaryKeys() { // Test that we get the correct memcmp result, // the bytes are in big-endian order in a key var schema = GetSchema(new TableBuilder() .AddColumn("key", KuduType.Int32, opt => opt.Key(true)) .AddColumn("key2", KuduType.String, opt => opt.Key(true))); var small = new PartialRow(schema); small.SetInt32("key", 20); small.SetString("key2", "data"); var smallPK = KeyEncoder.EncodePrimaryKey(small); Assert.Equal( new byte[] { 0x80, 0, 0, 20, (byte)'d', (byte)'a', (byte)'t', (byte)'a' }, smallPK); var big = new PartialRow(schema); big.SetInt32("key", 10000); big.SetString("key2", "data"); byte[] bigPK = KeyEncoder.EncodePrimaryKey(big); Assert.Equal( new byte[] { 0x80, 0, 0x27, 0x10, (byte)'d', (byte)'a', (byte)'t', (byte)'a' }, bigPK); Assert.True(smallPK.SequenceCompareTo(bigPK) < 0); }
/// <summary> /// Encodes the provided row into a range partition key. /// </summary> /// <param name="row">The row to encode.</param> /// <param name="rangeSchema">The range partition schema.</param> public static byte[] EncodeRangePartitionKey( PartialRow row, RangeSchema rangeSchema) { int maxSize = CalculateMaxPrimaryKeySize(row); Span <byte> buffer = stackalloc byte[maxSize]; EncodeColumns(row, rangeSchema.ColumnIds, buffer, out int bytesWritten); return(buffer.Slice(0, bytesWritten).ToArray()); }
public static int CalculateMaxPartitionKeySize( PartialRow row, PartitionSchema partitionSchema) { var hashSchemaSize = partitionSchema.HashBucketSchemas.Count * 4; // While the final partition key might not include any range columns, // we still need temporary space to encode the data to hash it. return(CalculateMaxPrimaryKeySize(row) + hashSchemaSize); }
public void TestSimplePrimaryKey() { var schema = GetSchema(new TableBuilder() .AddColumn("key", KuduType.String, opt => opt.Key(true))); var row = new PartialRow(schema); row.SetString("key", "foo"); Assert.Equal("foo".ToUtf8ByteArray(), KeyEncoder.EncodePrimaryKey(row)); }
public static int GetHashBucket(PartialRow row, HashBucketSchema hashSchema, int maxSize) { Span <byte> buffer = stackalloc byte[maxSize]; EncodeColumns(row, hashSchema.ColumnIds, buffer, out int bytesWritten); var slice = buffer.Slice(0, bytesWritten); var hash = Murmur2.Hash64(slice, hashSchema.Seed); var bucket = hash % (uint)hashSchema.NumBuckets; return((int)bucket); }
/// <summary> /// Creates a new partial row by deep-copying the data-fields of the /// provided partial row. /// </summary> /// <param name="row">The partial row to copy.</param> internal PartialRow(PartialRow row) { Schema = row.Schema; _rowAlloc = CloneArray(row._rowAlloc); _headerSize = row._headerSize; _nullOffset = row._nullOffset; _varLengthData = new byte[row._varLengthData.Length][]; for (int i = 0; i < _varLengthData.Length; i++) { _varLengthData[i] = CloneArray(row._varLengthData[i]); } }
/// <summary> /// Determine the partition index that the given row falls into. /// </summary> /// <param name="row">The row to be partitioned.</param> /// <returns> /// The resulting partition index. /// The result will be less than <see cref="NumPartitions"/>. /// </returns> public int PartitionRow(PartialRow row) { var result = GetResult(row); if (result.IsNonCoveredRange) { throw new NonCoveredRangeException( result.NonCoveredRangeStart, result.NonCoveredRangeEnd); } return(result.Index); }
/// <summary> /// Counts the partitions touched by a scan with optional primary key bounds. /// The table is assumed to have three INT8 columns as the primary key. /// </summary> /// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param> /// <param name="table">The table to scan.</param> /// <param name="partitions">The partitions of the table.</param> /// <param name="lowerBoundPrimaryKey">The optional lower bound primary key.</param> /// <param name="upperBoundPrimaryKey">The optional upper bound primary key.</param> private async Task CheckPartitionsPrimaryKeyAsync( int expectedTablets, KuduTable table, List <Partition> partitions, sbyte[] lowerBoundPrimaryKey, sbyte[] upperBoundPrimaryKey) { var scanBuilder = _client.NewScanTokenBuilder(table); if (lowerBoundPrimaryKey != null) { var lower = new PartialRow(table.Schema); for (int i = 0; i < 3; i++) { lower.SetSByte(i, lowerBoundPrimaryKey[i]); } scanBuilder.LowerBound(lower); } if (upperBoundPrimaryKey != null) { var upper = new PartialRow(table.Schema); for (int i = 0; i < 3; i++) { upper.SetSByte(i, upperBoundPrimaryKey[i]); } scanBuilder.ExclusiveUpperBound(upper); } var pruner = PartitionPruner.Create(scanBuilder); int scannedPartitions = 0; foreach (var partition in partitions) { if (!pruner.ShouldPruneForTests(partition)) { scannedPartitions++; } } // Check that the number of ScanTokens built for the scan matches. var tokens = await scanBuilder.BuildAsync(); Assert.Equal(expectedTablets, scannedPartitions); Assert.Equal(scannedPartitions, tokens.Count); Assert.Equal(expectedTablets == 0 ? 0 : 1, pruner.NumRangesRemaining); }
private FindTabletResult GetResult(PartialRow row) { var partitionSchema = _partitionSchema; int maxSize = KeyEncoder.CalculateMaxPartitionKeySize(row, partitionSchema); Span <byte> buffer = stackalloc byte[maxSize]; KeyEncoder.EncodePartitionKey( row, partitionSchema, buffer, out int bytesWritten); var partitionKey = buffer.Slice(0, bytesWritten); return(RemoteTabletExtensions.FindTablet(_tablets, partitionKey)); }
private static void EncodeColumns( PartialRow row, List <int> columnIds, Span <byte> destination, out int bytesWritten) { var numColumns = columnIds.Count; var written = 0; for (int i = 0; i < numColumns; i++) { bool isLast = i + 1 == numColumns; var columnIndex = row.Schema.GetColumnIndex(columnIds[i]); EncodeColumn(row, columnIndex, isLast, destination, out int localBytesWritten); destination = destination.Slice(localBytesWritten); written += localBytesWritten; } bytesWritten = written; }
private static void CheckPartitionKey( PartialRow row, PartitionSchema partitionSchema, byte[] expectedPartitionKey) { var maxSize = KeyEncoder.CalculateMaxPartitionKeySize(row, partitionSchema); Span <byte> buffer = stackalloc byte[maxSize]; KeyEncoder.EncodePartitionKey( row, partitionSchema, buffer, out int bytesWritten); var partitionKey = buffer.Slice(0, bytesWritten).ToArray(); Assert.Equal(expectedPartitionKey, partitionKey); }
/// <summary> /// Encodes the primary key of the row. /// </summary> /// <param name="row">The row to encode.</param> public static byte[] EncodePrimaryKey(PartialRow row) { var schema = row.Schema; int primaryKeyColumnCount = schema.PrimaryKeyColumnCount; int maxSize = CalculateMaxPrimaryKeySize(row); Span <byte> buffer = stackalloc byte[maxSize]; var slice = buffer; int bytesWritten = 0; for (int columnIdx = 0; columnIdx < primaryKeyColumnCount; columnIdx++) { bool isLast = columnIdx + 1 == primaryKeyColumnCount; EncodeColumn(row, columnIdx, isLast, slice, out int localBytesWritten); slice = slice.Slice(localBytesWritten); bytesWritten += localBytesWritten; } return(buffer.Slice(0, bytesWritten).ToArray()); }
public void ReadScore(string score) { string line = ""; List<PartialRow> rows = new List<PartialRow>(); StringReader sr = new StringReader(score); int idx = 0; while ((line = sr.ReadLine()) != null) { Debug.Log(line); string[] cells = line.Split(','); List<string> cs = new List<string>(); foreach(string c in cells){ if(c.Trim().Length > 0) cs.Add(c); } cells = cs.ToArray(); print(cells.Length); if(cells.Length <= 0) continue; PartialRow row = new PartialRow(); rows.Add(row); List<Partial> l = new List<Partial>(); for(int i=0; i<cells.Length-2; i+=2){ //print(""+cells[i]+":"+cells[i+1]); float t = float.Parse(cells[i].Trim()); float pow = float.Parse(cells[i+1].Trim()); float d = float.Parse(cells[i+2].Trim()) - t; Partial p = CreatePartial(t, d, pow); l.Add(p); } row.parts = l.ToArray(); } partials = rows.ToArray(); }
public void TestPrimaryKeyWithNull() { var schema = GetSchema(new TableBuilder() .AddColumn("key", KuduType.String, opt => opt.Key(true)) .AddColumn("key2", KuduType.String, opt => opt.Key(true))); var row1 = new PartialRow(schema); row1.SetString("key", "foo"); row1.SetString("key2", "bar"); Assert.Equal( "foo\0\0bar".ToUtf8ByteArray(), KeyEncoder.EncodePrimaryKey(row1)); var row2 = new PartialRow(schema); row2.SetString("key", "xxx\0yyy"); row2.SetString("key2", "bar"); Assert.Equal( "xxx\0\u0001yyy\0\0bar".ToUtf8ByteArray(), KeyEncoder.EncodePrimaryKey(row2)); }
public static void EncodePartitionKey( PartialRow row, PartitionSchema partitionSchema, Span <byte> destination, out int bytesWritten) { int localBytesWritten = 0; foreach (var hashSchema in partitionSchema.HashBucketSchemas) { var bucket = GetHashBucket(row, hashSchema, destination.Length); var slice = destination.Slice(0, 4); BinaryPrimitives.WriteInt32BigEndian(slice, bucket); destination = destination.Slice(4); localBytesWritten += 4; } var rangeColumns = partitionSchema.RangeSchema.ColumnIds; EncodeColumns(row, rangeColumns, destination, out int written); bytesWritten = localBytesWritten + written; }
public static int CalculateMaxPrimaryKeySize(PartialRow row) { var schema = row.Schema; var primaryKeyColumnCount = schema.PrimaryKeyColumnCount; int size = 0; for (int i = 0; i < primaryKeyColumnCount; i++) { var column = schema.GetColumn(i); if (column.IsFixedSize) { size += column.Size; } else { var isLast = i + 1 == primaryKeyColumnCount; var data = row.GetVarLengthColumn(i); if (isLast) { // The last column is copied directly without any special encoding. size += data.Length; } else { // For all other columns, the worst case would be a value that // contained all zeros, as each '0' would have to be encoded as '01'. // Additionally, each column is terminated by '00'. size += data.Length * 2 + 2; } } } return(size); }
/// <summary> /// Randomizes the fields in a given PartialRow. /// </summary> /// <param name="row">The PartialRow to randomize.</param> /// <param name="randomizeKeys">True if the key columns should be randomized.</param> public void RandomizeRow(PartialRow row, bool randomizeKeys) { var schema = row.Schema; var columns = schema.Columns; foreach (var col in columns) { if (col.IsKey && !randomizeKeys) { continue; } var type = col.Type; var name = col.Name; if (col.IsNullable && _random.NextDouble() <= _nullRate) { // Sometimes set nullable columns to null. row.SetNull(name); continue; } if (col.DefaultValue != null && !col.IsKey && _random.NextDouble() <= _defaultRate) { // Sometimes use the column default value. continue; } switch (type) { case KuduType.Bool: row.SetBool(name, _random.NextBool()); break; case KuduType.Int8: row.SetByte(name, (byte)_random.Next()); break; case KuduType.Int16: row.SetInt16(name, (short)_random.Next()); break; case KuduType.Int32: row.SetInt32(name, _random.Next()); break; case KuduType.Date: row.SetDateTime(name, RandomDate(_random)); break; case KuduType.Int64: case KuduType.UnixtimeMicros: row.SetInt64(name, _random.NextInt64()); break; case KuduType.Float: row.SetFloat(name, (float)_random.NextDouble()); break; case KuduType.Double: row.SetDouble(name, _random.NextDouble()); break; case KuduType.Decimal32: case KuduType.Decimal64: case KuduType.Decimal128: row.SetDecimal(name, RandomDecimal(col.TypeAttributes, _random)); break; case KuduType.String: row.SetString(name, RandomString(_stringLength, _random)); break; case KuduType.Varchar: row.SetString(name, RandomString(col.TypeAttributes.Length.Value, _random)); break; case KuduType.Binary: row.SetBinary(name, RandomBinary(_binaryLength, _random)); break; default: throw new NotSupportedException($"Unsupported type {type}"); } } }
public void IncrementColumn() { PartialRow row = GetPartialRowWithAllTypes(); // Boolean int boolIndex = row.Schema.GetColumnIndex("bool"); row.SetBool(boolIndex, false); Assert.True(row.IncrementColumn(boolIndex)); Assert.True(row.GetBool(boolIndex)); Assert.False(row.IncrementColumn(boolIndex)); // Int8 int int8Index = row.Schema.GetColumnIndex("int8"); row.SetSByte(int8Index, sbyte.MaxValue - 1); Assert.True(row.IncrementColumn(int8Index)); Assert.Equal(sbyte.MaxValue, row.GetSByte(int8Index)); Assert.False(row.IncrementColumn(int8Index)); // Int16 int int16Index = row.Schema.GetColumnIndex("int16"); row.SetInt16(int16Index, short.MaxValue - 1); Assert.True(row.IncrementColumn(int16Index)); Assert.Equal(short.MaxValue, row.GetInt16(int16Index)); Assert.False(row.IncrementColumn(int16Index)); // Int32 int int32Index = row.Schema.GetColumnIndex("int32"); row.SetInt32(int32Index, int.MaxValue - 1); Assert.True(row.IncrementColumn(int32Index)); Assert.Equal(int.MaxValue, row.GetInt32(int32Index)); Assert.False(row.IncrementColumn(int32Index)); // Int64 int int64Index = row.Schema.GetColumnIndex("int64"); row.SetInt64(int64Index, long.MaxValue - 1); Assert.True(row.IncrementColumn(int64Index)); Assert.Equal(long.MaxValue, row.GetInt64(int64Index)); Assert.False(row.IncrementColumn(int64Index)); // Date int dateIndex = row.Schema.GetColumnIndex("date"); row.SetDateTime(dateIndex, EpochTime.FromUnixTimeDays(EpochTime.MaxDateValue - 1)); Assert.True(row.IncrementColumn(dateIndex)); Assert.Equal(EpochTime.FromUnixTimeDays(EpochTime.MaxDateValue), row.GetDateTime(dateIndex)); Assert.False(row.IncrementColumn(dateIndex)); // Float int floatIndex = row.Schema.GetColumnIndex("float"); row.SetFloat(floatIndex, float.MaxValue); Assert.True(row.IncrementColumn(floatIndex)); Assert.Equal(float.PositiveInfinity, row.GetFloat(floatIndex)); Assert.False(row.IncrementColumn(floatIndex)); // Double int doubleIndex = row.Schema.GetColumnIndex("double"); row.SetDouble(doubleIndex, double.MaxValue); Assert.True(row.IncrementColumn(doubleIndex)); Assert.Equal(double.PositiveInfinity, row.GetDouble(doubleIndex)); Assert.False(row.IncrementColumn(doubleIndex)); // Decimal32 int decimalIndex32 = row.Schema.GetColumnIndex("decimal32"); // Decimal with precision 5, scale 3 has a max of 99.999 row.SetDecimal(decimalIndex32, 99.998m); Assert.True(row.IncrementColumn(decimalIndex32)); Assert.Equal(99.999m, row.GetDecimal(decimalIndex32)); Assert.False(row.IncrementColumn(decimalIndex32)); // Decimal64 int decimalIndex64 = row.Schema.GetColumnIndex("decimal64"); // Decimal with precision 5, scale 3 has a max of 99.999 row.SetDecimal(decimalIndex64, 99.998m); Assert.True(row.IncrementColumn(decimalIndex64)); Assert.Equal(99.999m, row.GetDecimal(decimalIndex64)); Assert.False(row.IncrementColumn(decimalIndex64)); // Decimal128 int decimalIndex128 = row.Schema.GetColumnIndex("decimal128"); // Decimal with precision 5, scale 3 has a max of 99.999 row.SetDecimal(decimalIndex128, 99.998m); Assert.True(row.IncrementColumn(decimalIndex128)); Assert.Equal(99.999m, row.GetDecimal(decimalIndex128)); Assert.False(row.IncrementColumn(decimalIndex128)); // String int stringIndex = row.Schema.GetColumnIndex("string"); row.SetString(stringIndex, "hello"); Assert.True(row.IncrementColumn(stringIndex)); Assert.Equal("hello\0", row.GetString(stringIndex)); // Binary int binaryIndex = row.Schema.GetColumnIndex("binary"); row.SetBinary(binaryIndex, new byte[] { 0, 1, 2, 3, 4 }); Assert.True(row.IncrementColumn(binaryIndex)); Assert.Equal(new byte[] { 0, 1, 2, 3, 4, 0 }, row.GetBinary(binaryIndex)); // Varchar int varcharIndex = row.Schema.GetColumnIndex("varchar"); row.SetString(varcharIndex, "hello"); Assert.True(row.IncrementColumn(varcharIndex)); Assert.Equal("hello\0", row.GetString(varcharIndex)); }
/// <summary> /// Add an upper bound (exclusive) primary key for the scan. /// If any bound is already added, this bound is intersected with that one. /// </summary> /// <param name="row">A partial row with specified key columns.</param> public TBuilder ExclusiveUpperBound(PartialRow row) { byte[] endPrimaryKey = KeyEncoder.EncodePrimaryKey(row); return(ExclusiveUpperBoundRaw(endPrimaryKey)); }
/// <summary> /// Add a lower bound (inclusive) primary key for the scan. /// If any bound is already added, this bound is intersected with that one. /// </summary> /// <param name="row">A partial row with specified key columns.</param> public TBuilder LowerBound(PartialRow row) { byte[] startPrimaryKey = KeyEncoder.EncodePrimaryKey(row); return(LowerBoundRaw(startPrimaryKey)); }
public void TestPartitionKeyEncoding() { var builder = new TableBuilder() .AddColumn("a", KuduType.Int32, opt => opt.Key(true)) .AddColumn("b", KuduType.String, opt => opt.Key(true)) .AddColumn("c", KuduType.String, opt => opt.Key(true)); var schema = GetSchema(builder); var partitionSchema = new PartitionSchema( new RangeSchema(new List <int> { 0, 1, 2 }), new List <HashBucketSchema> { new HashBucketSchema(new List <int> { 0, 1 }, 32, 0), new HashBucketSchema(new List <int> { 2 }, 32, 42) }, schema); var rowA = new PartialRow(schema); rowA.SetInt32(0, 0); rowA.SetString(1, ""); rowA.SetString(2, ""); CheckPartitionKey(rowA, partitionSchema, new byte[] { 0, 0, 0, 0, // hash(0, "") 0, 0, 0, 0x14, // hash("") 0x80, 0, 0, 0, // a = 0 0, 0 // b = ""; c is elided }); var rowB = new PartialRow(schema); rowB.SetInt32(0, 1); rowB.SetString(1, ""); rowB.SetString(2, ""); CheckPartitionKey(rowB, partitionSchema, new byte[] { 0, 0, 0, 0x5, // hash(1, "") 0, 0, 0, 0x14, // hash("") 0x80, 0, 0, 1, // a = 1 0, 0 // b = ""; c is elided }); var rowC = new PartialRow(schema); rowC.SetInt32(0, 0); rowC.SetString(1, "b"); rowC.SetString(2, "c"); CheckPartitionKey(rowC, partitionSchema, new byte[] { 0, 0, 0, 0x1A, // hash(0, "b") 0, 0, 0, 0x1D, // hash("c") 0x80, 0, 0, 0, // a = 0 (byte)'b', 0, 0, // b = "b" (byte)'c' // c = "c" }); var rowD = new PartialRow(schema); rowD.SetInt32(0, 1); rowD.SetString(1, "b"); rowD.SetString(2, "c"); CheckPartitionKey(rowD, partitionSchema, new byte[] { 0, 0, 0, 0, // hash(1, "b") 0, 0, 0, 0x1D, // hash("c") 0x80, 0, 0, 1, // a = 1 (byte)'b', 0, 0, // b = "b" (byte)'c' // c = "c" }); }
private async Task TestPartitionSchemaAsync(TableBuilder tableBuilder) { var table = await _client.CreateTableAsync(tableBuilder); var schema = table.Schema; var rows = CreateRows(); await InsertRowsAsync(table, rows); var tableScanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .Build(); // Full table scan Assert.Equal(rows, await CollectRowsAsync(tableScanner)); { // Lower bound var minRow = new Row("1", "3", "5"); var lowerBound = new PartialRow(schema); minRow.FillPartialRow(lowerBound); var expected = rows .Where(r => r.CompareTo(minRow) >= 0) .ToHashSet(); var scanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBound(lowerBound) .Build(); var results = await CollectRowsAsync(scanner); Assert.Equal(expected, results); var scanTokenBuilder = _client.NewScanTokenBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBound(lowerBound); var tokenResults = await CollectRowsAsync(scanTokenBuilder); Assert.Equal(expected, tokenResults); } { // Upper bound var maxRow = new Row("1", "3", "5"); var upperBound = new PartialRow(schema); maxRow.FillPartialRow(upperBound); var expected = rows .Where(r => r.CompareTo(maxRow) < 0) .ToHashSet(); var scanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .ExclusiveUpperBound(upperBound) .Build(); var results = await CollectRowsAsync(scanner); Assert.Equal(expected, results); var scanTokenBuilder = _client.NewScanTokenBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .ExclusiveUpperBound(upperBound); var tokenResults = await CollectRowsAsync(scanTokenBuilder); Assert.Equal(expected, tokenResults); } { // Lower & Upper bounds var minRow = new Row("1", "3", "5"); var maxRow = new Row("2", "4", ""); var lowerBound = new PartialRow(schema); minRow.FillPartialRow(lowerBound); var upperBound = new PartialRow(schema); maxRow.FillPartialRow(upperBound); var expected = rows .Where(r => r.CompareTo(minRow) >= 0 && r.CompareTo(maxRow) < 0) .ToHashSet(); var scanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBound(lowerBound) .ExclusiveUpperBound(upperBound) .Build(); var results = await CollectRowsAsync(scanner); Assert.Equal(expected, results); var scanTokenBuilder = _client.NewScanTokenBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBound(lowerBound) .ExclusiveUpperBound(upperBound); var tokenResults = await CollectRowsAsync(scanTokenBuilder); Assert.Equal(expected, tokenResults); } var tablets = await _client.GetTableLocationsAsync( table.TableId, null, 100); { // Per-tablet scan var results = new HashSet <Row>(); foreach (var tablet in tablets) { var scanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBoundPartitionKeyRaw(tablet.Partition.PartitionKeyStart) .ExclusiveUpperBoundPartitionKeyRaw(tablet.Partition.PartitionKeyEnd) .Build(); var tabletResults = await CollectRowsAsync(scanner); var intersection = results.Intersect(tabletResults); Assert.Empty(intersection); foreach (var row in tabletResults) { results.Add(row); } } Assert.Equal(rows, results); } { // Per-tablet scan with lower & upper bounds var minRow = new Row("1", "3", "5"); var maxRow = new Row("2", "4", ""); var lowerBound = new PartialRow(schema); minRow.FillPartialRow(lowerBound); var upperBound = new PartialRow(schema); maxRow.FillPartialRow(upperBound); var expected = rows .Where(r => r.CompareTo(minRow) >= 0 && r.CompareTo(maxRow) < 0) .ToHashSet(); var results = new HashSet <Row>(); foreach (var tablet in tablets) { var scanner = _client.NewScanBuilder(table) .SetReadMode(ReadMode.ReadYourWrites) .LowerBound(lowerBound) .ExclusiveUpperBound(upperBound) .LowerBoundPartitionKeyRaw(tablet.Partition.PartitionKeyStart) .ExclusiveUpperBoundPartitionKeyRaw(tablet.Partition.PartitionKeyEnd) .Build(); var tabletResults = await CollectRowsAsync(scanner); var intersection = results.Intersect(tabletResults); Assert.Empty(intersection); foreach (var row in tabletResults) { results.Add(row); } } Assert.Equal(expected, results); } }
public void ReadScore(string score) { string line = ""; int maxCol = 0; int rowNum = 0; StringReader sr1 = new StringReader(score); while ((line = sr1.ReadLine()) != null) { string[] cells = line.Split(','); if(cells.Length <= 0 || cells[0].Contains("//")) continue; maxCol = Mathf.Max(maxCol, cells.Length); rowNum++; } sr1.Close(); partials = new PartialRow[rowNum]; StringReader sr = new StringReader(score); Dictionary<string, List<Partial>> dic = new Dictionary<string, List<Partial>>(); int idx = 0; while ((line = sr.ReadLine()) != null) { //Debug.Log(line); string[] cells = line.Split(','); if(cells.Length <= 0 || cells[0].Contains("//")) continue; string id = cells[0]; if(!dic.ContainsKey(id)){ dic.Add(id, new List<Partial>()); } for(int i=1;i<cells.Length;i++){ if(cells[i].Trim().Length > 0){ if(i == cells.Length-1){ //last cell Debug.Log(""+id+":"+i+"-"+maxCol); Debug.Log(""+cells[i]); string[] cols = cells[i].Split(';'); Partial p = CreatePartial(id, i-1, maxCol-1, cols); dic[id].Add(p); }else { for(int j=i+1;i<cells.Length;j++){ if(cells[j].Trim().Length > 0 || j == cells.Length-1){ Debug.Log(""+id+":"+i+"-"+j); Debug.Log(""+cells[i]); string[] cols = cells[i].Split(';'); Partial p = CreatePartial(id, i-1, j-1, cols); dic[id].Add(p); i = j-1; break; } } } } } } for (int i=0; i<rowNum; i++) { partials[i] = new PartialRow(); partials[i].num = i; partials[i].parts = dic[""+i].ToArray(); } }
public void TestPrimaryKeyEncoding() { var schema = GetSchema(new TableBuilder() .AddColumn("int8", KuduType.Int8, opt => opt.Key(true)) .AddColumn("int16", KuduType.Int16, opt => opt.Key(true)) .AddColumn("int32", KuduType.Int32, opt => opt.Key(true)) .AddColumn("int64", KuduType.Int64, opt => opt.Key(true)) .AddColumn("decimal32", KuduType.Decimal32, opt => opt.Key(true) .DecimalAttributes(DecimalUtil.MaxDecimal32Precision, 0)) .AddColumn("decimal64", KuduType.Decimal64, opt => opt.Key(true) .DecimalAttributes(DecimalUtil.MaxDecimal64Precision, 0)) .AddColumn("decimal128", KuduType.Decimal128, opt => opt.Key(true) .DecimalAttributes(DecimalUtil.MaxDecimal128Precision, 0)) .AddColumn("varchar", KuduType.Varchar, opt => opt.Key(true) .VarcharAttributes(10)) .AddColumn("string", KuduType.String, opt => opt.Key(true)) .AddColumn("binary", KuduType.Binary, opt => opt.Key(true))); var rowA = new PartialRow(schema); rowA.SetSByte("int8", sbyte.MinValue); rowA.SetInt16("int16", short.MinValue); rowA.SetInt32("int32", int.MinValue); rowA.SetInt64("int64", long.MinValue); // Note: The decimal value is not the minimum of the underlying int32, // int64, int128 type so we don't use "minimum" values in the test. rowA.SetDecimal("decimal32", 5m); rowA.SetDecimal("decimal64", 6m); rowA.SetDecimal("decimal128", 7m); rowA.SetString("varchar", ""); rowA.SetString("string", ""); rowA.SetBinary("binary", Array.Empty <byte>()); var rowAEncoded = KeyEncoder.EncodePrimaryKey(rowA); Assert.Equal(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 5, 0x80, 0, 0, 0, 0, 0, 0, 6, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0 }, rowAEncoded); var rowB = new PartialRow(schema); rowB.SetSByte("int8", sbyte.MaxValue); rowB.SetInt16("int16", short.MaxValue); rowB.SetInt32("int32", int.MaxValue); rowB.SetInt64("int64", long.MaxValue); // Note: The decimal value is not the maximum of the underlying int32, // int64, int128 type so we don't use "maximum" values in the test. rowB.SetDecimal("decimal32", 5m); rowB.SetDecimal("decimal64", 6m); rowB.SetDecimal("decimal128", 7m); rowB.SetString("varchar", "abc\u0001\0defghij"); rowB.SetString("string", "abc\u0001\0def"); rowB.SetBinary("binary", "\0\u0001binary".ToUtf8ByteArray()); var rowBEncoded = KeyEncoder.EncodePrimaryKey(rowB); Assert.Equal(ToByteArray(new int[] { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0, 0, 5, 0x80, 0, 0, 0, 0, 0, 0, 6, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 'g', 'h', 0, 0, 'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 0, 0, 0, 1, 'b', 'i', 'n', 'a', 'r', 'y' }), rowBEncoded); var rowC = new PartialRow(schema); rowC.SetSByte("int8", 1); rowC.SetInt16("int16", 2); rowC.SetInt32("int32", 3); rowC.SetInt64("int64", 4); rowC.SetDecimal("decimal32", 5m); rowC.SetDecimal("decimal64", 6m); rowC.SetDecimal("decimal128", 7m); rowC.SetString("varchar", "abc\n12345678"); rowC.SetString("string", "abc\n123"); rowC.SetBinary("binary", "\0\u0001\u0002\u0003\u0004\u0005".ToUtf8ByteArray()); var rowCEncoded = KeyEncoder.EncodePrimaryKey(rowC); Assert.Equal(ToByteArray(new int[] { 0x81, 0x80, 2, 0x80, 0, 0, 3, 0x80, 0, 0, 0, 0, 0, 0, 4, 0x80, 0, 0, 5, 0x80, 0, 0, 0, 0, 0, 0, 6, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 'a', 'b', 'c', '\n', '1', '2', '3', '4', '5', '6', 0, 0, 'a', 'b', 'c', '\n', '1', '2', '3', 0, 0, 0, 1, 2, 3, 4, 5 }), rowCEncoded); var rowD = new PartialRow(schema); rowD.SetSByte("int8", -1); rowD.SetInt16("int16", -2); rowD.SetInt32("int32", -3); rowD.SetInt64("int64", -4); rowD.SetDecimal("decimal32", -5m); rowD.SetDecimal("decimal64", -6m); rowD.SetDecimal("decimal128", -7m); rowD.SetString("varchar", "\0abc\n\u0001\u0001\0 123\u0001\0"); rowD.SetString("string", "\0abc\n\u0001\u0001\0 123\u0001\0"); rowD.SetBinary("binary", "\0\u0001\u0002\u0003\u0004\u0005\0".ToUtf8ByteArray()); var rowDEncoded = KeyEncoder.EncodePrimaryKey(rowD); Assert.Equal(ToByteArray(new int[] { 127, 127, -2, 127, -1, -1, -3, 127, -1, -1, -1, -1, -1, -1, -4, 127, -1, -1, -5, 127, -1, -1, -1, -1, -1, -1, -6, 127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -7, 0, 1, 'a', 'b', 'c', '\n', 1, 1, 0, 1, ' ', '1', 0, 0, 0, 1, 'a', 'b', 'c', '\n', 1, 1, 0, 1, ' ', '1', '2', '3', 1, 0, 1, 0, 0, 0, 1, 2, 3, 4, 5, 0, }), rowDEncoded); }
/// <summary> /// Determine if the given row falls into a valid partition. /// </summary> /// <param name="row">The row to check.</param> public bool IsCovered(PartialRow row) { var result = GetResult(row); return(result.IsCoveredRange); }
public void FillPartialRow(PartialRow row) { row.SetString("a", ValA); row.SetString("b", ValB); row.SetString("c", ValC); }
/// <summary> /// Randomizes the fields in a given PartialRow. /// </summary> /// <param name="row">The PartialRow to randomize.</param> public void RandomizeRow(PartialRow row) { RandomizeRow(row, true); }