/// <summary>
/// Load a table of default schema with the specified number of records, in ascending key order.
/// </summary>
public static async Task LoadDefaultTableAsync(KuduClient client, KuduTable table, int numRows)
{
var rows = Enumerable.Range(0, numRows)
.Select(i => CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows);
}
public async Task TestAlterModifyColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// Check for expected defaults.
ColumnSchema col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.DefaultCompression, col.Compression);
Assert.Equal(EncodingType.AutoEncoding, col.Encoding);
Assert.Null(col.DefaultValue);
// Alter the table.
await _client.AlterTableAsync(new AlterTableBuilder(table)
.ChangeCompressionAlgorithm(col.Name, CompressionType.Snappy)
.ChangeEncoding(col.Name, EncodingType.Rle)
.ChangeDefault(col.Name, 0));
// Check for new values.
table = await _client.OpenTableAsync(_tableName);
col = table.Schema.GetColumn(1);
Assert.Equal(CompressionType.Snappy, col.Compression);
Assert.Equal(EncodingType.Rle, col.Encoding);
Assert.Equal(0, col.DefaultValue);
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
var options = _harness.CreateClientBuilder().BuildOptions();
var securityContext = new SecurityContext();
var systemClock = new SystemClock();
var connectionFactory = new KuduConnectionFactory(
options, securityContext, NullLoggerFactory.Instance);
_testConnectionFactory = new TestConnectionFactory(connectionFactory);
_client = new KuduClient(
options,
securityContext,
_testConnectionFactory,
systemClock,
NullLoggerFactory.Instance);
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("chaos_test_table")
.SetNumReplicas(3)
.CreateBasicRangePartition();
_table = await _client.CreateTableAsync(builder);
}
public static KuduOperation CreateBasicSchemaDeleteIgnore(KuduTable table, int key)
{
var row = table.NewDeleteIgnore();
row.SetInt32(0, key);
return(row);
}
private async Task <long> SetupTableForDiffScansAsync(KuduTable table, int numRows)
{
for (int i = 0; i < numRows / 2; i++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
// Grab the timestamp, then add more data so there's a diff.
long timestamp = _client.LastPropagatedTimestamp;
for (int i = numRows / 2; i < numRows; i++)
{
var row = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
// Delete some data so the is_deleted column can be tested.
for (int i = 0; i < numRows / 4; i++)
{
var row = table.NewDelete();
row.SetInt32(0, i);
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
return(timestamp);
}
public static async Task <List <string> > ScanTableToStringsAsync(
KuduClient client, KuduTable table, params KuduPredicate[] predicates)
{
var rowStrings = new List <string>();
var scanBuilder = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
var scanner = scanBuilder.Build();
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
rowStrings.Add(row.ToString());
}
}
rowStrings.Sort();
return(rowStrings);
}
public AbstractKuduScannerBuilder(KuduClient client, KuduTable table)
{
Client = client;
Table = table;
Predicates = new Dictionary <string, KuduPredicate>();
ScanRequestTimeout = -1; // TODO: Pull this from the client.
}
public KuduScanEnumerator(
ILogger logger,
KuduClient client,
KuduTable table,
List <ColumnSchemaPB> projectedColumnsPb,
KuduSchema projectionSchema,
OrderMode orderMode,
ReadMode readMode,
ReplicaSelection replicaSelection,
bool isFaultTolerant,
Dictionary <string, KuduPredicate> predicates,
long limit,
bool cacheBlocks,
byte[] startPrimaryKey,
byte[] endPrimaryKey,
long startTimestamp,
long htTimestamp,
int batchSizeBytes,
PartitionPruner partitionPruner,
CancellationToken cancellationToken)
{
_logger = logger;
_client = client;
_table = table;
_partitionPruner = partitionPruner;
_orderMode = orderMode;
_readMode = readMode;
_columns = projectedColumnsPb;
_schema = projectionSchema;
_predicates = predicates;
_replicaSelection = replicaSelection;
_isFaultTolerant = isFaultTolerant;
_limit = limit;
_cacheBlocks = cacheBlocks;
_startPrimaryKey = startPrimaryKey ?? Array.Empty <byte>();
_endPrimaryKey = endPrimaryKey ?? Array.Empty <byte>();
_startTimestamp = startTimestamp;
SnapshotTimestamp = htTimestamp;
_batchSizeBytes = batchSizeBytes;
_scannerId = ByteString.Empty;
_lastPrimaryKey = ByteString.Empty;
_cancellationToken = cancellationToken;
ResourceMetrics = new ResourceMetrics();
// If the partition pruner has pruned all partitions, then the scan can be
// short circuited without contacting any tablet servers.
if (!_partitionPruner.HasMorePartitionKeyRanges)
{
_closed = true;
}
// For READ_YOUR_WRITES scan mode, get the latest observed timestamp
// and store it. Always use this one as the propagated timestamp for
// the duration of the scan to avoid unnecessary wait.
if (readMode == ReadMode.ReadYourWrites)
{
_lowerBoundPropagationTimestamp = client.LastPropagatedTimestamp;
}
}
/// <summary>
/// Checks the number of tablets and pruner ranges generated for a scan with
/// predicates and optional partition key bounds.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="expectedPrunerRanges">The expected number of generated partition pruner ranges.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="lowerBoundPartitionKey">An optional lower bound partition key.</param>
/// <param name="upperBoundPartitionKey">An optional upper bound partition key.</param>
/// <param name="predicates">The predicates to apply to the scan.</param>
private async Task CheckPartitionsAsync(
int expectedTablets,
int expectedPrunerRanges,
KuduTable table,
List <Partition> partitions,
byte[] lowerBoundPartitionKey,
byte[] upperBoundPartitionKey,
params KuduPredicate[] predicates)
{
// Partition key bounds can't be applied to the ScanTokenBuilder.
var scanBuilder = _client.NewScanBuilder(table);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
if (lowerBoundPartitionKey != null)
{
scanBuilder.LowerBoundPartitionKeyRaw(lowerBoundPartitionKey);
}
if (upperBoundPartitionKey != null)
{
scanBuilder.ExclusiveUpperBoundPartitionKeyRaw(upperBoundPartitionKey);
}
var pruner = PartitionPruner.Create(scanBuilder);
int scannedPartitions = 0;
foreach (var partition in partitions)
{
if (!pruner.ShouldPruneForTests(partition))
{
scannedPartitions++;
}
}
Assert.Equal(expectedTablets, scannedPartitions);
Assert.Equal(expectedPrunerRanges, pruner.NumRangesRemaining);
// Check that the scan token builder comes up with the same amount.
// The scan token builder does not allow for upper/lower partition keys.
if (lowerBoundPartitionKey == null && upperBoundPartitionKey == null)
{
var tokenBuilder = _client.NewScanTokenBuilder(table);
foreach (var predicate in predicates)
{
tokenBuilder.AddPredicate(predicate);
}
// Check that the number of ScanTokens built for the scan matches.
var tokens = await tokenBuilder.BuildAsync();
Assert.Equal(expectedTablets, tokens.Count);
}
}
public static ValueTask <long> CountRowsAsync(KuduClient client, KuduTable table)
{
var scanner = client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites)
.SetReplicaSelection(ReplicaSelection.LeaderOnly)
.Build();
return(scanner.CountAsync());
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
await using var session = _client.NewSession();
// Create a 4-tablets table for scanning.
var builder = new TableBuilder(_tableName)
.AddColumn("key1", KuduType.String, opt => opt.Key(true))
.AddColumn("key2", KuduType.String, opt => opt.Key(true))
.AddColumn("val", KuduType.String)
.SetRangePartitionColumns("key1", "key2");
for (int i = 1; i < 4; i++)
{
builder.AddSplitRow(splitRow =>
{
splitRow.SetString("key1", i.ToString());
splitRow.SetString("key2", "");
});
}
var table = await _client.CreateTableAsync(builder);
// The data layout ends up like this:
// tablet '', '1': no rows
// tablet '1', '2': '111', '122', '133'
// tablet '2', '3': '211', '222', '233'
// tablet '3', '': '311', '322', '333'
var keys = new[] { "1", "2", "3" };
foreach (var key1 in keys)
{
foreach (var key2 in keys)
{
var insert = table.NewInsert();
insert.SetString(0, key1);
insert.SetString(1, key2);
insert.SetString(2, key2);
await session.EnqueueAsync(insert);
await session.FlushAsync();
}
}
_beforeWriteTimestamp = _client.LastPropagatedTimestamp;
// Reset the client in order to clear the propagated timestamp.
_newClient = _harness.CreateClient();
// Reopen the table using the new client.
_table = await _newClient.OpenTableAsync(_tableName);
_schema = _table.Schema;
}
private async Task InsertRowsAsync(KuduTable table, IEnumerable <Row> rows)
{
var operations = rows.Select(r =>
{
var insert = table.NewInsert();
r.FillPartialRow(insert);
return(insert);
});
await _client.WriteAsync(operations);
}
public static KuduOperation CreateBasicSchemaInsert(KuduTable table, int key)
{
var row = table.NewInsert();
row.SetInt32(0, key);
row.SetInt32(1, 2);
row.SetInt32(2, 3);
row.SetString(3, "a string");
row.SetBool(4, true);
return(row);
}
public AlterTableBuilder(KuduTable table)
{
_table = table;
_request = new AlterTableRequestPB
{
Table = new TableIdentifierPB
{
TableId = _table.SchemaPb.TableId
}
};
}
private static async ValueTask InsertRowsAsync(
IKuduSession session, KuduTable table, int startRow, int numRows)
{
var end = startRow + numRows;
for (var i = startRow; i < end; i++)
{
var insert = ClientTestUtil.CreateBasicSchemaInsert(table, i);
await session.EnqueueAsync(insert);
}
}
public static KuduOperation CreateAllNullsInsert(KuduTable table, int key)
{
var numColumns = table.Schema.Columns.Count;
var row = table.NewUpsert();
row.SetInt32(0, key);
for (int i = 1; i < numColumns; i++)
{
row.SetNull(i);
}
return(row);
}
public async Task TestRenameKeyColumn()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("c0", "c0Key"));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
// Scanning with the old schema.
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("c0", "c1")
.Build();
await foreach (var resultSet in scanner)
{
}
});
Assert.True(exception.Status.IsInvalidArgument);
Assert.Contains(
"Some columns are not present in the current schema: c0",
exception.Status.Message);
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("c0Key", table.Schema.GetColumn(0).Name);
Assert.Equal(2, table.Schema.Columns.Count);
// Add a row
var insert = table.NewInsert();
insert.SetInt32("c0Key", 101);
insert.SetInt32("c1", 101);
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
var scanner2 = _client.NewScanBuilder(table)
.SetProjectedColumns("c0Key", "c1")
.Build();
var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>();
Assert.Equal(101, rows.Count);
Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1));
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
// Use one tablet because multiple tablets don't work: we could jump
// from one tablet to another which could change the logical clock.
var builder = new TableBuilder("HybridTimeTest")
.AddColumn("key", KuduType.String, opt => opt.Key(true))
.SetRangePartitionColumns("key");
_table = await _client.CreateTableAsync(builder);
}
public async Task TestAlterAddColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddColumn("addNonNull", KuduType.Int32, opt => opt
.Nullable(false)
.DefaultValue(100))
.AddColumn("addNullable", KuduType.Int32)
.AddColumn("addNullableDef", KuduType.Int32, opt => opt
.DefaultValue(200)));
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal(5, table.Schema.Columns.Count);
// Add a row with addNullableDef=null
var insert = table.NewInsert();
insert.SetInt32("c0", 101);
insert.SetInt32("c1", 101);
insert.SetInt32("addNonNull", 101);
insert.SetInt32("addNullable", 101);
insert.SetNull("addNullableDef");
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
// Check defaults applied, and that row key=101
var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var expected = new List <string>(101);
for (int i = 0; i < 100; i++)
{
expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " +
"INT32 addNullable=NULL, INT32 addNullableDef=200");
}
expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " +
"INT32 addNullable=101, INT32 addNullableDef=NULL");
Assert.Equal(
expected.OrderBy(r => r),
results.OrderBy(r => r));
}
private ValueTask <long> CountRowsAsync(KuduTable table, params KuduPredicate[] predicates)
{
var scanBuilder = _client.NewScanBuilder(table)
.SetReadMode(ReadMode.ReadYourWrites);
foreach (var predicate in predicates)
{
scanBuilder.AddPredicate(predicate);
}
var scanner = scanBuilder.Build();
return(scanner.CountAsync());
}
public async Task TestAlterExtraConfigs()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
// 1. Check for expected defaults.
table = await _client.OpenTableAsync(_tableName);
Assert.DoesNotContain("kudu.table.history_max_age_sec", table.ExtraConfig);
// 2. Alter history max age second to 3600
var alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "3600" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("3600", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 3. Alter history max age second to 7200
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "7200" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("7200", table.ExtraConfig["kudu.table.history_max_age_sec"]);
// 4. Reset history max age second to default
alterExtraConfigs = new Dictionary <string, string>
{
{ "kudu.table.history_max_age_sec", "" }
};
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AlterExtraConfigs(alterExtraConfigs));
table = await _client.OpenTableAsync(_tableName);
Assert.Empty(table.ExtraConfig);
}
/// <summary>
/// Counts the partitions touched by a scan with optional primary key bounds.
/// The table is assumed to have three INT8 columns as the primary key.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="lowerBoundPrimaryKey">The optional lower bound primary key.</param>
/// <param name="upperBoundPrimaryKey">The optional upper bound primary key.</param>
private async Task CheckPartitionsPrimaryKeyAsync(
int expectedTablets,
KuduTable table,
List <Partition> partitions,
sbyte[] lowerBoundPrimaryKey,
sbyte[] upperBoundPrimaryKey)
{
var scanBuilder = _client.NewScanTokenBuilder(table);
if (lowerBoundPrimaryKey != null)
{
var lower = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
lower.SetSByte(i, lowerBoundPrimaryKey[i]);
}
scanBuilder.LowerBound(lower);
}
if (upperBoundPrimaryKey != null)
{
var upper = new PartialRow(table.Schema);
for (int i = 0; i < 3; i++)
{
upper.SetSByte(i, upperBoundPrimaryKey[i]);
}
scanBuilder.ExclusiveUpperBound(upper);
}
var pruner = PartitionPruner.Create(scanBuilder);
int scannedPartitions = 0;
foreach (var partition in partitions)
{
if (!pruner.ShouldPruneForTests(partition))
{
scannedPartitions++;
}
}
// Check that the number of ScanTokens built for the scan matches.
var tokens = await scanBuilder.BuildAsync();
Assert.Equal(expectedTablets, scannedPartitions);
Assert.Equal(scannedPartitions, tokens.Count);
Assert.Equal(expectedTablets == 0 ? 0 : 1, pruner.NumRangesRemaining);
}
/// <summary>
/// Checks the number of tablets and pruner ranges generated for a scan.
/// </summary>
/// <param name="expectedTablets">The expected number of tablets to satisfy the scan.</param>
/// <param name="expectedPrunerRanges">The expected number of generated partition pruner ranges.</param>
/// <param name="table">The table to scan.</param>
/// <param name="partitions">The partitions of the table.</param>
/// <param name="predicates">The predicates to apply to the scan.</param>
private Task CheckPartitionsAsync(
int expectedTablets,
int expectedPrunerRanges,
KuduTable table,
List <Partition> partitions,
params KuduPredicate[] predicates)
{
return(CheckPartitionsAsync(
expectedTablets,
expectedPrunerRanges,
table,
partitions,
null,
null,
predicates));
}
private static IEnumerable <KuduOperation> CreateRows(KuduTable table, int numRows)
{
var hosts = new[] { "host1.example.com", "host2.example.com" };
var metrics = new[] { "cpuload.avg1", "cpuload.avg5", "cpuload.avg15" };
var now = DateTime.UtcNow;
for (int i = 0; i < numRows; i++)
{
var row = table.NewInsert();
row.SetString("host", hosts[i % hosts.Length]);
row.SetString("metric", metrics[i % metrics.Length]);
row.SetDateTime("timestamp", now.AddSeconds(i));
row.SetDouble("value", Random.Shared.NextDouble());
yield return(row);
}
}
public async Task TestAlterRangePartitioningExclusiveInclusive()
{
// Create initial table with single range partition covering (-1, 99].
var builder = new TableBuilder(_tableName)
.SetNumReplicas(1)
.AddColumn("c0", KuduType.Int32, opt => opt.Key(true))
.AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false))
.SetRangePartitionColumns("c0")
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -1);
upper.SetInt32("c0", 99);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive);
KuduTable table = await _client.CreateTableAsync(builder);
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
// Insert some rows, and then drop the partition and ensure that the table is empty.
await InsertRowsAsync(table, 0, 100);
await InsertRowsAsync(table, 200, 300);
Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
}
public static KuduOperation CreateBasicSchemaUpsert(
KuduTable table, int key, int secondVal, bool hasNull)
{
var row = table.NewUpsert();
row.SetInt32(0, key);
row.SetInt32(1, secondVal);
row.SetInt32(2, 3);
if (hasNull)
{
row.SetNull(3);
}
else
{
row.SetString(3, "a string");
}
row.SetBool(4, true);
return(row);
}
private async Task CheckPredicatesAsync <T>(
KuduTable table,
SortedSet <T> values,
List <T> testValues)
{
var col = table.Schema.GetColumn("value");
Assert.Equal(values.Count + 1, await CountRowsAsync(table));
foreach (var v in testValues)
{
// value = v
var equal = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Equal, (dynamic)v);
Assert.Equal(values.Contains(v) ? 1 : 0, await CountRowsAsync(table, equal));
// value >= v
var greaterEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.GreaterEqual, (dynamic)v);
Assert.Equal(values.TailSet(v).Count, await CountRowsAsync(table, greaterEqual));
// value <= v
var lessEqual = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.LessEqual, (dynamic)v);
Assert.Equal(values.HeadSet(v, true).Count, await CountRowsAsync(table, lessEqual));
// value > v
var greater = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Greater, (dynamic)v);
Assert.Equal(values.TailSet(v, false).Count, await CountRowsAsync(table, greater));
// value < v
var less = KuduPredicate.NewComparisonPredicate(col, ComparisonOp.Less, (dynamic)v);
Assert.Equal(values.HeadSet(v).Count, await CountRowsAsync(table, less));
}
var isNotNull = KuduPredicate.NewIsNotNullPredicate(col);
Assert.Equal(values.Count, await CountRowsAsync(table, isNotNull));
var isNull = KuduPredicate.NewIsNullPredicate(col);
Assert.Equal(1, await CountRowsAsync(table, isNull));
}
public static KuduOperation CreateAllTypesInsert(KuduTable table, int key)
{
var row = table.NewUpsert();
row.SetInt32("key", key);
row.SetByte("int8", 42);
row.SetInt16("int16", 43);
row.SetInt32("int32", 44);
row.SetInt64("int64", 45);
row.SetBool("bool", true);
row.SetFloat("float", 52.35f);
row.SetDouble("double", 53.35);
row.SetString("string", "fun with ütf\0");
row.SetString("varchar", "árvíztűrő tükörfúrógép");
row.SetBinary("binary", new byte[] { 0, 1, 2, 3, 4 });
row.SetDateTime("timestamp", DateTime.Parse("8/19/2020 7:50 PM"));
row.SetDateTime("date", DateTime.Parse("8/19/2020"));
row.SetDecimal("decimal32", 12.345m);
row.SetDecimal("decimal64", 12.346m);
row.SetDecimal("decimal128", 12.347m);
return(row);
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("ScannerFaultToleranceTests")
.AddHashPartitions(_numTablets, "key");
_table = await _client.CreateTableAsync(builder);
var random = new Random();
_keys = new HashSet <int>();
while (_keys.Count < _numRows)
{
_keys.Add(random.Next());
}
var rows = _keys
.Select((key, i) =>
{
var insert = _table.NewInsert();
insert.SetInt32(0, key);
insert.SetInt32(1, i);
insert.SetInt32(2, i++);
insert.SetString(3, DataGenerator.RandomString(1024, random));
insert.SetBool(4, true);
return(insert);
})
.Chunk(1000);
foreach (var batch in rows)
{
await _client.WriteAsync(batch);
}
}
/// <summary>
/// Generates a list of random mutation operations. Any unique row, identified by
/// it's key, could have a random number of operations/mutations. However, the
/// target count of numInserts, numUpdates and numDeletes will always be achieved
/// if the entire list of operations is processed.
/// </summary>
/// <param name="table">The table to generate operations for.</param>
/// <param name="numInserts">The number of row mutations to end with an insert.</param>
/// <param name="numUpdates">The number of row mutations to end with an update.</param>
/// <param name="numDeletes">The number of row mutations to end with an delete.</param>
private List <KuduOperation> GenerateMutationOperations(
KuduTable table, int numInserts, int numUpdates, int numDeletes)
{
var results = new List <KuduOperation>();
var unfinished = new List <MutationState>();
int minMutationsBound = 5;
// Generate Operations to initialize all of the row with inserts.
var changeCounts = new List <(RowOperation type, int count)>
{
(RowOperation.Insert, numInserts),
(RowOperation.Update, numUpdates),
(RowOperation.Delete, numDeletes)
};
foreach (var(type, count) in changeCounts)
{
for (int i = 0; i < count; i++)
{
// Generate a random insert.
var insert = table.NewInsert();
_generator.RandomizeRow(insert);
var key = insert.GetInt32(0);
// Add the insert to the results.
results.Add(insert);
// Initialize the unfinished MutationState.
unfinished.Add(new MutationState(key, type, _random.Next(minMutationsBound)));
}
}
// Randomly pull from the unfinished list, mutate it and add that operation to
// the results. If it has been mutated at least the minimum number of times,
// remove it from the unfinished list.
while (unfinished.Count > 0)
{
// Get a random row to mutate.
int index = _random.Next(unfinished.Count);
MutationState state = unfinished[index];
// If the row is done, remove it from unfinished and continue.
if (state.NumMutations >= state.MinMutations && state.CurrentType == state.EndType)
{
unfinished.RemoveAt(index);
continue;
}
// Otherwise, generate an operation to mutate the row based on its current ChangeType.
// insert -> update|delete
// update -> update|delete
// delete -> insert
KuduOperation op;
if (state.CurrentType == RowOperation.Insert || state.CurrentType == RowOperation.Update)
{
op = _random.NextBool() ? table.NewUpdate() : table.NewDelete();
}
else
{
// Must be a delete, so we need an insert next.
op = table.NewInsert();
}
op.SetInt32(0, state.Key);
if (op.Operation != RowOperation.Delete)
{
_generator.RandomizeRow(op, randomizeKeys: false);
}
results.Add(op);
state.CurrentType = op.Operation;
state.NumMutations++;
}
return(results);
}
