public static KuduOperation CreateBasicSchemaInsert(KuduTable table, int key)
{
var row = table.NewInsert();
row.SetInt32(0, key);
row.SetInt32(1, 2);
row.SetInt32(2, 3);
row.SetString(3, "a string");
row.SetBool(4, true);
return(row);
}
private async Task InsertRowsAsync(KuduTable table, IEnumerable <Row> rows)
{
var operations = rows.Select(r =>
{
var insert = table.NewInsert();
r.FillPartialRow(insert);
return(insert);
});
await _client.WriteAsync(operations);
}
public async Task TestRenameKeyColumn()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.RenameColumn("c0", "c0Key"));
var exception = await Assert.ThrowsAsync <NonRecoverableException>(async() =>
{
// Scanning with the old schema.
var scanner = _client.NewScanBuilder(table)
.SetProjectedColumns("c0", "c1")
.Build();
await foreach (var resultSet in scanner)
{
}
});
Assert.True(exception.Status.IsInvalidArgument);
Assert.Contains(
"Some columns are not present in the current schema: c0",
exception.Status.Message);
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal("c0Key", table.Schema.GetColumn(0).Name);
Assert.Equal(2, table.Schema.Columns.Count);
// Add a row
var insert = table.NewInsert();
insert.SetInt32("c0Key", 101);
insert.SetInt32("c1", 101);
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
var scanner2 = _client.NewScanBuilder(table)
.SetProjectedColumns("c0Key", "c1")
.Build();
var rows = await scanner2.ScanToListAsync <(int c0Key, int c1)>();
Assert.Equal(101, rows.Count);
Assert.All(rows, row => Assert.Equal(row.c0Key, row.c1));
}
public async Task TestAlterAddColumns()
{
KuduTable table = await CreateTableAsync();
await InsertRowsAsync(table, 0, 100);
Assert.Equal(100, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddColumn("addNonNull", KuduType.Int32, opt => opt
.Nullable(false)
.DefaultValue(100))
.AddColumn("addNullable", KuduType.Int32)
.AddColumn("addNullableDef", KuduType.Int32, opt => opt
.DefaultValue(200)));
// Reopen table for the new schema.
table = await _client.OpenTableAsync(_tableName);
Assert.Equal(5, table.Schema.Columns.Count);
// Add a row with addNullableDef=null
var insert = table.NewInsert();
insert.SetInt32("c0", 101);
insert.SetInt32("c1", 101);
insert.SetInt32("addNonNull", 101);
insert.SetInt32("addNullable", 101);
insert.SetNull("addNullableDef");
await _session.EnqueueAsync(insert);
await _session.FlushAsync();
// Check defaults applied, and that row key=101
var results = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var expected = new List <string>(101);
for (int i = 0; i < 100; i++)
{
expected.Add($"INT32 c0={i}, INT32 c1={i}, INT32 addNonNull=100, " +
"INT32 addNullable=NULL, INT32 addNullableDef=200");
}
expected.Add("INT32 c0=101, INT32 c1=101, INT32 addNonNull=101, " +
"INT32 addNullable=101, INT32 addNullableDef=NULL");
Assert.Equal(
expected.OrderBy(r => r),
results.OrderBy(r => r));
}
private static IEnumerable <KuduOperation> CreateRows(KuduTable table, int numRows)
{
var hosts = new[] { "host1.example.com", "host2.example.com" };
var metrics = new[] { "cpuload.avg1", "cpuload.avg5", "cpuload.avg15" };
var now = DateTime.UtcNow;
for (int i = 0; i < numRows; i++)
{
var row = table.NewInsert();
row.SetString("host", hosts[i % hosts.Length]);
row.SetString("metric", metrics[i % metrics.Length]);
row.SetDateTime("timestamp", now.AddSeconds(i));
row.SetDouble("value", Random.Shared.NextDouble());
yield return(row);
}
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("ScannerFaultToleranceTests")
.AddHashPartitions(_numTablets, "key");
_table = await _client.CreateTableAsync(builder);
var random = new Random();
_keys = new HashSet <int>();
while (_keys.Count < _numRows)
{
_keys.Add(random.Next());
}
var rows = _keys
.Select((key, i) =>
{
var insert = _table.NewInsert();
insert.SetInt32(0, key);
insert.SetInt32(1, i);
insert.SetInt32(2, i++);
insert.SetString(3, DataGenerator.RandomString(1024, random));
insert.SetBool(4, true);
return(insert);
})
.Chunk(1000);
foreach (var batch in rows)
{
await _client.WriteAsync(batch);
}
}
/// <summary>
/// Generates a list of random mutation operations. Any unique row, identified by
/// it's key, could have a random number of operations/mutations. However, the
/// target count of numInserts, numUpdates and numDeletes will always be achieved
/// if the entire list of operations is processed.
/// </summary>
/// <param name="table">The table to generate operations for.</param>
/// <param name="numInserts">The number of row mutations to end with an insert.</param>
/// <param name="numUpdates">The number of row mutations to end with an update.</param>
/// <param name="numDeletes">The number of row mutations to end with an delete.</param>
private List <KuduOperation> GenerateMutationOperations(
KuduTable table, int numInserts, int numUpdates, int numDeletes)
{
var results = new List <KuduOperation>();
var unfinished = new List <MutationState>();
int minMutationsBound = 5;
// Generate Operations to initialize all of the row with inserts.
var changeCounts = new List <(RowOperation type, int count)>
{
(RowOperation.Insert, numInserts),
(RowOperation.Update, numUpdates),
(RowOperation.Delete, numDeletes)
};
foreach (var(type, count) in changeCounts)
{
for (int i = 0; i < count; i++)
{
// Generate a random insert.
var insert = table.NewInsert();
_generator.RandomizeRow(insert);
var key = insert.GetInt32(0);
// Add the insert to the results.
results.Add(insert);
// Initialize the unfinished MutationState.
unfinished.Add(new MutationState(key, type, _random.Next(minMutationsBound)));
}
}
// Randomly pull from the unfinished list, mutate it and add that operation to
// the results. If it has been mutated at least the minimum number of times,
// remove it from the unfinished list.
while (unfinished.Count > 0)
{
// Get a random row to mutate.
int index = _random.Next(unfinished.Count);
MutationState state = unfinished[index];
// If the row is done, remove it from unfinished and continue.
if (state.NumMutations >= state.MinMutations && state.CurrentType == state.EndType)
{
unfinished.RemoveAt(index);
continue;
}
// Otherwise, generate an operation to mutate the row based on its current ChangeType.
// insert -> update|delete
// update -> update|delete
// delete -> insert
KuduOperation op;
if (state.CurrentType == RowOperation.Insert || state.CurrentType == RowOperation.Update)
{
op = _random.NextBool() ? table.NewUpdate() : table.NewDelete();
}
else
{
// Must be a delete, so we need an insert next.
op = table.NewInsert();
}
op.SetInt32(0, state.Key);
if (op.Operation != RowOperation.Delete)
{
_generator.RandomizeRow(op, randomizeKeys: false);
}
results.Add(op);
state.CurrentType = op.Operation;
state.NumMutations++;
}
return(results);
}
