/// <summary>
/// Applies a list of operations and returns the final change type for each key.
/// </summary>
private async Task <Dictionary <int, RowOperation> > ApplyOperationsAsync(
KuduClient client, List <KuduOperation> operations)
{
var results = new Dictionary <int, RowOperation>();
// If there are no operations, return early.
if (operations.Count == 0)
{
return(results);
}
// On some runs, wait long enough to flush at the start.
if (_random.NextBool())
{
await Task.Delay(2000);
}
// Pick an int as a flush indicator so we flush once on average while applying operations.
int flushInt = _random.Next(operations.Count);
foreach (var operation in operations)
{
// On some runs, wait long enough to flush while applying operations.
if (_random.Next(operations.Count) == flushInt)
{
await Task.Delay(2000);
}
await client.WriteAsync(new[] { operation });
results[operation.GetInt32(0)] = operation.Operation;
}
return(results);
}
/// <summary>
/// Load a table of default schema with the specified number of records, in ascending key order.
/// </summary>
public static async Task LoadDefaultTableAsync(KuduClient client, KuduTable table, int numRows)
{
var rows = Enumerable.Range(0, numRows)
.Select(i => CreateBasicSchemaInsert(table, i));
await client.WriteAsync(rows);
}
private async Task SendAsync(List <KuduOperation> queue)
{
try
{
await _client.WriteAsync(queue, _options.ExternalConsistencyMode, _txnId)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.ExceptionFlushingSessionData(ex, queue.Count, _txnId);
var exceptionHandler = _options.ExceptionHandler;
if (exceptionHandler is not null)
{
var queueCopy = new List <KuduOperation>(queue);
var exceptionContext = new SessionExceptionContext(ex, queueCopy);
try
{
await exceptionHandler(exceptionContext).ConfigureAwait(false);
}
catch { }
}
}
}
public async Task TestInsertIgnoreAfterInsertHasNoRowError()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestInsertIgnoreAfterInsertHasNoRowError));
var table = await _client.CreateTableAsync(builder);
var rows = new[]
{
ClientTestUtil.CreateBasicSchemaInsert(table, 1),
ClientTestUtil.CreateBasicSchemaUpsert(table, 1, 1, false),
ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1)
};
await _client.WriteAsync(rows);
var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var rowString = Assert.Single(rowStrings);
Assert.Equal(
"INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " +
"STRING column3_s=a string, BOOL column4_b=True", rowString);
}
public async Task TestConstructorWithProperties()
{
var builder = new TableBuilder(nameof(TestConstructorWithProperties))
.AddColumn("Key", KuduType.Int32, opt => opt.Key(true))
.AddColumn("Column1", KuduType.String, opt => opt.Nullable(false))
.AddColumn("Column2", KuduType.Int32, opt => opt.Nullable(false))
.AddColumn("Column3", KuduType.String, opt => opt.Nullable(false));
var values = new[]
{
new MixedRecord(1, "val-1")
{
Column2 = 100, Column3 = "val-2"
},
new MixedRecord(2, "val-3")
{
Column2 = 200, Column3 = "val-4"
}
};
var table = await _client.CreateTableAsync(builder);
var rowsToInsert = values.Select(value =>
{
var insert = table.NewInsert();
insert.SetInt32("Key", value.Key);
insert.SetString("Column1", value.Column1);
insert.SetInt32("Column2", value.Column2);
insert.SetString("Column3", value.Column3);
return(insert);
});
await _client.WriteAsync(rowsToInsert);
var rows = await ScanAsync <MixedRecord>(table);
Assert.Equal(values, rows);
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("ScannerFaultToleranceTests")
.AddHashPartitions(_numTablets, "key");
_table = await _client.CreateTableAsync(builder);
var random = new Random();
_keys = new HashSet <int>();
while (_keys.Count < _numRows)
{
_keys.Add(random.Next());
}
var rows = _keys
.Select((key, i) =>
{
var insert = _table.NewInsert();
insert.SetInt32(0, key);
insert.SetInt32(1, i);
insert.SetInt32(2, i++);
insert.SetString(3, DataGenerator.RandomString(1024, random));
insert.SetBool(4, true);
return(insert);
})
.Chunk(1000);
foreach (var batch in rows)
{
await _client.WriteAsync(batch);
}
}
public async Task TestAllPrimaryKeyTypes()
{
var tableBuilder = new TableBuilder(nameof(TestAllPrimaryKeyTypes))
.AddColumn("int8", KuduType.Int8, opt => opt.Key(true))
.AddColumn("int16", KuduType.Int16, opt => opt.Key(true))
.AddColumn("int32", KuduType.Int32, opt => opt.Key(true))
.AddColumn("int64", KuduType.Int64, opt => opt.Key(true))
.AddColumn("string", KuduType.String, opt => opt.Key(true))
.AddColumn("binary", KuduType.Binary, opt => opt.Key(true))
.AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
.AddColumn("decimal32", KuduType.Decimal32, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal32Precision, 0))
.AddColumn("decimal64", KuduType.Decimal64, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal64Precision, 0))
.AddColumn("decimal128", KuduType.Decimal128, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal128Precision, 0))
.AddColumn("varchar", KuduType.Varchar, opt => opt.Key(true)
.VarcharAttributes(10))
.AddColumn("date", KuduType.Date, opt => opt.Key(true))
.AddColumn("bool", KuduType.Bool) // not primary key type
.AddColumn("float", KuduType.Float) // not primary key type
.AddColumn("double", KuduType.Double); // not primary key type
var table = await _client.CreateTableAsync(tableBuilder);
var insert = table.NewInsert();
insert.SetSByte("int8", 1);
insert.SetInt16("int16", 2);
insert.SetInt32("int32", 3);
insert.SetInt64("int64", 4);
insert.SetString("string", "foo");
insert.SetBinary("binary", "bar".ToUtf8ByteArray());
insert.SetInt64("timestamp", 6);
insert.SetDecimal("decimal32", DecimalUtil.MaxUnscaledDecimal32);
insert.SetDecimal("decimal64", DecimalUtil.MaxUnscaledDecimal64);
insert.SetDecimal("decimal128", decimal.Truncate(decimal.MaxValue));
insert.SetString("varchar", "varchar bar");
insert.SetDateTime("date", EpochTime.FromUnixTimeDays(0));
insert.SetBool("bool", true);
insert.SetFloat("float", 7.8f);
insert.SetDouble("double", 9.9);
await _client.WriteAsync(new[] { insert });
var scanner = _client.NewScanBuilder(table).Build();
var scannedRows = 0;
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
scannedRows++;
Assert.Equal(1, row.GetSByte("int8"));
Assert.Equal(2, row.GetInt16("int16"));
Assert.Equal(3, row.GetInt32("int32"));
Assert.Equal(4, row.GetInt64("int64"));
Assert.Equal("foo", row.GetString("string"));
Assert.Equal("bar".ToUtf8ByteArray(), row.GetBinary("binary"));
Assert.Equal(6, row.GetInt64("timestamp"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal32, row.GetDecimal("decimal32"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal64, row.GetDecimal("decimal64"));
Assert.Equal(decimal.Truncate(decimal.MaxValue), row.GetDecimal("decimal128"));
Assert.Equal("varchar ba", row.GetString("varchar"));
Assert.Equal(0, row.GetInt32("date"));
Assert.True(row.GetBool("bool"));
Assert.Equal(7.8f, row.GetFloat("float"));
Assert.Equal(9.9, row.GetDouble("double"));
}
}
Assert.Equal(1, scannedRows);
}
public async Task TestScanTokenWithWrongUuidSerialization()
{
int buckets = 8;
var builder = ClientTestUtil.CreateManyStringsSchema()
.SetTableName(_tableName)
.AddHashPartitions(buckets, "key");
var table = await _client.CreateTableAsync(builder);
int totalRows = 100;
var inserts = Enumerable.Range(0, totalRows).Select(i =>
{
var insert = table.NewInsert();
insert.SetString("key", $"key_{i:00}");
insert.SetString("c1", $"c1_{i}");
insert.SetString("c2", $"c2_{i}");
return(insert);
});
await _client.WriteAsync(inserts);
var tokens = await _client.NewScanTokenBuilder(table)
.SetEmptyProjection()
.IncludeTableMetadata(true)
.IncludeTabletMetadata(true)
.BuildAsync();
Assert.Equal(buckets, tokens.Count);
// Create a new client, open the newly created kudu table, and new scanners.
await using var newClient = _harness.CreateClient();
var newTable = await newClient.OpenTableAsync(_tableName);
var tabletIds = new List <string>();
var scanners = new List <KuduScanner>();
foreach (var token in tokens)
{
tabletIds.Add(token.Tablet.TabletId);
var scanBuilder = await newClient.NewScanBuilderFromTokenAsync(token);
scanners.Add(scanBuilder.Build());
}
// Step down all tablet leaders.
var masters = string.Join(',', _harness.GetMasterServers());
foreach (var tabletId in tabletIds)
{
await MiniKuduCluster.RunKuduToolAsync(
"tablet",
"leader_step_down",
masters,
tabletId);
await Task.Delay(1000);
}
// Delete all rows first through the new client.
var deletes = Enumerable.Range(0, totalRows).Select(i =>
{
var delete = table.NewDelete();
delete.SetString("key", $"key_{i:00}");
return(delete);
});
await newClient.WriteAsync(deletes);
// Insert all rows again through the new client.
var newInserts = Enumerable.Range(0, totalRows).Select(i =>
{
var insert = table.NewInsert();
insert.SetString("key", $"key_{i:00}");
insert.SetString("c1", $"c1_{i}");
insert.SetString("c2", $"c2_{i}");
return(insert);
});
await newClient.WriteAsync(newInserts);
// Verify all the row count.
long rowCount = 0;
foreach (var scanner in scanners)
{
rowCount += await scanner.CountAsync();
}
Assert.Equal(totalRows, rowCount);
}