public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
var options = _harness.CreateClientBuilder().BuildOptions();
var securityContext = new SecurityContext();
var systemClock = new SystemClock();
var connectionFactory = new KuduConnectionFactory(
options, securityContext, NullLoggerFactory.Instance);
_testConnectionFactory = new TestConnectionFactory(connectionFactory);
_client = new KuduClient(
options,
securityContext,
_testConnectionFactory,
systemClock,
NullLoggerFactory.Instance);
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("chaos_test_table")
.SetNumReplicas(3)
.CreateBasicRangePartition();
_table = await _client.CreateTableAsync(builder);
}
public async Task TestAlterRangePartitioningExclusiveInclusive()
{
// Create initial table with single range partition covering (-1, 99].
var builder = new TableBuilder(_tableName)
.SetNumReplicas(1)
.AddColumn("c0", KuduType.Int32, opt => opt.Key(true))
.AddColumn("c1", KuduType.Int32, opt => opt.Nullable(false))
.SetRangePartitionColumns("c0")
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", -1);
upper.SetInt32("c0", 99);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive);
KuduTable table = await _client.CreateTableAsync(builder);
await _client.AlterTableAsync(new AlterTableBuilder(table)
.AddRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
// Insert some rows, and then drop the partition and ensure that the table is empty.
await InsertRowsAsync(table, 0, 100);
await InsertRowsAsync(table, 200, 300);
Assert.Equal(200, await ClientTestUtil.CountRowsAsync(_client, table));
await _client.AlterTableAsync(new AlterTableBuilder(table)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 0);
upper.SetInt32("c0", 100);
}, RangePartitionBound.Inclusive, RangePartitionBound.Exclusive)
.DropRangePartition((lower, upper) =>
{
lower.SetInt32("c0", 199);
upper.SetInt32("c0", 299);
}, RangePartitionBound.Exclusive, RangePartitionBound.Inclusive));
Assert.Equal(0, await ClientTestUtil.CountRowsAsync(_client, table));
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
await using var session = _client.NewSession();
// Create a 4-tablets table for scanning.
var builder = new TableBuilder(_tableName)
.AddColumn("key1", KuduType.String, opt => opt.Key(true))
.AddColumn("key2", KuduType.String, opt => opt.Key(true))
.AddColumn("val", KuduType.String)
.SetRangePartitionColumns("key1", "key2");
for (int i = 1; i < 4; i++)
{
builder.AddSplitRow(splitRow =>
{
splitRow.SetString("key1", i.ToString());
splitRow.SetString("key2", "");
});
}
var table = await _client.CreateTableAsync(builder);
// The data layout ends up like this:
// tablet '', '1': no rows
// tablet '1', '2': '111', '122', '133'
// tablet '2', '3': '211', '222', '233'
// tablet '3', '': '311', '322', '333'
var keys = new[] { "1", "2", "3" };
foreach (var key1 in keys)
{
foreach (var key2 in keys)
{
var insert = table.NewInsert();
insert.SetString(0, key1);
insert.SetString(1, key2);
insert.SetString(2, key2);
await session.EnqueueAsync(insert);
await session.FlushAsync();
}
}
_beforeWriteTimestamp = _client.LastPropagatedTimestamp;
// Reset the client in order to clear the propagated timestamp.
_newClient = _harness.CreateClient();
// Reopen the table using the new client.
_table = await _newClient.OpenTableAsync(_tableName);
_schema = _table.Schema;
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
// Use one tablet because multiple tablets don't work: we could jump
// from one tablet to another which could change the logical clock.
var builder = new TableBuilder("HybridTimeTest")
.AddColumn("key", KuduType.String, opt => opt.Key(true))
.SetRangePartitionColumns("key");
_table = await _client.CreateTableAsync(builder);
}
public async Task TestInsertIgnoreAfterInsertHasNoRowError()
{
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestInsertIgnoreAfterInsertHasNoRowError));
var table = await _client.CreateTableAsync(builder);
var rows = new[]
{
ClientTestUtil.CreateBasicSchemaInsert(table, 1),
ClientTestUtil.CreateBasicSchemaUpsert(table, 1, 1, false),
ClientTestUtil.CreateBasicSchemaInsertIgnore(table, 1)
};
await _client.WriteAsync(rows);
var rowStrings = await ClientTestUtil.ScanTableToStringsAsync(_client, table);
var rowString = Assert.Single(rowStrings);
Assert.Equal(
"INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " +
"STRING column3_s=a string, BOOL column4_b=True", rowString);
}
public async Task TestExceptionCallback()
{
int numCallbacks = 0;
SessionExceptionContext sessionContext = null;
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName(nameof(TestExceptionCallback));
var table = await _client.CreateTableAsync(builder);
var row1 = ClientTestUtil.CreateBasicSchemaInsert(table, 1);
var row2 = ClientTestUtil.CreateBasicSchemaInsert(table, 1);
var sessionOptions = new KuduSessionOptions
{
ExceptionHandler = HandleSessionExceptionAsync
};
await using var session = _client.NewSession(sessionOptions);
await session.EnqueueAsync(row1);
await session.FlushAsync();
await session.EnqueueAsync(row2);
await session.FlushAsync();
ValueTask HandleSessionExceptionAsync(SessionExceptionContext context)
{
numCallbacks++;
sessionContext = context;
return(new ValueTask());
}
Assert.Equal(1, numCallbacks);
var errorRow = Assert.Single(sessionContext.Rows);
Assert.Same(row2, errorRow);
var exception = Assert.IsType <KuduWriteException>(sessionContext.Exception);
var exceptionRow = Assert.Single(exception.PerRowErrors);
Assert.True(exceptionRow.IsAlreadyPresent);
}
public async Task TestConstructorWithProperties()
{
var builder = new TableBuilder(nameof(TestConstructorWithProperties))
.AddColumn("Key", KuduType.Int32, opt => opt.Key(true))
.AddColumn("Column1", KuduType.String, opt => opt.Nullable(false))
.AddColumn("Column2", KuduType.Int32, opt => opt.Nullable(false))
.AddColumn("Column3", KuduType.String, opt => opt.Nullable(false));
var values = new[]
{
new MixedRecord(1, "val-1")
{
Column2 = 100, Column3 = "val-2"
},
new MixedRecord(2, "val-3")
{
Column2 = 200, Column3 = "val-4"
}
};
var table = await _client.CreateTableAsync(builder);
var rowsToInsert = values.Select(value =>
{
var insert = table.NewInsert();
insert.SetInt32("Key", value.Key);
insert.SetString("Column1", value.Column1);
insert.SetInt32("Column2", value.Column2);
insert.SetString("Column3", value.Column3);
return(insert);
});
await _client.WriteAsync(rowsToInsert);
var rows = await ScanAsync <MixedRecord>(table);
Assert.Equal(values, rows);
}
public async Task TestMultipleSessions()
{
int numTasks = 60;
var tasks = new List <Task>(numTasks);
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("TestMultipleSessions")
.CreateBasicRangePartition();
var table = await _client.CreateTableAsync(builder);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var token = cts.Token;
for (int i = 0; i < numTasks; i++)
{
var task = Task.Run(async() =>
{
while (!token.IsCancellationRequested)
{
await using var session = _client.NewSession();
for (int j = 0; j < 100; j++)
{
var row = table.NewUpsert();
row.SetInt32(0, j);
row.SetInt32(1, 12345);
row.SetInt32(2, 3);
row.SetNull(3);
row.SetBool(4, false);
await session.EnqueueAsync(row);
}
}
});
tasks.Add(task);
}
await Task.WhenAll(tasks);
}
public async Task InitializeAsync()
{
_harness = await new MiniKuduClusterBuilder().BuildHarnessAsync();
_client = _harness.CreateClient();
var builder = ClientTestUtil.GetBasicSchema()
.SetTableName("ScannerFaultToleranceTests")
.AddHashPartitions(_numTablets, "key");
_table = await _client.CreateTableAsync(builder);
var random = new Random();
_keys = new HashSet <int>();
while (_keys.Count < _numRows)
{
_keys.Add(random.Next());
}
var rows = _keys
.Select((key, i) =>
{
var insert = _table.NewInsert();
insert.SetInt32(0, key);
insert.SetInt32(1, i);
insert.SetInt32(2, i++);
insert.SetString(3, DataGenerator.RandomString(1024, random));
insert.SetBool(4, true);
return(insert);
})
.Chunk(1000);
foreach (var batch in rows)
{
await _client.WriteAsync(batch);
}
}
public async Task TestPrimaryKeyRangePruning()
{
// CREATE TABLE t
// (a INT8, b INT8, c INT8)
// PRIMARY KEY (a, b, c))
// PARTITION BY RANGE (a, b, c)
// (PARTITION VALUES < (0, 0, 0),
// PARTITION (0, 0, 0) <= VALUES < (10, 10, 10)
// PARTITION (10, 10, 10) <= VALUES);
var tableName = nameof(TestPrimaryKeyRangePruning);
var tableBuilder = new TableBuilder(tableName)
.AddColumn("a", KuduType.Int8, opt => opt.Key(true))
.AddColumn("b", KuduType.Int8, opt => opt.Key(true))
.AddColumn("c", KuduType.Int8, opt => opt.Key(true))
.SetRangePartitionColumns("a", "b", "c")
.AddSplitRow(row =>
{
row.SetByte("a", 0);
row.SetByte("b", 0);
row.SetByte("c", 0);
})
.AddSplitRow(row =>
{
row.SetByte("a", 10);
row.SetByte("b", 10);
row.SetByte("c", 10);
});
var table = await _client.CreateTableAsync(tableBuilder);
var partitions = await GetTablePartitionsAsync(table);
var min = sbyte.MinValue;
// No bounds
await CheckPartitionsPrimaryKeyAsync(3, table, partitions,
null, null);
// PK < (-1, min, min)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
null, new sbyte[] { -1, min, min });
// PK < (0, 0, 0)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
null, new sbyte[] { 0, 0, 0 });
// PK < (0, 0, min)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
null, new sbyte[] { 0, 0, min });
// PK < (10, 10, 10)
await CheckPartitionsPrimaryKeyAsync(2, table, partitions,
null, new sbyte[] { 10, 10, 10 });
// PK < (100, min, min)
await CheckPartitionsPrimaryKeyAsync(3, table, partitions,
null, new sbyte[] { 100, min, min });
// PK >= (-10, -10, -10)
await CheckPartitionsPrimaryKeyAsync(3, table, partitions,
new sbyte[] { -10, -10, -10 }, null);
// PK >= (0, 0, 0)
await CheckPartitionsPrimaryKeyAsync(2, table, partitions,
new sbyte[] { 0, 0, 0 }, null);
// PK >= (100, 0, 0)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
new sbyte[] { 100, 0, 0 }, null);
// PK >= (-10, 0, 0)
// PK < (100, 0, 0)
await CheckPartitionsPrimaryKeyAsync(3, table, partitions,
new sbyte[] { -10, 0, 0 }, new sbyte[] { 100, 0, 0 });
// PK >= (0, 0, 0)
// PK < (10, 10, 10)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
new sbyte[] { 0, 0, 0 }, new sbyte[] { 10, 0, 0 });
// PK >= (0, 0, 0)
// PK < (10, 10, 11)
await CheckPartitionsPrimaryKeyAsync(1, table, partitions,
new sbyte[] { 0, 0, 0 }, new sbyte[] { 10, 0, 0 });
// PK < (0, 0, 0)
// PK >= (10, 10, 11)
await CheckPartitionsPrimaryKeyAsync(0, table, partitions,
new sbyte[] { 10, 0, 0 }, new sbyte[] { 0, 0, 0 });
}
public async Task TestAllPrimaryKeyTypes()
{
var tableBuilder = new TableBuilder(nameof(TestAllPrimaryKeyTypes))
.AddColumn("int8", KuduType.Int8, opt => opt.Key(true))
.AddColumn("int16", KuduType.Int16, opt => opt.Key(true))
.AddColumn("int32", KuduType.Int32, opt => opt.Key(true))
.AddColumn("int64", KuduType.Int64, opt => opt.Key(true))
.AddColumn("string", KuduType.String, opt => opt.Key(true))
.AddColumn("binary", KuduType.Binary, opt => opt.Key(true))
.AddColumn("timestamp", KuduType.UnixtimeMicros, opt => opt.Key(true))
.AddColumn("decimal32", KuduType.Decimal32, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal32Precision, 0))
.AddColumn("decimal64", KuduType.Decimal64, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal64Precision, 0))
.AddColumn("decimal128", KuduType.Decimal128, opt => opt.Key(true)
.DecimalAttributes(DecimalUtil.MaxDecimal128Precision, 0))
.AddColumn("varchar", KuduType.Varchar, opt => opt.Key(true)
.VarcharAttributes(10))
.AddColumn("date", KuduType.Date, opt => opt.Key(true))
.AddColumn("bool", KuduType.Bool) // not primary key type
.AddColumn("float", KuduType.Float) // not primary key type
.AddColumn("double", KuduType.Double); // not primary key type
var table = await _client.CreateTableAsync(tableBuilder);
var insert = table.NewInsert();
insert.SetSByte("int8", 1);
insert.SetInt16("int16", 2);
insert.SetInt32("int32", 3);
insert.SetInt64("int64", 4);
insert.SetString("string", "foo");
insert.SetBinary("binary", "bar".ToUtf8ByteArray());
insert.SetInt64("timestamp", 6);
insert.SetDecimal("decimal32", DecimalUtil.MaxUnscaledDecimal32);
insert.SetDecimal("decimal64", DecimalUtil.MaxUnscaledDecimal64);
insert.SetDecimal("decimal128", decimal.Truncate(decimal.MaxValue));
insert.SetString("varchar", "varchar bar");
insert.SetDateTime("date", EpochTime.FromUnixTimeDays(0));
insert.SetBool("bool", true);
insert.SetFloat("float", 7.8f);
insert.SetDouble("double", 9.9);
await _client.WriteAsync(new[] { insert });
var scanner = _client.NewScanBuilder(table).Build();
var scannedRows = 0;
await foreach (var resultSet in scanner)
{
foreach (var row in resultSet)
{
scannedRows++;
Assert.Equal(1, row.GetSByte("int8"));
Assert.Equal(2, row.GetInt16("int16"));
Assert.Equal(3, row.GetInt32("int32"));
Assert.Equal(4, row.GetInt64("int64"));
Assert.Equal("foo", row.GetString("string"));
Assert.Equal("bar".ToUtf8ByteArray(), row.GetBinary("binary"));
Assert.Equal(6, row.GetInt64("timestamp"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal32, row.GetDecimal("decimal32"));
Assert.Equal(DecimalUtil.MaxUnscaledDecimal64, row.GetDecimal("decimal64"));
Assert.Equal(decimal.Truncate(decimal.MaxValue), row.GetDecimal("decimal128"));
Assert.Equal("varchar ba", row.GetString("varchar"));
Assert.Equal(0, row.GetInt32("date"));
Assert.True(row.GetBool("bool"));
Assert.Equal(7.8f, row.GetFloat("float"));
Assert.Equal(9.9, row.GetDouble("double"));
}
}
Assert.Equal(1, scannedRows);
}
public async Task TestBoolPredicates()
{
var builder = GetDefaultTableBuilder()
.SetTableName("bool-table")
.AddColumn("value", KuduType.Bool);
var table = await _client.CreateTableAsync(builder);
var values = new SortedSet <bool> {
true, false
};
var testValues = new List <bool> {
true, false
};
long i = 0;
foreach (var value in values)
{
var insert = table.NewInsert();
insert.SetInt64("key", i++);
insert.SetBool("value", value);
await _session.EnqueueAsync(insert);
}
var nullInsert = table.NewInsert();
nullInsert.SetInt64("key", i);
nullInsert.SetNull("value");
await _session.EnqueueAsync(nullInsert);
await _session.FlushAsync();
await CheckPredicatesAsync(table, values, testValues);
}
public async Task TestScanTokens()
{
var builder = ClientTestUtil.CreateManyStringsSchema()
.SetTableName(_tableName)
.AddHashPartitions(8, "key")
.CreateBasicRangePartition()
.AddSplitRow(row => row.SetString("key", "key_50"));
var table = await _client.CreateTableAsync(builder);
for (int i = 0; i < 100; i++)
{
var row = table.NewInsert();
row.SetString("key", $"key_{i}");
row.SetString("c1", $"c1_{i}");
row.SetString("c2", $"c2_{i}");
await _session.EnqueueAsync(row);
}
await _session.FlushAsync();
var tokenBuilder = _client.NewScanTokenBuilder(table)
.SetEmptyProjection()
// For this test, make sure that we cover the case that not all tablets
// are returned in a single batch.
.SetFetchTabletsPerRangeLookup(4);
List <KuduScanToken> tokens = await tokenBuilder.BuildAsync();
Assert.Equal(16, tokens.Count);
await using var newClient = _harness.CreateClient();
var rowCount = await CountScanTokenRowsAsync(newClient, tokens);
Assert.Equal(100, rowCount);
}