public IObservable <BigRow.Sample> ObserveSampleRowKeys(BigTable table, CancellationToken cancellationToken, Action <IObservable <BigRow.Sample> > blockingSubscriber)
{
return(BigRowSampleObservable(table, new SampleRowKeysRequest
{
TableName = table.Name.ToTableId(ClusterId)
}));
}
public IObservable <BigRow> ObserveUnsortedRows(BigTable table, RowFilter filter)
{
return(BigRowObservable(table, new ReadRowsRequest
{
TableName = table.Name.ToTableId(ClusterId),
AllowRowInterleaving = true,
Filter = filter
}));
}
public IObservable <BigRow> ObserveRows(BigTable table, RowFilter filter, long rowLimit = 0)
{
return(BigRowObservable(table, new ReadRowsRequest
{
TableName = table.Name.ToTableId(ClusterId),
NumRowsLimit = rowLimit,
Filter = filter
}));
}
public IObservable <BigRow> ObserveUnsortedRows(BigTable table, byte[] startKey = null, byte[] endKey = null)
{
return(BigRowObservable(table, new ReadRowsRequest
{
TableName = table.Name.ToTableId(ClusterId),
AllowRowInterleaving = true,
RowRange = new RowRange
{
StartKey = (startKey ?? new byte[0]).ToByteString(),
EndKey = (endKey ?? new byte[0]).ToByteString(),
}
}));
}
public IObservable <BigRow> ObserveUnsortedRows(BigTable table, string startKey = "", string endKey = "")
{
return(BigRowObservable(table, new ReadRowsRequest
{
TableName = table.Name.ToTableId(ClusterId),
AllowRowInterleaving = true,
RowRange = new RowRange
{
StartKey = startKey.ToByteString(table.Encoding),
EndKey = endKey.ToByteString(table.Encoding),
}
}));
}
public IObservable <BigRow> ObserveRows(BigTable table, byte[] startKey = null, byte[] endKey = null, long rowLimit = 0)
{
return(BigRowObservable(table, new ReadRowsRequest
{
TableName = table.Name.ToTableId(ClusterId),
NumRowsLimit = rowLimit,
RowRange = new RowRange
{
StartKey = startKey.ToByteString(),
EndKey = endKey.ToByteString(),
}
}));
}
private IObservable <BigRow.Sample> BigRowSampleObservable(BigTable table, SampleRowKeysRequest request)
{
// Return an observable for response
return(new BigtableObservable <SampleRowKeysResponse, BigRow.Sample>(async token =>
{
// Send read rows request
var response = _client.SampleRowKeys(request);
// Await initial response
await response.ResponseHeadersAsync;
return response.ResponseStream;
}, row => new BigRow.Sample(table, row.RowKey, row.OffsetBytes)));
}
public async Task <bool> WriteWhenAsync(BigTable table, byte[] key, RowFilter filter, IEnumerable <Mutation> whenFilterIsTrue, IEnumerable <Mutation> whenFilterIsFalse, CancellationToken cancellationToken = default(CancellationToken))
{
return(await WriteWhenAsync(table.Name, key, filter, whenFilterIsTrue, whenFilterIsFalse, cancellationToken));
}
public async Task <IEnumerable <BigRow> > WriteRowAsync(BigTable table, byte[] key, CancellationToken cancellationToken, IEnumerable <BigChange.FromRead> changes)
{
return(await WriteRowAsync(table.Name, key, cancellationToken, changes, table.Encoding));
}
public async Task <IEnumerable <BigRow> > GetRowsAsync(BigTable table, string startKey = "", string endKey = "", long rowLimit = 0, CancellationToken cancellationToken = default(CancellationToken))
{
return(await GetRowsAsync(table.Name, startKey, endKey, rowLimit, table.Encoding, cancellationToken));
}
/// <summary>
/// Creates a table based on the prototype <see cref="BigTable"/>.
/// The initial split-keys can be passed in to specify tablets that should be pre-initialized.
/// In general, it is best to allow the table to split automatically unless you understand BigTable indexing thoroughly.
/// </summary>
/// <param name="table"></param>
/// <param name="splitKeys"></param>
/// <returns></returns>
public async Task <BigTable> CreateTableAsync(BigTable table, IEnumerable <string> splitKeys = null)
{
return(await CreateTableAsync(table.Name, table.Families, table.Encoding, splitKeys));
}
public async Task <BigFamily> UpdateFamilyAsync(BigTable table, BigFamily family, RetentionPolicy policy)
{
return(await UpdateFamilyAsync(table.Name, family.Name, policy));
}
public async Task <IEnumerable <BigRow> > GetUnsortedRowsAsync(BigTable table, byte[] startKey = null, byte[] endKey = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(await GetUnsortedRowsAsync(table.Name, startKey, endKey, table.Encoding, cancellationToken));
}
public async Task <BigRow> GetRowAsync(BigTable table, byte[] key, CancellationToken cancellationToken = default(CancellationToken))
{
return(await GetRowAsync(table.Name, key, table.Encoding, cancellationToken));
}
public async Task <IEnumerable <BigRow.Sample> > SampleRowKeysAsync(BigTable table, CancellationToken cancellationToken)
{
return(await SampleRowKeysAsync(table.Name, table.Encoding, cancellationToken));
}
public async Task <IEnumerable <BigRow.Sample> > SampleRowKeysAsync(BigTable table)
{
return(await SampleRowKeysAsync(table.Name));
}
public async Task <IEnumerable <BigRow> > GetUnsortedRowsAsync(BigTable table, RowFilter filter, CancellationToken cancellationToken = default(CancellationToken))
{
return(await GetUnsortedRowsAsync(table.Name, filter, table.Encoding, cancellationToken));
}
// Functionality
#region - Private Functionality -
private IObservable <BigRow> BigRowObservable(BigTable table, ReadRowsRequest request)
{
// Return an observable for response
return(new BigtableObservable <ReadRowsResponse, BigRow>(async token => await ReadRows(request, token), row => new BigRow(table, row)));
}
public async Task <BigTable> GetTableAsync(BigTable table)
{
return(await GetTableAsync(table.Name, table.Encoding));
}
public async Task RenameTableAsync(BigTable table, string name)
{
await RenameTableAsync(table.Name, name);
table.Name = name;
}
public BigModule(IClientFactory clientFactory, ITableNameCache tableNameCache)
{
Get["/big/tables"] = _ =>
{
var model = new TablesModel();
using (var adminClient = clientFactory.GetAdminClient())
{
var tables = adminClient.ListTablesAsync().Result;
model.Tables.AddRange(tables.Select(x => new TablesEntryModel
{
Name = x.Name
}));
}
return(View["Tables.sshtml", model]);
};
Get["/big/data/list"] = _ =>
{
var model = new DataListModel
{
Table = Request.Query["table"],
StartKey = Request.Query["skey"],
CurrentKey = Request.Query["ckey"],
EndKey = Request.Query["ekey"]
};
model.SetTableNames(tableNameCache.Names);
model.SetMaxRows(Request.Query["maxrows"]);
if (string.IsNullOrEmpty(model.CurrentKey))
{
model.CurrentKey = model.StartKey;
}
if (!string.IsNullOrEmpty(model.Table) && !string.IsNullOrEmpty(model.StartKey) && !string.IsNullOrEmpty(model.EndKey))
{
using (var dataClient = clientFactory.GetDataClient())
{
var table = new BigTable(model.Table);
var rows = dataClient.GetRowsAsync(table, model.CurrentKey, model.EndKey, model.MaxRows + 1).Result;
foreach (var row in rows)
{
if (model.KeyCount < model.MaxRows)
{
model.AddRow(row);
}
else
{
model.NextKey = row.KeyString;
}
}
}
}
return(View["DataList.sshtml", model]);
};
Get["/big/data/sample"] = _ =>
{
const string timeSpanFormat = @"hh\:mm\:ss\.ff";
var model = new DataSampleModel
{
Table = Request.Query["table"]
};
model.SetTableNames(tableNameCache.Names);
model.SetMaxRows(Request.Query["maxrows"]);
if (!string.IsNullOrEmpty(model.Table))
{
using (var dataClient = clientFactory.GetDataClient())
{
var table = new BigTable(model.Table);
Stopwatch watch = Stopwatch.StartNew();
var keys = dataClient.SampleRowKeysAsync(table).Result.ToList();
watch.Stop();
model.SampleElapsed = watch.Elapsed.ToString(timeSpanFormat);
model.SampleCount = keys.Count;
watch = Stopwatch.StartNew();
foreach (var batch in keys.Take(model.MaxRows).Batch(50))
{
var rowTasks = batch
.Select(x => new
{
Task = dataClient.GetRowsAsync(table, x.Key, null, 1),
x.KeyString
})
.ToArray();
Task.WhenAll(rowTasks.Select(x => x.Task)).Wait(); // silly syntax to avoid resharper warning
foreach (var rowset in rowTasks)
{
if (rowset.Task.Result != null)
{
var row = rowset.Task.Result.FirstOrDefault();
if (row != null)
{
model.AddRow(row);
}
}
}
}
watch.Stop();
model.GetRowsElapsed = watch.Elapsed.ToString(timeSpanFormat);
}
}
return(View["DataSample.sshtml", model]);
};
}
public async Task <IEnumerable <BigRow> > GetRowsAsync(BigTable table, RowFilter filter, long rowLimit = 0, CancellationToken cancellationToken = default(CancellationToken))
{
return(await GetRowsAsync(table.Name, filter, rowLimit, table.Encoding, cancellationToken));
}
public async Task DeleteTableAsync(BigTable table)
{
await DeleteTableAsync(table.Name);
}
// Write signatures
#region - Write Signatures -
// -- // -- Write Operations -- // -- //
public async Task WriteRowAsync(BigTable table, string key, IEnumerable <BigChange> changes, Encoding encoding = null, CancellationToken cancellationToken = default(CancellationToken))
{
await WriteRowAsync(table.Name, key, changes, table.Encoding, cancellationToken);
}
public async Task <BigFamily> CreateFamilyAsync(BigTable table, string name, RetentionPolicy policy)
{
return(await CreateFamilyAsync(table.Name, name, policy));
}
public async Task WriteRowAsync(BigTable table, byte[] key, IEnumerable <BigChange> changes, CancellationToken cancellationToken = default(CancellationToken))
{
await WriteRowAsync(table.Name, key, changes, cancellationToken);
}
public async Task DeleteFamilyAsync(BigTable table, BigFamily family)
{
await DeleteFamilyAsync(table.Name, family.Name);
}
public async Task <IEnumerable <BigRow> > WriteRowAsync(BigTable table, byte[] key, IEnumerable <BigChange.FromRead> changes)
{
return(await WriteRowAsync(table, key, CancellationToken.None, changes));
}
public static async Task Run()
{
// Disable Grpc logging
GrpcEnvironment.DisableLogging();
try
{
// Create config
var config = Utilities.GetConfig();
// Create credentials
var credentials = await BigtableCredentials.UseApplicationDefaultCredentialsAsync();
// Admin client
using (var adminClient = new BigAdminClient(credentials, config))
{
// Get tables
var tables = (await adminClient.ListTablesAsync()).ToArray();
// Ensure pricing table exists
if (tables.All(t => t.Name != Constants.PricingTable))
{
// Inform user
CommandLine.InformUser("Setup", "Missing example table. Please run the Examples.Bootstrap project.");
// Hard stop
return;
}
// Show user
CommandLine.DisplayTables(tables);
}
// Wait for keypress
CommandLine.WaitForUserAndThen("scan for rows");
// Data client
using (var dataClient = new BigDataClient(credentials, config))
{
// Target
var pricingTable = new BigTable(Constants.PricingTable);
// Scan pricing table
var pricing = await dataClient.GetRowsAsync(pricingTable, Constants.ScanKeyStart, Constants.ScanKeyEnd, Constants.ScanLimit);
// Show the user
CommandLine.DisplayRows(pricing);
// Wait for keypress
CommandLine.WaitForUserAndThen("observe rows");
// Test observables
var waitSource = new CancellationTokenSource();
// ReSharper disable once MethodSupportsCancellation
var waitForObservation = Task.Run(() => Task.Delay(-1, waitSource.Token));
// Create a subscriber
var subscriber = new TestSubscriber(() => waitSource.Cancel());
// ReSharper disable once MethodSupportsCancellation
var observable = dataClient.ObserveRows(pricingTable, Constants.ScanKeyStart, Constants.ScanKeyEnd, Constants.ScanLimit);
using (observable.Subscribe(subscriber))
{
//await Task.Delay(1000);
Task.WaitAny(waitForObservation);
}
// Wait for keypress
CommandLine.WaitForUserAndThen("seek for row");
// Seek for data
var row = await dataClient.GetRowAsync(pricingTable, Constants.SeekKey);
// Show the user
CommandLine.DisplayRows(new[] { row });
}
}
catch (Exception exception)
{
CommandLine.InformUser("Oops", "Example didn't work out as planned");
CommandLine.RenderException(exception);
}
finally
{
// All done
CommandLine.WaitForUserAndThen("exit");
}
}
private BigChange.FromRead CreateIncrementRule <T, TParameter>(ReflectionCache reflection, BigTable table, Expression <Func <T, TParameter> > field, long value)
where TParameter : IBigTableField
{
var columnName = ExtractColumnName(reflection, field);
var familyName = ExtractFamilyName <T, TParameter>(reflection, field);
return(BigChange.FromRead.CreateCellIncrement(familyName, columnName, value, table.Encoding));
}
