public static BigtableClient CreateMutateRowsMockClient(
MutateRowsRequest initialRequest,
MutateRowsResponse.Types.Entry[] entriesForInitialStream,
MutateRowsResponse.Types.Entry[][] entriesForRetryStreams = null) =>
CreateMockClientForStreamingRpc(
initialRequest,
c => c.MutateRows(
It.Is <MutateRowsRequest>(r => ReferenceEquals(r, initialRequest)),
It.IsAny <CallSettings>()),
c => c.MutateRows(
It.IsAny <MutateRowsRequest>(),
It.IsAny <CallSettings>()),
entriesForInitialStream,
entriesForRetryStreams,
itemsToStream: entries => new MockMutateRowsStream(new MutateRowsResponse {
Entries = { entries }
}),
validator: (request, response) =>
{
// Make sure the request is properly formulated for the mock stream being returned.
if (request.Entries.Count != response.Responses.SelectMany(r => r.Entries).Count())
{
throw new InvalidOperationException("The specified request is invalid for the mock stream about to be returned.");
}
});
public static BigtableClient CreateClientForMutateRowsRetries(
MutateRowsRequest initialRequest,
MutateRowsResponse.Types.Entry[] entriesForInitialResponse,
MutateRowsResponse.Types.Entry[][] entriesForRetryResponses)
{
var mock = new Mock <BigtableServiceApiClient>();
if (entriesForRetryResponses != null)
{
var retryResponses = new Queue <MockMutateRowsStream>(entriesForRetryResponses.Select(StreamFromEntries));
mock.Setup(c => c.MutateRows(It.IsAny <MutateRowsRequest>(), It.IsAny <CallSettings>()))
.Returns(retryResponses.Dequeue);
}
// Setup the initial response last so the catch-all setup doesn't overwrite it.
mock.Setup(c => c.MutateRows(initialRequest, It.IsAny <CallSettings>()))
.Returns(StreamFromEntries(entriesForInitialResponse));
return(new BigtableClientImpl(new[] { mock.Object }, appProfileId: null));
MockMutateRowsStream StreamFromEntries(MutateRowsResponse.Types.Entry[] entries) =>
new MockMutateRowsStream(new MutateRowsResponse {
Entries = { entries }
});
}
/// <summary>
/// Add Test Data into Given Google Cloud Bigtable
/// </summary>
public void InsertTestDataToTable()
{
try
{
if (!IsTableExists())
{
throw new Exception("No Such Table Found");
}
var bigtableClient = BigtableClient.Create();
List <Mutation> Cols = new List <Mutation>();
TableName _table = new TableName(ProjectId, InstanceId, TableName);
var request = new MutateRowsRequest
{
TableNameAsTableName = _table,
};
request.Entries.Add(Mutations.CreateEntry(Guid.NewGuid().ToString(),
Mutations.SetCell(ColumnFamily, "TestColumnName", "Test Column Value")));
bigtableClient.MutateRows(request);
}
catch (RpcException ex)
{
throw ex;
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>Snippet for MutateRows</summary>
public async Task MutateRowsRequestObject()
{
// Snippet: MutateRows(MutateRowsRequest, CallSettings)
// Create client
BigtableClient bigtableClient = BigtableClient.Create();
// Initialize request argument(s)
MutateRowsRequest request = new MutateRowsRequest
{
TableNameAsTableName = TableName.FromProjectInstanceTable("[PROJECT]", "[INSTANCE]", "[TABLE]"),
Entries =
{
new MutateRowsRequest.Types.Entry(),
},
AppProfileId = "",
};
// Make the request, returning a streaming response
BigtableClient.MutateRowsStream response = bigtableClient.MutateRows(request);
// Read streaming responses from server until complete
// Note that C# 8 code can use await foreach
AsyncResponseStream <MutateRowsResponse> responseStream = response.GetResponseStream();
while (await responseStream.MoveNextAsync())
{
MutateRowsResponse responseItem = responseStream.Current;
// Do something with streamed response
}
// The response stream has completed
// End snippet
}
/// <summary>Snippet for MutateRows</summary>
public async Task MutateRows()
{
// Snippet: MutateRows(MutateRowsRequest,CallSettings)
// Create client
BigtableServiceApiClient bigtableServiceApiClient = BigtableServiceApiClient.Create();
// Initialize request argument
MutateRowsRequest request = new MutateRowsRequest
{
TableNameAsTableName = new TableName("[PROJECT]", "[INSTANCE]", "[TABLE]"),
Entries = { },
};
// Make the request, returning a streaming response
BigtableServiceApiClient.MutateRowsStream streamingResponse = bigtableServiceApiClient.MutateRows(request);
// Read streaming responses from server until complete
IAsyncEnumerator <MutateRowsResponse> responseStream = streamingResponse.ResponseStream;
while (await responseStream.MoveNext())
{
MutateRowsResponse response = responseStream.Current;
// Do something with streamed response
}
// The response stream has completed
// End snippet
}
public async Task RetryAfterTotalExpiration()
{
var settings = new BigtableServiceApiSettings();
// Don't allow for any time to retry.
settings.MutateRowsSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.Zero));
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow())
}
};
var client = Utilities.CreateMutateRowsMockClient(
request,
entriesForInitialStream: new[]
{
Utilities.CreateMutateRowsResponseEntry(0, Code.Ok),
Utilities.CreateMutateRowsResponseEntry(1, Code.DeadlineExceeded),
Utilities.CreateMutateRowsResponseEntry(2, Code.Ok)
},
entriesForRetryStreams: new[]
{
// 1st retry response entries
new[] { Utilities.CreateMutateRowsResponseEntry(0, Code.Ok) }
},
settings: settings);
var exception = await Assert.ThrowsAsync <RpcException>(() => client.MutateRowsAsync(request));
Assert.Equal(StatusCode.DeadlineExceeded, exception.StatusCode);
}
public async Task RetryBeforeTotalExpiration()
{
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow())
}
};
var client = Utilities.CreateMutateRowsMockClient(
request,
entriesForInitialStream: new[]
{
Utilities.CreateMutateRowsResponseEntry(0, Code.Ok),
Utilities.CreateMutateRowsResponseEntry(1, Code.DeadlineExceeded),
Utilities.CreateMutateRowsResponseEntry(2, Code.Ok)
},
entriesForRetryStreams: new[]
{
null, // A null entry will throw an Unavailable RpcException
// 2nd retry response entries
new[] { Utilities.CreateMutateRowsResponseEntry(0, Code.Ok) }
});
await client.MutateRowsAsync(request);
}
public async Task SimpleRetry()
{
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow())
}
};
var client = Utilities.CreateMutateRowsMockClient(
request,
entriesForInitialStream: new[]
{
Utilities.CreateMutateRowsResponseEntry(0, Code.Ok),
Utilities.CreateMutateRowsResponseEntry(1, Code.DeadlineExceeded),
Utilities.CreateMutateRowsResponseEntry(2, Code.Ok)
},
entriesForRetryStreams: new[]
{
// 1st retry response entries
new[] { Utilities.CreateMutateRowsResponseEntry(0, Code.Ok) }
});
var entries = (await client.MutateRowsAsync(request)).Entries.OrderBy(e => e.Index).ToList();
Assert.Equal(3, entries.Count);
var entry = entries[0];
Assert.Equal(0, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[1];
Assert.Equal(1, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[2];
Assert.Equal(2, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
}
public async Task InsertRowsAsync(
TableName tableName,
IEnumerable <BigtableByteString> rowKeys,
string familyName = null,
BigtableByteString?qualifierName = null,
BigtableByteString?valuePrefix = null,
BigtableVersion?version = null)
{
BigtableByteString rowKey = Guid.NewGuid().ToString();
familyName = familyName ?? ColumnFamily1;
qualifierName = qualifierName ?? "row_index";
valuePrefix = valuePrefix ?? "";
// TODO: Use cleaner API when available.
int counter = 0;
var request = new MutateRowsRequest
{
TableNameAsTableName = tableName,
Entries =
{
rowKeys.Select(k => new MutateRowsRequest.Types.Entry
{
RowKey = k.Value,
Mutations =
{
Mutations.SetCell(
familyName,
qualifierName.Value,
valuePrefix.Value.Value.Concat(new BigtableByteString(counter++).Value),
version)
}
})
}
};
var response = TableClient.MutateRows(request);
while (await response.ResponseStream.MoveNext(default))
public async Task NonRetryableErrorCodePreventsRetry()
{
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow())
}
};
var client = Utilities.CreateMutateRowsMockClient(
request,
entriesForInitialStream: new[]
{
Utilities.CreateMutateRowsResponseEntry(0, Code.Ok),
Utilities.CreateMutateRowsResponseEntry(1, Code.FailedPrecondition),
Utilities.CreateMutateRowsResponseEntry(2, Code.DeadlineExceeded)
},
entriesForRetryStreams: null // No retries should be made
);
var entries = (await client.MutateRowsAsync(request)).Entries.OrderBy(e => e.Index).ToList();
Assert.Equal(3, entries.Count);
var entry = entries[0];
Assert.Equal(0, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[1];
Assert.Equal(1, entry.Index);
Assert.Equal((int)Code.FailedPrecondition, entry.Status.Code);
entry = entries[2];
Assert.Equal(2, entry.Index);
Assert.Equal((int)Code.DeadlineExceeded, entry.Status.Code);
}
/// <summary>
/// Insert Test Data into Google Cloud Bigtable
/// </summary>
/// <returns></returns>
public bool InsertTestData(string columnName, string columnValue)
{
try
{
var bigtableClient = BigtableClient.Create();
List <Mutation> Cols = new List <Mutation>();
TableName _table = new TableName(ProjectId, InstanceId, TableName);
var request = new MutateRowsRequest
{
TableNameAsTableName = _table,
};
request.Entries.Add(Mutations.CreateEntry(Guid.NewGuid().ToString(),
Mutations.SetCell(ColumnFamily, columnName, columnValue)));
bigtableClient.MutateRows(request);
}
catch (Exception ex)
{
throw ex;
}
return(true);
}
internal void DoHelloWorld()
{
#region 1. Create table with column family.
try
{
Console.WriteLine($"Create new table: {_settings.TableName} with column family {_settings.ColumnFamily}, Instance: {_settings.InstanceId}");
// Check whether a table with given TableName already exists.
if (!DoesTableExist(_bigtableTableAdminClient, _tableNameAdmin))
{
_bigtableTableAdminClient.CreateTable(
new InstanceName(_settings.ProjectId, _settings.InstanceId),
_settings.TableName,
new Table
{
Granularity = Table.Types.TimestampGranularity.Millis,
ColumnFamilies =
{
{
_settings.ColumnFamily, new ColumnFamily
{
GcRule = new GcRule
{
MaxNumVersions = 1
}
}
}
}
});
Console.WriteLine(DoesTableExist(_bigtableTableAdminClient, _tableNameAdmin)
? $"Table {_settings.TableName} created succsessfully\n"
: $"There was a problem creating a table {_settings.TableName}");
}
else
{
Console.WriteLine($"Table: {_settings.TableName} already exist");
}
#endregion
#region 2. Insert multiple rows.
/* Each row has a unique row key.
*
* Note: This example uses sequential numeric IDs for simplicity, but
* this can result in poor performance in a production application.
* Since rows are stored in sorted order by key, sequential keys can
* result in poor distribution of operations across nodes.
*
* For more information about how to design a Bigtable schema for the
* best performance, see the documentation:
*
* https://cloud.google.com/bigtable/docs/schema-design */
Console.WriteLine($"Write some greetings to the table {_settings.TableName}");
// Insert 1 row using MutateRow()
_greetingIndex = 0;
try
{
_bigtableClient.MutateRow(_tableNameClient, _settings.RowKeyPrefix + _greetingIndex, MutationBuilder(_greetingIndex));
Console.WriteLine($"\tGreeting: --{_settings.Greetings[_greetingIndex]}-- written successfully");
}
catch (Exception ex)
{
Console.WriteLine($"\tFailed to write Greeting: --{_settings.Greetings[_greetingIndex]}");
Console.WriteLine(ex.Message);
throw;
}
// Insert multiple rows using MutateRows()
// Build a MutateRowsRequest (contains table name and a collection of entries).
MutateRowsRequest request = new MutateRowsRequest
{
TableNameAsTableName = _tableNameClient
};
_mapToOriginalGreetingIndex = new List <int>();
while (++_greetingIndex < _settings.Greetings.Length)
{
_mapToOriginalGreetingIndex.Add(_greetingIndex);
// Build an entry for every greeting (contains of row key and a collection of mutations).
request.Entries.Add(Mutations.CreateEntry(_settings.RowKeyPrefix + _greetingIndex,
MutationBuilder(_greetingIndex)));
}
BigtableClient.MutateRowsStream response = _bigtableClient.MutateRows(request);
Task write = CheckWrittenAsync(response);
write.Wait();
#endregion
#region Read the first row
Console.WriteLine("Read the first row");
int rowIndex = 0;
Row rowRead = _bigtableClient.ReadRow(_tableNameClient, _settings.RowKeyPrefix + rowIndex);
Console.WriteLine(
$"\tRow key: {rowRead.Key.ToStringUtf8()}, Value: {rowRead.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()}");
#endregion
#region Read all rows.
BigtableClient.ReadRowsStream responseRead = _bigtableClient.ReadRows(_tableNameClient);
Task printRead = PrintReadRowsAsync(responseRead);
printRead.Wait();
#endregion
#region Delete table.
Console.WriteLine($"Delete table: {_settings.TableName}");
_bigtableTableAdminClient.DeleteTable(_tableNameAdmin);
if (DoesTableExist(_bigtableTableAdminClient, _tableNameAdmin) == false)
{
Console.WriteLine($"Table: {_settings.TableName} deleted succsessfully");
}
#endregion
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
private static void DoHelloWorld()
{
try
{
// [START bigtable_hw_connect]
// BigtableTableAdminClient API lets us create, manage and delete tables.
BigtableTableAdminClient bigtableTableAdminClient = BigtableTableAdminClient.Create();
// BigtableClient API lets us read and write to a table.
BigtableClient bigtableClient = BigtableClient.Create();
// [END bigtable_hw_connect]
// [START bigtable_hw_create_table]
// Create a table with a single column family.
Console.WriteLine($"Create new table: {tableId} with column family: {columnFamily}, instance: {instanceId}");
// Check whether a table with given TableName already exists.
if (!TableExist(bigtableTableAdminClient))
{
bigtableTableAdminClient.CreateTable(
new InstanceName(projectId, instanceId),
tableId,
new Table
{
Granularity = Table.Types.TimestampGranularity.Millis,
ColumnFamilies =
{
{
columnFamily, new ColumnFamily
{
GcRule = new GcRule
{
MaxNumVersions = 1
}
}
}
}
});
// Confirm that table was created successfully.
Console.WriteLine(TableExist(bigtableTableAdminClient)
? $"Table {tableId} created successfully\n"
: $"There was a problem creating a table {tableId}");
}
else
{
Console.WriteLine($"Table: {tableId} already exists");
}
// [END bigtable_hw_create_table]
// [START bigtable_hw_write_rows]
// Initialize Google.Cloud.Bigtable.V2.TableName object.
Google.Cloud.Bigtable.Common.V2.TableName tableName = new Google.Cloud.Bigtable.Common.V2.TableName(projectId, instanceId, tableId);
// Write some rows
/* Each row has a unique row key.
*
* Note: This example uses sequential numeric IDs for simplicity, but
* this can result in poor performance in a production application.
* Since rows are stored in sorted order by key, sequential keys can
* result in poor distribution of operations across nodes.
*
* For more information about how to design a Bigtable schema for the
* best performance, see the documentation:
*
* https://cloud.google.com/bigtable/docs/schema-design */
Console.WriteLine($"Write some greetings to the table {tableId}");
// Insert 1 row using MutateRow()
s_greetingIndex = 0;
try
{
bigtableClient.MutateRow(tableName, rowKeyPrefix + s_greetingIndex, MutationBuilder());
Console.WriteLine($"\tGreeting: -- {s_greetings[s_greetingIndex],-18}-- written successfully");
}
catch (Exception ex)
{
Console.WriteLine($"\tFailed to write greeting: --{s_greetings[s_greetingIndex]}");
Console.WriteLine(ex.Message);
throw;
}
// Insert multiple rows using MutateRows()
// Build a MutateRowsRequest (contains table name and a collection of entries).
MutateRowsRequest request = new MutateRowsRequest
{
TableNameAsTableName = tableName
};
s_mapToOriginalGreetingIndex = new List <int>();
while (++s_greetingIndex < s_greetings.Length)
{
s_mapToOriginalGreetingIndex.Add(s_greetingIndex);
// Build an entry for every greeting (consists of rowkey and a collection of mutations).
string rowKey = rowKeyPrefix + s_greetingIndex;
request.Entries.Add(Mutations.CreateEntry(rowKey, MutationBuilder()));
}
// Make the request to write multiple rows.
MutateRowsResponse response = bigtableClient.MutateRows(request);
// Check the status code of each entry to ensure that it was written successfully.
foreach (MutateRowsResponse.Types.Entry entry in response.Entries)
{
s_greetingIndex = s_mapToOriginalGreetingIndex[(int)entry.Index];
if (entry.Status.Code == 0)
{
Console.WriteLine($"\tGreeting: -- {s_greetings[s_greetingIndex],-18}-- written successfully");
}
else
{
Console.WriteLine($"\tFailed to write greeting: --{s_greetings[s_greetingIndex]}");
Console.WriteLine(entry.Status.Message);
}
}
Mutation MutationBuilder() =>
Mutations.SetCell(columnFamily, columnName, s_greetings[s_greetingIndex], new BigtableVersion(DateTime.UtcNow));
//[END bigtable_hw_write_rows]
// [START bigtable_hw_create_filter]
RowFilter filter = RowFilters.CellsPerRowLimit(1);
// [END bigtable_hw_create_filter]
// [START bigtable_hw_get_with_filter]
// Read from the table.
Console.WriteLine("Read the first row");
int rowIndex = 0;
// Read a specific row. Apply a filter to return latest only cell value accross entire row.
Row rowRead = bigtableClient.ReadRow(
tableName, rowKey: rowKeyPrefix + rowIndex, filter: filter);
Console.WriteLine(
$"\tRow key: {rowRead.Key.ToStringUtf8()} " +
$" -- Value: {rowRead.Families[0].Columns[0].Cells[0].Value.ToStringUtf8(),-16} " +
$" -- Time Stamp: {rowRead.Families[0].Columns[0].Cells[0].TimestampMicros}");
// [END bigtable_hw_get_with_filter]
// [START bigtable_hw_scan_with_filter]
Console.WriteLine("Read all rows using streaming");
// stream the content of the whole table. Apply a filter to return latest only cell values accross all rows.
ReadRowsStream responseRead = bigtableClient.ReadRows(tableName, filter: filter);
Task printRead = PrintReadRowsAsync();
printRead.Wait();
async Task PrintReadRowsAsync()
{
await responseRead.ForEachAsync(row =>
{
Console.WriteLine($"\tRow key: {row.Key.ToStringUtf8()} " +
$" -- Value: {row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8(),-16} " +
$" -- Time Stamp: {row.Families[0].Columns[0].Cells[0].TimestampMicros}");
});
}
// [END bigtable_hw_scan_with_filter]
// [START bigtable_hw_delete_table]
// Clean up. Delete the table.
Console.WriteLine($"Delete table: {tableId}");
bigtableTableAdminClient.DeleteTable(name: tableName);
if (!TableExist(bigtableTableAdminClient))
{
Console.WriteLine($"Table: {tableId} deleted successfully");
}
// [END bigtable_hw_delete_table]
}
catch (Exception ex)
{
Console.WriteLine($"Exception while running HelloWorld: {ex.Message}");
}
}
public async Task MultipleRetries()
{
var request = new MutateRowsRequest
{
Entries =
{
Mutations.CreateEntry("a", Mutations.DeleteFromRow()),
Mutations.CreateEntry("b", Mutations.DeleteFromRow()),
Mutations.CreateEntry("c", Mutations.DeleteFromRow()),
Mutations.CreateEntry("d", Mutations.DeleteFromRow()),
Mutations.CreateEntry("e", Mutations.DeleteFromRow())
}
};
var client = MutateRowsStreamUtilities.CreateClientForMutateRowsRetries(
entriesForInitialResponse: new[]
{
MutateRowsStreamUtilities.CreateEntry(0, Code.Ok),
MutateRowsStreamUtilities.CreateEntry(1, Code.DeadlineExceeded),
MutateRowsStreamUtilities.CreateEntry(2, Code.Ok),
MutateRowsStreamUtilities.CreateEntry(3, Code.DeadlineExceeded),
MutateRowsStreamUtilities.CreateEntry(4, Code.DeadlineExceeded)
},
entriesForRetryResponses: new[]
{
// 1st retry response entries
new[]
{
// Original entry 1
MutateRowsStreamUtilities.CreateEntry(0, Code.Ok),
// Original entry 3
MutateRowsStreamUtilities.CreateEntry(1, Code.DeadlineExceeded),
// Original entry 4
MutateRowsStreamUtilities.CreateEntry(2, Code.Ok)
},
// 2nd retry response entries
new[]
{
// Original entry 3
MutateRowsStreamUtilities.CreateEntry(0, Code.FailedPrecondition)
}
});
var entries = (await client.MutateRowsAsync(request)).Entries.OrderBy(e => e.Index).ToList();
Assert.Equal(5, entries.Count);
var entry = entries[0];
Assert.Equal(0, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[1];
Assert.Equal(1, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[2];
Assert.Equal(2, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
entry = entries[3];
Assert.Equal(3, entry.Index);
Assert.Equal((int)Code.FailedPrecondition, entry.Status.Code);
entry = entries[4];
Assert.Equal(4, entry.Index);
Assert.Equal((int)Code.Ok, entry.Status.Code);
}
public override MutateRowsStream MutateRows(MutateRowsRequest request, CallSettings callSettings = null) => _mockMutateRowsStreams.Count != 0 ? _mockMutateRowsStreams.Dequeue() : throw new RequestMadeException();
public override Task <MutateRowsResponse> MutateRowsAsync(MutateRowsRequest request, CallSettings callSettings = null) => ConvertResult(request, callSettings, GetUnderlyingClient().MutateRows(request, callSettings));
public override MutateRowsResponse MutateRows(MutateRowsRequest request, CallSettings callSettings = null) => Task.Run(() => MutateRowsAsync(request, callSettings)).ResultWithUnwrappedExceptions();
public override MutateRowsStream MutateRows(MutateRowsRequest request, CallSettings callSettings = null) => throw new RequestMadeException();
