public async Task RetryingBeforeTotalExpiration() { var request = new ReadRowsRequest { Rows = RowSet.FromRowKeys("a", "b", "c") }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } } }, responsesForRetryStreams: new[] { null, // A null entry will throw an Unavailable RpcException new [] { new ReadRowsResponse { Chunks = { CreateChunk("b", "cf1", "column2", "value2", commitRow: true) } } } }); await client.ReadRows(request).ToListAsync(); }
/// <summary>Snippet for ReadRows</summary> public async Task ReadRowsRequestObject() { // Snippet: ReadRows(ReadRowsRequest, CallSettings) // Create client BigQueryReadClient bigQueryReadClient = BigQueryReadClient.Create(); // Initialize request argument(s) ReadRowsRequest request = new ReadRowsRequest { ReadStreamAsReadStreamName = ReadStreamName.FromProjectLocationSessionStream("[PROJECT]", "[LOCATION]", "[SESSION]", "[STREAM]"), Offset = 0L, }; // Make the request, returning a streaming response BigQueryReadClient.ReadRowsStream response = bigQueryReadClient.ReadRows(request); // Read streaming responses from server until complete // Note that C# 8 code can use await foreach AsyncResponseStream <ReadRowsResponse> responseStream = response.GetResponseStream(); while (await responseStream.MoveNextAsync()) { ReadRowsResponse responseItem = responseStream.Current; // Do something with streamed response } // The response stream has completed // End snippet }
/// <summary>Snippet for ReadRows</summary> public async Task ReadRowsRequestObject() { // Snippet: ReadRows(ReadRowsRequest, CallSettings) // Create client BigtableClient bigtableClient = BigtableClient.Create(); // Initialize request argument(s) ReadRowsRequest request = new ReadRowsRequest { TableNameAsTableName = TableName.FromProjectInstanceTable("[PROJECT]", "[INSTANCE]", "[TABLE]"), Rows = new RowSet(), Filter = new RowFilter(), RowsLimit = 0L, AppProfileId = "", }; // Make the request, returning a streaming response BigtableClient.ReadRowsStream response = bigtableClient.ReadRows(request); // Read streaming responses from server until complete // Note that C# 8 code can use await foreach AsyncResponseStream <ReadRowsResponse> responseStream = response.GetResponseStream(); while (await responseStream.MoveNextAsync()) { ReadRowsResponse responseItem = responseStream.Current; // Do something with streamed response } // The response stream has completed // End snippet }
/// <summary>Snippet for ReadRows</summary> public async Task ReadRowsRequestObject() { // Snippet: ReadRows(ReadRowsRequest, CallSettings) // Create client BigQueryStorageClient bigQueryStorageClient = BigQueryStorageClient.Create(); // Initialize request argument(s) ReadRowsRequest request = new ReadRowsRequest { ReadPosition = new StreamPosition(), }; // Make the request, returning a streaming response BigQueryStorageClient.ReadRowsStream response = bigQueryStorageClient.ReadRows(request); // Read streaming responses from server until complete // Note that C# 8 code can use await foreach AsyncResponseStream <ReadRowsResponse> responseStream = response.GetResponseStream(); while (await responseStream.MoveNextAsync()) { ReadRowsResponse responseItem = responseStream.Current; // Do something with streamed response } // The response stream has completed // End snippet }
private async Task <TimedOperation <ScanResults> > ExecuteScan(ReadRowsRequest scanRequest) { try { return(await new TimedOperation <ScanResults>().MeasureAsync(async() => { var service = new BigtableService.BigtableServiceClient(_channel); var results = new ScanResults(); using (var scanResults = service.ReadRows(scanRequest)) using (var responseStream = scanResults.ResponseStream) { while (await responseStream.MoveNext()) { AccumulateRows(responseStream.Current, results); } } return results; })); } catch (Exception exception) { ReportServiceFault(exception); throw; } }
public async Task ErrorAtVeryEndDoesntCauseRetry() { var request = new ReadRowsRequest { Rows = RowSet.FromRowKeys("a") }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } } }, errorAtEndOfLastStream: true); var rows = await client.ReadRows(request).ToList(); Assert.Equal(1, rows.Count); var row = rows[0]; Assert.Equal("a", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); }
/// <summary>Snippet for ReadRows</summary> public async Task ReadRows() { // Snippet: ReadRows(ReadRowsRequest,CallSettings) // Create client BigtableServiceApiClient bigtableServiceApiClient = BigtableServiceApiClient.Create(); // Initialize request argument ReadRowsRequest request = new ReadRowsRequest { TableNameAsTableName = new TableName("[PROJECT]", "[INSTANCE]", "[TABLE]"), }; // Make the request, returning a streaming response BigtableServiceApiClient.ReadRowsStream streamingResponse = bigtableServiceApiClient.ReadRows(request); // Read streaming responses from server until complete IAsyncEnumerator <ReadRowsResponse> responseStream = streamingResponse.ResponseStream; while (await responseStream.MoveNext()) { ReadRowsResponse response = responseStream.Current; // Do something with streamed response } // The response stream has completed // End snippet }
protected override async Task DeleteAsync(string persistenceId, SnapshotSelectionCriteria criteria) { var filter = RowFilters.Chain ( RowFilters.ColumnQualifierExact(SnapshotMetaDataColumnQualifier), // this filter ensures that we only download snapshot metadata RowFilters.TimestampRange( ToUtc(criteria.MinTimestamp), ToUtc(criteria.MaxTimeStamp)?.AddMilliseconds(1) // add a milliseconds since the upper bound is exclusive ), RowFilters.CellsPerColumnLimit(1) ); var readRowsRequest = new ReadRowsRequest { TableNameAsTableName = _tableName, Filter = filter, Rows = GetRowSet(persistenceId, criteria.MinSequenceNr, criteria.MaxSequenceNr) }; var deleteMutations = await _bigtableClient .ReadRows(readRowsRequest) .Select(SnapshotMetadataFromBigtableRow) .Where(metadata => SatisfiesTimestampCriteria(criteria, metadata)) .Select(metadata => Mutations.CreateEntry(GetRowKey(persistenceId, metadata.SequenceNr), Mutations.DeleteFromRow())) .ToList() .ConfigureAwait(false); if (deleteMutations.Count > 0) { await _bigtableClient.MutateRowsAsync(_tableName, deleteMutations).ConfigureAwait(false); } }
/// <summary>Snippet for ReadRows</summary> public async Task ReadRows_RequestObject() { // Snippet: ReadRows(ReadRowsRequest, CallSettings) // Create client BigQueryReadClient bigQueryReadClient = BigQueryReadClient.Create(); // Initialize request argument(s) ReadRowsRequest request = new ReadRowsRequest { ReadStreamAsReadStreamName = new ReadStreamName("[PROJECT]", "[LOCATION]", "[SESSION]", "[STREAM]"), Offset = 0L, }; // Make the request, returning a streaming response BigQueryReadClient.ReadRowsStream response = bigQueryReadClient.ReadRows(request); // Read streaming responses from server until complete IAsyncEnumerator <ReadRowsResponse> responseStream = response.ResponseStream; while (await responseStream.MoveNext()) { ReadRowsResponse responseItem = responseStream.Current; // Do something with streamed response } // The response stream has completed // End snippet }
public void TestFilterRowsRowRangesKeyInMiddle() { BigtableByteString key1 = "row1"; BigtableByteString key2 = "row2"; BigtableByteString key3 = "row3"; BigtableByteString lastFoundKey = "row1a"; RowSet fullRowSet = RowSet.FromRowKeys(key1, key2, key3); fullRowSet.RowRanges.Add(new[] { RowRange.OpenClosed(null, key1), // should be filtered out RowRange.Open(null, key1), // should be filtered out RowRange.Open(key1, key2), // should be converted (lastFoundKey, key2) RowRange.ClosedOpen(key1, key2), // should be converted (lastFoundKey, key2) RowRange.Closed(key1, key2), // should be converted (lastFoundKey, key2] RowRange.Open(key2, key3), // should stay RowRange.ClosedOpen(key2, key3) // should stay }); RowSet filteredRowSet = RowSet.FromRowKeys(key2, key3); filteredRowSet.RowRanges.Add(new[] { RowRange.Open(lastFoundKey, key2), // should be converted (lastFoundKey, key2) RowRange.Open(lastFoundKey, key2), // should be converted (lastFoundKey, key2) RowRange.OpenClosed(lastFoundKey, key2), // should be converted (lastFoundKey, key2] RowRange.Open(key2, key3), // should stay RowRange.ClosedOpen(key2, key3) // should stay }); ReadRowsRequest originalRequest = new ReadRowsRequest { Rows = fullRowSet }; ReadRowsRequest filteredRequest = new ReadRowsRequest { Rows = filteredRowSet }; BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); underTest.LastFoundKey = lastFoundKey; ReadRowsRequest originalRequestClone = originalRequest.Clone(); // Assert that originalRequest and originalRequestClone are different objects. Assert.False(ReferenceEquals(originalRequest, originalRequestClone)); // Assert that originalRequest and originalRequestClone have same value before calling BuildUpdatedRequest. Assert.Equal(originalRequest, originalRequestClone); Assert.Equal(filteredRequest, underTest.BuildUpdatedRequest()); // Assert that BuildUpdatedRequest did not modify the original RowSet. Assert.Equal(originalRequest, originalRequestClone); }
public void TestFilterRowsTestAllRange() { BigtableByteString key1 = "row1"; ReadRowsRequest originalRequest = CreateRowRangeRequest(RowRange.ClosedOpen(null, null)); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); underTest.LastFoundKey = key1; Assert.Equal(CreateRowRangeRequest(RowRange.Open(key1, null)), underTest.BuildUpdatedRequest()); }
// Used by all read signatures including observables private async Task <IAsyncEnumerator <ReadRowsResponse> > ReadRows(ReadRowsRequest request, CancellationToken cancellationToken = default(CancellationToken)) { // Read rows with native client var response = _client.ReadRows(request, cancellationToken: cancellationToken); // Wait for response to have started await response.ResponseHeadersAsync; // Return the response stream return(response.ResponseStream); }
// Used by single row private async Task <BigRow> ConvertRow(string table, Encoding encoding, ReadRowsRequest request, CancellationToken cancellationToken) { // Chain var results = await ConvertRows(table, encoding, request, cancellationToken); // Return to caller context await Task.Yield(); // Return first result return(results.FirstOrDefault()); }
public void TestAppProfileId() { var originalRequest = new ReadRowsRequest { Rows = RowSet.FromRowKey("a"), AppProfileId = "xyz" }; BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); }
public async Task RetryWithLastScannedKey() { var request = new ReadRowsRequest { Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z")) }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } }, new ReadRowsResponse { LastScannedRowKey = ByteString.CopyFromUtf8("m") } }, responsesForRetryStreams: new[] { new [] { new ReadRowsResponse { Chunks = { CreateChunk("w", "cf1", "column2", "value2", commitRow: true) } } } }); var rows = await client.ReadRows(request).ToListAsync(); Assert.Equal(2, rows.Count); var row = rows[0]; Assert.Equal("a", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); row = rows[1]; Assert.Equal("w", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column2", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value2", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); }
public async Task RetryPartialRowAfterCompleteRow() { var request = new ReadRowsRequest { Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z")) }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true), CreateChunk("b", "cf1", "column2", "value2") } } }, responsesForRetryStreams: new[] { new [] { new ReadRowsResponse { Chunks = { CreateChunk("b", "cf1", "column2", "value2"), CreateChunk("b", "cf1", "column3", "value3", commitRow: true) } } } }); var rows = await client.ReadRows(request).ToList(); Assert.Equal(2, rows.Count); var row = rows[0]; Assert.Equal("a", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); row = rows[1]; Assert.Equal("b", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column2", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value2", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); Assert.Equal("column3", row.Families[0].Columns[1].Qualifier.ToStringUtf8()); Assert.Equal("value3", row.Families[0].Columns[1].Cells[0].Value.ToStringUtf8()); }
public void TestDefaultRequest() { BigtableByteString lastFoundKey = "row015"; ReadRowsRequest originalRequest = new ReadRowsRequest(); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); underTest.LastFoundKey = lastFoundKey; Assert.Equal(CreateRowRangeRequest(RowRange.Open(lastFoundKey, null)), underTest.BuildUpdatedRequest()); }
public void TestFilterRowsUnsignedRows() { BigtableByteString key1 = new byte[] { 0x7f }; BigtableByteString key2 = new byte[] { 0x80 }; ReadRowsRequest originalRequest = CreateRowKeysRequest(key1, key2); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); underTest.LastFoundKey = key1; Assert.Equal(CreateRowKeysRequest(key2), underTest.BuildUpdatedRequest()); }
public void TestRowFilter() { var rowFilter = RowFilters.Chain( RowFilters.CellsPerColumnLimit(1), RowFilters.CellsPerRowOffset(2), RowFilters.CellsPerRowLimit(10)); ReadRowsRequest originalRequest = CreateRowFilterRequest(rowFilter); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); }
public void TestFilterRowsUnsignedRange() { BigtableByteString key1 = new byte[] { 0x7f }; BigtableByteString key2 = new byte[] { 0x80 }; ReadRowsRequest originalRequest = CreateRowRangeRequest(RowRange.ClosedOpen(key1, null)); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); underTest.LastFoundKey = key2; Assert.Equal(CreateRowRangeRequest(RowRange.Open(key2, null)), underTest.BuildUpdatedRequest()); }
public async Task ReadRows(ReadRowsTest testCase) { var request = new ReadRowsRequest { Rows = RowSet.FromRowRanges(RowRange.Closed("", null)) }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { testCase.Chunks } } }); var stream = client.ReadRows(request); List <Row> responses; if (testCase.Results.LastOrDefault()?.Error ?? false) { // Do not use ToList() here. We want to get all results before the first failure. responses = new List <Row>(); await Assert.ThrowsAsync <InvalidOperationException>( () => stream.ForEachAsync(row => responses.Add(row))); } else { responses = await stream.ToListAsync(); } var results = from row in responses from family in row.Families from column in family.Columns from cell in column.Cells select new ReadRowsTest.Types.Result { RowKey = row.Key.ToStringUtf8(), FamilyName = family.Name, Qualifier = column.Qualifier.ToStringUtf8(), TimestampMicros = cell.TimestampMicros, Value = cell.Value.ToStringUtf8(), Label = cell.Labels.FirstOrDefault() ?? "", Error = false }; var expectedResults = testCase.Results.Where(r => !r.Error); Assert.True(expectedResults.SequenceEqual(results), testCase.Description); }
public async Task SimpleRetryWithKeys() { var request = new ReadRowsRequest { Rows = RowSet.FromRowKeys("a", "b", "c") }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } } }, responsesForRetryStreams: new[] { new [] { new ReadRowsResponse { Chunks = { CreateChunk("b", "cf1", "column2", "value2", commitRow: true) } } } }); var rows = await client.ReadRows(request).ToListAsync(); Assert.Equal(2, rows.Count); var row = rows[0]; Assert.Equal("a", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); row = rows[1]; Assert.Equal("b", row.Key.ToStringUtf8()); Assert.Equal("cf1", row.Families[0].Name); Assert.Equal("column2", row.Families[0].Columns[0].Qualifier.ToStringUtf8()); Assert.Equal("value2", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8()); }
public void TestFilterRowsRowKeys() { BigtableByteString key1 = "row1"; BigtableByteString key2 = "row2"; BigtableByteString key3 = "row3"; ReadRowsRequest originalRequest = CreateRowKeysRequest(key1, key2, key3); BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); underTest.LastFoundKey = key1; Assert.Equal(CreateRowKeysRequest(key2, key3), underTest.BuildUpdatedRequest()); }
public async Task <IEnumerable <BigRow> > GetUnsortedRowsAsync(string tableName, RowFilter filter, Encoding encoding = null, CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken = cancellationToken == default(CancellationToken) ? CancellationToken.None : cancellationToken; encoding = encoding ?? BigModel.DefaultEncoding; var request = new ReadRowsRequest { TableName = tableName.ToTableId(ClusterId), AllowRowInterleaving = true, Filter = filter, }; return(await ConvertRows(tableName, encoding, request, cancellationToken)); }
public async Task <BigRow> GetRowAsync(string tableName, string key, Encoding encoding = null, CancellationToken cancellationToken = default(CancellationToken)) { // Defaultable parameters cancellationToken = cancellationToken == default(CancellationToken) ? CancellationToken.None : cancellationToken; encoding = encoding ?? BigModel.DefaultEncoding; // Create request var request = new ReadRowsRequest { TableName = tableName.ToTableId(ClusterId), RowKey = key.ToByteString(encoding) }; // Chain return(await ConvertRow(tableName, encoding, request, cancellationToken)); }
public void TestRowFilter() { var rowFilter = RowFilters.Chain( RowFilters.CellsPerColumnLimit(1), RowFilters.CellsPerRowOffset(2), RowFilters.CellsPerRowLimit(10)); var originalRequest = new ReadRowsRequest { Rows = RowSet.FromRowKey("a"), Filter = rowFilter }; BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest); Assert.Equal(originalRequest, underTest.BuildUpdatedRequest()); }
public async Task RetryWithLastScannedKey_Error() { var request = new ReadRowsRequest { Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z")) }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } }, new ReadRowsResponse { LastScannedRowKey = ByteString.CopyFromUtf8("b") } }, responsesForRetryStreams: new[] { new [] { new ReadRowsResponse { Chunks = { CreateChunk("b", "cf1", "column2", "value2", commitRow: true) } } } }); var stream = client.ReadRows(request); // We should be able to read one full row and then there should be an exception in the request validator // of Utilities.CreateReadRowsMockClient because the response from the retry stream with the "b" row key // should be outside the range of (b-z] requested during retry. int rowCount = 0; await Assert.ThrowsAsync <InvalidOperationException>(() => stream.ForEachAsync(row => rowCount++)); Assert.Equal(1, rowCount); }
public async Task <IEnumerable <BigRow> > GetRowsAsync(string tableName, RowFilter filter, long rowLimit = 0, Encoding encoding = null, CancellationToken cancellationToken = default(CancellationToken)) { // Defaultable parameters cancellationToken = cancellationToken == default(CancellationToken) ? CancellationToken.None : cancellationToken; encoding = encoding ?? BigModel.DefaultEncoding; // Create request var request = new ReadRowsRequest { TableName = tableName.ToTableId(ClusterId), NumRowsLimit = rowLimit, Filter = filter, }; // Chain return(await ConvertRows(tableName, encoding, request, cancellationToken)); }
public async Task RetryingAfterTotalExpiration() { var settings = new BigtableServiceApiSettings(); // Don't allow for any time to retry. settings.ReadRowsRetrySettings = settings.ReadRowsRetrySettings.WithTotalExpiration( Expiration.FromTimeout(TimeSpan.Zero)); var request = new ReadRowsRequest { Rows = RowSet.FromRowKeys("a", "b", "c") }; var client = Utilities.CreateReadRowsMockClient( request, initialStreamResponse: new[] { new ReadRowsResponse { Chunks = { CreateChunk("a", "cf1", "column1", "value1", commitRow: true) } } }, responsesForRetryStreams: new[] { null, // A null entry will throw an Unavailable RpcException new [] { new ReadRowsResponse { Chunks = { CreateChunk("b", "cf1", "column2", "value2", commitRow: true) } } } }, settings: settings); var exception = await Assert.ThrowsAsync <RpcException>(() => client.ReadRows(request).ToList()); Assert.Equal(StatusCode.Unavailable, exception.StatusCode); }
public async Task <IEnumerable <BigRow> > GetUnsortedRowsAsync(string tableName, byte[] startKey = null, byte[] endKey = null, Encoding encoding = null, CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken = cancellationToken == default(CancellationToken) ? CancellationToken.None : cancellationToken; encoding = encoding ?? BigModel.DefaultEncoding; var request = new ReadRowsRequest { TableName = tableName.ToTableId(ClusterId), AllowRowInterleaving = true, RowRange = new RowRange { StartKey = (startKey ?? new byte[0]).ToByteString(), EndKey = (endKey ?? new byte[0]).ToByteString(), } }; return(await ConvertRows(tableName, encoding, request, cancellationToken)); }