public void Open_Valid()
{
var range = RowRange.Open("a", "z");
Assert.True(range.StartKeyClosed.IsEmpty);
Assert.Equal("a", range.StartKeyOpen.ToStringUtf8());
Assert.True(range.EndKeyClosed.IsEmpty);
Assert.Equal("z", range.EndKeyOpen.ToStringUtf8());
range = RowRange.Open(null, new BigtableByteString(0x12, 0xFE));
Assert.True(range.StartKeyClosed.IsEmpty);
Assert.True(range.StartKeyOpen.IsEmpty);
Assert.True(range.EndKeyClosed.IsEmpty);
Assert.Equal(new byte[] { 0x12, 0xFE }, range.EndKeyOpen.ToByteArray());
range = RowRange.Open(ByteString.CopyFrom(0x00, 0x01, 0xFF), null);
Assert.True(range.StartKeyClosed.IsEmpty);
Assert.Equal(new byte[] { 0x00, 0x01, 0xFF }, range.StartKeyOpen.ToByteArray());
Assert.True(range.EndKeyClosed.IsEmpty);
Assert.True(range.EndKeyOpen.IsEmpty);
range = RowRange.Open(null, null);
Assert.True(range.StartKeyClosed.IsEmpty);
Assert.True(range.StartKeyOpen.IsEmpty);
Assert.True(range.EndKeyClosed.IsEmpty);
Assert.True(range.EndKeyOpen.IsEmpty);
}
public void FromRowRanges_Valid()
{
var set = RowSet.FromRowRanges(RowRange.Closed("a", "d"), RowRange.Open("x", "z"));
Assert.Equal(0, set.RowKeys.Count);
Assert.Equal(new[] { RowRange.Closed("a", "d"), RowRange.Open("x", "z") }, set.RowRanges);
}
public void TestFilterRowsRowRangesKeyInMiddle()
{
BigtableByteString key1 = "row1";
BigtableByteString key2 = "row2";
BigtableByteString key3 = "row3";
BigtableByteString lastFoundKey = "row1a";
RowSet fullRowSet = RowSet.FromRowKeys(key1, key2, key3);
fullRowSet.RowRanges.Add(new[]
{
RowRange.OpenClosed(null, key1), // should be filtered out
RowRange.Open(null, key1), // should be filtered out
RowRange.Open(key1, key2), // should be converted (lastFoundKey, key2)
RowRange.ClosedOpen(key1, key2), // should be converted (lastFoundKey, key2)
RowRange.Closed(key1, key2), // should be converted (lastFoundKey, key2]
RowRange.Open(key2, key3), // should stay
RowRange.ClosedOpen(key2, key3) // should stay
});
RowSet filteredRowSet = RowSet.FromRowKeys(key2, key3);
filteredRowSet.RowRanges.Add(new[]
{
RowRange.Open(lastFoundKey, key2), // should be converted (lastFoundKey, key2)
RowRange.Open(lastFoundKey, key2), // should be converted (lastFoundKey, key2)
RowRange.OpenClosed(lastFoundKey, key2), // should be converted (lastFoundKey, key2]
RowRange.Open(key2, key3), // should stay
RowRange.ClosedOpen(key2, key3) // should stay
});
ReadRowsRequest originalRequest = new ReadRowsRequest {
Rows = fullRowSet
};
ReadRowsRequest filteredRequest = new ReadRowsRequest {
Rows = filteredRowSet
};
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
Assert.Equal(originalRequest, underTest.BuildUpdatedRequest());
underTest.LastFoundKey = lastFoundKey;
ReadRowsRequest originalRequestClone = originalRequest.Clone();
// Assert that originalRequest and originalRequestClone are different objects.
Assert.False(ReferenceEquals(originalRequest, originalRequestClone));
// Assert that originalRequest and originalRequestClone have same value before calling BuildUpdatedRequest.
Assert.Equal(originalRequest, originalRequestClone);
Assert.Equal(filteredRequest, underTest.BuildUpdatedRequest());
// Assert that BuildUpdatedRequest did not modify the original RowSet.
Assert.Equal(originalRequest, originalRequestClone);
}
public void TestFilterRowsTestAllRange()
{
BigtableByteString key1 = "row1";
ReadRowsRequest originalRequest = CreateRowRangeRequest(RowRange.ClosedOpen(null, null));
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
underTest.LastFoundKey = key1;
Assert.Equal(CreateRowRangeRequest(RowRange.Open(key1, null)), underTest.BuildUpdatedRequest());
}
public async Task RetryWithLastScannedKey()
{
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z"))
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "value1", commitRow: true)
}
},
new ReadRowsResponse
{
LastScannedRowKey = ByteString.CopyFromUtf8("m")
}
},
responsesForRetryStreams: new[]
{
new []
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("w", "cf1", "column2", "value2", commitRow: true)
}
}
}
});
var rows = await client.ReadRows(request).ToListAsync();
Assert.Equal(2, rows.Count);
var row = rows[0];
Assert.Equal("a", row.Key.ToStringUtf8());
Assert.Equal("cf1", row.Families[0].Name);
Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8());
Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8());
row = rows[1];
Assert.Equal("w", row.Key.ToStringUtf8());
Assert.Equal("cf1", row.Families[0].Name);
Assert.Equal("column2", row.Families[0].Columns[0].Qualifier.ToStringUtf8());
Assert.Equal("value2", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8());
}
public async Task RetryPartialRowAfterCompleteRow()
{
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z"))
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "value1", commitRow: true),
CreateChunk("b", "cf1", "column2", "value2")
}
}
},
responsesForRetryStreams: new[]
{
new []
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("b", "cf1", "column2", "value2"),
CreateChunk("b", "cf1", "column3", "value3", commitRow: true)
}
}
}
});
var rows = await client.ReadRows(request).ToList();
Assert.Equal(2, rows.Count);
var row = rows[0];
Assert.Equal("a", row.Key.ToStringUtf8());
Assert.Equal("cf1", row.Families[0].Name);
Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8());
Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8());
row = rows[1];
Assert.Equal("b", row.Key.ToStringUtf8());
Assert.Equal("cf1", row.Families[0].Name);
Assert.Equal("column2", row.Families[0].Columns[0].Qualifier.ToStringUtf8());
Assert.Equal("value2", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8());
Assert.Equal("column3", row.Families[0].Columns[1].Qualifier.ToStringUtf8());
Assert.Equal("value3", row.Families[0].Columns[1].Cells[0].Value.ToStringUtf8());
}
public void TestDefaultRequest()
{
BigtableByteString lastFoundKey = "row015";
ReadRowsRequest originalRequest = new ReadRowsRequest();
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
Assert.Equal(originalRequest, underTest.BuildUpdatedRequest());
underTest.LastFoundKey = lastFoundKey;
Assert.Equal(CreateRowRangeRequest(RowRange.Open(lastFoundKey, null)), underTest.BuildUpdatedRequest());
}
public override async Task <long> ReadHighestSequenceNrAsync(string persistenceId, long fromSequenceNr)
{
var rowRange = RowRange.ClosedOpen(
ToRowKeyBigtableByteString(persistenceId, fromSequenceNr),
ToRowKeyBigtableByteString(persistenceId, long.MaxValue));
var rows = RowSet.FromRowRanges(rowRange);
var stream = _bigtableClient.ReadRows(
_tableName,
rows: rows,
filter: RowFilters.Chain(RowFilters.CellsPerRowLimit(1), RowFilters.StripValueTransformer()));
var lastRow = await stream.LastOrDefault().ConfigureAwait(false);
return(lastRow == null ? 0L : GetSequenceNumber(lastRow));
}
// [END bigtable_reads_row_range]
// [START bigtable_reads_row_ranges]
/// <summary>
/// /// Reads multiple ranges of rows from an existing table.
///</summary>
/// <param name="projectId">Your Google Cloud Project ID.</param>
/// <param name="instanceId">Your Google Cloud Bigtable Instance ID.</param>
/// <param name="tableId">Your Google Cloud Bigtable table ID.</param>
public string readRowRanges(string projectId = "YOUR-PROJECT-ID", string instanceId = "YOUR-INSTANCE-ID", string tableId = "YOUR-TABLE-ID")
{
BigtableClient bigtableClient = BigtableClient.Create();
TableName tableName = new TableName(projectId, instanceId, tableId);
RowSet rowSet = RowSet.FromRowRanges(RowRange.ClosedOpen("phone#4c410523#20190501", "phone#4c410523#20190601"),
RowRange.ClosedOpen("phone#5c10102#20190501", "phone#5c10102#20190601"));
ReadRowsStream readRowsStream = bigtableClient.ReadRows(tableName, rowSet);
string result = "";
readRowsStream.ForEach(row => result += printRow(row));
return(result);
}
public void TestFilterRowsUnsignedRange()
{
BigtableByteString key1 = new byte[] { 0x7f };
BigtableByteString key2 = new byte[] { 0x80 };
ReadRowsRequest originalRequest = CreateRowRangeRequest(RowRange.ClosedOpen(key1, null));
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
underTest.LastFoundKey = key2;
Assert.Equal(CreateRowRangeRequest(RowRange.Open(key2, null)),
underTest.BuildUpdatedRequest());
}
public async Task ReadRows(ReadRowsTest testCase)
{
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(RowRange.Closed("", null))
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks = { testCase.Chunks }
}
});
var stream = client.ReadRows(request);
List <Row> responses;
if (testCase.Results.LastOrDefault()?.Error ?? false)
{
// Do not use ToList() here. We want to get all results before the first failure.
responses = new List <Row>();
await Assert.ThrowsAsync <InvalidOperationException>(
() => stream.ForEachAsync(row => responses.Add(row)));
}
else
{
responses = await stream.ToListAsync();
}
var results = from row in responses
from family in row.Families
from column in family.Columns
from cell in column.Cells
select new ReadRowsTest.Types.Result
{
RowKey = row.Key.ToStringUtf8(),
FamilyName = family.Name,
Qualifier = column.Qualifier.ToStringUtf8(),
TimestampMicros = cell.TimestampMicros,
Value = cell.Value.ToStringUtf8(),
Label = cell.Labels.FirstOrDefault() ?? "",
Error = false
};
var expectedResults = testCase.Results.Where(r => !r.Error);
Assert.True(expectedResults.SequenceEqual(results), testCase.Description);
}
public async Task RetryWithLastScannedKey_Error()
{
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z"))
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "value1", commitRow: true)
}
},
new ReadRowsResponse
{
LastScannedRowKey = ByteString.CopyFromUtf8("b")
}
},
responsesForRetryStreams: new[]
{
new []
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("b", "cf1", "column2", "value2", commitRow: true)
}
}
}
});
var stream = client.ReadRows(request);
// We should be able to read one full row and then there should be an exception in the request validator
// of Utilities.CreateReadRowsMockClient because the response from the retry stream with the "b" row key
// should be outside the range of (b-z] requested during retry.
int rowCount = 0;
await Assert.ThrowsAsync <InvalidOperationException>(() => stream.ForEachAsync(row => rowCount++));
Assert.Equal(1, rowCount);
}
private RowSet GetRowSet(string persistenceId, long minSequenceNr, long maxSequenceNr)
{
var from = GetRowKey(persistenceId, minSequenceNr);
var to = GetRowKey(persistenceId, maxSequenceNr);
RowSet rowSet;
if (minSequenceNr == maxSequenceNr)
{
rowSet = RowSet.FromRowKey(from);
}
else
{
rowSet = RowSet.FromRowRanges(RowRange.Closed(from, to));
}
return(rowSet);
}
public async Task RetryDuringSplitCell()
{
var request = new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(RowRange.Closed("a", "z"))
};
var client = Utilities.CreateReadRowsMockClient(
request,
initialStreamResponse: new[]
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "valu", valueSize: 6)
}
}
},
responsesForRetryStreams: new[]
{
new []
{
new ReadRowsResponse
{
Chunks =
{
CreateChunk("a", "cf1", "column1", "valu", valueSize: 6),
CreateContinuationChunk("e1", commitRow: true)
}
}
}
});
var rows = await client.ReadRows(request).ToList();
Assert.Equal(1, rows.Count);
var row = rows[0];
Assert.Equal("a", row.Key.ToStringUtf8());
Assert.Equal("cf1", row.Families[0].Name);
Assert.Equal("column1", row.Families[0].Columns[0].Qualifier.ToStringUtf8());
Assert.Equal("value1", row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8());
}
public void TestRowsLimitOnlyRequest()
{
BigtableByteString lastFoundKey = "row015";
ReadRowsRequest originalRequest = new ReadRowsRequest {
RowsLimit = 1000
};
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
Assert.Equal(originalRequest, underTest.BuildUpdatedRequest());
underTest.LastFoundKey = lastFoundKey;
underTest.IncrementRowsReadSoFar(10);
ReadRowsRequest updatedRequest = CreateRowRangeRequest(RowRange.Open(lastFoundKey, null));
updatedRequest.RowsLimit = 990;
Assert.Equal(updatedRequest, underTest.BuildUpdatedRequest());
}
// [END bigtable_reads_row_ranges]
// [START bigtable_reads_prefix]
/// <summary>
/// /// Reads rows starting with a prefix from an existing table.
///</summary>
/// <param name="projectId">Your Google Cloud Project ID.</param>
/// <param name="instanceId">Your Google Cloud Bigtable Instance ID.</param>
/// <param name="tableId">Your Google Cloud Bigtable table ID.</param>
public string readPrefix(string projectId = "YOUR-PROJECT-ID", string instanceId = "YOUR-INSTANCE-ID", string tableId = "YOUR-TABLE-ID")
{
BigtableClient bigtableClient = BigtableClient.Create();
TableName tableName = new TableName(projectId, instanceId, tableId);
String prefix = "phone";
Char prefixEndChar = prefix[prefix.Length - 1];
prefixEndChar++;
String end = prefix.Substring(0, prefix.Length - 1) + prefixEndChar;
RowSet rowSet = RowSet.FromRowRanges(RowRange.Closed(prefix, end));
ReadRowsStream readRowsStream = bigtableClient.ReadRows(tableName, rowSet);
string result = "";
readRowsStream.ForEach(row => result += printRow(row));
return(result);
}
public static void DeleteRows(string tableName, RowRange rowRange)
{
var bigtableClient = BigtableClient.Create();
var stream = bigtableClient.ReadRows(Google.Cloud.Bigtable.Common.V2.TableName.Parse(tableName), RowSet.FromRowRanges(rowRange));
var deleteRows = new List <MutateRowsRequest.Types.Entry>();
using (var enumerator = stream.GetEnumerator())
{
while (enumerator.MoveNext().GetAwaiter().GetResult())
{
deleteRows.Add(Mutations.CreateEntry(enumerator.Current.Key, Mutations.DeleteFromRow()));
}
}
if (deleteRows.Any())
{
bigtableClient.MutateRows(Google.Cloud.Bigtable.Common.V2.TableName.Parse(tableName), deleteRows);
}
}
public void TestRowsLimit()
{
BigtableByteString startKeyOpenOriginal = "row050";
BigtableByteString lastFoundKey = "row125";
ReadRowsRequest originalRequest = CreateRowRangeRequest(RowRange.OpenClosed(startKeyOpenOriginal, null));
originalRequest.RowsLimit = 100;
ReadRowsRequest updatedRequest = CreateRowRangeRequest(RowRange.OpenClosed(lastFoundKey, null));
updatedRequest.RowsLimit = 25;
BigtableReadRowsRequestManager underTest = new BigtableReadRowsRequestManager(originalRequest);
Assert.Equal(originalRequest, underTest.BuildUpdatedRequest());
underTest.LastFoundKey = lastFoundKey;
underTest.IncrementRowsReadSoFar(75);
Assert.Equal(updatedRequest, underTest.BuildUpdatedRequest());
}
protected void DeleteRows(string tableName, string pid)
{
var rowRange = RowRange.Closed(new BigtableByteString($"{pid}"), new BigtableByteString($"{pid}~"));
var bigtableClient = BigtableClient.Create();
var stream = bigtableClient.ReadRows(TableName.Parse(tableName), RowSet.FromRowRanges(rowRange));
var deleteRows = new List <MutateRowsRequest.Types.Entry>();
using (var enumerator = stream.GetEnumerator())
{
while (enumerator.MoveNext().GetAwaiter().GetResult())
{
deleteRows.Add(Mutations.CreateEntry(enumerator.Current.Key, Mutations.DeleteFromRow()));
}
}
if (deleteRows.Any())
{
bigtableClient.MutateRows(TableName.Parse(tableName), deleteRows);
}
}
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr)
{
var rows = RowSet.FromRowRanges(RowRange.Closed(ToRowKeyBigtableByteString(persistenceId, 0), ToRowKeyBigtableByteString(persistenceId, toSequenceNr)));
var stream = _bigtableClient.ReadRows(_tableName, rows: rows);
var deleteEntries = new List <MutateRowsRequest.Types.Entry>();
using (var enumerator = stream.GetEnumerator())
{
while (await enumerator.MoveNext().ConfigureAwait(false))
{
deleteEntries.Add(Mutations.CreateEntry(enumerator.Current.Key, Mutations.DeleteFromRow()));
}
}
if (deleteEntries.Any())
{
var last = deleteEntries.LastOrDefault();
deleteEntries.RemoveAt(deleteEntries.Count - 1);
deleteEntries.Add(Mutations.CreateEntry(last.RowKey, Mutations.SetCell(_family, PayloadColumnQualifier, ByteString.Empty, new BigtableVersion(-1))));
await _bigtableClient.MutateRowsAsync(_tableName, deleteEntries);
}
}
public override async Task ReplayMessagesAsync(IActorContext context, string persistenceId, long fromSequenceNr, long toSequenceNr, long max, Action <IPersistentRepresentation> recoveryCallback)
{
if (max <= 0 || toSequenceNr < fromSequenceNr)
{
return;
}
var startKey = ToRowKeyBigtableByteString(persistenceId, fromSequenceNr);
var endKey = ToRowKeyBigtableByteString(persistenceId, toSequenceNr);
RowSet rowSet;
if (fromSequenceNr == toSequenceNr)
{
rowSet = RowSet.FromRowKey(startKey);
}
else
{
rowSet = RowSet.FromRowRanges(RowRange.Closed(startKey, endKey));
}
var stream = _bigtableClient.ReadRows(_tableName,
rows: rowSet,
filter: RowFilters.CellsPerColumnLimit(1),
rowsLimit: max);
using (var asyncEnumerator = stream.GetEnumerator())
{
while (await asyncEnumerator.MoveNext().ConfigureAwait(false))
{
var persitentRepresentation = ToPersistentRepresentation(asyncEnumerator.Current);
if (persitentRepresentation != null)
{
recoveryCallback.Invoke(persitentRepresentation);
}
}
}
}
private static ReadRowsRequest CreateRowRangeRequest(RowRange rowRange) =>
new ReadRowsRequest
{
Rows = RowSet.FromRowRanges(rowRange)
};
private void ClearTable()
{
var rowRange = RowRange.Closed(new BigtableByteString($"{Pid}"), new BigtableByteString($"{Pid}~"));
BigtableTestUtils.DeleteRows(TableName, rowRange);
}
public void FromRowRanges_Invalid()
{
Assert.Throws <ArgumentNullException>(() => RowSet.FromRowRanges(null));
Assert.Throws <ArgumentException>(() => RowSet.FromRowRanges(RowRange.Closed("a", "z"), null));
}
protected override void PreparePersistenceId(string pid)
{
var rowRange = RowRange.Closed(new BigtableByteString($"{pid}"), new BigtableByteString($"{pid}~"));
BigtableTestUtils.DeleteRows(TableName, rowRange);
}
