public async Task <BigtableByteString> InsertRowAsync( TableName tableName, string familyName = null, BigtableByteString?qualifierName = null, BigtableByteString?value = null, BigtableVersion?version = null) { BigtableByteString rowKey = Guid.NewGuid().ToString(); familyName = familyName ?? DefaultColumnFamily; qualifierName = qualifierName ?? DefaultColumnQualifier; value = value ?? DefaultValue; await TableClient.MutateRowAsync( tableName, rowKey, Mutations.SetCell( familyName, qualifierName.Value, value.Value, version)); await BigtableAssert.HasSingleValueAsync( TableClient, tableName, rowKey, familyName, qualifierName.Value, value.Value, version); return(rowKey); }
/// <summary> /// Add Test Data into Given Google Cloud Bigtable /// </summary> public void InsertTestDataToTable() { try { if (!IsTableExists()) { throw new Exception("No Such Table Found"); } var bigtableClient = BigtableClient.Create(); List <Mutation> Cols = new List <Mutation>(); TableName _table = new TableName(ProjectId, InstanceId, TableName); var request = new MutateRowsRequest { TableNameAsTableName = _table, }; request.Entries.Add(Mutations.CreateEntry(Guid.NewGuid().ToString(), Mutations.SetCell(ColumnFamily, "TestColumnName", "Test Column Value"))); bigtableClient.MutateRows(request); } catch (RpcException ex) { throw ex; } catch (Exception ex) { throw ex; } }
public async Task InsertRowsAsync( TableName tableName, IEnumerable <BigtableByteString> rowKeys, string familyName = null, BigtableByteString?qualifierName = null, BigtableByteString?valuePrefix = null, BigtableVersion?version = null) { BigtableByteString rowKey = Guid.NewGuid().ToString(); familyName = familyName ?? DefaultColumnFamily; qualifierName = qualifierName ?? "row_index"; valuePrefix = valuePrefix ?? ""; int counter = 0; var response = await TableClient.MutateRowsAsync( tableName, rowKeys.Select(k => Mutations.CreateEntry( k.Value, Mutations.SetCell( familyName, qualifierName.Value, valuePrefix.Value.Value.Concat( new BigtableByteString(counter++).Value), version))).ToArray()); var entries = response.Entries.OrderBy(e => e.Index); Assert.True(entries.All(e => e.Status.Code == (int)Code.Ok)); }
public void IdempotentCheckOnMutateRowRequest() { var setCell = Mutations.SetCell("abc", "cq1", "value", new BigtableVersion(-1)); var deleteFromColumn = Mutations.DeleteFromColumn("abc", "def", new BigtableVersionRange(-1, -1)); var request = new MutateRowRequest { Mutations = { Mutations.DeleteFromRow(), setCell, deleteFromColumn, Mutations.DeleteFromFamily("abc") } }; Assert.True(request.IsIdempotent()); deleteFromColumn.DeleteFromColumn.TimeRange.StartTimestampMicros = -1; deleteFromColumn.DeleteFromColumn.TimeRange.EndTimestampMicros = -1; Assert.True(request.IsIdempotent()); setCell.SetCell.TimestampMicros = -1; Assert.False(request.IsIdempotent()); }
public BigtableClientFixture() { BigtableClient bigtableClient = BigtableClient.Create(); _bigtableTableAdminClient = BigtableTableAdminClient.Create(); Table table = new Table { Granularity = Table.Types.TimestampGranularity.Millis }; table.ColumnFamilies.Add("stats_summary", new ColumnFamily()); table.ColumnFamilies.Add("cell_plan", new ColumnFamily()); CreateTableRequest createTableRequest = new CreateTableRequest { ParentAsInstanceName = new InstanceName(projectId, instanceId), Table = table, TableId = tableId, }; _bigtableTableAdminClient.CreateTable(createTableRequest); TableName tableName = new TableName(projectId, instanceId, tableId); BigtableVersion timestamp = new BigtableVersion(new DateTime(2020, 1, 10, 14, 0, 0, DateTimeKind.Utc)); BigtableVersion timestamp_minus_hr = new BigtableVersion(new DateTime(2020, 1, 10, 13, 0, 0, DateTimeKind.Utc)); MutateRowsRequest.Types.Entry[] entries = { Mutations.CreateEntry(new BigtableByteString("phone#4c410523#20190501"), Mutations.SetCell("cell_plan", "data_plan_01gb", "false", timestamp), Mutations.SetCell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr), Mutations.SetCell("cell_plan", "data_plan_05gb", "true", timestamp), Mutations.SetCell("stats_summary", "connected_cell", "1", timestamp), Mutations.SetCell("stats_summary", "connected_wifi", "1", timestamp), Mutations.SetCell("stats_summary", "os_build", "PQ2A.190405.003", timestamp)), Mutations.CreateEntry(new BigtableByteString("phone#4c410523#20190502"), Mutations.SetCell("cell_plan", "data_plan_05gb", "true", timestamp), Mutations.SetCell("stats_summary", "connected_cell", "1", timestamp), Mutations.SetCell("stats_summary", "connected_wifi", "1", timestamp), Mutations.SetCell("stats_summary", "os_build", "PQ2A.190405.004", timestamp)), Mutations.CreateEntry(new BigtableByteString("phone#4c410523#20190505"), Mutations.SetCell("cell_plan", "data_plan_05gb", "true", timestamp), Mutations.SetCell("stats_summary", "connected_cell", "0", timestamp), Mutations.SetCell("stats_summary", "connected_wifi", "1", timestamp), Mutations.SetCell("stats_summary", "os_build", "PQ2A.190406.000", timestamp)), Mutations.CreateEntry(new BigtableByteString("phone#5c10102#20190501"), Mutations.SetCell("cell_plan", "data_plan_10gb", "true", timestamp), Mutations.SetCell("stats_summary", "connected_cell", "1", timestamp), Mutations.SetCell("stats_summary", "connected_wifi", "1", timestamp), Mutations.SetCell("stats_summary", "os_build", "PQ2A.190401.002", timestamp)), Mutations.CreateEntry(new BigtableByteString("phone#5c10102#20190502"), Mutations.SetCell("cell_plan", "data_plan_10gb", "true", timestamp), Mutations.SetCell("stats_summary", "connected_cell", "1", timestamp), Mutations.SetCell("stats_summary", "connected_wifi", "0", timestamp), Mutations.SetCell("stats_summary", "os_build", "PQ2A.190406.000", timestamp)) }; bigtableClient.MutateRows(tableName, entries); }
public void SetCell_NonIdempotent() { var mutation = Mutations.SetCell("abc", "cq1", "value", new BigtableVersion(-1)); Assert.True(mutation.IsIdempotent()); mutation.SetCell.TimestampMicros = -1; Assert.False(mutation.IsIdempotent()); }
public void SetCell_String(string familyName) { var mutation = Mutations.SetCell(familyName, "cq1", "value", new BigtableVersion(3)); Assert.NotNull(mutation.SetCell); Assert.Equal(familyName, mutation.SetCell.FamilyName); Assert.Equal("cq1", mutation.SetCell.ColumnQualifier.ToStringUtf8()); Assert.Equal("value", mutation.SetCell.Value.ToStringUtf8()); Assert.Equal(3000, mutation.SetCell.TimestampMicros); }
private CheckAndMutateRowRequest ToMutateRowIfNotExistsRequest(IPersistentRepresentation persistent, ActorSystem system) { var request = new CheckAndMutateRowRequest(); request.TableNameAsTableName = _tableName; var payload = PersistentToBytes(persistent, system); request.PredicateFilter = RowFilters.PassAllFilter(); request.RowKey = ByteString.CopyFromUtf8(ToRowKeyString(persistent.PersistenceId, persistent.SequenceNr)); request.FalseMutations.Add(Mutations.SetCell(_family, PayloadColumnQualifier, payload, new BigtableVersion(-1))); return(request); }
public void SetCell_Bytes(string familyName) { var mutation = Mutations.SetCell(familyName, "cq1", new BigtableByteString(1, 2, 3, 4, 5), new BigtableVersion(3)); Assert.NotNull(mutation.SetCell); Assert.Equal(familyName, mutation.SetCell.FamilyName); Assert.Equal("cq1", mutation.SetCell.ColumnQualifier.ToStringUtf8()); var bytes = mutation.SetCell.Value.ToByteArray(); Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, bytes); Assert.Equal(3000, mutation.SetCell.TimestampMicros); }
protected override async Task SaveAsync(SnapshotMetadata metadata, object snapshot) { var snapshotBytes = SnapshotToBytes(metadata, snapshot, Context.System); byte[] snapshotMetadataBytes = SnapshotMetadataToBytes(metadata); var request = new MutateRowRequest(); var version = new BigtableVersion(metadata.Timestamp.ToUniversalTime()); request.TableNameAsTableName = _tableName; request.Mutations.Add(Mutations.SetCell(_family, SnapshotColumnQualifier, ByteString.CopyFrom(snapshotBytes), version)); request.Mutations.Add(Mutations.SetCell(_family, SnapshotMetaDataColumnQualifier, ByteString.CopyFrom(snapshotMetadataBytes), version)); request.RowKey = GetRowKey(metadata.PersistenceId, metadata.SequenceNr); await _bigtableClient.MutateRowAsync(request).ConfigureAwait(false); }
public void SetCell_Long(string familyName) { var mutation = Mutations.SetCell(familyName, "cq1", 12345, new BigtableVersion(3)); Assert.NotNull(mutation.SetCell); Assert.Equal(familyName, mutation.SetCell.FamilyName); Assert.Equal("cq1", mutation.SetCell.ColumnQualifier.ToStringUtf8()); var bytes = mutation.SetCell.Value.ToByteArray(); if (BitConverter.IsLittleEndian) { Array.Reverse(bytes); } Assert.Equal(12345, BitConverter.ToInt64(bytes, 0)); Assert.Equal(3000, mutation.SetCell.TimestampMicros); }
/// <summary> /// Mutate multiple rows in an existing table and column family. Updates multiple cells within each row. ///</summary> /// <param name="projectId">Your Google Cloud Project ID.</param> /// <param name="instanceId">Your Google Cloud Bigtable Instance ID.</param> /// <param name="tableId">Your Google Cloud Bigtable table ID.</param> public string writeBatch( string projectId = "YOUR-PROJECT-ID", string instanceId = "YOUR-INSTANCE-ID", string tableId = "YOUR-TABLE-ID") { BigtableClient bigtableClient = BigtableClient.Create(); TableName tableName = new TableName(projectId, instanceId, tableId); BigtableVersion timestamp = new BigtableVersion(DateTime.UtcNow); String COLUMN_FAMILY = "stats_summary"; MutateRowsRequest.Types.Entry mutations1 = Mutations.CreateEntry(new BigtableByteString("tablet#a0b81f74#20190501"), Mutations.SetCell(COLUMN_FAMILY, "connected_cell", 1, timestamp), Mutations.SetCell(COLUMN_FAMILY, "os_build", "12155.0.0-rc1", timestamp) ); MutateRowsRequest.Types.Entry mutations2 = Mutations.CreateEntry(new BigtableByteString("tablet#a0b81f74#20190502"), Mutations.SetCell(COLUMN_FAMILY, "connected_cell", 1, timestamp), Mutations.SetCell(COLUMN_FAMILY, "os_build", "12145.0.0-rc6", timestamp) ); MutateRowsRequest.Types.Entry[] entries = { mutations1, mutations2 }; MutateRowsResponse mutateRowResponse = bigtableClient.MutateRows(tableName, entries); foreach (MutateRowsResponse.Types.Entry entry in mutateRowResponse.Entries) { if (entry.Status.Code == 0) { Console.WriteLine($"Row {entry.Index} written successfully"); } else { Console.WriteLine($"\tFailed to write row {entry.Index}"); Console.WriteLine(entry.Status.Message); return(entry.Status.Message); } } return("Successfully wrote 2 rows"); }
/// <summary> /// Insert Test Data into Google Cloud Bigtable /// </summary> /// <returns></returns> public bool InsertTestData(string columnName, string columnValue) { try { var bigtableClient = BigtableClient.Create(); List <Mutation> Cols = new List <Mutation>(); TableName _table = new TableName(ProjectId, InstanceId, TableName); var request = new MutateRowsRequest { TableNameAsTableName = _table, }; request.Entries.Add(Mutations.CreateEntry(Guid.NewGuid().ToString(), Mutations.SetCell(ColumnFamily, columnName, columnValue))); bigtableClient.MutateRows(request); } catch (Exception ex) { throw ex; } return(true); }
public async Task InsertRowsAsync( TableName tableName, IEnumerable <BigtableByteString> rowKeys, string familyName = null, BigtableByteString?qualifierName = null, BigtableByteString?valuePrefix = null, BigtableVersion?version = null) { BigtableByteString rowKey = Guid.NewGuid().ToString(); familyName = familyName ?? ColumnFamily1; qualifierName = qualifierName ?? "row_index"; valuePrefix = valuePrefix ?? ""; // TODO: Use cleaner API when available. int counter = 0; var request = new MutateRowsRequest { TableNameAsTableName = tableName, Entries = { rowKeys.Select(k => new MutateRowsRequest.Types.Entry { RowKey = k.Value, Mutations = { Mutations.SetCell( familyName, qualifierName.Value, valuePrefix.Value.Value.Concat(new BigtableByteString(counter++).Value), version) } }) } }; var response = TableClient.MutateRows(request); while (await response.ResponseStream.MoveNext(default))
protected override async Task DeleteMessagesToAsync(string persistenceId, long toSequenceNr) { var rows = RowSet.FromRowRanges(RowRange.Closed(ToRowKeyBigtableByteString(persistenceId, 0), ToRowKeyBigtableByteString(persistenceId, toSequenceNr))); var stream = _bigtableClient.ReadRows(_tableName, rows: rows); var deleteEntries = new List <MutateRowsRequest.Types.Entry>(); using (var enumerator = stream.GetEnumerator()) { while (await enumerator.MoveNext().ConfigureAwait(false)) { deleteEntries.Add(Mutations.CreateEntry(enumerator.Current.Key, Mutations.DeleteFromRow())); } } if (deleteEntries.Any()) { var last = deleteEntries.LastOrDefault(); deleteEntries.RemoveAt(deleteEntries.Count - 1); deleteEntries.Add(Mutations.CreateEntry(last.RowKey, Mutations.SetCell(_family, PayloadColumnQualifier, ByteString.Empty, new BigtableVersion(-1)))); await _bigtableClient.MutateRowsAsync(_tableName, deleteEntries); } }
/// <summary> /// Mutate one row in an existing table and column family. Updates multiple cells within that row using one API call. ///</summary> /// <param name="projectId">Your Google Cloud Project ID.</param> /// <param name="instanceId">Your Google Cloud Bigtable Instance ID.</param> /// <param name="tableId">Your Google Cloud Bigtable table ID.</param> public string writeSimple( string projectId = "YOUR-PROJECT-ID", string instanceId = "YOUR-INSTANCE-ID", string tableId = "YOUR-TABLE-ID") { BigtableClient bigtableClient = BigtableClient.Create(); TableName tableName = new TableName(projectId, instanceId, tableId); BigtableByteString rowkey = new BigtableByteString("phone#4c410523#20190501"); BigtableVersion timestamp = new BigtableVersion(DateTime.UtcNow); String COLUMN_FAMILY = "stats_summary"; Mutation[] mutations = { Mutations.SetCell(COLUMN_FAMILY, "connected_cell", 1, timestamp), Mutations.SetCell(COLUMN_FAMILY, "connected_wifi", 1, timestamp), Mutations.SetCell(COLUMN_FAMILY, "os_build", "PQ2A.190405.003", timestamp) }; MutateRowResponse mutateRowResponse = bigtableClient.MutateRow(tableName, rowkey, mutations); Console.WriteLine(mutateRowResponse); return($"Successfully wrote row {rowkey}"); }
/// <summary> /// Check if a row has a certain value then mutate the row if it does. ///</summary> /// <param name="projectId">Your Google Cloud Project ID.</param> /// <param name="instanceId">Your Google Cloud Bigtable Instance ID.</param> /// <param name="tableId">Your Google Cloud Bigtable table ID.</param> public string writeConditional( string projectId = "YOUR-PROJECT-ID", string instanceId = "YOUR-INSTANCE-ID", string tableId = "YOUR-TABLE-ID") { BigtableClient bigtableClient = BigtableClient.Create(); TableName tableName = new TableName(projectId, instanceId, tableId); BigtableByteString rowkey = new BigtableByteString("phone#4c410523#20190501"); BigtableVersion timestamp = new BigtableVersion(DateTime.UtcNow); String COLUMN_FAMILY = "stats_summary"; CheckAndMutateRowResponse checkAndMutateRowResponse = bigtableClient.CheckAndMutateRow( tableName, rowkey, RowFilters.Chain( RowFilters.FamilyNameExact(COLUMN_FAMILY), RowFilters.ColumnQualifierExact("os_build"), RowFilters.ValueRegex("PQ2A\\..*")), Mutations.SetCell(COLUMN_FAMILY, "os_name", "android", timestamp)); return($"Successfully updated row's os_name: {checkAndMutateRowResponse.PredicateMatched}"); }
/// <summary> /// Builds a <see cref="Mutation"/> for <see cref="MutateRowRequest"/> or an <see cref="MutateRowsRequest.Types.Entry"/> /// </summary> /// <param name="greetingNumber"> an index of a greeting</param> private Mutation MutationBuilder(int greetingNumber) => Mutations.SetCell(_settings.ColumnFamily, $"field{greetingNumber}", _settings.Greetings[greetingNumber]);
// Builds a <see cref="Mutation"/> for <see cref="MutateRowRequest"/> or an <see cref="MutateRowsRequest.Types.Entry"/> private static Mutation MutationBuilder(int greetingNumber) => Mutations.SetCell(columnFamily, columnName, s_greetings[greetingNumber], new BigtableVersion(DateTime.UtcNow));
public void SetCellInvalidFamilyName(string familyName) { Assert.Throws <ArgumentException>( () => Mutations.SetCell(familyName, default(string), default(string), null)); }
private static void DoHelloWorld() { try { // [START bigtable_hw_connect] // BigtableTableAdminClient API lets us create, manage and delete tables. BigtableTableAdminClient bigtableTableAdminClient = BigtableTableAdminClient.Create(); // BigtableClient API lets us read and write to a table. BigtableClient bigtableClient = BigtableClient.Create(); // [END bigtable_hw_connect] // [START bigtable_hw_create_table] // Create a table with a single column family. Console.WriteLine($"Create new table: {tableId} with column family: {columnFamily}, instance: {instanceId}"); // Check whether a table with given TableName already exists. if (!TableExist(bigtableTableAdminClient)) { bigtableTableAdminClient.CreateTable( new InstanceName(projectId, instanceId), tableId, new Table { Granularity = Table.Types.TimestampGranularity.Millis, ColumnFamilies = { { columnFamily, new ColumnFamily { GcRule = new GcRule { MaxNumVersions = 1 } } } } }); // Confirm that table was created successfully. Console.WriteLine(TableExist(bigtableTableAdminClient) ? $"Table {tableId} created successfully\n" : $"There was a problem creating a table {tableId}"); } else { Console.WriteLine($"Table: {tableId} already exists"); } // [END bigtable_hw_create_table] // [START bigtable_hw_write_rows] // Initialize Google.Cloud.Bigtable.V2.TableName object. Google.Cloud.Bigtable.Common.V2.TableName tableName = new Google.Cloud.Bigtable.Common.V2.TableName(projectId, instanceId, tableId); // Write some rows /* Each row has a unique row key. * * Note: This example uses sequential numeric IDs for simplicity, but * this can result in poor performance in a production application. * Since rows are stored in sorted order by key, sequential keys can * result in poor distribution of operations across nodes. * * For more information about how to design a Bigtable schema for the * best performance, see the documentation: * * https://cloud.google.com/bigtable/docs/schema-design */ Console.WriteLine($"Write some greetings to the table {tableId}"); // Insert 1 row using MutateRow() s_greetingIndex = 0; try { bigtableClient.MutateRow(tableName, rowKeyPrefix + s_greetingIndex, MutationBuilder()); Console.WriteLine($"\tGreeting: -- {s_greetings[s_greetingIndex],-18}-- written successfully"); } catch (Exception ex) { Console.WriteLine($"\tFailed to write greeting: --{s_greetings[s_greetingIndex]}"); Console.WriteLine(ex.Message); throw; } // Insert multiple rows using MutateRows() // Build a MutateRowsRequest (contains table name and a collection of entries). MutateRowsRequest request = new MutateRowsRequest { TableNameAsTableName = tableName }; s_mapToOriginalGreetingIndex = new List <int>(); while (++s_greetingIndex < s_greetings.Length) { s_mapToOriginalGreetingIndex.Add(s_greetingIndex); // Build an entry for every greeting (consists of rowkey and a collection of mutations). string rowKey = rowKeyPrefix + s_greetingIndex; request.Entries.Add(Mutations.CreateEntry(rowKey, MutationBuilder())); } // Make the request to write multiple rows. MutateRowsResponse response = bigtableClient.MutateRows(request); // Check the status code of each entry to ensure that it was written successfully. foreach (MutateRowsResponse.Types.Entry entry in response.Entries) { s_greetingIndex = s_mapToOriginalGreetingIndex[(int)entry.Index]; if (entry.Status.Code == 0) { Console.WriteLine($"\tGreeting: -- {s_greetings[s_greetingIndex],-18}-- written successfully"); } else { Console.WriteLine($"\tFailed to write greeting: --{s_greetings[s_greetingIndex]}"); Console.WriteLine(entry.Status.Message); } } Mutation MutationBuilder() => Mutations.SetCell(columnFamily, columnName, s_greetings[s_greetingIndex], new BigtableVersion(DateTime.UtcNow)); //[END bigtable_hw_write_rows] // [START bigtable_hw_create_filter] RowFilter filter = RowFilters.CellsPerRowLimit(1); // [END bigtable_hw_create_filter] // [START bigtable_hw_get_with_filter] // Read from the table. Console.WriteLine("Read the first row"); int rowIndex = 0; // Read a specific row. Apply a filter to return latest only cell value accross entire row. Row rowRead = bigtableClient.ReadRow( tableName, rowKey: rowKeyPrefix + rowIndex, filter: filter); Console.WriteLine( $"\tRow key: {rowRead.Key.ToStringUtf8()} " + $" -- Value: {rowRead.Families[0].Columns[0].Cells[0].Value.ToStringUtf8(),-16} " + $" -- Time Stamp: {rowRead.Families[0].Columns[0].Cells[0].TimestampMicros}"); // [END bigtable_hw_get_with_filter] // [START bigtable_hw_scan_with_filter] Console.WriteLine("Read all rows using streaming"); // stream the content of the whole table. Apply a filter to return latest only cell values accross all rows. ReadRowsStream responseRead = bigtableClient.ReadRows(tableName, filter: filter); Task printRead = PrintReadRowsAsync(); printRead.Wait(); async Task PrintReadRowsAsync() { await responseRead.ForEachAsync(row => { Console.WriteLine($"\tRow key: {row.Key.ToStringUtf8()} " + $" -- Value: {row.Families[0].Columns[0].Cells[0].Value.ToStringUtf8(),-16} " + $" -- Time Stamp: {row.Families[0].Columns[0].Cells[0].TimestampMicros}"); }); } // [END bigtable_hw_scan_with_filter] // [START bigtable_hw_delete_table] // Clean up. Delete the table. Console.WriteLine($"Delete table: {tableId}"); bigtableTableAdminClient.DeleteTable(name: tableName); if (!TableExist(bigtableTableAdminClient)) { Console.WriteLine($"Table: {tableId} deleted successfully"); } // [END bigtable_hw_delete_table] } catch (Exception ex) { Console.WriteLine($"Exception while running HelloWorld: {ex.Message}"); } }
Enumerable.Range(0, entryCount).Select(i => Mutations.CreateEntry("Row" + i, Mutations.SetCell("Family" + i, default, default)))