public override void Add(Log log, ref int insertCount) { var app = log.App.UpdateParty; if (app.Type == 0) // todo: const? enum? { var row = new BigQueryInsertRow { { "party_id", app.GroupId }, // GroupId と party_id は実質同じものなのでこれでOK { "player_id", app.PlayerId.Hash() }, { "party_entry_datetime", DateTimeParseToUtc(app.Date) }, }; RowsEntry.Add(row); insertCount++; } else { var row = new BigQueryInsertRow { { "party_id", app.GroupId }, { "player_id", app.PlayerId.Hash() }, { "party_exit_datetime", DateTimeParseToUtc(app.Date) }, }; RowsExit.Add(row); insertCount++; } }
public static Func <object, BigQueryInsertRow> GetValueToBigQueryFunction(Type type) { if (!IsContractType(type)) { throw new NotImplementedException($"Type {type.FullName} is not supported by BigQuerier Contract (maybe QuerierContract attribute is missing?)"); } var propertyMappers = FilterValidTypeProperties(type) .Select(p => Property.GetBigQueryRecordFieldAppendFunction(p)) .ToArray(); return(value => { if (value == null) { return null; } var row = new BigQueryInsertRow(); foreach (var mapper in propertyMappers) { mapper(value, row); } return row; }); }
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Silent() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "year", BigQueryDbType.Int64 } }.Build()); var rows = new BigQueryInsertRow[] { new BigQueryInsertRow { { "year", 2019 } }, new BigQueryInsertRow { { "noSuchField", 10 } }, new BigQueryInsertRow { { "year", "Unknown" } }, }; var options = new InsertOptions { AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = true }; // Now two rows are inserted, we are ignoring unknown fields so only the last row is bad. var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2); Assert.Equal(1, insertResult.OriginalRowsWithErrors); Assert.Equal(3, insertResult.InsertAttemptRowCount); Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, insertResult.Status); Assert.Contains(insertResult.Errors, e => e.OriginalRowIndex == 2); }
public void InsertRow_BadData_IgnoreBadData() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "name", BigQueryDbType.String } }.Build()); Assert.Equal(0, table.ListRows().Count()); var row = new BigQueryInsertRow { { "noSuchField", 10 } }; table.InsertRow(row, new InsertOptions { AllowUnknownFields = true }); // Check that we get the row. Use WaitForRows as // sometimes this seems to be not-completely-immediate. var command = new BigQueryCommand($"SELECT * FROM {table}"); Assert.Equal(1, WaitForRows(client, command).Count()); }
public void InsertRow_BadData_Throws(InsertOptions options, int[] errorRowsIndexes) { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "year", BigQueryDbType.Int64 } }.Build()); var rows = new BigQueryInsertRow[] { new BigQueryInsertRow { { "noSuchField", 10 } }, new BigQueryInsertRow { { "year", "Unknown" } } }; var exception = Assert.Throws <GoogleApiException>(() => table.InsertRows(rows, options)); Assert.Equal(errorRowsIndexes.Length, exception.Error.Errors.Count); foreach (var index in errorRowsIndexes) { Assert.Contains(exception.Error.Errors, e => e.Message.ToLower().Contains($"in row {index}")); } }
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Throws() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "year", BigQueryDbType.Int64 } }.Build()); var rows = new BigQueryInsertRow[] { new BigQueryInsertRow { { "year", 2019 } }, new BigQueryInsertRow { { "noSuchField", 10 } }, new BigQueryInsertRow { { "year", "Unknown" } } }; var options = new InsertOptions { AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = false }; var exception = Assert.Throws <GoogleApiException>(() => table.InsertRows(rows, options)); Assert.Equal(1, exception.Error.Errors.Count); Assert.Contains("Row 2", exception.Error.Errors[0].Message); }
public void InsertRow_BadData_Silent(InsertOptions options, int[] errorRowsIndexes) { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "year", BigQueryDbType.Int64 } }.Build()); var rows = new BigQueryInsertRow[] { new BigQueryInsertRow { { "noSuchField", 10 } }, new BigQueryInsertRow { { "year", "Unknown" } } }; var insertResult = table.InsertRows(rows, options); Assert.Equal(errorRowsIndexes.Length, insertResult.OriginalRowsWithErrors); Assert.Equal(errorRowsIndexes, insertResult.Errors. Select(e => (int?)e.OriginalRowIndex ?? -1). OrderBy(index => index). ToArray()); }
public override void Add(Log log, ref int insertCount) { var battle = log.Battle.kill_history; var row = new BigQueryInsertRow { { "match_id", battle.match_id }, { "kill_datetime", DateTimeParseToUtc(battle.kill_datetime) }, { "kill_team_cd", battle.kill_team_cd }, { "kill_offense_defense_cd", battle.kill_offense_defense_cd }, { "kill_player_id", battle.kill_player_id.Hash() }, { "kill_unit_id", battle.kill_unit_id }, { "kill_armed_id", battle.kill_armed_id }, { "kill_point_x", battle.kill_point?.X }, { "kill_point_y", battle.kill_point?.Y }, { "kill_point_z", battle.kill_point?.Z }, { "killed_team_cd", battle.killed_team_cd }, { "killed_offense_defense_cd", battle.killed_offense_defense_cd }, { "killed_player_id", battle.killed_player_id.Hash() }, { "killed_unit_id", battle.killed_unit_id }, { "killed_point_x", battle.killed_point.X }, { "killed_point_y", battle.killed_point.Y }, { "killed_point_z", battle.killed_point.Z }, { "environment_kill_flag", battle.environment_kill_flag }, }; Rows.Add(row); insertCount++; }
public void InsertRow_RecordField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = IdGenerator.FromGuid(); var row = new BigQueryInsertRow { ["guid"] = guid, ["position"] = new BigQueryInsertRow { ["x"] = 10L, ["y"] = 20L } }; _fixture.InsertAndWait(table, () => table.InsertRow(row), 1); string sql = $"SELECT guid, position.x, position.y FROM {table} WHERE guid=@guid"; var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) }; var resultRows = client.ExecuteQuery(sql, parameters) .Select(r => new { Guid = (string)r["guid"], X = (long)r["x"], Y = (long)r["y"] }) .ToList(); var expectedResults = new[] { new { Guid = guid, X = 10L, Y = 20L } }; Assert.Equal(expectedResults, resultRows); }
public void InsertRow_RepeatedField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = IdGenerator.FromGuid(); var row = new BigQueryInsertRow { ["guid"] = guid, ["tags"] = new[] { "a", "b" } }; _fixture.InsertAndWait(table, () => table.InsertRow(row), 1); string sql = $"SELECT guid, tag FROM {table}, UNNEST(tags) AS tag WHERE guid=@guid ORDER BY tag"; var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) }; var resultRows = client.ExecuteQuery(sql, parameters) .Select(r => new { Guid = (string)r["guid"], Tag = (string)r["tag"] }) .ToList(); var expectedResults = new[] { new { Guid = guid, Tag = "a" }, new { Guid = guid, Tag = "b" } }; Assert.Equal(expectedResults, resultRows); }
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Silent() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); // Don't insert into a table used by other tests... var table = dataset.CreateTable( _fixture.CreateTableId(), new TableSchemaBuilder { { "year", BigQueryDbType.Int64 } }.Build()); var rows = new BigQueryInsertRow[] { new BigQueryInsertRow { { "year", 2019 } }, new BigQueryInsertRow { { "noSuchField", 10 } }, new BigQueryInsertRow { { "year", "Unknown" } }, }; var options = new InsertOptions { AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = true }; // Now two rows are inserted, we are ignoring unknown fields so only the last row is bad. _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2); }
public void InsertRow_RepeatedField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = Guid.NewGuid().ToString(); var row = new BigQueryInsertRow { ["guid"] = guid, ["tags"] = new[] { "a", "b" } }; table.InsertRow(row); var command = new BigQueryCommand($"SELECT guid, tag FROM {table}, UNNEST(tags) AS tag WHERE guid=@guid ORDER BY tag") { Parameters = { { "guid", BigQueryDbType.String, guid } } }; var resultRows = WaitForRows(client, command) .Select(r => new { Guid = (string)r["guid"], Tag = (string)r["tag"] }) .ToList(); var expectedResults = new[] { new { Guid = guid, Tag = "a" }, new { Guid = guid, Tag = "b" } }; Assert.Equal(expectedResults, resultRows); }
public void InsertRow_RecordField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = Guid.NewGuid().ToString(); var row = new BigQueryInsertRow { ["guid"] = guid, ["position"] = new BigQueryInsertRow { ["x"] = 10L, ["y"] = 20L } }; table.InsertRow(row); var command = new BigQueryCommand($"SELECT guid, position.x, position.y FROM {table} WHERE guid=@guid") { Parameters = { { "guid", BigQueryDbType.String, guid } } }; var resultRows = WaitForRows(client, command) .Select(r => new { Guid = (string)r["guid"], X = (long)r["x"], Y = (long)r["y"] }) .ToList(); var expectedResults = new[] { new { Guid = guid, X = 10L, Y = 20L } }; Assert.Equal(expectedResults, resultRows); }
public void TableInsertRows( string projectId = "your-project-id", string datasetId = "your_dataset_id", string tableId = "your_table_id" ) { BigQueryClient client = BigQueryClient.Create(projectId); BigQueryInsertRow[] rows = new BigQueryInsertRow[] { // The insert ID is optional, but can avoid duplicate data // when retrying inserts. new BigQueryInsertRow(insertId: "row1") { { "name", "Washington" }, { "post_abbr", "WA" } }, new BigQueryInsertRow(insertId: "row2") { { "name", "Colorado" }, { "post_abbr", "CO" } } }; client.InsertRows(datasetId, tableId, rows); }
public override void Add(Log log, ref int insertCount) { var battle = log.Battle.damage_history; var row = new BigQueryInsertRow { { "match_id", battle.match_id }, { "round_start_datetime", DateTimeParseToUtc(battle.round_start_datetime) }, { "round_count", battle.round_count }, { "inflict_damage_team_cd", battle.inflict_damage_team_cd }, { "inflict_damage_offense_defense_cd", battle.inflict_damage_offense_defense_cd }, { "inflict_damage_player_id", battle.inflict_damage_player_id.Hash() }, { "inflict_damage_unit_id", battle.inflict_damage_unit_id }, { "inflict_damage_armed_id", battle.inflict_damage_armed_id }, { "take_damage_team_cd", battle.take_damage_team_cd }, { "take_damage_offense_defense_cd", battle.take_damage_offense_defense_cd }, { "take_damage_player_id", battle.take_damage_player_id.Hash() }, { "take_damage_unit_id", battle.take_damage_unit_id }, { "inflict_damage_quantity", battle.inflict_damage_quantity }, { "inflict_damage_quantity_shield", battle.inflict_damage_quantity_shield }, }; Rows.Add(row); insertCount++; }
public override void Add(Log log, ref int insertCount) { var battle = log.Battle.match_use_armed_history; var row = new BigQueryInsertRow { { "match_id", battle.match_id }, { "team_cd", battle.team_cd }, { "round_start_datetime", DateTimeParseToUtc(battle.round_start_datetime) }, { "round_count", battle.round_count }, { "offense_defense_cd", battle.offense_defense_cd }, { "player_id", battle.player_id.Hash() }, { "unit_id", battle.unit_id }, { "armed_id", battle.armed_id }, { "use_armed_count", battle.use_armed_count }, { "bullet_fire_count", battle.bullet_fire_count }, { "bullet_hit_count", battle.bullet_hit_count }, { "bullet_headshot_count", battle.bullet_headshot_count }, { "kill_count", battle.kill_count }, { "heal_quantity", battle.heal_quantity }, { "block_damage_quantity", battle.block_damage_quantity }, { "brake_shield_count", battle.brake_shield_count }, { "using_armed_time_second", battle.using_armed_time_second }, { "brake_placement_object_count", battle.brake_placement_object_count }, { "heal_assist_count", battle.heal_assist_count }, { "enhance_assist_count", battle.enhance_assist_count }, { "weakness_assist_count", battle.weakness_assist_count }, }; Rows.Add(row); insertCount++; }
public void InsertRow_RecordRepeatedField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = IdGenerator.FromGuid(); var row = new BigQueryInsertRow { ["guid"] = guid, ["job"] = new BigQueryInsertRow { ["company"] = "Pet Store", ["roles"] = new[] { "cashier", "manager" } }, }; _fixture.InsertAndWait(table, () => table.InsertRow(row), 1); string sql = $"SELECT job FROM {table} WHERE guid=@guid"; var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) }; var fetchedRow = client.ExecuteQuery(sql, parameters) .Single(); var job = (Dictionary <string, object>)fetchedRow["job"]; Assert.Equal("Pet Store", (string)job["company"]); Assert.Equal(new[] { "cashier", "manager" }, (string[])job["roles"]); }
public void InsertRow_RecordRepeatedField() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.ComplexTypesTableId); var guid = Guid.NewGuid().ToString(); var row = new BigQueryInsertRow { ["guid"] = guid, ["job"] = new BigQueryInsertRow { ["company"] = "Pet Store", ["roles"] = new[] { "cashier", "manager" } }, }; table.InsertRow(row); var command = new BigQueryCommand($"SELECT job FROM {table} WHERE guid=@guid") { Parameters = { { "guid", BigQueryDbType.String, guid } } }; var fetchedRow = WaitForRows(client, command).Single(); var job = (Dictionary <string, object>)fetchedRow["job"]; Assert.Equal("Pet Store", (string)job["company"]); Assert.Equal(new[] { "cashier", "manager" }, (string[])job["roles"]); }
public void RepeatedValue_NullRejectedOnConversion() { var row = new BigQueryInsertRow { { "names", new[] { "a", null, "b" } } }; Assert.Throws <InvalidOperationException>(() => row.ToRowsData()); }
public void RepeatedValue() { var row = new BigQueryInsertRow { { "numbers", new[] { 1, 2 } } }; var rowData = row.ToRowsData(); Assert.Equal(new object[] { 1, 2 }, rowData.Json["numbers"]); }
private void AssertInvalid(string name, object value) { var row = new BigQueryInsertRow(); Assert.Throws <ArgumentException>(() => row.Add(name, value)); Assert.Throws <ArgumentException>(() => row[name] = value); Assert.Throws <ArgumentException>(() => row.Add(new Dictionary <string, object> { { name, value } })); }
public void GetEnumerator() { var row = new BigQueryInsertRow { { "field1", 10 }, { "field2", "text" } }; Assert.Equal(2, row.Cast <object>().Count()); }
public void TimespanFormatting() { var row = new BigQueryInsertRow { { "field", new TimeSpan(1, 2, 3) }, }; var rowData = row.ToRowsData(); Assert.Equal("01:02:03", rowData.Json["field"]); }
public void DateTimeFormatting() { var row = new BigQueryInsertRow { { "field", new DateTime(2000, 1, 1, 5, 0, 0, DateTimeKind.Utc) }, }; var rowData = row.ToRowsData(); Assert.Equal("2000-01-01 05:00:00Z", rowData.Json["field"]); }
public void Numeric_Json() { object value = BigQueryNumeric.Parse("123.456"); var row = new BigQueryInsertRow { { "field", value } }; var rowData = row.ToRowsData(); Assert.Equal("123.456", rowData.Json["field"]); }
public void SupportedValueTypes_Passthrough(Type type) { object value = Activator.CreateInstance(type); var row = new BigQueryInsertRow { { "field", value } }; var rowData = row.ToRowsData(); Assert.Equal(value, rowData.Json["field"]); }
public void UnspecifiedDateTimeFormatting() { var row = new BigQueryInsertRow { { "field", new DateTime(2000, 1, 1, 5, 0, 0, DateTimeKind.Unspecified) }, }; var rowData = row.ToRowsData(false); Assert.Equal("2000-01-01T05:00:00", rowData.Json["field"]); }
public void DateTimeOffsetFormatting() { var row = new BigQueryInsertRow { // 3am UTC { "field", new DateTimeOffset(2000, 1, 1, 5, 0, 0, TimeSpan.FromHours(2)) }, }; var rowData = row.ToRowsData(); Assert.Equal("2000-01-01 03:00:00Z", rowData.Json["field"]); }
public void InsertRow_BadData() { var client = BigQueryClient.Create(_fixture.ProjectId); var dataset = client.GetDataset(_fixture.DatasetId); var table = dataset.GetTable(_fixture.HighScoreTableId); var row = new BigQueryInsertRow { { "noSuchField", 10 } }; Assert.Throws <GoogleApiException>(() => table.InsertRow(row)); }
public void AddDictionary() { var dictionary = new Dictionary <string, object> { { "field1", "value1" } }; var row = new BigQueryInsertRow { dictionary }; Assert.Equal("value1", row["field1"]); }