public override void Add(Log log, ref int insertCount)
{
var app = log.App.UpdateParty;
if (app.Type == 0) // todo: const? enum?
{
var row = new BigQueryInsertRow
{
{ "party_id", app.GroupId }, // GroupId と party_id は実質同じものなのでこれでOK
{ "player_id", app.PlayerId.Hash() },
{ "party_entry_datetime", DateTimeParseToUtc(app.Date) },
};
RowsEntry.Add(row);
insertCount++;
}
else
{
var row = new BigQueryInsertRow
{
{ "party_id", app.GroupId },
{ "player_id", app.PlayerId.Hash() },
{ "party_exit_datetime", DateTimeParseToUtc(app.Date) },
};
RowsExit.Add(row);
insertCount++;
}
}
public static Func <object, BigQueryInsertRow> GetValueToBigQueryFunction(Type type)
{
if (!IsContractType(type))
{
throw new NotImplementedException($"Type {type.FullName} is not supported by BigQuerier Contract (maybe QuerierContract attribute is missing?)");
}
var propertyMappers = FilterValidTypeProperties(type)
.Select(p => Property.GetBigQueryRecordFieldAppendFunction(p))
.ToArray();
return(value =>
{
if (value == null)
{
return null;
}
var row = new BigQueryInsertRow();
foreach (var mapper in propertyMappers)
{
mapper(value, row);
}
return row;
});
}
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Silent()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "year", BigQueryDbType.Int64 }
}.Build());
var rows = new BigQueryInsertRow[]
{
new BigQueryInsertRow {
{ "year", 2019 }
},
new BigQueryInsertRow {
{ "noSuchField", 10 }
},
new BigQueryInsertRow {
{ "year", "Unknown" }
},
};
var options = new InsertOptions {
AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = true
};
// Now two rows are inserted, we are ignoring unknown fields so only the last row is bad.
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
Assert.Equal(1, insertResult.OriginalRowsWithErrors);
Assert.Equal(3, insertResult.InsertAttemptRowCount);
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, insertResult.Status);
Assert.Contains(insertResult.Errors, e => e.OriginalRowIndex == 2);
}
public void InsertRow_BadData_IgnoreBadData()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "name", BigQueryDbType.String }
}.Build());
Assert.Equal(0, table.ListRows().Count());
var row = new BigQueryInsertRow {
{ "noSuchField", 10 }
};
table.InsertRow(row, new InsertOptions {
AllowUnknownFields = true
});
// Check that we get the row. Use WaitForRows as
// sometimes this seems to be not-completely-immediate.
var command = new BigQueryCommand($"SELECT * FROM {table}");
Assert.Equal(1, WaitForRows(client, command).Count());
}
public void InsertRow_BadData_Throws(InsertOptions options, int[] errorRowsIndexes)
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "year", BigQueryDbType.Int64 }
}.Build());
var rows = new BigQueryInsertRow[]
{
new BigQueryInsertRow {
{ "noSuchField", 10 }
},
new BigQueryInsertRow {
{ "year", "Unknown" }
}
};
var exception = Assert.Throws <GoogleApiException>(() => table.InsertRows(rows, options));
Assert.Equal(errorRowsIndexes.Length, exception.Error.Errors.Count);
foreach (var index in errorRowsIndexes)
{
Assert.Contains(exception.Error.Errors, e => e.Message.ToLower().Contains($"in row {index}"));
}
}
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Throws()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "year", BigQueryDbType.Int64 }
}.Build());
var rows = new BigQueryInsertRow[]
{
new BigQueryInsertRow {
{ "year", 2019 }
},
new BigQueryInsertRow {
{ "noSuchField", 10 }
},
new BigQueryInsertRow {
{ "year", "Unknown" }
}
};
var options = new InsertOptions {
AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = false
};
var exception = Assert.Throws <GoogleApiException>(() => table.InsertRows(rows, options));
Assert.Equal(1, exception.Error.Errors.Count);
Assert.Contains("Row 2", exception.Error.Errors[0].Message);
}
public void InsertRow_BadData_Silent(InsertOptions options, int[] errorRowsIndexes)
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "year", BigQueryDbType.Int64 }
}.Build());
var rows = new BigQueryInsertRow[]
{
new BigQueryInsertRow {
{ "noSuchField", 10 }
},
new BigQueryInsertRow {
{ "year", "Unknown" }
}
};
var insertResult = table.InsertRows(rows, options);
Assert.Equal(errorRowsIndexes.Length, insertResult.OriginalRowsWithErrors);
Assert.Equal(errorRowsIndexes,
insertResult.Errors.
Select(e => (int?)e.OriginalRowIndex ?? -1).
OrderBy(index => index).
ToArray());
}
public override void Add(Log log, ref int insertCount)
{
var battle = log.Battle.kill_history;
var row = new BigQueryInsertRow
{
{ "match_id", battle.match_id },
{ "kill_datetime", DateTimeParseToUtc(battle.kill_datetime) },
{ "kill_team_cd", battle.kill_team_cd },
{ "kill_offense_defense_cd", battle.kill_offense_defense_cd },
{ "kill_player_id", battle.kill_player_id.Hash() },
{ "kill_unit_id", battle.kill_unit_id },
{ "kill_armed_id", battle.kill_armed_id },
{ "kill_point_x", battle.kill_point?.X },
{ "kill_point_y", battle.kill_point?.Y },
{ "kill_point_z", battle.kill_point?.Z },
{ "killed_team_cd", battle.killed_team_cd },
{ "killed_offense_defense_cd", battle.killed_offense_defense_cd },
{ "killed_player_id", battle.killed_player_id.Hash() },
{ "killed_unit_id", battle.killed_unit_id },
{ "killed_point_x", battle.killed_point.X },
{ "killed_point_y", battle.killed_point.Y },
{ "killed_point_z", battle.killed_point.Z },
{ "environment_kill_flag", battle.environment_kill_flag },
};
Rows.Add(row);
insertCount++;
}
public void InsertRow_RecordField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = IdGenerator.FromGuid();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["position"] = new BigQueryInsertRow {
["x"] = 10L, ["y"] = 20L
}
};
_fixture.InsertAndWait(table, () => table.InsertRow(row), 1);
string sql = $"SELECT guid, position.x, position.y FROM {table} WHERE guid=@guid";
var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) };
var resultRows = client.ExecuteQuery(sql, parameters)
.Select(r => new { Guid = (string)r["guid"], X = (long)r["x"], Y = (long)r["y"] })
.ToList();
var expectedResults = new[]
{
new { Guid = guid, X = 10L, Y = 20L }
};
Assert.Equal(expectedResults, resultRows);
}
public void InsertRow_RepeatedField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = IdGenerator.FromGuid();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["tags"] = new[] { "a", "b" }
};
_fixture.InsertAndWait(table, () => table.InsertRow(row), 1);
string sql = $"SELECT guid, tag FROM {table}, UNNEST(tags) AS tag WHERE guid=@guid ORDER BY tag";
var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) };
var resultRows = client.ExecuteQuery(sql, parameters)
.Select(r => new { Guid = (string)r["guid"], Tag = (string)r["tag"] })
.ToList();
var expectedResults = new[]
{
new { Guid = guid, Tag = "a" },
new { Guid = guid, Tag = "b" }
};
Assert.Equal(expectedResults, resultRows);
}
public void InsertRow_BadData_IgnoreUnknownAndBadRows_Silent()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
// Don't insert into a table used by other tests...
var table = dataset.CreateTable(
_fixture.CreateTableId(),
new TableSchemaBuilder {
{ "year", BigQueryDbType.Int64 }
}.Build());
var rows = new BigQueryInsertRow[]
{
new BigQueryInsertRow {
{ "year", 2019 }
},
new BigQueryInsertRow {
{ "noSuchField", 10 }
},
new BigQueryInsertRow {
{ "year", "Unknown" }
},
};
var options = new InsertOptions {
AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = true
};
// Now two rows are inserted, we are ignoring unknown fields so only the last row is bad.
_fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
}
public void InsertRow_RepeatedField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = Guid.NewGuid().ToString();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["tags"] = new[] { "a", "b" }
};
table.InsertRow(row);
var command = new BigQueryCommand($"SELECT guid, tag FROM {table}, UNNEST(tags) AS tag WHERE guid=@guid ORDER BY tag")
{
Parameters = { { "guid", BigQueryDbType.String, guid } }
};
var resultRows = WaitForRows(client, command)
.Select(r => new { Guid = (string)r["guid"], Tag = (string)r["tag"] })
.ToList();
var expectedResults = new[]
{
new { Guid = guid, Tag = "a" },
new { Guid = guid, Tag = "b" }
};
Assert.Equal(expectedResults, resultRows);
}
public void InsertRow_RecordField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = Guid.NewGuid().ToString();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["position"] = new BigQueryInsertRow {
["x"] = 10L, ["y"] = 20L
}
};
table.InsertRow(row);
var command = new BigQueryCommand($"SELECT guid, position.x, position.y FROM {table} WHERE guid=@guid")
{
Parameters = { { "guid", BigQueryDbType.String, guid } }
};
var resultRows = WaitForRows(client, command)
.Select(r => new { Guid = (string)r["guid"], X = (long)r["x"], Y = (long)r["y"] })
.ToList();
var expectedResults = new[]
{
new { Guid = guid, X = 10L, Y = 20L }
};
Assert.Equal(expectedResults, resultRows);
}
public void TableInsertRows(
string projectId = "your-project-id",
string datasetId = "your_dataset_id",
string tableId = "your_table_id"
)
{
BigQueryClient client = BigQueryClient.Create(projectId);
BigQueryInsertRow[] rows = new BigQueryInsertRow[]
{
// The insert ID is optional, but can avoid duplicate data
// when retrying inserts.
new BigQueryInsertRow(insertId: "row1")
{
{ "name", "Washington" },
{ "post_abbr", "WA" }
},
new BigQueryInsertRow(insertId: "row2")
{
{ "name", "Colorado" },
{ "post_abbr", "CO" }
}
};
client.InsertRows(datasetId, tableId, rows);
}
public override void Add(Log log, ref int insertCount)
{
var battle = log.Battle.damage_history;
var row = new BigQueryInsertRow
{
{ "match_id", battle.match_id },
{ "round_start_datetime", DateTimeParseToUtc(battle.round_start_datetime) },
{ "round_count", battle.round_count },
{ "inflict_damage_team_cd", battle.inflict_damage_team_cd },
{ "inflict_damage_offense_defense_cd", battle.inflict_damage_offense_defense_cd },
{ "inflict_damage_player_id", battle.inflict_damage_player_id.Hash() },
{ "inflict_damage_unit_id", battle.inflict_damage_unit_id },
{ "inflict_damage_armed_id", battle.inflict_damage_armed_id },
{ "take_damage_team_cd", battle.take_damage_team_cd },
{ "take_damage_offense_defense_cd", battle.take_damage_offense_defense_cd },
{ "take_damage_player_id", battle.take_damage_player_id.Hash() },
{ "take_damage_unit_id", battle.take_damage_unit_id },
{ "inflict_damage_quantity", battle.inflict_damage_quantity },
{ "inflict_damage_quantity_shield", battle.inflict_damage_quantity_shield },
};
Rows.Add(row);
insertCount++;
}
public override void Add(Log log, ref int insertCount)
{
var battle = log.Battle.match_use_armed_history;
var row = new BigQueryInsertRow
{
{ "match_id", battle.match_id },
{ "team_cd", battle.team_cd },
{ "round_start_datetime", DateTimeParseToUtc(battle.round_start_datetime) },
{ "round_count", battle.round_count },
{ "offense_defense_cd", battle.offense_defense_cd },
{ "player_id", battle.player_id.Hash() },
{ "unit_id", battle.unit_id },
{ "armed_id", battle.armed_id },
{ "use_armed_count", battle.use_armed_count },
{ "bullet_fire_count", battle.bullet_fire_count },
{ "bullet_hit_count", battle.bullet_hit_count },
{ "bullet_headshot_count", battle.bullet_headshot_count },
{ "kill_count", battle.kill_count },
{ "heal_quantity", battle.heal_quantity },
{ "block_damage_quantity", battle.block_damage_quantity },
{ "brake_shield_count", battle.brake_shield_count },
{ "using_armed_time_second", battle.using_armed_time_second },
{ "brake_placement_object_count", battle.brake_placement_object_count },
{ "heal_assist_count", battle.heal_assist_count },
{ "enhance_assist_count", battle.enhance_assist_count },
{ "weakness_assist_count", battle.weakness_assist_count },
};
Rows.Add(row);
insertCount++;
}
public void InsertRow_RecordRepeatedField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = IdGenerator.FromGuid();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["job"] = new BigQueryInsertRow {
["company"] = "Pet Store", ["roles"] = new[] { "cashier", "manager" }
},
};
_fixture.InsertAndWait(table, () => table.InsertRow(row), 1);
string sql = $"SELECT job FROM {table} WHERE guid=@guid";
var parameters = new[] { new BigQueryParameter("guid", BigQueryDbType.String, guid) };
var fetchedRow = client.ExecuteQuery(sql, parameters)
.Single();
var job = (Dictionary <string, object>)fetchedRow["job"];
Assert.Equal("Pet Store", (string)job["company"]);
Assert.Equal(new[] { "cashier", "manager" }, (string[])job["roles"]);
}
public void InsertRow_RecordRepeatedField()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.ComplexTypesTableId);
var guid = Guid.NewGuid().ToString();
var row = new BigQueryInsertRow
{
["guid"] = guid,
["job"] = new BigQueryInsertRow {
["company"] = "Pet Store", ["roles"] = new[] { "cashier", "manager" }
},
};
table.InsertRow(row);
var command = new BigQueryCommand($"SELECT job FROM {table} WHERE guid=@guid")
{
Parameters = { { "guid", BigQueryDbType.String, guid } }
};
var fetchedRow = WaitForRows(client, command).Single();
var job = (Dictionary <string, object>)fetchedRow["job"];
Assert.Equal("Pet Store", (string)job["company"]);
Assert.Equal(new[] { "cashier", "manager" }, (string[])job["roles"]);
}
public void RepeatedValue_NullRejectedOnConversion()
{
var row = new BigQueryInsertRow {
{ "names", new[] { "a", null, "b" } }
};
Assert.Throws <InvalidOperationException>(() => row.ToRowsData());
}
public void RepeatedValue()
{
var row = new BigQueryInsertRow {
{ "numbers", new[] { 1, 2 } }
};
var rowData = row.ToRowsData();
Assert.Equal(new object[] { 1, 2 }, rowData.Json["numbers"]);
}
private void AssertInvalid(string name, object value)
{
var row = new BigQueryInsertRow();
Assert.Throws <ArgumentException>(() => row.Add(name, value));
Assert.Throws <ArgumentException>(() => row[name] = value);
Assert.Throws <ArgumentException>(() => row.Add(new Dictionary <string, object> {
{ name, value }
}));
}
public void GetEnumerator()
{
var row = new BigQueryInsertRow
{
{ "field1", 10 },
{ "field2", "text" }
};
Assert.Equal(2, row.Cast <object>().Count());
}
public void TimespanFormatting()
{
var row = new BigQueryInsertRow
{
{ "field", new TimeSpan(1, 2, 3) },
};
var rowData = row.ToRowsData();
Assert.Equal("01:02:03", rowData.Json["field"]);
}
public void DateTimeFormatting()
{
var row = new BigQueryInsertRow
{
{ "field", new DateTime(2000, 1, 1, 5, 0, 0, DateTimeKind.Utc) },
};
var rowData = row.ToRowsData();
Assert.Equal("2000-01-01 05:00:00Z", rowData.Json["field"]);
}
public void Numeric_Json()
{
object value = BigQueryNumeric.Parse("123.456");
var row = new BigQueryInsertRow {
{ "field", value }
};
var rowData = row.ToRowsData();
Assert.Equal("123.456", rowData.Json["field"]);
}
public void SupportedValueTypes_Passthrough(Type type)
{
object value = Activator.CreateInstance(type);
var row = new BigQueryInsertRow {
{ "field", value }
};
var rowData = row.ToRowsData();
Assert.Equal(value, rowData.Json["field"]);
}
public void UnspecifiedDateTimeFormatting()
{
var row = new BigQueryInsertRow
{
{ "field", new DateTime(2000, 1, 1, 5, 0, 0, DateTimeKind.Unspecified) },
};
var rowData = row.ToRowsData(false);
Assert.Equal("2000-01-01T05:00:00", rowData.Json["field"]);
}
public void DateTimeOffsetFormatting()
{
var row = new BigQueryInsertRow
{
// 3am UTC
{ "field", new DateTimeOffset(2000, 1, 1, 5, 0, 0, TimeSpan.FromHours(2)) },
};
var rowData = row.ToRowsData();
Assert.Equal("2000-01-01 03:00:00Z", rowData.Json["field"]);
}
public void InsertRow_BadData()
{
var client = BigQueryClient.Create(_fixture.ProjectId);
var dataset = client.GetDataset(_fixture.DatasetId);
var table = dataset.GetTable(_fixture.HighScoreTableId);
var row = new BigQueryInsertRow {
{ "noSuchField", 10 }
};
Assert.Throws <GoogleApiException>(() => table.InsertRow(row));
}
public void AddDictionary()
{
var dictionary = new Dictionary <string, object> {
{ "field1", "value1" }
};
var row = new BigQueryInsertRow {
dictionary
};
Assert.Equal("value1", row["field1"]);
}