public Task InsertRowsAsync(IEnumerable <GoogleBigQueryRow> rows, CancellationToken cancellationToken)
{
if (rows != null && rows.Count() > 0)
{
return(GetTableAsync(cancellationToken)
.ContinueWith((tableTask) =>
{
BigQueryTable table = tableTask.Result;
var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties)).ToArray();
return table.InsertRowsAsync(bigQueryRows, new InsertOptions()
{
AllowUnknownFields = false
}, cancellationToken)
.ContinueWith((insertRowsTask) =>
{
if (insertRowsTask.IsFaulted)
{
throw insertRowsTask.Exception.InnerExceptions.First();
}
});
}, cancellationToken).Unwrap());
}
return(Task.CompletedTask);
}
public async Task InsertRows(BigQueryInsertRow[] rows, CancellationToken ct)
{
try
{
await _table.InsertRowsAsync(
rows,
new InsertOptions
{
AllowUnknownFields = true,
},
cancellationToken : ct
);
}
catch (Exception ex)
{
throw new BigQuerierException($"Failed to insert row to {_table.FullyQualifiedId}", ex);
}
}
public async Task SaveEventBatch(IList <OutputEvent> outputEvent)
{
var insertList = outputEvent.Select(x => new BigQueryInsertRow()
{
{ "botIdentifier", x.botIdentifier },
{ "ownerIdentity", x.ownerIdentity },
{ "identity", x.identity },
{ "messageId", x.messageId },
{ "storageDate", x.storageDate },
{ "category", x.category },
{ "action", x.action },
{ "extras", x.extras },
{ "externalId", x.externalId },
{ "group", x.group },
{ "source", x.source },
{ "value", x.value },
{ "label", x.label }
}).ToList();
await _table.InsertRowsAsync(insertList);
_logger.LogInformation("Inserted to Messages. Qt: {Qt}", insertList.Count);
}
public async Task SaveMessageBatch(IList <OutputMessage> ouputMessage)
{
var insertList = ouputMessage.Select(x => new BigQueryInsertRow()
{
{ "botIdentifier", x.botIdentifier },
{ "type", x.type },
{ "id", x.id },
{ "from", x.from },
{ "to", x.to },
{ "metadata", x.metadata },
{ "content", x.content },
{ "target", x.target },
{ "uri", x.uri },
{ "previewUri", x.previewUri },
{ "title", x.title },
{ "text", x.text },
{ "storageDate", x.storageDate },
{ "direction", x.direction }
}).ToList();
await _table.InsertRowsAsync(insertList);
_logger.LogInformation("Inserted to Messages. Qt: {Qt}", insertList.Count);
}
public Task InsertRowsAsync(DateTime date, IEnumerable <GoogleBigQueryRow> rows, CancellationToken cancellationToken)
{
if (rows != null && rows.Count() > 0)
{
int dateDiff = (date - DateTime.UtcNow.Date).Days;
if (dateDiff >= -31 && dateDiff <= 16)
{
var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties));
return(GetTableAsync(date, cancellationToken)
.ContinueWith((tableTask) =>
{
BigQueryTable table = tableTask.Result;
return table.InsertRowsAsync(bigQueryRows, new InsertOptions()
{
AllowUnknownFields = true
}, cancellationToken)
.ContinueWith((insertRowsTask) =>
{
if (insertRowsTask.IsFaulted)
{
throw insertRowsTask.Exception.InnerExceptions.First();
}
});
}, cancellationToken).Unwrap());
}
else
{
throw new ArgumentOutOfRangeException("BigQuery streamming API don't allow to write data in DAY partioned tabled outside 31 days in the past and 16 days in the future.");
}
}
return(Task.CompletedTask);
}