public Task InsertRowsAsync(IEnumerable <GoogleBigQueryRow> rows, CancellationToken cancellationToken) { if (rows != null && rows.Count() > 0) { return(GetTableAsync(cancellationToken) .ContinueWith((tableTask) => { BigQueryTable table = tableTask.Result; var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties)).ToArray(); return table.InsertRowsAsync(bigQueryRows, new InsertOptions() { AllowUnknownFields = false }, cancellationToken) .ContinueWith((insertRowsTask) => { if (insertRowsTask.IsFaulted) { throw insertRowsTask.Exception.InnerExceptions.First(); } }); }, cancellationToken).Unwrap()); } return(Task.CompletedTask); }
public async Task InsertRows(BigQueryInsertRow[] rows, CancellationToken ct) { try { await _table.InsertRowsAsync( rows, new InsertOptions { AllowUnknownFields = true, }, cancellationToken : ct ); } catch (Exception ex) { throw new BigQuerierException($"Failed to insert row to {_table.FullyQualifiedId}", ex); } }
public async Task SaveEventBatch(IList <OutputEvent> outputEvent) { var insertList = outputEvent.Select(x => new BigQueryInsertRow() { { "botIdentifier", x.botIdentifier }, { "ownerIdentity", x.ownerIdentity }, { "identity", x.identity }, { "messageId", x.messageId }, { "storageDate", x.storageDate }, { "category", x.category }, { "action", x.action }, { "extras", x.extras }, { "externalId", x.externalId }, { "group", x.group }, { "source", x.source }, { "value", x.value }, { "label", x.label } }).ToList(); await _table.InsertRowsAsync(insertList); _logger.LogInformation("Inserted to Messages. Qt: {Qt}", insertList.Count); }
public async Task SaveMessageBatch(IList <OutputMessage> ouputMessage) { var insertList = ouputMessage.Select(x => new BigQueryInsertRow() { { "botIdentifier", x.botIdentifier }, { "type", x.type }, { "id", x.id }, { "from", x.from }, { "to", x.to }, { "metadata", x.metadata }, { "content", x.content }, { "target", x.target }, { "uri", x.uri }, { "previewUri", x.previewUri }, { "title", x.title }, { "text", x.text }, { "storageDate", x.storageDate }, { "direction", x.direction } }).ToList(); await _table.InsertRowsAsync(insertList); _logger.LogInformation("Inserted to Messages. Qt: {Qt}", insertList.Count); }
public Task InsertRowsAsync(DateTime date, IEnumerable <GoogleBigQueryRow> rows, CancellationToken cancellationToken) { if (rows != null && rows.Count() > 0) { int dateDiff = (date - DateTime.UtcNow.Date).Days; if (dateDiff >= -31 && dateDiff <= 16) { var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties)); return(GetTableAsync(date, cancellationToken) .ContinueWith((tableTask) => { BigQueryTable table = tableTask.Result; return table.InsertRowsAsync(bigQueryRows, new InsertOptions() { AllowUnknownFields = true }, cancellationToken) .ContinueWith((insertRowsTask) => { if (insertRowsTask.IsFaulted) { throw insertRowsTask.Exception.InnerExceptions.First(); } }); }, cancellationToken).Unwrap()); } else { throw new ArgumentOutOfRangeException("BigQuery streamming API don't allow to write data in DAY partioned tabled outside 31 days in the past and 16 days in the future."); } } return(Task.CompletedTask); }