Task IAsyncCollector <GoogleBigQueryRow> .FlushAsync(CancellationToken cancellationToken) { var tasks = new List <Task>(); if (items.Count > 0) { Type itemType = items.First().GetType(); var bqService = new BigQueryService(googleBigQueryAttribute, itemType); //items without date { var rows = items.Where(c => !c.Date.HasValue); tasks.Add(bqService.InsertRowsAsync(rows, cancellationToken)); } //items with date { var groups = items.Where(c => c.Date.HasValue).GroupBy(c => c.Date.Value.Date); foreach (var group in groups) { tasks.Add(bqService.InsertRowsAsync(group.Key, group, cancellationToken)); } } } return(Task.WhenAll(tasks) .ContinueWith((allTasks) => { if (allTasks.IsFaulted) { throw allTasks.Exception.InnerException; } })); }
public static BigQueryService GetPublisherClient(GoogleBigQueryAttribute googleBigQueryAttribute, Type itemType) { var key = $"{googleBigQueryAttribute.GetHashCode()}-{itemType.GetType().FullName}".GetHashCode(); if (publisherClientCache.ContainsKey(key)) { var expiringBigQueryService = publisherClientCache[key]; if ((DateTime.UtcNow - expiringBigQueryService.CreatedUtc).TotalHours > 1) { var bigQueryService = new BigQueryService(googleBigQueryAttribute, itemType); var expiringBigQueryService1 = new ExpiringBigQueryService(DateTime.UtcNow, bigQueryService); publisherClientCache.AddOrUpdate(key, expiringBigQueryService1, (newkey, oldValue) => expiringBigQueryService1); return(bigQueryService); } return(expiringBigQueryService.BigQueryService); } else { var bigQueryService = new BigQueryService(googleBigQueryAttribute, itemType); var expiringBigQueryService = new ExpiringBigQueryService(DateTime.UtcNow, bigQueryService); publisherClientCache.AddOrUpdate(key, expiringBigQueryService, (newkey, oldValue) => expiringBigQueryService); return(bigQueryService); } }
public Task CreateTableAsync <T>(bool timePartitioning, CancellationToken cancellationToken) { var service = new BigQueryService(googleBigQueryManagementAttribute, typeof(T)); return(service.CreateTableAsync(timePartitioning, cancellationToken)); }
public Task DeleteTableAsync(string tableName, CancellationToken cancellationToken) { var service = new BigQueryService(googleBigQueryManagementAttribute, null); return(service.DeleteTableAsync(tableName, cancellationToken)); }
public ExpiringBigQueryService(DateTime createdUtc, BigQueryService bigQueryService) { CreatedUtc = createdUtc; BigQueryService = bigQueryService; }