Task IAsyncCollector <GoogleBigQueryRow> .FlushAsync(CancellationToken cancellationToken)
{
var tasks = new List <Task>();
if (items.Count > 0)
{
Type itemType = items.First().GetType();
var bqService = new BigQueryService(googleBigQueryAttribute, itemType);
//items without date
{
var rows = items.Where(c => !c.Date.HasValue);
tasks.Add(bqService.InsertRowsAsync(rows, cancellationToken));
}
//items with date
{
var groups = items.Where(c => c.Date.HasValue).GroupBy(c => c.Date.Value.Date);
foreach (var group in groups)
{
tasks.Add(bqService.InsertRowsAsync(group.Key, group, cancellationToken));
}
}
}
return(Task.WhenAll(tasks)
.ContinueWith((allTasks) =>
{
if (allTasks.IsFaulted)
{
throw allTasks.Exception.InnerException;
}
}));
}
public static BigQueryService GetPublisherClient(GoogleBigQueryAttribute googleBigQueryAttribute, Type itemType)
{
var key = $"{googleBigQueryAttribute.GetHashCode()}-{itemType.GetType().FullName}".GetHashCode();
if (publisherClientCache.ContainsKey(key))
{
var expiringBigQueryService = publisherClientCache[key];
if ((DateTime.UtcNow - expiringBigQueryService.CreatedUtc).TotalHours > 1)
{
var bigQueryService = new BigQueryService(googleBigQueryAttribute, itemType);
var expiringBigQueryService1 = new ExpiringBigQueryService(DateTime.UtcNow, bigQueryService);
publisherClientCache.AddOrUpdate(key, expiringBigQueryService1, (newkey, oldValue) => expiringBigQueryService1);
return(bigQueryService);
}
return(expiringBigQueryService.BigQueryService);
}
else
{
var bigQueryService = new BigQueryService(googleBigQueryAttribute, itemType);
var expiringBigQueryService = new ExpiringBigQueryService(DateTime.UtcNow, bigQueryService);
publisherClientCache.AddOrUpdate(key, expiringBigQueryService, (newkey, oldValue) => expiringBigQueryService);
return(bigQueryService);
}
}
public Task CreateTableAsync <T>(bool timePartitioning, CancellationToken cancellationToken)
{
var service = new BigQueryService(googleBigQueryManagementAttribute, typeof(T));
return(service.CreateTableAsync(timePartitioning, cancellationToken));
}
public Task DeleteTableAsync(string tableName, CancellationToken cancellationToken)
{
var service = new BigQueryService(googleBigQueryManagementAttribute, null);
return(service.DeleteTableAsync(tableName, cancellationToken));
}
public ExpiringBigQueryService(DateTime createdUtc, BigQueryService bigQueryService)
{
CreatedUtc = createdUtc;
BigQueryService = bigQueryService;
}
