private static async Task <PartProgress> ProcessMetas(string connectionString, string tagIdName, string tableName, string retentionPolicy,
string measurement, TagTypes tagType, string defaultTime, bool doAdHocResampling, bool skipLastPoint, int period, List <TsMetadata> metas, int batchSize, int parallelism,
InfluxQueryExecutor influxQueryExecutor, EntityCreator entityCreator, int previouslyProcessedTags, string statusMessage, int count, int totalCount, TraceWriter log)
{
var progress = new PartProgress();
if (metas.Count == 0)
{
return(progress);
}
var batchBuilder = new StringBuilder();
var entities = await LoadEntities(retentionPolicy, measurement, tagType, defaultTime, doAdHocResampling, skipLastPoint, period,
metas, batchBuilder, batchSize, parallelism, progress, influxQueryExecutor, entityCreator, log);
while (progress.Skip < metas.Count || entities.Count > 0)
{
Task bulkInsertTask = Task.CompletedTask;
if (entities.Count > 0)
{
log.Info($"Entities Created: {entities.Count}");
progress.ProcessedPoints += entities.Count;
bulkInsertTask = Timer.Time(() => SqlBulkInserter.BulkInsert(connectionString, entities, tagIdName, tableName), $"Bulk insert of {entities.Count}", log);
}
else
{
log.Info($"No Entities Found.");
}
Task <List <TS> > entitiesTask = Task.FromResult(new List <TS>());
if (progress.Skip < metas.Count)
{
entitiesTask = LoadEntities(retentionPolicy, measurement, tagType, defaultTime, doAdHocResampling, skipLastPoint, period,
metas, batchBuilder, batchSize, parallelism, progress, influxQueryExecutor, entityCreator, log);
}
await Task.WhenAll(bulkInsertTask, entitiesTask);
entities = entitiesTask.Result;
log.Info($"{progress.ProcessedTags}/{count} processed of {statusMessage}.");
log.Info($"{progress.ProcessedTags + previouslyProcessedTags}/{totalCount} processed of total.");
}
return(progress);
}
public void Add(PartProgress partProgress)
{
ProcessedTags += partProgress.ProcessedTags;
ProcessedPoints += partProgress.ProcessedPoints;
}
private static async Task <List <TS> > LoadEntities(string retentionPolicy, string measurement, TagTypes tagType, string defaultTime, bool doAdHocResampling, bool skipLastPoint, int period, List <TsMetadata> metas, StringBuilder batchBuilder, int batchSize, int parallelism, PartProgress progress, InfluxQueryExecutor influxQueryExecutor, EntityCreator entityCreator, TraceWriter log)
{
var tasks = new List <Task <string> >(parallelism);
var tags = new List <string>();
for (int a = 0; a < parallelism; a++)
{
if (progress.Skip >= metas.Count)
{
break;
}
var batch = metas.Skip(progress.Skip).Take(batchSize);
progress.Skip += batchSize;
foreach (var tagMeta in batch)
{
string tag = TagCleaner.Clean(tagMeta.Tag);
tags.Add(tagMeta.Tag + " cleaned: " + tag);
string query = QueryBuilder.CreateMainQuery(
retentionPolicy, measurement, tagType, tagMeta.Watermark, tagMeta.LastTimestampBeforeWatermark,
tagMeta.LastTimestampAfterWatermark, defaultTime, tag, doAdHocResampling);
batchBuilder.Append(query);
progress.ProcessedTags++;
}
//tasks.Add(Timer.Time(() => influxQueryExecutor.Query(batchBuilder.ToString(), log), $"Querying influx #{a}", log));
tasks.Add(influxQueryExecutor.Query(batchBuilder.ToString(), log));
batchBuilder.Clear();
}
try
{
await Timer.Time(() => Task.WhenAll(tasks), "Getting aggregates", log);
}
catch (Exception ex)
{
//log.Error("One of these tags failed:\r\n" + string.Join("\r\n", tags), ex);
throw;
}
var metasDict = metas.ToDictionary(k => k.Tag, v => v);
List <TS> entities = Timer.TimeSync(() => entityCreator.CreateEntities <TS>(tasks.Select(t => t.Result), metasDict, period, skipLastPoint), "Deserializing resultsets and creating entities", log);
return(entities);
}
