private static async Task <PartProgress> ProcessMetas(string connectionString, string tagIdName, string tableName, string retentionPolicy, string measurement, TagTypes tagType, string defaultTime, bool doAdHocResampling, bool skipLastPoint, int period, List <TsMetadata> metas, int batchSize, int parallelism, InfluxQueryExecutor influxQueryExecutor, EntityCreator entityCreator, int previouslyProcessedTags, string statusMessage, int count, int totalCount, TraceWriter log) { var progress = new PartProgress(); if (metas.Count == 0) { return(progress); } var batchBuilder = new StringBuilder(); var entities = await LoadEntities(retentionPolicy, measurement, tagType, defaultTime, doAdHocResampling, skipLastPoint, period, metas, batchBuilder, batchSize, parallelism, progress, influxQueryExecutor, entityCreator, log); while (progress.Skip < metas.Count || entities.Count > 0) { Task bulkInsertTask = Task.CompletedTask; if (entities.Count > 0) { log.Info($"Entities Created: {entities.Count}"); progress.ProcessedPoints += entities.Count; bulkInsertTask = Timer.Time(() => SqlBulkInserter.BulkInsert(connectionString, entities, tagIdName, tableName), $"Bulk insert of {entities.Count}", log); } else { log.Info($"No Entities Found."); } Task <List <TS> > entitiesTask = Task.FromResult(new List <TS>()); if (progress.Skip < metas.Count) { entitiesTask = LoadEntities(retentionPolicy, measurement, tagType, defaultTime, doAdHocResampling, skipLastPoint, period, metas, batchBuilder, batchSize, parallelism, progress, influxQueryExecutor, entityCreator, log); } await Task.WhenAll(bulkInsertTask, entitiesTask); entities = entitiesTask.Result; log.Info($"{progress.ProcessedTags}/{count} processed of {statusMessage}."); log.Info($"{progress.ProcessedTags + previouslyProcessedTags}/{totalCount} processed of total."); } return(progress); }
public void Add(PartProgress partProgress) { ProcessedTags += partProgress.ProcessedTags; ProcessedPoints += partProgress.ProcessedPoints; }
private static async Task <List <TS> > LoadEntities(string retentionPolicy, string measurement, TagTypes tagType, string defaultTime, bool doAdHocResampling, bool skipLastPoint, int period, List <TsMetadata> metas, StringBuilder batchBuilder, int batchSize, int parallelism, PartProgress progress, InfluxQueryExecutor influxQueryExecutor, EntityCreator entityCreator, TraceWriter log) { var tasks = new List <Task <string> >(parallelism); var tags = new List <string>(); for (int a = 0; a < parallelism; a++) { if (progress.Skip >= metas.Count) { break; } var batch = metas.Skip(progress.Skip).Take(batchSize); progress.Skip += batchSize; foreach (var tagMeta in batch) { string tag = TagCleaner.Clean(tagMeta.Tag); tags.Add(tagMeta.Tag + " cleaned: " + tag); string query = QueryBuilder.CreateMainQuery( retentionPolicy, measurement, tagType, tagMeta.Watermark, tagMeta.LastTimestampBeforeWatermark, tagMeta.LastTimestampAfterWatermark, defaultTime, tag, doAdHocResampling); batchBuilder.Append(query); progress.ProcessedTags++; } //tasks.Add(Timer.Time(() => influxQueryExecutor.Query(batchBuilder.ToString(), log), $"Querying influx #{a}", log)); tasks.Add(influxQueryExecutor.Query(batchBuilder.ToString(), log)); batchBuilder.Clear(); } try { await Timer.Time(() => Task.WhenAll(tasks), "Getting aggregates", log); } catch (Exception ex) { //log.Error("One of these tags failed:\r\n" + string.Join("\r\n", tags), ex); throw; } var metasDict = metas.ToDictionary(k => k.Tag, v => v); List <TS> entities = Timer.TimeSync(() => entityCreator.CreateEntities <TS>(tasks.Select(t => t.Result), metasDict, period, skipLastPoint), "Deserializing resultsets and creating entities", log); return(entities); }