protected override void Handle(TrafficEvent t)
{
//初始化第一个
if (_trafficEvent.DateTime == DateTime.MinValue)
{
if (t.DateTime != DateTime.MinValue)
{
_trafficEvent.DateTime = t.DateTime;
_trafficEvent.EndTime = t.DateTime;
_trafficEvent.DataId = t.DataId;
_insertBlock?.Post(new TrafficEvent
{
DateTime = new DateTime(_trafficEvent.DateTime.Year, _trafficEvent.DateTime.Month, _trafficEvent.DateTime.Day, _trafficEvent.DateTime.Hour, _trafficEvent.DateTime.Minute, _trafficEvent.DateTime.Second),
EndTime = null,
DataId = _trafficEvent.DataId
});
_logger.LogDebug((int)LogEvent.事件数据块, $"初始化事件 {t.DataId} {t.DateTime}");
}
}
else if (_trafficEvent.EndTime.HasValue)
{
//30秒内只处理有效数据
if ((t.DateTime - _trafficEvent.EndTime.Value).TotalSeconds <= 30)
{
if (t.DateTime != DateTime.MinValue)
{
_logger.LogDebug((int)LogEvent.事件数据块, $"更新结束时间 {t.DataId} {_trafficEvent.DateTime} {_trafficEvent.EndTime}->{t.DateTime}");
_trafficEvent.EndTime = t.DateTime;
}
}
//30秒外入库
//然后保存新数据
else
{
TrafficEvent trafficEvent = new TrafficEvent
{
DateTime = new DateTime(_trafficEvent.DateTime.Year, _trafficEvent.DateTime.Month, _trafficEvent.DateTime.Day, _trafficEvent.DateTime.Hour, _trafficEvent.DateTime.Minute, _trafficEvent.DateTime.Second),
EndTime = new DateTime(_trafficEvent.EndTime.Value.Year, _trafficEvent.EndTime.Value.Month, _trafficEvent.EndTime.Value.Day, _trafficEvent.EndTime.Value.Hour, _trafficEvent.EndTime.Value.Minute, _trafficEvent.EndTime.Value.Second),
DataId = _trafficEvent.DataId
};
_logger.LogDebug((int)LogEvent.事件数据块, $"确认事件 {trafficEvent.DataId} {trafficEvent.DateTime} {trafficEvent.EndTime}");
_updateBlock?.Post(trafficEvent);
_trafficEvent.DateTime = t.DateTime;
_trafficEvent.EndTime = t.DateTime;
_insertBlock?.Post(new TrafficEvent
{
DateTime = new DateTime(_trafficEvent.DateTime.Year, _trafficEvent.DateTime.Month, _trafficEvent.DateTime.Day, _trafficEvent.DateTime.Hour, _trafficEvent.DateTime.Minute, _trafficEvent.DateTime.Second),
EndTime = null,
DataId = _trafficEvent.DataId
});
_logger.LogDebug((int)LogEvent.事件数据块, $"初始化事件 {t.DataId} {t.DateTime}");
}
}
}
public virtual void Log(Action <StringBuilder> buildAction, [CallerMemberName] string member = null, [CallerLineNumber] int line = -1, [CallerFilePath] string filePath = null)
{
var sb = new StringBuilder();
var msg = this.GetIoCManager().DefualtContainer.Resolve <LoggingMessage>();
msg.StringBuilderAction = buildAction;
msg.FilePath = filePath;
msg.Line = line;
msg.Member = member;
msg.MessageResult = null;
msg.Level = LoggerBase <TLevel> .GetOrCreateCachedString(Level);
_startBlock?.Post(msg);
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var line in _rawLines)
{
target.Post(new GraphiteLine(line.Name, line.Value, line.Timestamp ?? Epoch));
}
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var gauge in Gauges)
{
target.Post(new GraphiteLine(RootNamespace + gauge.Key, gauge.Value, Epoch));
}
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var set in Sets)
{
foreach (var item in set.Value)
{
target.Post(new GraphiteLine(RootNamespace + set.Key, 1, Epoch));
}
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var latency in Latencies)
{
var lines = MakeGraphiteLines(latency);
foreach (var line in lines)
{
target.Post(line);
}
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary<string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + ( percentileName ?? ( "p" + percentile ) );
var incoming = new ActionBlock<StatsdMessage>( p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return new DatapointBox(maxItemsPerBucket, latency.ValueMS);
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return bag;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary<int, bool>();
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
},
(key, setDict) =>
{
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
int percentileValue;
foreach (var measurements in Timings)
{
if (TryComputePercentile(measurements, out percentileValue))
{
target.Post(
new GraphiteLine(
RootNamespace + measurements.Key + PercentileName,
percentileValue,
Epoch));
}
}
}
private void DownloadPage(ITargetBlock<Parse> targetAscTransform)
{
foreach (var q in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true))
{
var cars = _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id));
var emailss = cars.Where(e => e.HtmlDownloaded == false);
foreach (var car in cars)
{
var parse = new Parse
{
CarId = car.Id,
Link = car.Link,
UserAgent = _dataAccess.GetRandomUserAgent()
};
targetAscTransform.Post(parse);
}
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
IIntervalService intervalService)
{
var rawLines = new ConcurrentStack<Raw>();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
rawLines.Push(p as Raw);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (rawLines.Count == 0)
{
return;
}
var lines = rawLines.ToArray();
rawLines.Clear();
var bucket = new RawBucket(lines, e.Epoch);
target.Post(bucket);
};
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var counter = p as Counter;
counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
private static void PostOrThrow <T>(this ITargetBlock <T> actionBlock, T input)
{
bool success = actionBlock.Post(input);
Contract.Assert(success, "Could not post to ActionBlock");
}
public void GetPostData()
{
try
{
_webHelper.WaitForElement(By.CssSelector(".eo2As"), 2000);
if (MultiSrcPostChevron != null)
{
if (VideoSourceClass.Any())
{
foreach (var webElement in VideoSourceClass)
{
_tempLinkList.Add(webElement.GetAttribute("src"));
}
}
else if (ImageSourceClass.Any())
{
foreach (var webElement in ImageSourceClass)
{
if (!webElement.GetAttribute("srcset").Contains("1080w"))
{
continue;
}
var stringList = webElement.GetAttribute("srcset").Split(',');
var index = Array.FindIndex(stringList, row => row.Contains("1080w"));
_tempLinkList.Add(stringList[index].Remove(stringList[index].Length - 6));
}
}
MultiSrcPostChevron.Click();
GetPostData();
}
else
{
if (VideoSourceClass.Any())
{
foreach (var webElement in VideoSourceClass)
{
_tempLinkList.Add(webElement.GetAttribute("src"));
}
}
else if (ImageSourceClass.Any())
{
foreach (var webElement in ImageSourceClass)
{
if (!webElement.GetAttribute("srcset").Contains("1080w"))
{
continue;
}
var stringList = webElement.GetAttribute("srcset").Split(',');
var index = Array.FindIndex(stringList, row => row.Contains("1080w"));
_tempLinkList.Add(stringList[index].Remove(stringList[index].Length - 6));
}
}
_tempLinkList = _tempLinkList.Distinct().ToList();
var timeStamp = _webHelper.RefineTimeStamp();
if (_targetText != null)
{
GetComments(timeStamp);
}
for (var i = 0; i < _tempLinkList.Count; i++)
{
_targetMedia.Post(new KeyValuePair <string, string>(timeStamp + " " + (_tempLinkList.Count - i),
_tempLinkList[i]));
}
if (NextPostPaginationArrow != null)
{
NextPostPaginationArrow.Click();
_tempLinkList.Clear();
if (_targetText != null)
{
new PostPage(_driver, _targetMedia).GetPostData();
}
else
{
new PostPage(_driver, _targetMedia, _targetText).GetPostData();
}
}
else
{
_targetMedia.Complete();
_targetText?.Complete();
Logger.Info("Finished capture of post data");
}
}
}
catch (StaleElementReferenceException)
{
Logger.Error("Stale Element, Retrying");
GetPostData();
}
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception?exception, Func <TState, Exception?, string> formatter)
{
_targetBlock.Post($"{Name}: " + formatter(state, exception));
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<string, int>>();
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, bool>();
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
}
);
}
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, double>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, double>();
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
}
);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
private void DownloadPage(ITargetBlock<Parse> targetAscTransform)
{
var totalPages = 0;
foreach (var parse in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true).Select(q => _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id))).SelectMany(cars => cars.Select(car => new Parse
{
CarId = car.Id,
Link = car.Link,
UserAgent = _dataAccess.GetRandomUserAgent()
})))
{
targetAscTransform.Post(parse);
totalPages++;
}
Logger.Debug("Total for this run is {0}", totalPages);
}
public void PostFile(string file, ITargetBlock <string> batchReadBlock)
{
batchReadBlock.Post(file);
}
private async Task ProduceObjects(IAsyncEnumerable <IUser> users, ITargetBlock <IUser> target)
{
await users.ForEachAsync(t => target.Post(t));
target.Complete();
}
private async Task ProduceObjects(IAsyncEnumerable <IGroup> groups, ITargetBlock <IGroup> target)
{
await groups.ForEachAsync(t => target.Post(t));
target.Complete();
}
public bool Post(Task <string> readTask)
{
return(_firstBlock.Post(readTask));
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary <string, ConcurrentDictionary <MetricInfo, double> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var blockKey = new MetricInfo(metricName, calendargram.Tags);
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary <MetricInfo, double>();
window.AddOrUpdate(blockKey, (key2) => 1, (key2, oldValue) => 1);
return(window);
},
(key, window) =>
{
window.AddOrUpdate(blockKey, (key2) => 1, (key2, oldValue) => 1);
return(window);
}
);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary <MetricInfo, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary <MetricInfo, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Name.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
var newBlockKey = new MetricInfo(metricName, metricsAndValues[index].Key.Tags);
if (metrics.ContainsKey(newBlockKey))
{
metrics[newBlockKey] += 1;
}
else
{
metrics[newBlockKey] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return(new KeyValuePair <MetricInfo, double>(
metric.Key,
metric.Value
));
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
/// <summary>
/// Send message to pipeline
/// </summary>
/// <param name="value">message to send</param>
/// <returns>true if message is sent</returns>
public bool Post(T value)
{
ITargetBlock <T> target = Root as ITargetBlock <T> ?? throw new InvalidOperationException("First block is not a target to send");
return(target.Post(value));
}
public bool Post(OperationMessage message) => _startBlock.Post(message);
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <CounterBucket> target,
string rootNamespace,
IIntervalService intervalService)
{
var sets = new ConcurrentDictionary <string, ConcurrentDictionary <string, int> >();
var windows = new ConcurrentDictionary <string, ConcurrentDictionary <string, bool> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary <string, bool>();
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return(window);
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return(window);
}
);
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary <String, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary <String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return(new KeyValuePair <string, double>(
metric.Key,
metric.Value
));
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static void PostAssert <T>(ITargetBlock <T> target, T item)
{
bool retval = target.Post(item);
Contracts.Assert(retval);
}
public bool Post(OperationMessage message)
{
return(_startBlock.Post(message));
}