public async Task WorkAsync(
long?lastSubmissionId,
ITargetBlock <Submission> target,
ITargetBlock <CrawlerError> errors,
CancellationToken cancellationToken)
{
try
{
await DoWork(lastSubmissionId, target, errors, cancellationToken);
}
catch (OperationCanceledException e)
{
// save current result
_logger.LogInformation("Crawler cancelled");
target.Complete();
errors.Complete();
}
catch (Exception e)
{
// error in crawling won't break existing records
// so we can just save crawled results.
_logger.LogError(e, "Error occured when crawling zoj, lastSubmissionId {0}", lastSubmissionId);
target.Complete();
errors.Complete();
}
}
internal static bool TestComplete <T>(ITargetBlock <T> target)
{
Console.WriteLine("* TestComplete");
//test multiple call doesn't throw exceptions
target.Complete();
target.Complete();
return(true);
}
/// <summary>
/// Produces stealth LDAP targets
/// </summary>
/// <param name="queue"></param>
/// <returns></returns>
protected override async Task ProduceLdap(ITargetBlock <SearchResultEntry> queue)
{
var token = Helpers.GetCancellationToken();
//If we haven't generated our stealth targets, we'll build it now
if (!_stealthTargetsBuilt)
{
Console.WriteLine("[+] Finding Stealth Targets from LDAP Properties");
Console.WriteLine();
var targetSids = await FindPathTargetSids();
SetStealthTargetSids(targetSids);
_stealthTargetsBuilt = true;
OutputTasks.StartOutputTimer();
//Output our stealth targets to the queue
foreach (var searchResult in Searcher.QueryLdap(Query, Props, SearchScope.Subtree, Options.Instance.SearchBase))
{
if (token.IsCancellationRequested)
{
Console.WriteLine("[-] Terminating Producer as cancellation was requested. Waiting for pipeline to finish");
break;
}
await queue.SendAsync(searchResult);
}
queue.Complete();
}
else
{
// We've already built our stealth targets, and we're doing a loop
OutputTasks.StartOutputTimer();
var targets = new List <SearchResultEntry>();
targets.AddRange(_stealthTargetSids.Values);
if (!Options.Instance.ExcludeDomainControllers)
{
targets.AddRange(DomainControllerSids.Values);
}
foreach (var searchResult in targets)
{
if (token.IsCancellationRequested)
{
break;
}
await queue.SendAsync(searchResult);
}
queue.Complete();
}
}
// Demonstrates the production end of the producer and consumer pattern.
public void Produce(ITargetBlock <byte[]> target)
{
// Create a Random object to generate random data.
Random rand = new Random();
// In a loop, fill a buffer with random data and
// post the buffer to the target block.
for (int i = 0; i < 100; i++)
{
// Create an array to hold random byte data.
byte[] buffer = new byte[1024];
// Fill the buffer with random bytes.
rand.NextBytes(buffer);
// Post the result to the message block.
target.Post(buffer);
//My code for debugging
lock (producedLocker)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Posted the buffer: " + buffer[i]);
Console.ForegroundColor = ConsoleColor.White;
}
}
// Set the target to the completed state to signal to the consumer
// that no more data will be available.
target.Complete();
}
// ITargetBlock: Represents a dataflow block that is a target for data
static void ProduceAsync(ITargetBlock <byte[]> target)
{
// Create a Random object to generate random data.
Random rand = new Random();
// In a loop, fill a buffer with random data and
// post the buffer to target block.
for (int i = 0; i < 100; i++)
{
// Create an array to hold random byte data.
byte[] buffer = new byte[1024];
// Fill the buffer with random bytes.
rand.NextBytes(buffer);
// Post the result to the message block.
target.Post(buffer);
}
// Set the target to the completed state to signal to the consumer
// that no more data will be available.
// Signals to the IDataflowBlock that it should not accept nor
// produce any more messages nor consume any more postponed messages
target.Complete();
}
static void Produce(ITargetBlock <List <Vertex> > target)
{
var nodes = DataAccess.Nodes.GetAll().GetEnumerator();
while (nodes.MoveNext())
{
var buffer = new List <Vertex>();
for (int i = 0; i < 10000; i++)
{
var node = nodes.Current;
var vertex = NodeToVertex(node);
var l = Edges.SourceDictionary.GetValueOrDefault(vertex.Id);
if (l != null)
{
foreach (var sourceEdges in l)
{
var targetNode = DataAccess.Nodes.GetById(sourceEdges.Target);
if (targetNode != null)
{
var targetVertex = NodeToVertex(targetNode);
var relationship = EdgeToRelationShip(sourceEdges, targetVertex);
vertex.AddOutRelationship(relationship);
}
}
}
var r = Edges.TargetDictionary.GetValueOrDefault(vertex.Id);
if (r != null)
{
foreach (var targetEdge in r)
{
var sourceNode = DataAccess.Nodes.GetById(targetEdge.Source);
if (sourceNode != null)
{
var sourceVertex = NodeToVertex(sourceNode);
var relationship = EdgeToRelationShip(targetEdge, sourceVertex);
vertex.AddInRelationship(relationship);
}
}
}
buffer.Add(vertex);
if (!nodes.MoveNext())
{
break;
}
}
target.Post(buffer);
}
// Set the target to the completed state to signal to the consumer
// that no more data will be available.
target.Complete();
}
public static async Task SendAndCompleteAsync <T>(this ITargetBlock <T> target, IEnumerable <T> items, CancellationToken cancelToken = default(CancellationToken))
{
await Task.Yield();
try
{
foreach (var item in items)
{
if (target.Completion.IsFaulted)
{
throw target.Completion.Exception;
}
if (!await target.SendAsync(item, cancelToken))
{
throw new InvalidOperationException();
}
cancelToken.ThrowIfCancellationRequested();
}
target.Complete();
}
catch (Exception ex)
{
target.Fault(ex);
throw;
}
}
public async Task Producer(string category, int pageSize, ITargetBlock <UnexpandedArticle[]> targetBlock)
{
if (string.IsNullOrWhiteSpace(category))
{
throw new ArgumentException(nameof(category));
}
if (targetBlock == null)
{
throw new ArgumentException(nameof(targetBlock));
}
var nextBatch = await _wikiArticle.AlphabeticalList(new ArticleListRequestParameters { Category = Uri.EscapeDataString(category), Limit = pageSize });
bool isNextBatchAvailable;
do
{
await targetBlock.SendAsync(nextBatch.Items);
isNextBatchAvailable = !string.IsNullOrEmpty(nextBatch.Offset);
if (isNextBatchAvailable)
{
nextBatch = await _wikiArticle.AlphabeticalList(new ArticleListRequestParameters
{
Category = category,
Limit = pageSize,
Offset = nextBatch.Offset
});
}
} while (isNextBatchAvailable);
targetBlock.Complete();
}
private ITargetBlock <PhotoContext> CreateCheckDuplicateBlock(ITargetBlock <PhotoContext> analysis, ITargetBlock <PhotoContext> moveToRecycleBin)
{
var checkDuplicateBlock = new ActionBlock <PhotoContext>(context =>
{
var existingEntry = this.mediaItemCollection.FindOne(image => image.Hash == context.Hash);
if (existingEntry == null)
{
Log.Information($"New picture found: {context.Source.FullName}");
analysis.Post(context);
}
else
{
Log.Warning($"Possible duplicate picture found: {context.Source.FullName}. Matches: {existingEntry.FullName}");
moveToRecycleBin.Post(context);
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
checkDuplicateBlock.Completion.ContinueWith(task =>
{
analysis.Complete();
moveToRecycleBin.Complete();
});
return(checkDuplicateBlock);
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
// Completing the output block before the action block means any final messages that are currently being produced
// will not be sent out, which is what we want in this case.
_broadcastBlock?.Complete();
_dataSourceLink?.Dispose();
if (_actionBlock != null)
{
_actionBlock.Complete();
_threadingService.ExecuteSynchronously(async() =>
{
// Wait for any processing to finish so we don't fight over the cookies 🍪
await _actionBlock.Completion;
IVsAsyncFileChangeEx vsAsyncFileChangeEx = await _fileChangeService.GetValueAsync();
// Unsubscribe from all files
foreach (uint cookie in _fileWatcherCookies.Values)
{
await vsAsyncFileChangeEx.UnadviseFileChangeAsync(cookie);
}
});
}
}
base.Dispose(disposing);
}
public void Produce(ITargetBlock <Model[]> target, string fileName)
{
string resourceName = Path.GetFullPath(@"..\..\App_Data\") + fileName;
using (StreamReader reader = new StreamReader(File.OpenRead(resourceName), Encoding.UTF8))
{
CsvReader csv = new CsvReader(reader);
//csvReader.Configuration.BufferSize = 4096;
csv.Configuration.WillThrowOnMissingField = false;
var list = new List <Model>();
while (csv.Read())
{
var record = csv.GetRecord <Model>();
list.Add(record);
if (list.Count >= maxItems)
{
target.Post(list.ToArray());
list.Clear();
}
}
if (list.Count > 0)
{
target.Post(list.ToArray());
list.Clear();
}
}
// Set the target to the completed state to signal to the consumer
// that no more data will be available.
target.Complete();
}
public async Task StopAsync(CancellationToken cancellationToken)
{
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_listenerTask == null)
{
throw new InvalidOperationException("The server is not started");
}
await _transportListener.StopAsync(cancellationToken);
_listenerCts.Cancel();
_consumerBlock.Complete();
await Task.WhenAll(_listenerTask, _consumerBlock.Completion).ConfigureAwait(false);
_listenerCts.Dispose();
_listenerCts = null;
_listenerTask = null;
_consumerBlock = null;
}
finally
{
_semaphore.Release();
}
}
/// <summary>
/// Runs when all DataflowBlocks are finished
/// </summary>
/// <param name="finishedTask">Task representing the state of all DataflowBlocks</param>
private async Task OnAllBlocksFinished(Task finishedTask)
{
//Complete the persist block now all processing
//blocks have finished to ensure that all pending
//actions are persisted
_persistBlock.Complete();
await _persistBlock.Completion;
if (finishedTask.IsFaulted)
{
ServiceRegistration.Get <ILogger>().Error("FanArtActionBlock: Error processing fanart actions", finishedTask.Exception);
// ReSharper disable once AssignNullToNotNullAttribute
_tcs.SetException(finishedTask.Exception);
}
else
{
//Log the cancellation but set the TaskCompletionSource to RanToCompletion to
//avoid throwing exceptions if the Completion Task is awaited
if (finishedTask.IsCanceled)
{
ServiceRegistration.Get <ILogger>().Debug("FanArtActionBlock: Fanart actions cancelled");
}
_tcs.SetResult(null);
}
}
private async Task ProduceObjects(ITargetBlock <User> target)
{
string newDeltaLink = null;
if (this.context.InDelta)
{
if (!this.context.IncomingWatermark.Contains("user"))
{
throw new WarningNoWatermarkException();
}
Watermark watermark = this.context.IncomingWatermark["user"];
if (watermark.Value == null)
{
throw new WarningNoWatermarkException();
}
newDeltaLink = await GraphHelperUsers.GetUsersWithDelta(this.client, watermark.Value, target, this.token);
}
else
{
//await GraphHelperUsers.GetUsers(client, target, context.Token, "displayName", "onPremisesSamAccountName", "id", "userPrincipalName");
newDeltaLink = await GraphHelperUsers.GetUsersWithDelta(this.client, target, this.token, "displayName", "onPremisesSamAccountName", "id", "userPrincipalName");
}
if (newDeltaLink != null)
{
logger.Trace($"Got delta link {newDeltaLink}");
this.context.OutgoingWatermark.Add(new Watermark("user", newDeltaLink, "string"));
}
target.Complete();
}
private async Task ReadAsync(Stream stream, CancellationToken cancellationToken)
{
try
{
while (true)
{
int length;
if (!ReadHeader(stream, out length))
{
break;
}
var buffer = Buffer.ReadFrom(stream, length);
if (buffer.Size != length)
{
throw new IOException("Source stream is not well-formed");
}
await _TargetBlock.SendAsync(buffer, cancellationToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
}
finally
{
_TargetBlock.Complete();
}
}
/// <summary>
/// Handles the completion of all blocks up to this one.
/// </summary>
/// <param name="obj"></param>
private async Task HandleCompletion(Task obj)
{
//Link to null, so that the flow completes
if (_lastTransformerBlockOnCompletion != null)
{
var nullt = DataflowBlock.NullTarget <TOut>();
_lastTransformerBlockOnCompletion.LinkTo(nullt);
}
if (_transformerOnCompletion != null)
{
IEnumerable <TOut> elements = GetCollectedItems();
if (elements != null)
{
//Post all items to the first transformer
foreach (TOut element in elements)
{
if (element == null)
{
continue;
}
_transformerOnCompletion.SendAsync(element).Wait();
}
}
else
{
}
//Set it to complete
_transformerOnCompletion.Complete();
//Wait for the last transformer to complete
_lastTransformerBlockOnCompletion.Completion.Wait();
}
await WaitForContinuingTasks();
}
public void Complete()
{
_gauges.Complete();
_counters.Complete();
_timings.ForEach(p => p.Complete());
_raw.Complete();
}
IDisposable ISourceBlock <T> .LinkTo(ITargetBlock <T> target, DataflowLinkOptions?linkOptions)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
linkOptions ??= new DataflowLinkOptions();
LinkRegistration <T> registration;
lock (this.CompletionLock)
{
if (this.Completion.IsCompleted)
{
var exception = this.Completion.Exception;
if (exception == null)
{
target.Complete();
}
else
{
target.Fault(exception);
}
return(ActionDisposable.Nop);
}
registration = new LinkRegistration <T>(target, linkOptions.MaxMessages, linkOptions.PropagateCompletion, this.HandleUnlink);
this._linkManager.AddLink(registration, linkOptions.Append);
}
this._resetEvent.Set();
return(new ActionDisposable(registration.Unlink));
}
/// <summary>
/// Group delta imports have a few problems that need to be resolved.
/// 1. You can't currently filter on teams-type groups only. You have to get all groups and then filter yourself
/// 2. This isn't so much of a problem apart from that you have to specify your attribute selection on the initial query to include members and owners. This means you get all members and owners for all groups
/// 3. Membership information comes in chunks of 20 members, however chunks for each group can be returned in any order. This breaks the way FIM works, as we would have to hold all group objects in memory, wait to see if duplicates arrive in the stream, merge them, and only once we have all groups with all members, confidently pass them back to the sync engine in one massive batch
/// </summary>
/// <param name="context"></param>
/// <param name="target"></param>
/// <returns></returns>
private async Task ProduceObjectsDelta(ITargetBlock <Group> target)
{
string newDeltaLink;
if (this.context.InDelta)
{
if (!this.context.IncomingWatermark.Contains("group"))
{
throw new WarningNoWatermarkException();
}
Watermark watermark = this.context.IncomingWatermark["group"];
if (watermark.Value == null)
{
throw new WarningNoWatermarkException();
}
newDeltaLink = await GraphHelperGroups.GetGroups(this.client, watermark.Value, target, this.token);
}
else
{
newDeltaLink = await GraphHelperGroups.GetGroups(this.client, target, this.token, "displayName", "resourceProvisioningOptions", "id", "mailNickname", "description", "visibility", "members", "owners");
}
if (newDeltaLink != null)
{
logger.Trace($"Got delta link {newDeltaLink}");
this.context.OutgoingWatermark.Add(new Watermark("group", newDeltaLink, "string"));
}
target.Complete();
}
public CrawlerPropagator(
ITargetBlock <Submission> submissionOutput,
ITargetBlock <CrawlerError> errorOutput)
{
_submissionOutput = submissionOutput;
_errorOutput = errorOutput;
_input = InitInputBlock();
_queue = new Queue <CrawlerMessage>(BufferCapacity);
Completion = _input.Completion.ContinueWith(task =>
{
if (task.IsFaulted)
{
_submissionOutput.Fault(task.Exception);
_errorOutput.Fault(task.Exception);
}
else
{
// TODO: handle cancel
_submissionOutput.Complete();
_errorOutput.Complete();
}
});
}
/// <summary>
/// Adds a target block to send messages to.
/// </summary>
/// <returns>
/// An object that can be used to destroy the link to the added target.
/// </returns>
public IDisposable AddTarget(ITargetBlock <T> targetBlock, DataflowLinkOptions options)
{
CancellationTokenSource cancellationTokenSource = null;
if (options.PropagateCompletion)
{
cancellationTokenSource = new CancellationTokenSource();
block.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
targetBlock.Fault(t.Exception);
}
else
{
targetBlock.Complete();
}
}, cancellationTokenSource.Token);
}
var target = new Target(
this, targetBlock, options.MaxMessages, cancellationTokenSource);
TargetDictionary [targetBlock] = target;
if (options.Append)
{
appendQueue.Enqueue(target);
}
else
{
prependQueue.Enqueue(target);
}
return(target);
}
private async Task ProduceObjects(ITargetBlock <Beta.Group> target)
{
var client = ((GraphConnectionContext)this.context.ConnectionContext).BetaClient;
await GraphHelperGroups.GetGroups(client, target, this.context.ConfigParameters[ConfigParameterNames.FilterQuery].Value, this.token, "displayName", "resourceProvisioningOptions", "id", "mailNickname", "description", "visibility");
target.Complete();
}
static void Produce(int pages, ITargetBlock <Pokemon> target)
{
Parallel.For(1, pages + 1, (i) =>
{
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(Helper.GetLink(i));
var nodes = htmlDoc.DocumentNode.QuerySelectorAll("#cardResults > li a");
Parallel.ForEach(nodes, (item) =>
{
var htmlItem = web.Load(Helper.GetLink(0, item.GetAttributeValue("href", "")));
string description = htmlItem.DocumentNode.QuerySelector(".card-description h1").InnerText;
string expansion = htmlItem.DocumentNode.QuerySelector(".stats-footer a").InnerText;
string expansionNumber = htmlItem.DocumentNode.QuerySelector(".stats-footer span").InnerText;
string urlCardImage = htmlItem.DocumentNode.QuerySelector(".card-image img").GetAttributeValue("src", "");
string base64CardImage = Convert.ToBase64String(new HttpClient().GetByteArrayAsync(urlCardImage).Result);
target.Post(new Pokemon(description, expansionNumber, expansion, base64CardImage));
});
});
target.Complete();
}
/// <summary>
/// Links an action to this block. Usefull if it's an action block.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="target"></param>
/// <returns></returns>
public IFlowBlock BroadcastTo(ITargetBlock <TOut> target, bool linkToCompletion = true)
{
ItemProcessed += (x) =>
{
DataflowBlock.SendAsync(target, x).Wait();
};
if (linkToCompletion)
{
AddCompletionTask(target.Completion);
}
var actionBlock = this.GetProcessingBlock();
actionBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
//_lastLinkedBlock = target;
return(this);
}
/// <inheritdoc cref="ISourceBlock{TOutput}.LinkTo(ITargetBlock{TOutput}, DataflowLinkOptions)"/>
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
linkOptions ??= new DataflowLinkOptions();
LinkRegistration <TOutput> registration;
lock (this.Lock)
{
if (this._propagatedCompletion)
{
var exception = this.Completion.Exception;
if (exception == null)
{
target.Complete();
}
else
{
target.Fault(exception);
}
return(ActionDisposable.Nop);
}
registration = new LinkRegistration <TOutput>(target, linkOptions.MaxMessages, linkOptions.PropagateCompletion, this.HandleUnlink);
this._linkManager.AddLink(registration, linkOptions.Append);
}
this.OfferToTargets();
return(new ActionDisposable(registration.Unlink));
}
private async Task DoWork(
long?lastSubmissionId,
ITargetBlock <Submission> target,
ITargetBlock <CrawlerError> errors,
CancellationToken cancellationToken)
{
var after = lastSubmissionId ?? 1;
while (true)
{
using var json = await Request(after, cancellationToken);
var root = json.RootElement;
if (root.TryGetProperty("error", out _))
{
after = await TrySkipErrorSubmission(after, target, errors, cancellationToken);
}
else
{
after = await ParseSubmissions(
root.GetProperty("submissions").EnumerateArray(),
target, errors);
if (!root.GetProperty("hasAfter").GetBoolean())
{
break;
}
}
}
target.Complete();
errors.Complete();
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_subscription?.Dispose();
_targetBlock.Complete();
}
}
/// <summary>
/// Join both 2 producers - Parse and ReadAsync wait until both sources before saving to db
/// </summary>
/// <param name="target"></param>
/// <returns></returns>
static async Task ProduceAll(ITargetBlock <Match> target)
{
var producer1 = Parse(target);
var producer2 = ReadAsync(target);
await Task.WhenAll(producer1, producer2);
target.Complete();
}
public void OnlyScrapeStory()
{
LoginToAccount();
var storyPage = new ProfilePage(_driver, _scraperOptions.TargetAccount).EnterStory(_targetStory);
storyPage?.SaveStoryContent();
_targetStory.Complete();
}
protected override void Dispose(bool disposing)
{
_subscription?.Dispose();
_targetBlock.Complete();
// Disposed, notify anyone listening that we're never going to be active
OnCanceled();
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary<string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + ( percentileName ?? ( "p" + percentile ) );
var incoming = new ActionBlock<StatsdMessage>( p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return new DatapointBox(maxItemsPerBucket, latency.ValueMS);
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return bag;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary<int, bool>();
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
},
(key, setDict) =>
{
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public LinkFingerprintJoiner(ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> targetBlock)
{
if (null == targetBlock)
throw new ArgumentNullException(nameof(targetBlock));
_targetBlock = targetBlock;
AnnotatedPathsBlock = new ActionBlock<AnnotatedPath[]>(
aps => Task.WhenAll(aps.Select(SetAnnotatedPathAsync)),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
FileFingerprintBlock = new ActionBlock<IFileFingerprint>(SetFileFingerprintAsync,
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
var task = Task.WhenAll(AnnotatedPathsBlock.Completion, FileFingerprintBlock.Completion)
.ContinueWith(_ => { targetBlock.Complete(); });
TaskCollector.Default.Add(task, "Joiner completion");
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var counter = p as Counter;
counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
static Task PostAllFilePathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<AnnotatedPath> filePathTargetBlock, CancellationToken cancellationToken)
{
var scanTasks = paths
.Select<CollectionPath, Task>(path =>
Task.Factory.StartNew(
async () =>
{
foreach (var file in PathUtil.ScanDirectory(path.Path, filePredicate))
{
if (cancellationToken.IsCancellationRequested)
break;
var relativePath = PathUtil.MakeRelativePath(path.Path, file.FullName);
var annotatedPath = new AnnotatedPath { FileInfo = file, Collection = path.Collection ?? path.Path, RelativePath = relativePath };
await filePathTargetBlock.SendAsync(annotatedPath, cancellationToken).ConfigureAwait(false);
}
},
cancellationToken,
TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning | TaskCreationOptions.RunContinuationsAsynchronously,
TaskScheduler.Default));
var task = Task.WhenAll(scanTasks);
var completeTask = task.ContinueWith(_ => filePathTargetBlock.Complete());
TaskCollector.Default.Add(completeTask, "PostAllFilePathsAsync");
return task;
}
async Task TransformAnnotatedPathsToFileFingerprint(ISourceBlock<AnnotatedPath[]> annotatedPathSourceBlock,
ITargetBlock<IFileFingerprint> fileFingerprintTargetBlock, CancellationToken cancellationToken)
{
try
{
var targets = new ConcurrentDictionary<string, TransformBlock<AnnotatedPath, IFileFingerprint>>(StringComparer.InvariantCultureIgnoreCase);
var routeBlock = new ActionBlock<AnnotatedPath[]>(
async filenames =>
{
foreach (var filename in filenames)
{
if (null == filename)
continue;
var cachedBlob = GetCachedFileFingerprint(filename.FileInfo);
if (null != cachedBlob)
{
await fileFingerprintTargetBlock.SendAsync(cachedBlob, cancellationToken).ConfigureAwait(false);
continue;
}
var host = PathUtil.GetHost(filename.FileInfo.FullName);
TransformBlock<AnnotatedPath, IFileFingerprint> target;
while (!targets.TryGetValue(host, out target))
{
target = new TransformBlock<AnnotatedPath, IFileFingerprint>(annotatedPath => ProcessFileAsync(annotatedPath.FileInfo, cancellationToken),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5,
CancellationToken = cancellationToken
});
if (!targets.TryAdd(host, target))
continue;
Debug.WriteLine($"FileFingerprintManager.GenerateBlobsAsync() starting reader for host: '{host}'");
target.LinkTo(fileFingerprintTargetBlock, blob => null != blob);
target.LinkTo(DataflowBlock.NullTarget<IFileFingerprint>());
break;
}
//Debug.WriteLine($"FileFingerprintManager.GenerateFileFingerprintsAsync() Sending {annotatedPath} for host '{host}'");
await target.SendAsync(filename, cancellationToken).ConfigureAwait(false);
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 16, CancellationToken = cancellationToken });
var distinctPaths = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
var distinctBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>(
annotatedPaths =>
{
for (var i = 0; i < annotatedPaths.Length; ++i)
{
if (!distinctPaths.Add(annotatedPaths[i].FileInfo.FullName))
annotatedPaths[i] = null;
}
return annotatedPaths;
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, CancellationToken = cancellationToken });
distinctBlock.LinkTo(routeBlock, new DataflowLinkOptions { PropagateCompletion = true });
annotatedPathSourceBlock.LinkTo(distinctBlock, new DataflowLinkOptions { PropagateCompletion = true });
await routeBlock.Completion.ConfigureAwait(false);
foreach (var target in targets.Values)
target.Complete();
await Task.WhenAll(targets.Values.Select(target => target.Completion));
}
catch (Exception ex)
{
Console.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() failed: " + ex.Message);
}
finally
{
Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() is done");
fileFingerprintTargetBlock.Complete();
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, double>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, double>();
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
}
);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<string, int>>();
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, bool>();
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => true, (key2, oldValue) => true);
return window;
}
);
}
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public async Task PostList(ITargetBlock<Core.Entities.BankAccount> target)
{
Diag.ThreadPrint("PostList - start");
var transform = new TransformBlock<BankAccount, Core.Entities.BankAccount>(ef =>
mapper.Map<Core.Entities.BankAccount>(ef), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 });
transform.LinkTo(target);
await Task.Run(() =>
{
Diag.ThreadPrint("PostList - task start");
using (FinanceEntities context = factory.CreateContext())
{
(from b in context.BankAccounts.Include(a => a.Bank)
select b).AsParallel().ForAll(ef => transform.Post(ef));
//await transform.Completion;
transform.Completion.ContinueWith(t =>
{
if (t.IsFaulted) target.Fault(t.Exception);
else
{
Diag.ThreadPrint("PostList - task set target complete");
target.Complete();
}
});
transform.Complete();
}
Diag.ThreadPrint("PostList - task end");
}).ConfigureAwait(false);
Diag.ThreadPrint("PostList - end");
}
