public Logger(params AbstractWriter[] writers) { Contract.Requires(writers != null); Contract.Requires(writers.Length >= 1); cts = new CancellationTokenSource(); bufferBlocks = new List <BufferBlock <LogItem> >(); writerBlocks = new List <ActionBlock <LogItem[]> >(); foreach (var writer in writers) { writer.Setup(); var bufferBlock = new BufferBlock <LogItem>( new DataflowBlockOptions() { CancellationToken = cts.Token }); var batchBlock = new BatchBlock <LogItem>( writer.BatchSize, new GroupingDataflowBlockOptions() { Greedy = true, CancellationToken = cts.Token }); var timer = new Timer(state => batchBlock.TriggerBatch()); var timeoutBlock = new TransformBlock <LogItem, LogItem>( value => { timer.Change((int)writer.Timeout.TotalMilliseconds, Timeout.Infinite); return(value); }, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token, SingleProducerConstrained = true }); var writerBlock = new ActionBlock <LogItem[]>( async logItems => { await writer.WriteAsync(logItems); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, CancellationToken = cts.Token, SingleProducerConstrained = true }); writerBlock.Completion.ContinueWith(task => writer.Teardown()); timeoutBlock.LinkTo(batchBlock); bufferBlock.LinkTo(timeoutBlock); batchBlock.LinkTo(writerBlock); writerBlocks.Add(writerBlock); bufferBlocks.Add(bufferBlock); timeoutBlock.HandleCompletion(batchBlock); bufferBlock.HandleCompletion(timeoutBlock); batchBlock.HandleCompletion(writerBlock); } }
public static async void Fetch() { long fetched = 0; long skipped = 0; long errored = 0; var startedOn = DateTime.UtcNow; var imageRegex = new Regex("(?<=<IMG\\sSRC=\").*?(?=\"\\s.*?>)", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.CultureInvariant | RegexOptions.Compiled); List <Link> links; try { links = await GetLinks(); Status.Info.Log("Parsed {0:N0} URIs", links.Count); if (Properties.Settings.Default.MaxToFetch > 0) { links = links.Take(Properties.Settings.Default.MaxToFetch).ToList(); } Status.Info.Log("Queued {0:N0} images to be downloaded", links.Count); } catch (Exception error) { Status.BadGetUrls.Log(error.Message); PressAnyKeyToTerminate(); return; } var scraper = new TransformBlock <Link, Link>( async link => { try { Status.Scraping.Log(link.PageUri.AbsoluteUri); var html = await new HttpClient().GetStringAsync(link.PageUri); var src = imageRegex.Match(html).Value; if (string.IsNullOrWhiteSpace(src)) { Interlocked.Increment(ref skipped); Status.NoImage.Log(link.PageUri.AbsoluteUri); return(null); } link.ImageUri = new Uri(Properties.Settings.Default.BaseUri, src); Status.Scraped.Log(link.PageUri.AbsoluteUri); return(link); } catch (Exception error) { Interlocked.Increment(ref errored); Status.BadScrape.Log(error.Message); return(null); } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 2 }); var fetcher = new ActionBlock <Link>( async link => { try { var fileName = Path.Combine(Properties.Settings.Default.SaveToPath, Path.GetFileName(link.ImageUri.AbsoluteUri).ToLower()); var nameOnly = Path.GetFileName(fileName); if (!Properties.Settings.Default.OverwriteAll) { if (File.Exists(fileName)) { Interlocked.Increment(ref skipped); Status.DupImage.Log("{0:MM/dd/yyyy} - {1}", link.Date, nameOnly); return; } } Status.Fetching.Log("{0:MM/dd/yyyy} - {1}", link.Date, nameOnly); await link.ImageUri.Download(fileName); Interlocked.Increment(ref fetched); Status.Fetched.Log("{0:MM/dd/yyyy} - {1}", link.Date, nameOnly); } catch (Exception error) { Interlocked.Increment(ref errored); Status.BadFetch.Log(error.Message); } }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); scraper.LinkTo(fetcher, link => link != null); scraper.LinkTo(DataflowBlock.NullTarget <Link>()); scraper.HandleCompletion(fetcher); Status.Info.Log("Fetching APOD's archive list"); links.ForEach(link => scraper.Post(link)); scraper.Complete(); try { await fetcher.Completion; Status.Finished.Log("Fetched: {0:N0}, Skipped: {1:N0}, Errors: {2:N0}, Seconds: {3:N2}", fetched, skipped, errored, (DateTime.UtcNow - startedOn).TotalMilliseconds / 1000.0); } catch (AggregateException errors) { foreach (var error in errors.InnerExceptions) { Status.Failure.Log(error.Message); } } catch (TaskCanceledException) { Status.Cancelled.Log("The process was manually cancelled!"); } catch (Exception error) { Status.Failure.Log(error.Message); } PressAnyKeyToTerminate(); }