public async Task RunWithoutChannelsAsync() { // 1. Read file // 2. Download license // 3. Check content for known words // 4. Write to other file var stopwatch = new Stopwatch(); stopwatch.Start(); using var reader = new StreamReader(File.OpenRead("Licenses/input.txt")); await using var writer = new StreamWriter(File.OpenWrite("Licenses/withoutchannels-output.txt")); string?url; while ((url = await reader.ReadLineAsync()) != null) { url = url.Trim(); _outputHelper.WriteLine("Checking {0}...", url); var contents = await _httpClient.GetStringOrNullAsync(url); var licenseIdentifier = LicenseStrings.TryIdentify(contents ?? string.Empty); await writer.WriteLineAsync($"{url};{licenseIdentifier}"); } stopwatch.Stop(); _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed); }
public async Task RunWithChannelsAsync() { // 1. Read file // 2. Download license // 3. Check content for known words // 4. Write to other file var stopwatch = new Stopwatch(); stopwatch.Start(); await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannelextensions-output.txt")); await Channel .CreateBounded <string>(50000) .Source(ReadInput()) .PipeAsync( maxConcurrency: 100, capacity: 500, transform: async url => { var contents = await _httpClient.GetStringOrNullAsync(url); return(url, contents); }) .Pipe( maxConcurrency: 10, capacity: 50000, transform: tuple => { var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty); return($"{tuple.Item1};{licenseIdentifier}"); }) .ReadAllAsync(async line => { await writer.WriteLineAsync(line); }); stopwatch.Stop(); _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed); }
public async Task RunWithChannelsAsync() { // 1. Read file // 2. Download license // 3. Check content for known words // 4. Write to other file var downloadUriChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000) { SingleReader = false, SingleWriter = true }); var processContentChannel = Channel.CreateBounded <(string, string?)>(new BoundedChannelOptions(100) { SingleReader = false, SingleWriter = false }); var writeOutputChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000) { SingleReader = true, SingleWriter = false }); var stopwatch = new Stopwatch(); stopwatch.Start(); var tasks = new List <Task>(); // 1. Read file tasks.Add(Wrap(async() => { using var reader = new StreamReader(File.OpenRead("Licenses/input.txt")); string?url; while ((url = await reader.ReadLineAsync()) != null) { await downloadUriChannel.Writer.WriteAsync(url.Trim()); } downloadUriChannel.Writer.TryComplete(); })); // 2. Download license var downloadTasks = new List <Task>(); for (var i = 0; i < 100; i++) { downloadTasks.Add(Wrap(async() => { while (await downloadUriChannel.Reader.WaitToReadAsync()) { while (downloadUriChannel.Reader.TryRead(out var url)) { var contents = await _httpClient.GetStringOrNullAsync(url); await processContentChannel.Writer.WriteAsync((url, contents)); } } })); } tasks.Add(Wrap(async() => { await Task.WhenAll(downloadTasks); processContentChannel.Writer.TryComplete(); })); // 3. Check for known words var processTasks = new List <Task>(); for (var i = 0; i < 10; i++) { processTasks.Add(Wrap(async() => { while (await processContentChannel.Reader.WaitToReadAsync()) { while (processContentChannel.Reader.TryRead(out var tuple)) { var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty); await writeOutputChannel.Writer.WriteAsync($"{tuple.Item1};{licenseIdentifier}"); } } })); } tasks.Add(Wrap(async() => { await Task.WhenAll(processTasks); writeOutputChannel.Writer.TryComplete(); })); // 4. Write to output file tasks.Add(Wrap(async() => { await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannels-output.txt")); while (await writeOutputChannel.Reader.WaitToReadAsync()) { while (writeOutputChannel.Reader.TryRead(out var line)) { await writer.WriteLineAsync(line); } } })); await Task.WhenAll(tasks); stopwatch.Stop(); _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed); }