public async Task RunWithoutChannelsAsync()
{
// 1. Read file
// 2. Download license
// 3. Check content for known words
// 4. Write to other file
var stopwatch = new Stopwatch();
stopwatch.Start();
using var reader = new StreamReader(File.OpenRead("Licenses/input.txt"));
await using var writer = new StreamWriter(File.OpenWrite("Licenses/withoutchannels-output.txt"));
string?url;
while ((url = await reader.ReadLineAsync()) != null)
{
url = url.Trim();
_outputHelper.WriteLine("Checking {0}...", url);
var contents = await _httpClient.GetStringOrNullAsync(url);
var licenseIdentifier = LicenseStrings.TryIdentify(contents ?? string.Empty);
await writer.WriteLineAsync($"{url};{licenseIdentifier}");
}
stopwatch.Stop();
_outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
}
public async Task RunWithChannelsAsync()
{
// 1. Read file
// 2. Download license
// 3. Check content for known words
// 4. Write to other file
var stopwatch = new Stopwatch();
stopwatch.Start();
await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannelextensions-output.txt"));
await Channel
.CreateBounded <string>(50000)
.Source(ReadInput())
.PipeAsync(
maxConcurrency: 100,
capacity: 500,
transform: async url =>
{
var contents = await _httpClient.GetStringOrNullAsync(url);
return(url, contents);
})
.Pipe(
maxConcurrency: 10,
capacity: 50000,
transform: tuple =>
{
var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty);
return($"{tuple.Item1};{licenseIdentifier}");
})
.ReadAllAsync(async line =>
{
await writer.WriteLineAsync(line);
});
stopwatch.Stop();
_outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
}
public async Task RunWithChannelsAsync()
{
// 1. Read file
// 2. Download license
// 3. Check content for known words
// 4. Write to other file
var downloadUriChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000)
{
SingleReader = false, SingleWriter = true
});
var processContentChannel = Channel.CreateBounded <(string, string?)>(new BoundedChannelOptions(100)
{
SingleReader = false, SingleWriter = false
});
var writeOutputChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000)
{
SingleReader = true, SingleWriter = false
});
var stopwatch = new Stopwatch();
stopwatch.Start();
var tasks = new List <Task>();
// 1. Read file
tasks.Add(Wrap(async() =>
{
using var reader = new StreamReader(File.OpenRead("Licenses/input.txt"));
string?url;
while ((url = await reader.ReadLineAsync()) != null)
{
await downloadUriChannel.Writer.WriteAsync(url.Trim());
}
downloadUriChannel.Writer.TryComplete();
}));
// 2. Download license
var downloadTasks = new List <Task>();
for (var i = 0; i < 100; i++)
{
downloadTasks.Add(Wrap(async() =>
{
while (await downloadUriChannel.Reader.WaitToReadAsync())
{
while (downloadUriChannel.Reader.TryRead(out var url))
{
var contents = await _httpClient.GetStringOrNullAsync(url);
await processContentChannel.Writer.WriteAsync((url, contents));
}
}
}));
}
tasks.Add(Wrap(async() =>
{
await Task.WhenAll(downloadTasks);
processContentChannel.Writer.TryComplete();
}));
// 3. Check for known words
var processTasks = new List <Task>();
for (var i = 0; i < 10; i++)
{
processTasks.Add(Wrap(async() =>
{
while (await processContentChannel.Reader.WaitToReadAsync())
{
while (processContentChannel.Reader.TryRead(out var tuple))
{
var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty);
await writeOutputChannel.Writer.WriteAsync($"{tuple.Item1};{licenseIdentifier}");
}
}
}));
}
tasks.Add(Wrap(async() =>
{
await Task.WhenAll(processTasks);
writeOutputChannel.Writer.TryComplete();
}));
// 4. Write to output file
tasks.Add(Wrap(async() =>
{
await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannels-output.txt"));
while (await writeOutputChannel.Reader.WaitToReadAsync())
{
while (writeOutputChannel.Reader.TryRead(out var line))
{
await writer.WriteLineAsync(line);
}
}
}));
await Task.WhenAll(tasks);
stopwatch.Stop();
_outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
}