Beispiel #1
0
        public async Task RunWithoutChannelsAsync()
        {
            // 1. Read file
            // 2. Download license
            // 3. Check content for known words
            // 4. Write to other file

            var stopwatch = new Stopwatch();

            stopwatch.Start();

            using var reader       = new StreamReader(File.OpenRead("Licenses/input.txt"));
            await using var writer = new StreamWriter(File.OpenWrite("Licenses/withoutchannels-output.txt"));

            string?url;

            while ((url = await reader.ReadLineAsync()) != null)
            {
                url = url.Trim();

                _outputHelper.WriteLine("Checking {0}...", url);

                var contents = await _httpClient.GetStringOrNullAsync(url);

                var licenseIdentifier = LicenseStrings.TryIdentify(contents ?? string.Empty);

                await writer.WriteLineAsync($"{url};{licenseIdentifier}");
            }

            stopwatch.Stop();
            _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
        }
Beispiel #2
0
        public async Task RunWithChannelsAsync()
        {
            // 1. Read file
            // 2. Download license
            // 3. Check content for known words
            // 4. Write to other file

            var stopwatch = new Stopwatch();

            stopwatch.Start();

            await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannelextensions-output.txt"));

            await Channel
            .CreateBounded <string>(50000)
            .Source(ReadInput())
            .PipeAsync(
                maxConcurrency: 100,
                capacity: 500,
                transform: async url =>
            {
                var contents = await _httpClient.GetStringOrNullAsync(url);
                return(url, contents);
            })
            .Pipe(
                maxConcurrency: 10,
                capacity: 50000,
                transform: tuple =>
            {
                var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty);
                return($"{tuple.Item1};{licenseIdentifier}");
            })
            .ReadAllAsync(async line =>
            {
                await writer.WriteLineAsync(line);
            });

            stopwatch.Stop();
            _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
        }
        public async Task RunWithChannelsAsync()
        {
            // 1. Read file
            // 2. Download license
            // 3. Check content for known words
            // 4. Write to other file

            var downloadUriChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000)
            {
                SingleReader = false, SingleWriter = true
            });
            var processContentChannel = Channel.CreateBounded <(string, string?)>(new BoundedChannelOptions(100)
            {
                SingleReader = false, SingleWriter = false
            });
            var writeOutputChannel = Channel.CreateBounded <string>(new BoundedChannelOptions(50000)
            {
                SingleReader = true, SingleWriter = false
            });

            var stopwatch = new Stopwatch();

            stopwatch.Start();

            var tasks = new List <Task>();

            // 1. Read file
            tasks.Add(Wrap(async() =>
            {
                using var reader = new StreamReader(File.OpenRead("Licenses/input.txt"));

                string?url;
                while ((url = await reader.ReadLineAsync()) != null)
                {
                    await downloadUriChannel.Writer.WriteAsync(url.Trim());
                }

                downloadUriChannel.Writer.TryComplete();
            }));

            // 2. Download license
            var downloadTasks = new List <Task>();

            for (var i = 0; i < 100; i++)
            {
                downloadTasks.Add(Wrap(async() =>
                {
                    while (await downloadUriChannel.Reader.WaitToReadAsync())
                    {
                        while (downloadUriChannel.Reader.TryRead(out var url))
                        {
                            var contents = await _httpClient.GetStringOrNullAsync(url);
                            await processContentChannel.Writer.WriteAsync((url, contents));
                        }
                    }
                }));
            }
            tasks.Add(Wrap(async() =>
            {
                await Task.WhenAll(downloadTasks);
                processContentChannel.Writer.TryComplete();
            }));

            // 3. Check for known words
            var processTasks = new List <Task>();

            for (var i = 0; i < 10; i++)
            {
                processTasks.Add(Wrap(async() =>
                {
                    while (await processContentChannel.Reader.WaitToReadAsync())
                    {
                        while (processContentChannel.Reader.TryRead(out var tuple))
                        {
                            var licenseIdentifier = LicenseStrings.TryIdentify(tuple.Item2 ?? string.Empty);
                            await writeOutputChannel.Writer.WriteAsync($"{tuple.Item1};{licenseIdentifier}");
                        }
                    }
                }));
            }
            tasks.Add(Wrap(async() =>
            {
                await Task.WhenAll(processTasks);
                writeOutputChannel.Writer.TryComplete();
            }));

            // 4. Write to output file
            tasks.Add(Wrap(async() =>
            {
                await using var writer = new StreamWriter(File.OpenWrite("Licenses/withchannels-output.txt"));

                while (await writeOutputChannel.Reader.WaitToReadAsync())
                {
                    while (writeOutputChannel.Reader.TryRead(out var line))
                    {
                        await writer.WriteLineAsync(line);
                    }
                }
            }));

            await Task.WhenAll(tasks);

            stopwatch.Stop();
            _outputHelper.WriteLine("Completed in {0}", stopwatch.Elapsed);
        }