Example #1
0
        private async Task StrikeAsync()
        {
            _log.WriteLine("Start lightning strike for {0}...", _cursorFile);

            // Get batch range
            int batchStart;
            int batchEnd;

            using (var cursorStreamReader = new StreamReader(_cursorFile))
            {
                var batchRange = (await cursorStreamReader.ReadLineAsync()).Split(',');
                batchStart = int.Parse(batchRange[0]);
                batchEnd   = int.Parse(batchRange[1]);

                _log.WriteLine("Batch range: {0} - {1}", batchStart, batchEnd);
            }
            if (batchStart > batchEnd)
            {
                _log.WriteLine("Batch already finished.");
                return;
            }

            // Time to strike
            var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose);
            var collectorHttpClient       = new CollectorHttpClient(httpMessageHandlerFactory());
            var serviceProvider           = GetServiceProvider();
            var catalogClient             = serviceProvider.GetRequiredService <ICatalogClient>();
            var registrationUpdater       = serviceProvider.GetRequiredService <IRegistrationUpdater>();

            var startElement = string.Format("Element@{0}.", batchStart);
            var endElement   = string.Format("Element@{0}.", batchEnd + 1);

            using (var indexStreamReader = new StreamReader(_indexFile))
            {
                string line;

                // Skip entries that are not in the current batch bounds
                do
                {
                    line = await indexStreamReader.ReadLineAsync();
                }while (!line.Contains(startElement));

                // Run until we're outside the current batch bounds
                while (!string.IsNullOrEmpty(line) && !line.Contains(endElement) && !indexStreamReader.EndOfStream)
                {
                    _log.WriteLine(line);

                    try
                    {
                        var packageId = line.Split(new[] { ". " }, StringSplitOptions.None).Last().Trim();

                        IStrike strike;
                        switch (_driver)
                        {
                        case JsonDriver:
                            strike = new JsonStrike(catalogClient, registrationUpdater, packageId);
                            break;

                        default:
                            strike = new GraphStrike(this, collectorHttpClient, packageId);
                            break;
                        }

                        line = await indexStreamReader.ReadLineAsync();

                        while (!string.IsNullOrEmpty(line) && !line.Contains("Element@"))
                        {
                            var url = line.TrimEnd();
                            await strike.ProcessCatalogLeafUrlAsync(url);

                            // Read next line
                            line = await indexStreamReader.ReadLineAsync();
                        }

                        await strike.FinishAsync();

                        // Update cursor file so next time we have less work to do
                        batchStart++;
                        await UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd);
                    }
                    catch (Exception)
                    {
                        UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd).Wait();
                        throw;
                    }
                }
            }

            await UpdateCursorFileAsync("DONE" + _cursorFile, batchStart, batchEnd);

            _log.WriteLine("Finished lightning strike for {0}.", _cursorFile);
        }
Example #2
0
        private async Task StrikeAsync()
        {
            _log.WriteLine("Start lightning strike for {0}...", _cursorFile);

            // Get batch range
            int batchStart;
            int batchEnd;

            using (var cursorStreamReader = new StreamReader(_cursorFile))
            {
                var batchRange = (await cursorStreamReader.ReadLineAsync()).Split(',');
                batchStart = int.Parse(batchRange[0]);
                batchEnd   = int.Parse(batchRange[1]);

                _log.WriteLine("Batch range: {0} - {1}", batchStart, batchEnd);
            }
            if (batchStart > batchEnd)
            {
                _log.WriteLine("Batch already finished.");
                return;
            }

            // Time to strike
            var container           = GetAutofacContainer();
            var catalogClient       = container.Resolve <ICatalogClient>();
            var registrationUpdater = container.Resolve <IRegistrationUpdater>();

            var startElement = string.Format("Element@{0}.", batchStart);
            var endElement   = string.Format("Element@{0}.", batchEnd + 1);

            using (var indexStreamReader = new StreamReader(_indexFile))
            {
                string line;

                // Skip entries that are not in the current batch bounds
                do
                {
                    line = await indexStreamReader.ReadLineAsync();
                }while (!line.Contains(startElement));

                // Run until we're outside the current batch bounds
                while (!string.IsNullOrEmpty(line) && !line.Contains(endElement) && !indexStreamReader.EndOfStream)
                {
                    _log.WriteLine(line);

                    try
                    {
                        var packageId = line.Split(new[] { ". " }, StringSplitOptions.None).Last().Trim();

                        IStrike strike = new JsonStrike(catalogClient, registrationUpdater, packageId);

                        line = await indexStreamReader.ReadLineAsync();

                        while (!string.IsNullOrEmpty(line) && !line.Contains("Element@"))
                        {
                            var url = line.TrimEnd();
                            await strike.ProcessCatalogLeafUrlAsync(url);

                            // Read next line
                            line = await indexStreamReader.ReadLineAsync();
                        }

                        await strike.FinishAsync();

                        // Update cursor file so next time we have less work to do
                        batchStart++;
                        await UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd);
                    }
                    catch (Exception)
                    {
                        UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd).Wait();
                        throw;
                    }
                }
            }

            await UpdateCursorFileAsync("DONE" + _cursorFile, batchStart, batchEnd);

            _log.WriteLine("Finished lightning strike for {0}.", _cursorFile);
        }