private static void Dispose()
            {
                Log.LogMessage("Cleaning up...");

                if (_context != null)
                {
                    _context.Dispose();
                    _context = null;
                }

                if (_csvWriter != null)
                {
                    _csvWriter.Dispose();
                    _csvWriter = null;
                }

                if (_log != null)
                {
                    _log.Dispose();
                    _log = null;
                }

                if (_cursor != null)
                {
                    _cursor.Dispose();
                    _cursor = null;
                }

                if (_httpClient != null)
                {
                    _httpClient.Dispose();
                    _httpClient = null;
                }
            }
Ejemplo n.º 2
0
        private async Task CommitBatch(EntitiesContext context, FileCursor cursor, Logger logger, DateTime?cursorTime)
        {
            logger.Log("Committing batch...");

            var count = await context.SaveChangesAsync();

            if (cursorTime.HasValue)
            {
                await cursor.Write(cursorTime.Value);
            }

            logger.Log($"{count} packages saved.");
        }
            private static void Initialize(string metadataFileName, string connectionString)
            {
                var metadataFileReader = new StreamReader(metadataFileName);

                var configuration = new CsvHelper.Configuration.Configuration()
                {
                    HasHeaderRecord = false
                };

                configuration.RegisterClassMap <RepositoryMetadataLogMap>();
                _csvReader = new CsvReader(metadataFileReader, configuration);

                _context = new EntitiesContext(connectionString, readOnly: false);
                _cursor  = new FileCursor(CursorFileName);
                _log     = new Log(ErrorsFileName);
            }
            private static async Task Initialize(string connectionString, Uri serviceDiscoveryUri)
            {
                Log.LogMessage("Initializing");

                _context = new EntitiesContext(connectionString, readOnly: true);

                var metadataFileStreamWriter = new StreamWriter(RepositoryMetadataFileName, append: true);

                metadataFileStreamWriter.AutoFlush = true;
                metadataFileStreamWriter.BaseStream.Seek(0, SeekOrigin.End);
                _csvWriter = new CsvWriter(metadataFileStreamWriter);

                _log    = new Log(ErrorsFileName);
                _cursor = new FileCursor(CursorFileName);

                _httpClient       = new HttpClient();
                _flatContainerUri = await GetFlatContainerUriAsync(serviceDiscoveryUri);
            }
Ejemplo n.º 5
0
        public async override Task Run()
        {
            var token = new CancellationToken();

            try
            {
                var regionInformations = _configuration.RegionInformations;
                var instances          = new List <Instance>();

                foreach (var regionInformation in regionInformations)
                {
                    instances.AddRange(await _searchServiceClient.GetSearchEndpointsAsync(regionInformation, token));
                }

                var maxCommit = DateTimeOffset.MinValue;

                foreach (Instance instance in instances)
                {
                    try
                    {
                        var commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token);

                        maxCommit = commitDateTime > maxCommit ? commitDateTime : maxCommit;
                    }
                    catch (Exception e)
                    {
                        Logger.LogError("An exception was encountered so no HTTP response was returned. {Exception}", e);
                    }
                }

                if (maxCommit == DateTimeOffset.MinValue)
                {
                    Logger.LogError("Failed to retrieve a proper starting commit. Abandoning the current run.");
                    return;
                }

                var catalogLeafProcessor = new PackageLagCatalogLeafProcessor(instances, _searchServiceClient, _telemetryService, LoggerFactory.CreateLogger <PackageLagCatalogLeafProcessor>());
                if (_configuration.RetryLimit > 0)
                {
                    catalogLeafProcessor.RetryLimit = _configuration.RetryLimit;
                }

                if (_configuration.WaitBetweenRetrySeconds > 0)
                {
                    catalogLeafProcessor.WaitBetweenPolls = TimeSpan.FromSeconds(_configuration.WaitBetweenRetrySeconds);
                }

                var settings = new CatalogProcessorSettings
                {
                    ServiceIndexUrl           = _configuration.ServiceIndexUrl,
                    DefaultMinCommitTimestamp = maxCommit,
                    ExcludeRedundantLeaves    = false
                };

                var start = new FileCursor("cursor.json", LoggerFactory.CreateLogger <FileCursor>());
                await start.SetAsync(maxCommit.AddTicks(1));

                var catalogProcessor = new CatalogProcessor(start, _catalogClient, catalogLeafProcessor, settings, LoggerFactory.CreateLogger <CatalogProcessor>());

                bool success;
                int  retryCount = 0;
                do
                {
                    success = await catalogProcessor.ProcessAsync();

                    if (!success || !await catalogLeafProcessor.WaitForProcessing())
                    {
                        retryCount++;
                        Logger.LogError("Processing the catalog leafs failed. Retry Count {CatalogProcessRetryCount}", retryCount);
                    }
                }while (!success && retryCount < MAX_CATALOG_RETRY_COUNT);

                return;
            }
            catch (Exception e)
            {
                Logger.LogError("Exception Occured. {Exception}", e);
                return;
            }
        }
Ejemplo n.º 6
0
        public async Task Update(SqlConnection connection, string fileName)
        {
            if (!File.Exists(fileName))
            {
                throw new ArgumentException($"File '{fileName}' doesn't exist");
            }

            using (var context = new EntitiesContext(connection, readOnly: false))
                using (var cursor = new FileCursor(CursorFileName))
                    using (var logger = new Logger(ErrorsFileName))
                    {
                        var startTime = await cursor.Read();

                        logger.Log($"Starting database update - Cursor time: {startTime:u}");

                        var repository = new EntityRepository <Package>(context);

                        var packages = repository.GetAll().Include(p => p.PackageRegistration);

                        using (var csv = CreateCsvReader(fileName))
                        {
                            var counter         = 0;
                            var lastCreatedDate = default(DateTime?);

                            var result = await TryReadMetadata(csv);

                            while (result.Success)
                            {
                                var metadata = result.Metadata;

                                if (metadata.Created >= startTime)
                                {
                                    var package = packages.FirstOrDefault(p => p.PackageRegistration.Id == metadata.Id && p.NormalizedVersion == metadata.Version);

                                    if (package != null)
                                    {
                                        UpdatePackage(package, metadata.Metadata, context);
                                        logger.LogPackage(metadata.Id, metadata.Version, "Metadata updated.");

                                        counter++;

                                        if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created)
                                        {
                                            lastCreatedDate = metadata.Created;
                                        }
                                    }
                                    else
                                    {
                                        await logger.LogPackageError(metadata.Id, metadata.Version, "Could not find package in the database.");
                                    }
                                }

                                if (counter >= UpdateBatchSize)
                                {
                                    await CommitBatch(context, cursor, logger, metadata.Created);

                                    counter = 0;
                                }

                                result = await TryReadMetadata(csv);
                            }

                            if (counter > 0)
                            {
                                await CommitBatch(context, cursor, logger, lastCreatedDate);
                            }
                        }
                    }
        }
Ejemplo n.º 7
0
        public async Task Collect(SqlConnection connection, Uri serviceDiscoveryUri, DateTime?lastCreateTime, string fileName)
        {
            using (var context = new EntitiesContext(connection, readOnly: true))
                using (var cursor = new FileCursor(CursorFileName))
                    using (var logger = new Logger(ErrorsFileName))
                    {
                        context.SetCommandTimeout(300); // large query

                        var startTime = await cursor.Read();

                        logger.Log($"Starting metadata collection - Cursor time: {startTime:u}");

                        var repository = new EntityRepository <Package>(context);

                        var packages = repository.GetAll()
                                       .Include(p => p.PackageRegistration);
                        if (QueryIncludes != null)
                        {
                            packages = packages.Include(QueryIncludes);
                        }

                        packages = packages
                                   .Where(p => p.Created <lastCreateTime && p.Created> startTime)
                                   .Where(p => p.PackageStatusKey == PackageStatus.Available)
                                   .OrderBy(p => p.Created);
                        if (LimitTo > 0)
                        {
                            packages = packages.Take(LimitTo);
                        }

                        var flatContainerUri = await GetFlatContainerUri(serviceDiscoveryUri);

                        using (var csv = CreateCsvWriter(fileName))
                            using (var http = new HttpClient())
                            {
                                // We want these downloads ignored by stats pipelines - this user agent is automatically skipped.
                                // See https://github.com/NuGet/NuGet.Jobs/blob/262da48ed05d0366613bbf1c54f47879aad96dcd/src/Stats.ImportAzureCdnStatistics/StatisticsParser.cs#L41
                                http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",
                                                                                   "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)   Backfill Job: NuGet.Gallery GalleryTools");

                                var counter         = 0;
                                var lastCreatedDate = default(DateTime?);

                                foreach (var package in packages)
                                {
                                    var id             = package.PackageRegistration.Id;
                                    var version        = package.NormalizedVersion;
                                    var idLowered      = id.ToLowerInvariant();
                                    var versionLowered = version.ToLowerInvariant();

                                    try
                                    {
                                        var metadata = default(TMetadata);

                                        var nuspecUri =
                                            $"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.nuspec";
                                        using (var nuspecStream = await http.GetStreamAsync(nuspecUri))
                                        {
                                            var document = LoadDocument(nuspecStream);

                                            var nuspecReader = new NuspecReader(document);

                                            if (SourceType == MetadataSourceType.NuspecOnly)
                                            {
                                                metadata = ReadMetadata(nuspecReader);
                                            }
                                            else if (SourceType == MetadataSourceType.Nupkg)
                                            {
                                                var nupkgUri =
                                                    $"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.{versionLowered}.nupkg";
                                                metadata = await FetchMetadataAsync(http, nupkgUri, nuspecReader, id, version, logger);
                                            }
                                        }

                                        if (ShouldWriteMetadata(metadata))
                                        {
                                            var record = new PackageMetadata(id, version, metadata, package.Created);

                                            csv.WriteRecord(record);

                                            await csv.NextRecordAsync();

                                            logger.LogPackage(id, version, $"Metadata saved");
                                        }
                                    }
                                    catch (Exception e)
                                    {
                                        await logger.LogPackageError(id, version, e);
                                    }

                                    counter++;

                                    if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created)
                                    {
                                        lastCreatedDate = package.Created;
                                    }

                                    if (counter >= CollectBatchSize)
                                    {
                                        logger.Log($"Writing {package.Created:u} to cursor...");
                                        await cursor.Write(package.Created);

                                        counter = 0;
                                    }
                                }

                                if (counter > 0 && lastCreatedDate.HasValue)
                                {
                                    await cursor.Write(lastCreatedDate.Value);
                                }
                            }
                    }
        }
Ejemplo n.º 8
0
        public async Task Collect(string connectionString, Uri serviceDiscoveryUri, DateTime?lastCreateTime, string fileName)
        {
            using (var context = new EntitiesContext(connectionString, readOnly: true))
                using (var cursor = new FileCursor(CursorFileName))
                    using (var logger = new Logger(ErrorsFileName))
                    {
                        var startTime = await cursor.Read();

                        logger.Log($"Starting metadata collection - Cursor time: {startTime:u}");

                        var repository = new EntityRepository <Package>(context);

                        var packages = repository.GetAll()
                                       .Include(p => p.PackageRegistration)
                                       .Where(p => p.Created <lastCreateTime && p.Created> startTime)
                                       .Where(p => p.PackageStatusKey == PackageStatus.Available || p.PackageStatusKey == PackageStatus.Validating)
                                       .OrderBy(p => p.Created);

                        var flatContainerUri = await GetFlatContainerUri(serviceDiscoveryUri);

                        using (var csv = CreateCsvWriter(fileName))
                            using (var http = new HttpClient())
                            {
                                var counter         = 0;
                                var lastCreatedDate = default(DateTime?);

                                foreach (var package in packages)
                                {
                                    var id      = package.PackageRegistration.Id;
                                    var version = package.NormalizedVersion;

                                    var nuspecUri = $"{flatContainerUri}/{id.ToLowerInvariant()}/{version.ToLowerInvariant()}/{id.ToLowerInvariant()}.nuspec";

                                    try
                                    {
                                        var metadata = await FetchMetadata(http, nuspecUri);

                                        if (ShouldWriteMetadata(metadata))
                                        {
                                            var record = new PackageMetadata(id, version, metadata, package.Created);

                                            csv.WriteRecord(record);

                                            await csv.NextRecordAsync();

                                            logger.LogPackage(id, version, "Metadata saved.");
                                        }
                                    }
                                    catch (Exception e)
                                    {
                                        await logger.LogPackageError(id, version, e);
                                    }

                                    counter++;

                                    if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created)
                                    {
                                        lastCreatedDate = package.Created;
                                    }

                                    if (counter >= CollectBatchSize)
                                    {
                                        logger.Log($"Writing {package.Created:u} to cursor...");
                                        await cursor.Write(package.Created);

                                        counter = 0;
                                    }
                                }

                                if (counter > 0 && lastCreatedDate.HasValue)
                                {
                                    await cursor.Write(lastCreatedDate.Value);
                                }
                            }
                    }
        }