private async Task ArchiveBlobAsync(ILeasedLogFile logFile) { var stopwatch = Stopwatch.StartNew(); try { await EnsureCopiedToContainerAsync(logFile, _targetContainer); _jobEventSource.FinishingArchiveUpload(logFile.Uri); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob archiving duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedArchiveUpload(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString); _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber); _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform); _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName)); return(true); } catch (Exception exception) { ApplicationInsights.TrackException(exception); Trace.TraceError(exception.ToString()); } return(false); }
private async Task <IReadOnlyCollection <PackageStatistics> > ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } return(packageStatistics); }
public override async Task <bool> Run() { try { var stopwatch = Stopwatch.StartNew(); // build downloads.v1.json var targets = new List <StorageContainerTarget>(); targets.Add(new StorageContainerTarget(_cloudStorageAccount, _statisticsContainerName)); foreach (var dataContainerName in _dataContainerNames) { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } var downloadCountReport = new DownloadCountReport(targets, _statisticsDatabase, _galleryDatabase); await downloadCountReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(DownloadCountReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(DownloadCountReport.ReportName); stopwatch.Restart(); // build stats-totals.json var galleryTotalsReport = new GalleryTotalsReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await galleryTotalsReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(GalleryTotalsReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(GalleryTotalsReport.ReportName); // build tools.v1.json var toolsReport = new DownloadsPerToolVersionReport(_cloudStorageAccount, _statisticsContainerName, _statisticsDatabase, _galleryDatabase); await toolsReport.Run(); stopwatch.Stop(); ApplicationInsights.TrackMetric(DownloadsPerToolVersionReport.ReportName + " Generation Time (ms)", stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackReportProcessed(DownloadsPerToolVersionReport.ReportName); stopwatch.Restart(); return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
public override async Task <bool> Run() { try { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_cloudStorageContainerName); Trace.TraceInformation("Generating reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); if (string.IsNullOrEmpty(_reportName)) { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new ReportBuilder(ReportNames.NuGetClientVersion), new ReportDataCollector(_storedProcedures[ReportNames.NuGetClientVersion], _statisticsDatabase) }, { new ReportBuilder(ReportNames.Last6Months), new ReportDataCollector(_storedProcedures[ReportNames.Last6Months], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularity), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularity], _statisticsDatabase) }, { new ReportBuilder(ReportNames.RecentPopularityDetail), new ReportDataCollector(_storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { await ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report"); } await RebuildPackageReports(destinationContainer, reportGenerationTime); await CleanInactiveRecentPopularityDetailByPackageReports(destinationContainer, reportGenerationTime); } else { // generate only the specific report var reportBuilder = new ReportBuilder(_reportName); var reportDataCollector = new ReportDataCollector(_storedProcedures[_reportName], _statisticsDatabase); await ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } Trace.TraceInformation("Generated reports from {0}/{1} and saving to {2}/{3}", _statisticsDatabase.DataSource, _statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
private async Task <Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile) { var stopwatch = Stopwatch.StartNew(); try { _jobEventSource.BeginningOpenCompressedBlob(logFile.Uri); var memoryStream = new MemoryStream(); // decompress into memory (these are rolling log files and relatively small) using (var blobStream = await logFile.Blob.OpenReadAsync(AccessCondition.GenerateLeaseCondition(logFile.LeaseId), null, null)) { await blobStream.CopyToAsync(memoryStream); memoryStream.Position = 0; } stopwatch.Stop(); _jobEventSource.FinishedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackMetric("Open compressed blob duration (ms)", stopwatch.ElapsedMilliseconds, logFile.Blob.Name); // verify if the stream is gzipped or not if (await IsGzipCompressed(memoryStream)) { return(new GZipInputStream(memoryStream)); } else { return(memoryStream); } } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedOpenCompressedBlob(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } }
private async Task <string> TryAcquireLeaseAsync(ICloudBlob blob) { string leaseId; var blobUriString = blob.Uri.ToString(); try { var sourceBlobExists = await blob.ExistsAsync(); if (!sourceBlobExists) { return(null); } _jobEventSource.BeginningAcquireLease(blobUriString); leaseId = await blob.AcquireLeaseAsync(_defaultLeaseTime, null); _jobEventSource.FinishedAcquireLease(blobUriString); } catch (StorageException storageException) { // check if this is a 409 Conflict with a StatusDescription stating that "There is already a lease present." // or 404 NotFound (might have been removed by another other instance of this job) var webException = storageException.InnerException as WebException; if (webException != null) { var httpWebResponse = webException.Response as HttpWebResponse; if (httpWebResponse != null) { if ((httpWebResponse.StatusCode == HttpStatusCode.Conflict && httpWebResponse.StatusDescription == "There is already a lease present.") || httpWebResponse.StatusCode == HttpStatusCode.NotFound) { _jobEventSource.FailedAcquireLease(blobUriString); // no need to report these in Application Insights return(null); } } } _jobEventSource.FailedAcquireLease(blobUriString); ApplicationInsights.TrackException(storageException); throw; } return(leaseId); }
public async Task <IReadOnlyCollection <ILeasedLogFile> > LeaseNextLogFilesToBeProcessedAsync(string prefix) { try { _jobEventSource.BeginningBlobListing(prefix); var blobResultSegments = await _container.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.None, _maxListBlobResultSegments, null, null, null); _jobEventSource.FinishingBlobListing(prefix); var leasedFiles = new List <ILeasedLogFile>(); foreach (var logFile in blobResultSegments.Results) { if (leasedFiles.Count == _maxLeasesPerJobRun) { break; } // Get the source blob var blobName = logFile.Uri.Segments.Last(); var logFileBlob = _container.GetBlockBlobReference(blobName); // try to acquire a lease on the blob var leaseId = await TryAcquireLeaseAsync(logFileBlob); if (string.IsNullOrEmpty(leaseId)) { // the blob is already leased, ignore it and move on continue; } leasedFiles.Add(new LeasedLogFile(logFileBlob, leaseId)); } return(leasedFiles); } catch (Exception exception) { _jobEventSource.FailedBlobListing(prefix); ApplicationInsights.TrackException(exception); } return(Enumerable.Empty <ILeasedLogFile>().ToList()); }
private async Task DeleteSourceBlobAsync(ILeasedLogFile logFile) { if (await logFile.Blob.ExistsAsync()) { try { _jobEventSource.BeginningDelete(logFile.Uri); var accessCondition = AccessCondition.GenerateLeaseCondition(logFile.LeaseId); await logFile.Blob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, accessCondition, null, null); _jobEventSource.FinishedDelete(logFile.Uri); } catch (Exception exception) { _jobEventSource.FailedDelete(logFile.Uri); ApplicationInsights.TrackException(exception, logFile.Blob.Name); throw; } } }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString); var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); _galleryDatabase = new SqlConnectionStringBuilder(galleryDatabaseConnectionString); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount); _statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); _dataStorageAccount = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount); var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName) .Split(new [] { '|' }, StringSplitOptions.RemoveEmptyEntries); foreach (var containerName in containerNames) { ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName); } _dataContainerNames = containerNames; return(true); } catch (Exception exception) { Trace.TraceError(exception.ToString()); ApplicationInsights.TrackException(exception); return(false); } }
internal async Task InsertDownloadFactsAsync(DataTable downloadFacts, string logFileName) { Trace.WriteLine("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); using (var connection = await _targetDatabase.ConnectTo()) using (var transaction = connection.BeginTransaction(IsolationLevel.Serializable)) { var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, transaction); bulkCopy.DestinationTableName = downloadFacts.TableName; bulkCopy.BulkCopyTimeout = _defaultCommandTimeout; try { await bulkCopy.WriteToServerAsync(downloadFacts); transaction.Commit(); stopwatch.Stop(); ApplicationInsights.TrackMetric("Insert facts duration (ms)", stopwatch.ElapsedMilliseconds, logFileName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } ApplicationInsights.TrackException(exception, logFileName); transaction.Rollback(); throw; } } Trace.Write(" DONE"); }
private async Task <IReadOnlyCollection <T> > GetDimension <T>(string dimension, string logFileName, Func <SqlConnection, Task <IReadOnlyCollection <T> > > retrieve) { var stopwatch = Stopwatch.StartNew(); var count = _maxRetryCount; while (count > 0) { try { _jobEventSource.BeginningRetrieveDimension(dimension); IReadOnlyCollection <T> dimensions; using (var connection = await _targetDatabase.ConnectTo()) { dimensions = await retrieve(connection); } stopwatch.Stop(); _jobEventSource.FinishedRetrieveDimension(dimension, stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackRetrieveDimensionDuration(dimension, stopwatch.ElapsedMilliseconds, logFileName); return(dimensions); } catch (SqlException e) { --count; if (count <= 0) { throw; } if (e.Number == 1205) { Trace.TraceWarning("Deadlock, retrying..."); ApplicationInsights.TrackSqlException("SQL Deadlock", e, logFileName, dimension); } else if (e.Number == -2) { Trace.TraceWarning("Timeout, retrying..."); ApplicationInsights.TrackSqlException("SQL Timeout", e, logFileName, dimension); } else if (e.Number == 2601) { Trace.TraceWarning("Duplicate key, retrying..."); ApplicationInsights.TrackSqlException("SQL Duplicate Key", e, logFileName, dimension); } else { throw; } Task.Delay(_retryDelay).Wait(); } catch (Exception exception) { _jobEventSource.FailedRetrieveDimension(dimension); ApplicationInsights.TrackException(exception, logFileName); if (stopwatch.IsRunning) { stopwatch.Stop(); } throw; } } return(Enumerable.Empty <T>().ToList()); }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var dnxStatistics = new List <DnxStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } if (logEntry.RequestUrl.Contains("dist.asp.net")) { var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry); if (dnxInfo != null) { dnxStatistics.Add(dnxInfo); } } } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics); return(cdnStatistics); }