public MarkClean( IDictionary <string, string> jobArgsDictionary, ILogger <MarkClean> logger, CloudStorageAccount cloudStorageAccount, string containerName, NuGetV2Feed feed, PackageValidationAuditor packageValidationAuditor, string galleryBaseAddress) { _logger = logger; _cloudStorageAccount = cloudStorageAccount; _containerName = containerName; PackageId = JobConfigurationManager.GetArgument(jobArgsDictionary, CommandLineArguments.PackageId); PackageId = HttpUtility.UrlDecode(PackageId); PackageVersion = JobConfigurationManager.GetArgument(jobArgsDictionary, CommandLineArguments.PackageVersion); PackageVersion = HttpUtility.UrlDecode(PackageVersion); var validationIdStr = JobConfigurationManager.GetArgument(jobArgsDictionary, CommandLineArguments.ValidationId); _validationId = Guid.Parse(validationIdStr); _comment = JobConfigurationManager.GetArgument(jobArgsDictionary, CommandLineArguments.Comment); _alias = JobConfigurationManager.GetArgument(jobArgsDictionary, CommandLineArguments.Alias); _feed = feed; _packageValidationAuditor = packageValidationAuditor; _galleryBaseAddress = galleryBaseAddress; }
public VcsCallbackServerStartup() { // Configure to get values from keyvault var configurationService = new ConfigurationService(new SecretReaderFactory()); // Get configuration var cloudStorageAccount = CloudStorageAccount.Parse(configurationService.Get("DataStorageAccount").Result); var containerName = configurationService.Get("ContainerName").Result; string instrumentationKey = configurationService.Get("ApplicationInsightsInstrumentationKey").Result; Services.Logging.ApplicationInsights.Initialize(instrumentationKey); ILoggerFactory loggerFactory = Services.Logging.LoggingSetup.CreateLoggerFactory(); _logger = loggerFactory.CreateLogger <VcsCallbackServerStartup>(); // Services _packageValidationTable = new PackageValidationTable(cloudStorageAccount, containerName); _packageValidationAuditor = new PackageValidationAuditor(cloudStorageAccount, containerName, loggerFactory); _notificationService = new NotificationService(cloudStorageAccount, containerName); }
private async Task RunValidationsAsync(IValidator validator) { Logger.LogInformation($"{{{TraceConstant.EventName}}}: " + $"Checking the queue of {{{TraceConstant.ValidatorName}}}", "ValidatorQueueCheck", validator.Name); // Services var packageValidationTable = new PackageValidationTable(_cloudStorageAccount, _containerName); var packageValidationAuditor = new PackageValidationAuditor(_cloudStorageAccount, _containerName, LoggerFactory); var packageValidationQueue = new PackageValidationQueue(_cloudStorageAccount, _containerName, LoggerFactory); var notificationService = new NotificationService(_cloudStorageAccount, _containerName); // Get messages to process var messages = await packageValidationQueue.DequeueAsync(validator.Name, _batchSize, validator.VisibilityTimeout); foreach (var message in messages) { // Audit entry collection to which our validator can write var auditEntries = new List <PackageValidationAuditEntry>(); var validationResult = ValidationResult.Unknown; // Deadlettering if (message.DequeueCount > 10) { validationResult = ValidationResult.Deadlettered; auditEntries.Add(new PackageValidationAuditEntry { Timestamp = DateTimeOffset.UtcNow, ValidatorName = validator.Name, Message = $"Message has been attempted too many times and is being deadlettered. Aborting validator.", EventId = ValidationEvent.Deadlettered, }); } if (validationResult != ValidationResult.Deadlettered) { try { // Perform the validation Logger.LogInformation($"Starting validator {{{TraceConstant.ValidatorName}}} " + $"for validation {{{TraceConstant.ValidationId}}} " + $"- package {{{TraceConstant.PackageId}}} " + $"v. {{{TraceConstant.PackageVersion}}}...", validator.Name, message.ValidationId, message.PackageId, message.PackageVersion); validationResult = await validator.ValidateAsync(message, auditEntries); Logger.LogInformation($"Finished running validator {{{TraceConstant.ValidatorName}}} " + $"for validation {{{TraceConstant.ValidationId}}} " + $"- package {{{TraceConstant.PackageId}}} " + $"v. {{{TraceConstant.PackageVersion}}}. " + $"Result: {{{TraceConstant.ValidationResult}}}", validator.Name, message.ValidationId, message.PackageId, message.PackageVersion, validationResult); } catch (Exception ex) { // Audit the exception, but do not remove the message yet. // We want to retry validation on next run. auditEntries.Add(new PackageValidationAuditEntry { Timestamp = DateTimeOffset.UtcNow, ValidatorName = validator.Name, Message = $"Exception thrown during validation - {ex.Message}\r\n{ex.StackTrace}", EventId = ValidationEvent.ValidatorException, }); Logger.LogError(TraceEvent.ValidatorException, ex, $"Exception while running validator {{{TraceConstant.ValidatorName}}} " + $"for validation {{{TraceConstant.ValidationId}}} " + $"- package {{{TraceConstant.PackageId}}} " + $"v. {{{TraceConstant.PackageVersion}}}", validator.Name, message.ValidationId, message.PackageId, message.PackageVersion); } } // Process message if (validationResult != ValidationResult.Unknown) { TrackValidatorResult(validator.Name, message.ValidationId, validationResult.ToString(), message.PackageId, message.PackageVersion); // Update our tracking entity var packageValidationEntity = await packageValidationTable.GetValidationAsync(message.ValidationId); if (packageValidationEntity != null && validationResult != ValidationResult.Asynchronous) { packageValidationEntity.ValidatorCompleted(validator.Name, validationResult); await packageValidationTable.StoreAsync(packageValidationEntity); } // Remove the message await packageValidationQueue.DeleteAsync(validator.Name, message); } // Write audit entries await packageValidationAuditor.WriteAuditEntriesAsync(message.ValidationId, message.PackageId, message.PackageVersion, auditEntries); // Process failure if (validationResult == ValidationResult.Failed || validationResult == ValidationResult.Deadlettered) { var audit = await packageValidationAuditor.ReadAuditAsync(message.ValidationId, message.PackageId, message.PackageVersion); await notificationService.SendNotificationAsync( "validation", $"Validation {message.ValidationId} ({message.PackageId} {message.PackageVersion}) returned '{validationResult}'", audit.Humanize()); } } Logger.LogInformation($"Done checking the queue of {{{TraceConstant.ValidatorName}}}", validator.Name); }