public override void Init(IDictionary <string, string> jobArgsDictionary) { var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; _perPackageReportDegreeOfParallelism = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.PerPackageReportDegreeOfParallelism) ?? DefaultPerPackageReportDegreeOfParallelism; _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount); _statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName); _dataStorageAccount = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount); _reportName = ValidateReportName(JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.WarehouseReportName)); _statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString); _galleryDatabase = new SqlConnectionStringBuilder(galleryDatabaseConnectionString); var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName) .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries); foreach (var containerName in containerNames) { ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName); } _dataContainerNames = containerNames; }
public override void Init(IDictionary <string, string> jobArgsDictionary) { _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); _galleryDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _galleryBrand = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand); _galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl); _mailFrom = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.MailFrom); var smtpConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SmtpUri); var smtpUri = new SmtpUri(new Uri(smtpConnectionString)); _smtpClient = CreateSmtpClient(smtpUri); _warnDaysBeforeExpiration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.WarnDaysBeforeExpiration) ?? _warnDaysBeforeExpiration; _allowEmailResendAfterDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.AllowEmailResendAfterDays) ?? _allowEmailResendAfterDays; var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); var storageContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName); var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var storageFactory = new AzureStorageFactory(storageAccount, storageContainerName, LoggerFactory); _storage = storageFactory.Create(); }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { // Init member variables Source = new SqlConnectionStringBuilder( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase, EnvironmentVariableKeys.SqlGallery)); Destination = new SqlConnectionStringBuilder( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase, EnvironmentVariableKeys.SqlWarehouse)); // By default, keep 7 days of statistics DaysToKeep = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, "DaysToKeep") ?? 7; return(true); } catch (Exception ex) { Trace.TraceError(ex.ToString()); } return(false); }
public override void Init(IDictionary <string, string> jobArgsDictionary) { var retrievedMaxManifestSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MaxManifestSize); MaxManifestSize = retrievedMaxManifestSize == null ? DefaultMaxAllowedManifestBytes : Convert.ToInt64(retrievedMaxManifestSize); PackageDatabase = new SqlConnectionStringBuilder( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase)); Source = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceStorage)); Backups = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.BackupStorage)); SourceContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultSourceContainerName; BackupsContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.BackupContainerName) ?? DefaultBackupContainerName; ReadMeContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.ReadMeContainerName) ?? DefaultReadMeContainerName; SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName); BackupsContainer = Backups.CreateCloudBlobClient().GetContainerReference(BackupsContainerName); ReadMeContainer = Source.CreateCloudBlobClient().GetContainerReference(ReadMeContainerName); MaxTryCount = DefaultMaxRetryCount; }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); _loggerFactory = LoggingSetup.CreateLoggerFactory(); _logger = _loggerFactory.CreateLogger <Job>(); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, "MinAgeInDays") ?? DefaultMinAgeInDays; Trace.TraceInformation("Min age in days: " + _minAgeInDays); return(true); } catch (Exception exception) { _logger.LogCritical("Job failed to initialize. {Exception}", exception); } return(false); }
public override void Init(IDictionary <string, string> jobArgsDictionary) { var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); }
public override void Init(IDictionary <string, string> jobArgsDictionary) { _backupDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_BackupDays) ?? DefaultBackupDays; _destStorageAccountName = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_DestStorageAccountName); _destStorageKeyValue = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_DestStorageKeyValue); _sourceContainers = jobArgsDictionary. Where(kvp => kvp.Key.StartsWith(ArgumentNames.CopyAzureContainer_SourceContainerPrefix)). Select(kvp => new AzureContainerInfo(kvp.Value)); }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { JobScenario = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ScenarioArgumentName); if (JobScenario == null) { throw new ArgumentException("Argument '" + ScenarioArgumentName + "' is mandatory." + HelpMessage); } LogCount = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, LogCountArgumentName); return(true); }
public override void Init(IDictionary <string, string> jobArgsDictionary) { var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString); var destinationDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); _destinationDatabase = new SqlConnectionStringBuilder(destinationDatabaseConnectionString); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(ConsoleLogOnly); var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration); _logger = loggerFactory.CreateLogger <Job>(); _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); _galleryDatabase = new SqlConnectionStringBuilder(databaseConnectionString); _galleryBrand = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand); _galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl); _mailFrom = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.MailFrom); var smtpConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SmtpUri); var smtpUri = new SmtpUri(new Uri(smtpConnectionString)); _smtpClient = CreateSmtpClient(smtpUri); _warnDaysBeforeExpiration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.WarnDaysBeforeExpiration) ?? _warnDaysBeforeExpiration; _allowEmailResendAfterDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.AllowEmailResendAfterDays) ?? _allowEmailResendAfterDays; var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); var storageContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName); var csa = CloudStorageAccount.Parse(storageConnectionString); var storageFactory = new AzureStorageFactory(csa, storageContainerName, loggerFactory); _storage = storageFactory.Create(); } catch (Exception exception) { _logger.LogCritical(LogEvents.JobInitFailed, exception, "Failed to initialize job!"); return(false); } return(true); }
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary) { var cloudStorageAccountConnStringSource = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureAccountConnectionStringSource); var cloudStorageAccountConnStringDest = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureAccountConnectionStringDestination); _cloudStorageAccountSource = ValidateAzureCloudStorageAccount(cloudStorageAccountConnStringSource); _cloudStorageAccountDestination = ValidateAzureCloudStorageAccount(cloudStorageAccountConnStringDest); _cloudStorageContainerNameDestination = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureContainerNameDestination); _cloudStorageContainerNameSource = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureContainerNameSource); _destinationFilePrefix = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.DestinationFilePrefix); _executionTimeoutInSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ArgumentNames.ExecutionTimeoutInSeconds) ?? DefaultExecutionTimeoutInSeconds; var source = new AzureStatsLogSource(cloudStorageAccountConnStringSource, _cloudStorageContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess); var dest = new AzureStatsLogDestination(cloudStorageAccountConnStringDest, _cloudStorageContainerNameDestination); _chinaCollector = new ChinaStatsCollector(source, dest); }
private void AddConfiguration(IServiceCollection serviceCollection, IDictionary <string, string> jobArgsDictionary) { var configuration = new StatusAggregatorConfiguration() { StorageAccount = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount), StorageAccountSecondary = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccountSecondary), ContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName), TableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName), Environments = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment) .Split(';'), MaximumSeverity = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity) ?? int.MaxValue, TeamId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId), EventStartMessageDelayMinutes = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventStartMessageDelayMinutes) ?? _defaultEventStartMessageDelayMinutes, EventEndDelayMinutes = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventEndDelayMinutes) ?? _defaultEventEndDelayMinutes, EventVisibilityPeriodDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventVisibilityPeriodDays) ?? _defaultEventVisibilityPeriod, }; serviceCollection.AddSingleton(configuration); var incidentApiConfiguration = new IncidentApiConfiguration() { BaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)), Certificate = GetCertificateFromConfiguration( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate), Logger) }; serviceCollection.AddSingleton(incidentApiConfiguration); }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); ApplicationInsights.Initialize(instrumentationKey); var loggerFactory = LoggingSetup.CreateLoggerFactory(); _logger = loggerFactory.CreateLogger <Job>(); var retrievedMaxManifestSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MaxManifestSize); MaxManifestSize = retrievedMaxManifestSize == null ? DefaultMaxAllowedManifestBytes : Convert.ToInt64(retrievedMaxManifestSize); PackageDatabase = new SqlConnectionStringBuilder( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase)); Enum.TryParse(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StorageType), out StorageType storageType); switch (storageType) { case StorageType.FileSystem: FileStorage = new FileSystemStorage(jobArgsDictionary); break; default: FileStorage = new AzureStorage(jobArgsDictionary); break; } MaxTryCount = DefaultMaxRetryCount; } catch (Exception exception) { Trace.TraceError($"Failed to initalize job! {exception}"); return(false); } return(true); }
public override void Init(IDictionary <string, string> jobArgsDictionary) { _galleryBaseAddress = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryBaseAddress); var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); _cloudStorageAccount = CreateCloudStorageAccount(JobArgumentNames.DataStorageAccount, storageConnectionString); _containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName); _runValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RunValidationTasks).Split(';'); _requestValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RequestValidationTasks).Split(';'); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? DefaultBatchSize; // Add validators if (_runValidationTasks.Contains(VcsValidator.ValidatorName)) { var serviceUrl = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorServiceUrl)); var consumerCode = "DIRECT"; var callbackUrl = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorCallbackUrl)); var packageUrlTemplate = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageUrlTemplate); var submitterAlias = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorSubmitterAlias); // if contact alias set, use it, if not, use submitter alias. var contactAlias = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.VcsContactAlias) ?? submitterAlias; var scanningService = new VcsVirusScanningService( serviceUrl, consumerCode, contactAlias, submitterAlias, LoggerFactory); _validators.Add(new VcsValidator( callbackUrl, packageUrlTemplate, scanningService, LoggerFactory.CreateLogger <VcsValidator>())); } }