Exemple #1
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
            var statisticsDatabaseConnectionString  = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
            var galleryDatabaseConnectionString     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase);
            var dataStorageAccountConnectionString  = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);

            _sqlCommandTimeoutSeconds            = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds;
            _perPackageReportDegreeOfParallelism = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.PerPackageReportDegreeOfParallelism) ?? DefaultPerPackageReportDegreeOfParallelism;

            _cloudStorageAccount     = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount);
            _statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName);
            _dataStorageAccount      = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount);
            _reportName         = ValidateReportName(JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.WarehouseReportName));
            _statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString);
            _galleryDatabase    = new SqlConnectionStringBuilder(galleryDatabaseConnectionString);

            var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName)
                                 .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);

            foreach (var containerName in containerNames)
            {
                ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName);
            }

            _dataContainerNames = containerNames;
        }
Exemple #2
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf);

            var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase);

            _galleryDatabase = new SqlConnectionStringBuilder(databaseConnectionString);

            _galleryBrand      = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand);
            _galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl);

            _mailFrom = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.MailFrom);

            var smtpConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SmtpUri);
            var smtpUri = new SmtpUri(new Uri(smtpConnectionString));

            _smtpClient = CreateSmtpClient(smtpUri);

            _warnDaysBeforeExpiration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.WarnDaysBeforeExpiration)
                                        ?? _warnDaysBeforeExpiration;

            _allowEmailResendAfterDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.AllowEmailResendAfterDays)
                                         ?? _allowEmailResendAfterDays;

            var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);
            var storageContainerName    = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName);

            var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
            var storageFactory = new AzureStorageFactory(storageAccount, storageContainerName, LoggerFactory);

            _storage = storageFactory.Create();
        }
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                // Init member variables
                Source =
                    new SqlConnectionStringBuilder(
                        JobConfigurationManager.GetArgument(jobArgsDictionary,
                                                            JobArgumentNames.SourceDatabase,
                                                            EnvironmentVariableKeys.SqlGallery));
                Destination =
                    new SqlConnectionStringBuilder(
                        JobConfigurationManager.GetArgument(jobArgsDictionary,
                                                            JobArgumentNames.DestinationDatabase,
                                                            EnvironmentVariableKeys.SqlWarehouse));

                // By default, keep 7 days of statistics
                DaysToKeep = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, "DaysToKeep") ?? 7;

                return(true);
            }
            catch (Exception ex)
            {
                Trace.TraceError(ex.ToString());
            }
            return(false);
        }
Exemple #4
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            var retrievedMaxManifestSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MaxManifestSize);

            MaxManifestSize = retrievedMaxManifestSize == null
                ? DefaultMaxAllowedManifestBytes
                : Convert.ToInt64(retrievedMaxManifestSize);

            PackageDatabase = new SqlConnectionStringBuilder(
                JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase));

            Source = CloudStorageAccount.Parse(
                JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceStorage));
            Backups = CloudStorageAccount.Parse(
                JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.BackupStorage));
            SourceContainerName  = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultSourceContainerName;
            BackupsContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.BackupContainerName) ?? DefaultBackupContainerName;
            ReadMeContainerName  = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.ReadMeContainerName) ?? DefaultReadMeContainerName;

            SourceContainer  = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName);
            BackupsContainer = Backups.CreateCloudBlobClient().GetContainerReference(BackupsContainerName);
            ReadMeContainer  = Source.CreateCloudBlobClient().GetContainerReference(ReadMeContainerName);

            MaxTryCount = DefaultMaxRetryCount;
        }
Exemple #5
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                ApplicationInsights.Initialize(instrumentationKey);

                _loggerFactory = LoggingSetup.CreateLoggerFactory();
                _logger        = _loggerFactory.CreateLogger <Job>();

                var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
                _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString);

                _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, "MinAgeInDays") ?? DefaultMinAgeInDays;
                Trace.TraceInformation("Min age in days: " + _minAgeInDays);

                return(true);
            }
            catch (Exception exception)
            {
                _logger.LogCritical("Job failed to initialize. {Exception}", exception);
            }

            return(false);
        }
Exemple #6
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);

            _targetDatabase = new SqlConnectionStringBuilder(databaseConnectionString);

            _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays;
            Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays);
        }
 public override void Init(IDictionary <string, string> jobArgsDictionary)
 {
     _backupDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_BackupDays) ??
                   DefaultBackupDays;
     _destStorageAccountName = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_DestStorageAccountName);
     _destStorageKeyValue    = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.CopyAzureContainer_DestStorageKeyValue);
     _sourceContainers       = jobArgsDictionary.
                               Where(kvp => kvp.Key.StartsWith(ArgumentNames.CopyAzureContainer_SourceContainerPrefix)).
                               Select(kvp => new AzureContainerInfo(kvp.Value));
 }
Exemple #8
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            JobScenario = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ScenarioArgumentName);
            if (JobScenario == null)
            {
                throw new ArgumentException("Argument '" + ScenarioArgumentName + "' is mandatory." + HelpMessage);
            }

            LogCount = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, LogCountArgumentName);

            return(true);
        }
Exemple #9
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);

            _statisticsDatabase = new SqlConnectionStringBuilder(statisticsDatabaseConnectionString);

            var destinationDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase);

            _destinationDatabase = new SqlConnectionStringBuilder(destinationDatabaseConnectionString);

            _batchSize         = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize;
            _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds;
        }
Exemple #10
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                ApplicationInsights.Initialize(instrumentationKey);

                var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(ConsoleLogOnly);
                var loggerFactory       = LoggingSetup.CreateLoggerFactory(loggerConfiguration);
                _logger = loggerFactory.CreateLogger <Job>();

                _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf);

                var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase);
                _galleryDatabase = new SqlConnectionStringBuilder(databaseConnectionString);

                _galleryBrand      = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand);
                _galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl);

                _mailFrom = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.MailFrom);

                var smtpConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SmtpUri);
                var smtpUri = new SmtpUri(new Uri(smtpConnectionString));
                _smtpClient = CreateSmtpClient(smtpUri);

                _warnDaysBeforeExpiration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.WarnDaysBeforeExpiration)
                                            ?? _warnDaysBeforeExpiration;

                _allowEmailResendAfterDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, MyJobArgumentNames.AllowEmailResendAfterDays)
                                             ?? _allowEmailResendAfterDays;

                var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);
                var storageContainerName    = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName);

                var csa            = CloudStorageAccount.Parse(storageConnectionString);
                var storageFactory = new AzureStorageFactory(csa, storageContainerName, loggerFactory);
                _storage = storageFactory.Create();
            }
            catch (Exception exception)
            {
                _logger.LogCritical(LogEvents.JobInitFailed, exception, "Failed to initialize job!");

                return(false);
            }

            return(true);
        }
Exemple #11
0
        public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
        {
            var cloudStorageAccountConnStringSource = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureAccountConnectionStringSource);
            var cloudStorageAccountConnStringDest   = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureAccountConnectionStringDestination);

            _cloudStorageAccountSource            = ValidateAzureCloudStorageAccount(cloudStorageAccountConnStringSource);
            _cloudStorageAccountDestination       = ValidateAzureCloudStorageAccount(cloudStorageAccountConnStringDest);
            _cloudStorageContainerNameDestination = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureContainerNameDestination);
            _cloudStorageContainerNameSource      = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.AzureContainerNameSource);
            _destinationFilePrefix     = JobConfigurationManager.GetArgument(jobArgsDictionary, ArgumentNames.DestinationFilePrefix);
            _executionTimeoutInSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, ArgumentNames.ExecutionTimeoutInSeconds) ?? DefaultExecutionTimeoutInSeconds;

            var source = new AzureStatsLogSource(cloudStorageAccountConnStringSource, _cloudStorageContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess);
            var dest   = new AzureStatsLogDestination(cloudStorageAccountConnStringDest, _cloudStorageContainerNameDestination);

            _chinaCollector = new ChinaStatsCollector(source, dest);
        }
Exemple #12
0
        private void AddConfiguration(IServiceCollection serviceCollection, IDictionary <string, string> jobArgsDictionary)
        {
            var configuration = new StatusAggregatorConfiguration()
            {
                StorageAccount =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount),
                StorageAccountSecondary =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccountSecondary),
                ContainerName =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName),
                TableName =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName),
                Environments =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment)
                    .Split(';'),
                MaximumSeverity =
                    JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity)
                    ?? int.MaxValue,
                TeamId =
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId),
                EventStartMessageDelayMinutes =
                    JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventStartMessageDelayMinutes)
                    ?? _defaultEventStartMessageDelayMinutes,
                EventEndDelayMinutes =
                    JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventEndDelayMinutes)
                    ?? _defaultEventEndDelayMinutes,
                EventVisibilityPeriodDays =
                    JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventVisibilityPeriodDays)
                    ?? _defaultEventVisibilityPeriod,
            };

            serviceCollection.AddSingleton(configuration);

            var incidentApiConfiguration = new IncidentApiConfiguration()
            {
                BaseUri =
                    new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)),
                Certificate =
                    GetCertificateFromConfiguration(
                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate),
                        Logger)
            };

            serviceCollection.AddSingleton(incidentApiConfiguration);
        }
Exemple #13
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                var instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                ApplicationInsights.Initialize(instrumentationKey);

                var loggerFactory = LoggingSetup.CreateLoggerFactory();
                _logger = loggerFactory.CreateLogger <Job>();

                var retrievedMaxManifestSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MaxManifestSize);
                MaxManifestSize = retrievedMaxManifestSize == null
                    ? DefaultMaxAllowedManifestBytes
                    : Convert.ToInt64(retrievedMaxManifestSize);

                PackageDatabase = new SqlConnectionStringBuilder(
                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase));

                Enum.TryParse(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StorageType), out StorageType storageType);
                switch (storageType)
                {
                case StorageType.FileSystem:
                    FileStorage = new FileSystemStorage(jobArgsDictionary);
                    break;

                default:
                    FileStorage = new AzureStorage(jobArgsDictionary);
                    break;
                }

                MaxTryCount = DefaultMaxRetryCount;
            }
            catch (Exception exception)
            {
                Trace.TraceError($"Failed to initalize job! {exception}");
                return(false);
            }

            return(true);
        }
Exemple #14
0
        public override void Init(IDictionary <string, string> jobArgsDictionary)
        {
            _galleryBaseAddress = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryBaseAddress);

            var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);

            _cloudStorageAccount = CreateCloudStorageAccount(JobArgumentNames.DataStorageAccount, storageConnectionString);

            _containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName);

            _runValidationTasks     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RunValidationTasks).Split(';');
            _requestValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RequestValidationTasks).Split(';');
            _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? DefaultBatchSize;

            // Add validators
            if (_runValidationTasks.Contains(VcsValidator.ValidatorName))
            {
                var serviceUrl         = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorServiceUrl));
                var consumerCode       = "DIRECT";
                var callbackUrl        = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorCallbackUrl));
                var packageUrlTemplate = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageUrlTemplate);
                var submitterAlias     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorSubmitterAlias);

                // if contact alias set, use it, if not, use submitter alias.
                var contactAlias = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.VcsContactAlias) ?? submitterAlias;

                var scanningService = new VcsVirusScanningService(
                    serviceUrl,
                    consumerCode,
                    contactAlias,
                    submitterAlias,
                    LoggerFactory);

                _validators.Add(new VcsValidator(
                                    callbackUrl,
                                    packageUrlTemplate,
                                    scanningService,
                                    LoggerFactory.CreateLogger <VcsValidator>()));
            }
        }