protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var gallery            = arguments.GetOrThrow <string>(Arguments.Gallery);
            var index              = arguments.GetOrThrow <string>(Arguments.Index);
            var packageBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
            var source             = arguments.GetOrThrow <string>(Arguments.Source);
            var requireSignature   = arguments.GetOrDefault(Arguments.RequireSignature, false);
            var verbose            = arguments.GetOrDefault(Arguments.Verbose, false);

            CommandHelpers.AssertAzureStorage(arguments);

            var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
            var auditingStorageFactory   = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose);

            var endpointInputs        = CommandHelpers.GetEndpointFactoryInputs(arguments);
            var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);

            Logger.LogInformation(
                "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} endpoints: {Endpoints}",
                gallery, index, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name)));

            _packageValidator = new PackageValidatorFactory(LoggerFactory)
                                .Create(gallery, index, packageBaseAddress, auditingStorageFactory, endpointInputs, messageHandlerFactory, requireSignature, verbose);

            _queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);

            _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);

            _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger <LoggerMonitoringNotificationService>());

            _regResource = Repository.Factory.GetCoreV3(index).GetResource <RegistrationResourceV3>(cancellationToken);

            _client = new CollectorHttpClient(messageHandlerFactory());
        }
        private static CatalogStorageFactory CreateStorageFactoryImpl(IDictionary <string, string> arguments,
                                                                      IDictionary <string, string> argumentNameMap,
                                                                      bool verbose,
                                                                      bool compressed)
        {
            Uri storageBaseAddress    = null;
            var storageBaseAddressStr = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageBaseAddress]);

            if (!string.IsNullOrEmpty(storageBaseAddressStr))
            {
                storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/";

                storageBaseAddress = new Uri(storageBaseAddressStr);
            }

            var storageType = arguments.GetOrThrow <string>(Arguments.StorageType);

            if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase))
            {
                var storagePath = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StoragePath]);

                if (storageBaseAddress != null)
                {
                    return(new CatalogFileStorageFactory(storageBaseAddress, storagePath, verbose));
                }

                TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]);
                return(null);
            }

            if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase))
            {
                var storageAccountName = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageAccountName]);
                var storageKeyValue    = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageKeyValue]);
                var storageContainer   = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageContainer]);
                var storagePath        = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StoragePath]);
                var storageSuffix      = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageSuffix]);
                var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds]));
                var storageServerTimeout             = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageServerTimeoutInSeconds]));
                var storageUseServerSideCopy         = arguments.GetOrDefault <bool>(argumentNameMap[Arguments.StorageUseServerSideCopy]);

                var credentials = new StorageCredentials(storageAccountName, storageKeyValue);

                var account = string.IsNullOrEmpty(storageSuffix) ?
                              new CloudStorageAccount(credentials, useHttps: true) :
                              new CloudStorageAccount(credentials, storageSuffix, useHttps: true);

                return(new CatalogAzureStorageFactory(
                           account,
                           storageContainer,
                           storageOperationMaxExecutionTime,
                           storageServerTimeout,
                           storagePath,
                           storageBaseAddress,
                           storageUseServerSideCopy,
                           compressed,
                           verbose));
            }
            throw new ArgumentException($"Unrecognized storageType \"{storageType}\"");
        }
        private static ISecretInjector GetSecretInjector(IDictionary<string, string> arguments)
        {
            ISecretReader secretReader;

            var vaultName = arguments.GetOrDefault<string>(Arguments.VaultName);
            if (string.IsNullOrEmpty(vaultName))
            {
                secretReader = new EmptySecretReader();
            }
            else
            {
                var clientId = arguments.GetOrThrow<string>(Arguments.ClientId);
                var certificateThumbprint = arguments.GetOrThrow<string>(Arguments.CertificateThumbprint);
                var storeName = arguments.GetOrDefault(Arguments.StoreName, StoreName.My);
                var storeLocation = arguments.GetOrDefault(Arguments.StoreLocation, StoreLocation.LocalMachine);
                var shouldValidateCert = arguments.GetOrDefault(Arguments.ValidateCertificate, false);

                var keyVaultConfig = new KeyVaultConfiguration(vaultName, clientId, certificateThumbprint, storeName, storeLocation, shouldValidateCert);

                secretReader = new CachingSecretReader(new KeyVaultReader(keyVaultConfig),
                    arguments.GetOrDefault(Arguments.RefreshIntervalSec, CachingSecretReader.DefaultRefreshIntervalSec));
            }

            return new SecretInjector(secretReader);
        }
        private static ISecretInjector GetSecretInjector(IDictionary <string, string> arguments)
        {
            ISecretReader secretReader;

            var vaultName = arguments.GetOrDefault <string>(Arguments.VaultName);

            if (string.IsNullOrEmpty(vaultName))
            {
                secretReader = new EmptySecretReader();
            }
            else
            {
                var clientId = arguments.GetOrThrow <string>(Arguments.ClientId);
                var certificateThumbprint = arguments.GetOrThrow <string>(Arguments.CertificateThumbprint);
                var storeName             = arguments.GetOrDefault(Arguments.StoreName, StoreName.My);
                var storeLocation         = arguments.GetOrDefault(Arguments.StoreLocation, StoreLocation.LocalMachine);
                var shouldValidateCert    = arguments.GetOrDefault(Arguments.ValidateCertificate, true);

                var keyVaultCertificate = CertificateUtility.FindCertificateByThumbprint(storeName, storeLocation, certificateThumbprint, shouldValidateCert);
                var keyVaultConfig      = new KeyVaultConfiguration(vaultName, clientId, keyVaultCertificate);

                secretReader = new CachingSecretReader(new KeyVaultReader(keyVaultConfig),
                                                       arguments.GetOrDefault(Arguments.RefreshIntervalSec, CachingSecretReader.DefaultRefreshIntervalSec));
            }

            return(new SecretInjector(secretReader));
        }
Ejemplo n.º 5
0
 private void InitPrepare(IDictionary <string, string> arguments)
 {
     _outputFolder = arguments.GetOrThrow <string>(Arguments.OutputFolder);
     _catalogIndex = arguments.GetOrThrow <string>(Arguments.Source);
     _templateFile = arguments.GetOrThrow <string>(Arguments.TemplateFile);
     _batchSize    = arguments.GetOrThrow <string>(Arguments.BatchSize);
 }
Ejemplo n.º 6
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var gallery = arguments.GetOrThrow <string>(Arguments.Gallery);
            var index   = arguments.GetOrThrow <string>(Arguments.Index);
            var source  = arguments.GetOrThrow <string>(Arguments.Source);
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            CommandHelpers.AssertAzureStorage(arguments);

            var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments);

            var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);

            var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);

            var queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);

            Logger.LogInformation(
                "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} endpoints: {Endpoints}",
                gallery, index, monitoringStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name)));

            var context = _collectorFactory.Create(
                queue,
                source,
                monitoringStorageFactory,
                endpointInputs,
                TelemetryService,
                messageHandlerFactory);

            _collector = context.Collector;
            _front     = context.Front;
            _back      = context.Back;
        }
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            _connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString);
            _source           = arguments.GetOrThrow <string>(Arguments.Source);
            _path             = arguments.GetOrThrow <string>(Arguments.Path);

            _catalogIndexUrl = new Uri(_source);
        }
 protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
 {
     _gallery = arguments.GetOrThrow<string>(Arguments.Gallery);
     _verbose = arguments.GetOrDefault(Arguments.Verbose, false);
     _id = arguments.GetOrThrow<string>(Arguments.Id);
     _version = arguments.GetOrDefault<string>(Arguments.Version);
     _storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose);
 }
        public static Lucene.Net.Store.Directory GetLuceneDirectoryImpl(
            IDictionary <string, string> arguments,
            IDictionary <string, string> argumentNameMap,
            out string destination,
            bool required = true)
        {
            destination = null;

            try
            {
                var luceneDirectoryType = arguments.GetOrThrow <string>(argumentNameMap[Arguments.DirectoryType]);

                if (luceneDirectoryType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase))
                {
                    var lucenePath = arguments.GetOrThrow <string>(argumentNameMap[Arguments.Path]);

                    var directoryInfo = new DirectoryInfo(lucenePath);

                    destination = lucenePath;

                    if (directoryInfo.Exists)
                    {
                        return(new SimpleFSDirectory(directoryInfo));
                    }

                    directoryInfo.Create();
                    directoryInfo.Refresh();

                    return(new SimpleFSDirectory(directoryInfo));
                }
                if (luceneDirectoryType.Equals(Arguments.AzureStorageType, StringComparison.InvariantCultureIgnoreCase))
                {
                    var luceneStorageAccountName = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageAccountName]);

                    var luceneStorageKeyValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageKeyValue]);

                    var luceneStorageContainer = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageContainer]);

                    var credentials = new StorageCredentials(luceneStorageAccountName, luceneStorageKeyValue);
                    var account     = new CloudStorageAccount(credentials, useHttps: true);

                    destination = luceneStorageContainer;

                    return(new AzureDirectory(account, luceneStorageContainer));
                }
                Trace.TraceError("Unrecognized Lucene Directory Type \"{0}\"", luceneDirectoryType);
                return(null);
            }
            catch (ArgumentException)
            {
                if (required)
                {
                    throw;
                }

                return(null);
            }
        }
Ejemplo n.º 10
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var source  = arguments.GetOrThrow <string>(Arguments.Source);
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            var contentBaseAddress      = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress);
            var galleryBaseAddress      = arguments.GetOrDefault <string>(Arguments.GalleryBaseAddress);
            var isContentFlatContainer  = arguments.GetOrDefault <bool>(Arguments.ContentIsFlatContainer);
            var allIconsInFlatContainer = arguments.GetOrDefault <bool>(Arguments.AllIconsInFlatContainer);
            var maxConcurrentBatches    = MaxConcurrentBatches(arguments.GetOrDefault <int>(Arguments.MaxConcurrentBatches));

            // The term "legacy" here refers to the registration hives that do not contain any SemVer 2.0.0 packages.
            // In production, this is two registration hives:
            //   1) the first hive released, which is not gzipped and does not have SemVer 2.0.0 packages
            //   2) the secondary hive released, which is gzipped but does not have SemVer 2.0.0 packages
            var storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose);

            Logger.LogInformation(
                "CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"",
                source,
                storageFactories.LegacyStorageFactory);

            if (isContentFlatContainer)
            {
                var flatContainerCursorUriString = arguments.GetOrThrow <string>(Arguments.CursorUri);
                var flatContainerName            = arguments.GetOrThrow <string>(Arguments.FlatContainerName);
                RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName);
                // In case that the flat container is used as the packages' source the registration needs to wait for the flatcontainer cursor
                _back = new HttpReadCursor(new Uri(flatContainerCursorUriString));
            }
            else
            {
                RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider();
                _back = MemoryCursor.CreateMax();
            }

            _collector = new RegistrationCollector(
                new Uri(source),
                storageFactories.LegacyStorageFactory,
                storageFactories.SemVer2StorageFactory,
                contentBaseAddress == null ? null : new Uri(contentBaseAddress),
                galleryBaseAddress == null ? null : new Uri(galleryBaseAddress),
                allIconsInFlatContainer,
                TelemetryService,
                Logger,
                CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
                maxConcurrentBatches: maxConcurrentBatches);

            var cursorStorage = storageFactories.LegacyStorageFactory.Create();

            _front = new DurableCursor(cursorStorage.ResolveUri("cursor.json"), cursorStorage, MemoryCursor.MinValue);
            storageFactories.SemVer2StorageFactory?.Create();

            _destination = storageFactories.LegacyStorageFactory.DestinationAddress;
            TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination?.AbsoluteUri;
        }
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            _gallery = arguments.GetOrThrow <string>(Arguments.Gallery);
            _verbose = arguments.GetOrDefault(Arguments.Verbose, false);
            _id      = arguments.GetOrThrow <string>(Arguments.Id);
            _version = arguments.GetOrDefault <string>(Arguments.Version);

            var storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose);

            _storage = storageFactory.Create();
        }
Ejemplo n.º 12
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            PrintLightning();

            // Hard code against Azure storage.
            arguments[Arguments.StorageType] = Arguments.AzureStorageType;

            // Configure the package path provider.
            var useFlatContainerAsPackageContent = arguments.GetOrDefault <bool>(Arguments.ContentIsFlatContainer, false);

            if (!useFlatContainerAsPackageContent)
            {
                RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider();
            }
            else
            {
                var flatContainerName = arguments.GetOrThrow <string>(Arguments.FlatContainerName);
                RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName);
            }

            _command            = arguments.GetOrThrow <string>(Arguments.Command);
            _verbose            = arguments.GetOrDefault(Arguments.Verbose, false);
            _log                = _verbose ? Console.Out : new StringWriter();
            _contentBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
            _galleryBaseAddress = arguments.GetOrThrow <string>(Arguments.GalleryBaseAddress);
            _storageFactories   = CommandHelpers.CreateRegistrationStorageFactories(arguments, _verbose);
            _shouldIncludeSemVer2ForLegacyStorageFactory = RegistrationCollector.GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory);
            _postProcessGraphForLegacyStorageFactory     = RegistrationCollector.GetPostProcessGraphForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory);
            _forceIconsFromFlatContainer = arguments.GetOrDefault <bool>(Arguments.AllIconsInFlatContainer);
            _driver = arguments.GetOrDefault(Arguments.Driver, GraphDriver).ToLowerInvariant();
            // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments
            // used by "prepare" should be used when executing "strike".
            _arguments = arguments;

            if (_driver != GraphDriver && _driver != JsonDriver)
            {
                throw new NotSupportedException($"The lightning driver '{_driver}' is not supported.");
            }

            switch (_command.ToLowerInvariant())
            {
            case "charge":
            case "prepare":
                InitPrepare(arguments);
                break;

            case "strike":
                InitStrike(arguments);
                break;

            default:
                throw new NotSupportedException($"The lightning command '{_command}' is not supported.");
            }
        }
Ejemplo n.º 13
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            ServicePointManager.DefaultConnectionLimit = DegreeOfParallelism;

            var verbose              = arguments.GetOrDefault(Arguments.Verbose, false);
            var packageStorageBase   = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
            var failCacheTime        = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1));
            var auxStorageFactory    = CreateAuxStorageFactory(arguments, verbose);
            var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose);
            var packageStorage       = new AzureStorage(
                storageBaseUri: new Uri(packageStorageBase),
                maxExecutionTime: TimeSpan.FromMinutes(15),
                serverTimeout: TimeSpan.FromMinutes(10),
                useServerSideCopy: true,
                compressContent: false,
                verbose: true,
                throttle: null);
            var source               = arguments.GetOrThrow <string>(Arguments.Source);
            var iconProcessor        = new IconProcessor(TelemetryService, LoggerFactory.CreateLogger <IconProcessor>());
            var httpHandlerFactory   = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);
            var httpMessageHandler   = httpHandlerFactory();
            var httpClient           = new HttpClient(httpMessageHandler);
            var simpleHttpClient     = new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>());
            var catalogClient        = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger <CatalogClient>());
            var httpResponseProvider = new HttpClientWrapper(httpClient);
            var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger <ExternalIconContentProvider>());
            var iconCopyResultCache  = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger <IconCopyResultCache>());

            var leafProcessor = new CatalogLeafDataProcessor(
                packageStorage,
                iconProcessor,
                externalIconProvider,
                iconCopyResultCache,
                TelemetryService,
                LoggerFactory.CreateLogger <CatalogLeafDataProcessor>());

            _collector = new IconsCollector(
                new Uri(source),
                TelemetryService,
                targetStorageFactory,
                catalogClient,
                leafProcessor,
                iconCopyResultCache,
                auxStorageFactory,
                CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
                LoggerFactory.CreateLogger <IconsCollector>());
            var cursorStorage = auxStorageFactory.Create();

            _front = new DurableCursor(cursorStorage.ResolveUri("c2icursor.json"), cursorStorage, DateTime.MinValue.ToUniversalTime());
        }
Ejemplo n.º 14
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            _maxRequeueQueueSize = arguments.GetOrDefault(Arguments.MaxRequeueQueueSize, DefaultMaxQueueSize);

            CommandHelpers.AssertAzureStorage(arguments);

            var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);
            _packageValidatorContextQueue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);

            Logger.LogInformation(
                "CONFIG storage: {Storage}",
                monitoringStorageFactory);

            _monitoringCursor = ValidationFactory.GetFront(monitoringStorageFactory);
            _galleryCursor    = CreateCursor(monitoringStorageFactory, GalleryCursorFileName);
            _deletedCursor    = CreateCursor(monitoringStorageFactory, DeletedCursorFileName);

            var connectionString    = arguments.GetOrThrow <string>(Arguments.ConnectionString);
            var galleryDbConnection = new AzureSqlConnectionFactory(
                connectionString,
                SecretInjector,
                LoggerFactory.CreateLogger <AzureSqlConnectionFactory>());

            var packageContentUriBuilder = new PackageContentUriBuilder(
                arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat));

            var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300);

            _galleryDatabaseQueryService = new GalleryDatabaseQueryService(
                galleryDbConnection,
                packageContentUriBuilder,
                TelemetryService,
                timeoutInSeconds);

            var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
                "Auditing",
                arguments,
                verbose,
                new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));

            _auditingStorage = auditingStorageFactory.Create();

            var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);

            _client = new CollectorHttpClient(messageHandlerFactory());
        }
Ejemplo n.º 15
0
        public static EndpointConfiguration GetEndpointConfiguration(IDictionary <string, string> arguments)
        {
            var registrationCursorUri  = arguments.GetOrThrow <Uri>(Arguments.RegistrationCursorUri);
            var flatContainerCursorUri = arguments.GetOrThrow <Uri>(Arguments.FlatContainerCursorUri);

            var instanceNameToSearchBaseUri   = GetSuffixToUri(arguments, Arguments.SearchBaseUriPrefix);
            var instanceNameToSearchCursorUri = GetSuffixToUri(arguments, Arguments.SearchCursorUriPrefix);
            var instanceNameToSearchConfig    = new Dictionary <string, SearchEndpointConfiguration>();

            foreach (var pair in instanceNameToSearchBaseUri)
            {
                var instanceName = pair.Key;

                // Find all cursors with an instance name starting with the search base URI instance name. We do this
                // because there may be multiple potential cursors representing the state of a search service.
                var matchingCursors = instanceNameToSearchCursorUri.Keys.Where(x => x.StartsWith(instanceName)).ToList();

                if (!matchingCursors.Any())
                {
                    throw new ArgumentException(
                              $"The -{Arguments.SearchBaseUriPrefix}{instanceName} argument does not have any matching " +
                              $"-{Arguments.SearchCursorUriPrefix}{instanceName}* arguments.");
                }

                instanceNameToSearchConfig[instanceName] = new SearchEndpointConfiguration(
                    matchingCursors.Select(x => instanceNameToSearchCursorUri[x]).ToList(),
                    pair.Value);

                foreach (var key in matchingCursors)
                {
                    instanceNameToSearchCursorUri.Remove(key);
                }
            }

            // See if there are any search cursor URI arguments left over and error out. Better to fail than to ignore
            // an argument that the user expected to be relevant.
            if (instanceNameToSearchCursorUri.Any())
            {
                throw new ArgumentException(
                          $"There are -{Arguments.SearchCursorUriPrefix}* arguments without matching " +
                          $"-{Arguments.SearchBaseUriPrefix}* arguments. The unmatched suffixes were: {string.Join(", ", instanceNameToSearchCursorUri.Keys)}");
            }

            return(new EndpointConfiguration(
                       registrationCursorUri,
                       flatContainerCursorUri,
                       instanceNameToSearchConfig));
        }
        public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key, string messageWithFormat)
        {
            V value = dictionary.GetOrThrow(key, messageWithFormat);

            dictionary.Remove(key);
            return(value);
        }
        public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key, Func <K, Exception> exception)
        {
            V value = dictionary.GetOrThrow(key, exception);

            dictionary.Remove(key);
            return(value);
        }
Ejemplo n.º 18
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var source  = arguments.GetOrThrow <string>(Arguments.Source);
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            CommandHelpers.AssertAzureStorage(arguments);

            var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
            var endpointConfiguration    = CommandHelpers.GetEndpointConfiguration(arguments);
            var messageHandlerFactory    = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);
            var statusService            = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);
            var queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);

            Logger.LogInformation(
                "CONFIG storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}",
                monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri);

            _enqueuer = ValidationFactory.CreatePackageValidatorContextEnqueuer(
                queue,
                source,
                monitoringStorageFactory,
                endpointConfiguration,
                TelemetryService,
                messageHandlerFactory,
                LoggerFactory);
        }
Ejemplo n.º 19
0
 public static void AssertAzureStorage(IDictionary <string, string> arguments)
 {
     if (arguments.GetOrThrow <string>(Arguments.StorageType) != Arguments.AzureStorageType)
     {
         throw new ArgumentException("Only Azure storage is supported!");
     }
 }
Ejemplo n.º 20
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            PrintLightning();

            // Hard code against Azure storage.
            arguments[Arguments.StorageType] = Arguments.AzureStorageType;

            _command = arguments.GetOrThrow <string>(Arguments.Command);
            _verbose = arguments.GetOrDefault(Arguments.Verbose, false);
            _log     = _verbose ? Console.Out : new StringWriter();
            // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments
            // used by "prepare" should be used when executing "strike".
            _arguments = arguments;

            switch (_command.ToLowerInvariant())
            {
            case "charge":
            case "prepare":
                InitPrepare(arguments);
                break;

            case "strike":
                InitStrike(arguments);
                break;

            default:
                throw new NotSupportedException($"The lightning command '{_command}' is not supported.");
            }
        }
        protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
        {
            _directory = CommandHelpers.GetLuceneDirectory(arguments);
            _source = arguments.GetOrThrow<string>(Arguments.Source);
            _verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            _registration = arguments.GetOrDefault<string>(Arguments.Registration);
            if (_registration == null)
            {
                Logger.LogInformation("Lucene index will be created up to the end of the catalog (alternatively if you provide a registration it will not pass that)");
            }

            _catalogBaseAddress = arguments.GetOrDefault<string>(Arguments.CatalogBaseAddress);
            if (_catalogBaseAddress == null)
            {
                Logger.LogInformation("No catalogBaseAddress was specified so the Lucene index will NOT contain the storage paths");
            }

            _storageBaseAddress = arguments.GetOrDefault<string>(Arguments.StorageBaseAddress);

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" registration: \"{Registration}\"" +
                                   " catalogBaseAddress: \"{CatalogBaseAddress}\" storageBaseAddress: \"{StorageBaseAddress}\"",
                                   _source,
                                   _registration ?? "(null)",
                                   _catalogBaseAddress ?? "(null)",
                                   _storageBaseAddress ?? "(null)");

            _handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(_verbose, _catalogBaseAddress, _storageBaseAddress);
        }
Ejemplo n.º 22
0
        public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key) where K : object
        {
            V value = dictionary.GetOrThrow(key);

            dictionary.Remove(key);
            return(value);
        }
Ejemplo n.º 23
0
        public static IStorageQueue <T> CreateStorageQueue <T>(IDictionary <string, string> arguments, int version)
        {
            var storageType = arguments.GetOrThrow <string>(Arguments.StorageType);

            if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase))
            {
                var storageAccountName = arguments.GetOrThrow <string>(Arguments.StorageAccountName);
                var storageQueueName   = arguments.GetOrDefault <string>(Arguments.StorageQueueName);

                var account = GetCloudStorageAccount(storageAccountName, endpointSuffix: null, arguments, ArgumentNames);
                return(new StorageQueue <T>(new AzureStorageQueue(account, storageQueueName),
                                            new JsonMessageSerializer <T>(JsonSerializerUtility.SerializerSettings), version));
            }
            else
            {
                throw new NotImplementedException("Only Azure storage queues are supported!");
            }
        }
Ejemplo n.º 24
0
        private static Dictionary <string, Uri> GetSuffixToUri(IDictionary <string, string> arguments, string prefix)
        {
            var suffixToUri = new Dictionary <string, Uri>();

            foreach (var key in arguments.Keys.Where(x => x.StartsWith(prefix)))
            {
                var suffix = key.Substring(prefix.Length);
                suffixToUri[suffix] = arguments.GetOrThrow <Uri>(key);
            }

            return(suffixToUri);
        }
Ejemplo n.º 25
0
        private static CloudStorageAccount GetCloudStorageAccount(string storageAccountName, string endpointSuffix, IDictionary <string, string> arguments, IDictionary <string, string> argumentNameMap)
        {
            var storageKeyValue = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageKeyValue]);

            if (string.IsNullOrEmpty(storageKeyValue))
            {
                var storageSasValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageSasValue]);
                var credentialSas   = new StorageCredentials(storageSasValue);
                return(new CloudStorageAccount(credentialSas, storageAccountName, endpointSuffix, useHttps: true));
            }

            var credentialKey = new StorageCredentials(storageAccountName, storageKeyValue);

            return(new CloudStorageAccount(credentialKey, endpointSuffix, useHttps: true));
        }
Ejemplo n.º 26
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            _directory = CommandHelpers.GetLuceneDirectory(arguments, out var destination);
            _source    = arguments.GetOrThrow <string>(Arguments.Source);
            _verbose   = arguments.GetOrDefault(Arguments.Verbose, false);

            _registration = arguments.GetOrDefault <string>(Arguments.Registration);
            if (_registration == null)
            {
                Logger.LogInformation("Lucene index will be created up to the end of the catalog (alternatively if you provide a registration it will not pass that)");
            }

            _catalogBaseAddress = arguments.GetOrDefault <string>(Arguments.CatalogBaseAddress);
            if (_catalogBaseAddress == null)
            {
                Logger.LogInformation("No catalogBaseAddress was specified so the Lucene index will NOT contain the storage paths");
            }

            _storageBaseAddress = arguments.GetOrDefault <string>(Arguments.StorageBaseAddress);

            var commitTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.CommitTimeoutInSeconds);

            if (commitTimeoutInSeconds.HasValue)
            {
                _commitTimeout = TimeSpan.FromSeconds(commitTimeoutInSeconds.Value);
            }
            else
            {
                _commitTimeout = null;
            }

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" registration: \"{Registration}\"" +
                                  " catalogBaseAddress: \"{CatalogBaseAddress}\" storageBaseAddress: \"{StorageBaseAddress}\" commitTimeout: \"{CommmitTimeout}\"",
                                  _source,
                                  _registration ?? "(null)",
                                  _catalogBaseAddress ?? "(null)",
                                  _storageBaseAddress ?? "(null)",
                                  _commitTimeout?.ToString() ?? "(null)");

            _handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(
                TelemetryService,
                _verbose,
                _catalogBaseAddress,
                _storageBaseAddress);

            _destination = destination;
            TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination;
        }
Ejemplo n.º 27
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var source                     = arguments.GetOrThrow <string>(Arguments.Source);
            var verbose                    = arguments.GetOrDefault(Arguments.Verbose, false);
            var contentBaseAddress         = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress);
            var storageFactory             = CommandHelpers.CreateStorageFactory(arguments, verbose);
            var httpClientTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.HttpClientTimeoutInSeconds);
            var httpClientTimeout          = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null;

            StorageFactory preferredPackageSourceStorageFactory = null;
            IAzureStorage  preferredPackageSourceStorage        = null;

            var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false);

            if (preferAlternatePackageSourceStorage)
            {
                preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose);
                preferredPackageSourceStorage        = preferredPackageSourceStorageFactory.Create() as IAzureStorage;
            }

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
                                  source,
                                  storageFactory,
                                  preferredPackageSourceStorageFactory);
            Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout);

            MaxDegreeOfParallelism = 256;

            _collector = new DnxCatalogCollector(
                new Uri(source),
                storageFactory,
                preferredPackageSourceStorage,
                contentBaseAddress == null ? null : new Uri(contentBaseAddress),
                TelemetryService,
                Logger,
                MaxDegreeOfParallelism,
                httpClient => new CatalogClient(new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()), LoggerFactory.CreateLogger <CatalogClient>()),
                CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
                httpClientTimeout);

            var storage = storageFactory.Create();

            _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue);
            _back  = MemoryCursor.CreateMax();

            _destination = storageFactory.BaseAddress;
            TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri;
        }
        protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
        {
            var source = arguments.GetOrThrow<string>(Arguments.Source);
            var unlistShouldDelete = arguments.GetOrDefault(Arguments.UnlistShouldDelete, false);
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            var contentBaseAddress = arguments.GetOrDefault<string>(Arguments.ContentBaseAddress);

            StorageFactory storageFactoryToUse;

            var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
            var compressedStorageFactory = CommandHelpers.CreateCompressedStorageFactory(arguments, verbose);

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory);

            RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider();

            if (compressedStorageFactory != null)
            {
                var secondaryStorageBaseUrlRewriter = new SecondaryStorageBaseUrlRewriter(new List<KeyValuePair<string, string>>
                {
                    // always rewrite storage root url in seconary
                    new KeyValuePair<string, string>(storageFactory.BaseAddress.ToString(), compressedStorageFactory.BaseAddress.ToString())
                });

                var aggregateStorageFactory = new AggregateStorageFactory(
                    storageFactory,
                    new[] { compressedStorageFactory },
                    secondaryStorageBaseUrlRewriter.Rewrite);

                storageFactoryToUse = aggregateStorageFactory;
            }
            else
            {
                storageFactoryToUse = storageFactory;
            }

            _collector = new RegistrationCollector(new Uri(source), storageFactoryToUse, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
            {
                ContentBaseAddress = contentBaseAddress == null
                    ? null
                    : new Uri(contentBaseAddress)
            };

            var storage = storageFactoryToUse.Create();
            _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue);
            _back = MemoryCursor.CreateMax();
        }
        public static IEnumerable <EndpointFactory.Input> GetEndpointFactoryInputs(IDictionary <string, string> arguments)
        {
            return(arguments.GetOrThrow <string>(Arguments.EndpointsToTest).Split(';').Select(e =>
            {
                var endpointParts = e.Split('|');

                if (endpointParts.Count() < 2)
                {
                    throw new ArgumentException(
                        $"\"{e}\" is not a valid endpoint! Each endpoint must follow this format: " +
                        $"<endpoint-to-test-name>|<endpoint-to-test-cursor>.");
                }

                return new EndpointFactory.Input(endpointParts[0], new Uri(endpointParts[1]));
            }));
        }
        protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
        {
            Gallery = arguments.GetOrThrow<string>(Arguments.Gallery);
            Verbose = arguments.GetOrDefault(Arguments.Verbose, false);
            StartDate = arguments.GetOrDefault(Arguments.StartDate, DateTimeMinValueUtc);

            var catalogStorageFactory = CommandHelpers.CreateStorageFactory(arguments, Verbose);
            var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose);

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", Gallery, catalogStorageFactory);

            CatalogStorage = catalogStorageFactory.Create();
            AuditingStorage = auditingStorageFactory.Create();

            Top = 20;
            Timeout = TimeSpan.FromSeconds(300);
        }
Ejemplo n.º 31
0
        protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
        {
            var source = arguments.GetOrThrow<string>(Arguments.Source);
            var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
            var contentBaseAddress = arguments.GetOrDefault<string>(Arguments.ContentBaseAddress);
            var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory);

            _collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
            {
                ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
            };

            var storage = storageFactory.Create();
            _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue);
            _back = MemoryCursor.CreateMax();
        }
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            Gallery   = arguments.GetOrThrow <string>(Arguments.Gallery);
            Verbose   = arguments.GetOrDefault(Arguments.Verbose, false);
            StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc);
            SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false);

            StorageFactory preferredPackageSourceStorageFactory = null;

            var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false);

            if (preferAlternatePackageSourceStorage)
            {
                preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, Verbose);
            }

            var catalogStorageFactory  = CommandHelpers.CreateStorageFactory(arguments, Verbose);
            var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose);

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
                                  Gallery,
                                  catalogStorageFactory,
                                  preferredPackageSourceStorageFactory);

            CatalogStorage  = catalogStorageFactory.Create();
            AuditingStorage = auditingStorageFactory.Create();

            if (preferAlternatePackageSourceStorage)
            {
                PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create();
            }

            Destination = catalogStorageFactory.BaseAddress;
            TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri;

            Top     = 20;
            Timeout = TimeSpan.FromSeconds(300);
        }
Ejemplo n.º 33
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            Verbose   = arguments.GetOrDefault(Arguments.Verbose, false);
            StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc);
            Top       = arguments.GetOrDefault(Arguments.CursorSize, 20);
            SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false);

            StorageFactory preferredPackageSourceStorageFactory = null;

            var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false);

            if (preferAlternatePackageSourceStorage)
            {
                preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
                    "PreferredPackageSourceStorage",
                    arguments,
                    Verbose,
                    new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));
            }

            var catalogStorageFactory = CommandHelpers.CreateStorageFactory(
                arguments,
                Verbose,
                new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));

            var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
                "Auditing",
                arguments,
                Verbose,
                new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));

            Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
                                  GalleryDbConnection,
                                  catalogStorageFactory,
                                  preferredPackageSourceStorageFactory);

            CatalogStorage  = catalogStorageFactory.Create();
            AuditingStorage = auditingStorageFactory.Create();

            if (preferAlternatePackageSourceStorage)
            {
                PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create();
            }

            Destination = catalogStorageFactory.BaseAddress;
            TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri;

            // Setup gallery database access
            PackageContentUriBuilder = new PackageContentUriBuilder(
                arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat));

            var connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString);

            GalleryDbConnection = new AzureSqlConnectionFactory(
                connectionString,
                SecretInjector,
                LoggerFactory.CreateLogger <AzureSqlConnectionFactory>());

            var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300);

            Timeout = TimeSpan.FromSeconds(timeoutInSeconds);
            GalleryDatabaseQueryService = new GalleryDatabaseQueryService(
                GalleryDbConnection,
                PackageContentUriBuilder,
                TelemetryService,
                timeoutInSeconds);
        }
Ejemplo n.º 34
0
        protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
        {
            PrintLightning();

            _command = arguments.GetOrThrow<string>(Arguments.Command);
            _verbose = arguments.GetOrDefault(Arguments.Verbose, false);

            _log = _verbose ? Console.Out : new StringWriter();

            switch (_command.ToLowerInvariant())
            {
                case "charge":
                case "prepare":
                    InitPrepare(arguments);
                    break;
                case "strike":
                    InitStrike(arguments);
                    break;
                default:
                    throw new ArgumentNullException();
            }

            _contentBaseAddress = arguments.GetOrThrow<string>(Arguments.ContentBaseAddress);
            _storageAccount = arguments.GetOrThrow<string>(Arguments.StorageAccount);
            _storageContainer = arguments.GetOrThrow<string>(Arguments.StorageContainer);
            _storageBaseAddress = arguments.GetOrThrow<string>(Arguments.StorageBaseAddress);
            _compress = arguments.GetOrDefault(Arguments.Compress, false);
        }
Ejemplo n.º 35
0
 private void InitStrike(IDictionary <string, string> arguments)
 {
     _indexFile  = arguments.GetOrThrow <string>(Arguments.IndexFile);
     _cursorFile = arguments.GetOrThrow <string>(Arguments.CursorFile);
 }
Ejemplo n.º 36
0
 private void InitPrepare(IDictionary<string, string> arguments)
 {
     _outputFolder = arguments.GetOrThrow<string>(Arguments.OutputFolder);
     _catalogIndex = arguments.GetOrThrow<string>(Arguments.CatalogIndex);
     _templateFile = arguments.GetOrThrow<string>(Arguments.TemplateFile);
     _batchSize = arguments.GetOrThrow<string>(Arguments.BatchSize);
 }
Ejemplo n.º 37
0
 private void InitStrike(IDictionary<string, string> arguments)
 {
     _indexFile = arguments.GetOrThrow<string>(Arguments.IndexFile);
     _cursorFile = arguments.GetOrThrow<string>(Arguments.CursorFile);
 }
Ejemplo n.º 38
0
        private static StorageFactory CreateStorageFactoryImpl(IDictionary<string, string> arguments, IDictionary<string, string> argumentNameMap, bool verbose)
        {
            Uri storageBaseAddress = null;
            var storageBaseAddressStr = arguments.GetOrDefault<string>(argumentNameMap[Arguments.StorageBaseAddress]);
            if (!string.IsNullOrEmpty(storageBaseAddressStr))
            {
                storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/";

                storageBaseAddress = new Uri(storageBaseAddressStr);
            }

            var storageType = arguments.GetOrThrow<string>(Arguments.StorageType);

            if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase))
            {
                var storagePath = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StoragePath]);

                if (storageBaseAddress != null)
                {
                    return new FileStorageFactory(storageBaseAddress, storagePath) {Verbose = verbose};
                }

                TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]);
                return null;
            }

            if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase))
            {
                var storageAccountName = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageAccountName]);
                var storageKeyValue = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageKeyValue]);
                var storageContainer = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageContainer]);
                var storagePath = arguments.GetOrDefault<string>(argumentNameMap[Arguments.StoragePath]);

                var credentials = new StorageCredentials(storageAccountName, storageKeyValue);
                var account = new CloudStorageAccount(credentials, true);
                return new AzureStorageFactory(account, storageContainer, storagePath, storageBaseAddress) { Verbose = verbose };
            }
            
            throw new ArgumentException($"Unrecognized storageType \"{storageType}\"");
        }
Ejemplo n.º 39
0
 protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
 {
     _connectionString = arguments.GetOrThrow<string>(Arguments.ConnectionString);
     _path = arguments.GetOrThrow<string>(Arguments.Path);
 }
Ejemplo n.º 40
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var gallery                    = arguments.GetOrThrow <string>(Arguments.Gallery);
            var index                      = arguments.GetOrThrow <string>(Arguments.Index);
            var packageBaseAddress         = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress);
            var source                     = arguments.GetOrThrow <string>(Arguments.Source);
            var requireRepositorySignature = arguments.GetOrDefault(Arguments.RequireRepositorySignature, false);
            var verbose                    = arguments.GetOrDefault(Arguments.Verbose, false);

            var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300);
            var sqlTimeout       = TimeSpan.FromSeconds(timeoutInSeconds);

            var connectionString    = arguments.GetOrThrow <string>(Arguments.ConnectionString);
            var galleryDbConnection = new AzureSqlConnectionFactory(
                connectionString,
                SecretInjector,
                LoggerFactory.CreateLogger <AzureSqlConnectionFactory>());
            var packageContentUriBuilder = new PackageContentUriBuilder(
                arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat));
            var galleryDatabase = new GalleryDatabaseQueryService(
                galleryDbConnection,
                packageContentUriBuilder,
                TelemetryService,
                timeoutInSeconds);

            CommandHelpers.AssertAzureStorage(arguments);

            var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
            var auditingStorageFactory   = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose);

            var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments);
            var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);

            Logger.LogInformation(
                "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}",
                gallery, index, monitoringStorageFactory, auditingStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri);

            var validatorConfig = new ValidatorConfiguration(
                packageBaseAddress,
                requireRepositorySignature);

            _packageValidator = ValidationFactory.CreatePackageValidator(
                gallery,
                index,
                auditingStorageFactory,
                validatorConfig,
                endpointConfiguration,
                messageHandlerFactory,
                galleryDatabase,
                LoggerFactory);

            _queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version);

            _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);

            _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger <LoggerMonitoringNotificationService>());

            _client = new CollectorHttpClient(messageHandlerFactory());

            _queueLoopDuration = TimeSpan.FromHours(
                arguments.GetOrDefault(
                    Arguments.QueueLoopDurationHours,
                    DefaultQueueLoopDurationHours));

            _queueDelay = TimeSpan.FromSeconds(
                arguments.GetOrDefault(
                    Arguments.QueueDelaySeconds,
                    DefaultQueueDelaySeconds));

            _workerCount = arguments.GetOrDefault(Arguments.WorkerCount, DefaultWorkerCount);

            SetUserAgentString();
        }
Ejemplo n.º 41
0
        public static Lucene.Net.Store.Directory GetLuceneDirectoryImpl(IDictionary<string, string> arguments, IDictionary<string, string> argumentNameMap, bool required = true)
        {
            try
            {
                var luceneDirectoryType = arguments.GetOrThrow<string>(argumentNameMap[Arguments.DirectoryType]);

                if (luceneDirectoryType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase))
                {
                    var lucenePath = arguments.GetOrThrow<string>(argumentNameMap[Arguments.Path]);

                    var directoryInfo = new DirectoryInfo(lucenePath);

                    if (directoryInfo.Exists)
                    {
                        return new SimpleFSDirectory(directoryInfo);
                    }

                    directoryInfo.Create();
                    directoryInfo.Refresh();

                    return new SimpleFSDirectory(directoryInfo);
                }
                if (luceneDirectoryType.Equals(Arguments.AzureStorageType, StringComparison.InvariantCultureIgnoreCase))
                {
                    var luceneStorageAccountName = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageAccountName]);

                    var luceneStorageKeyValue = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageKeyValue]);

                    var luceneStorageContainer = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageContainer]);

                    var credentials = new StorageCredentials(luceneStorageAccountName, luceneStorageKeyValue);
                    var account = new CloudStorageAccount(credentials, true);
                    return new AzureDirectory(account, luceneStorageContainer);
                }
                Trace.TraceError("Unrecognized Lucene Directory Type \"{0}\"", luceneDirectoryType);
                return null;
            }
            catch (ArgumentException)
            {
                if (required)
                {
                    throw;
                }

                return null;
            }
        }
Ejemplo n.º 42
0
        protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
        {
            var source           = arguments.GetOrThrow <string>(Arguments.Source);
            var storageAccount   = arguments.GetOrThrow <string>(Arguments.StorageAccountName);
            var storageKey       = arguments.GetOrThrow <string>(Arguments.StorageKeyValue);
            var storageContainer = arguments.GetOrThrow <string>(Arguments.StorageContainer);
            var verify           = arguments.GetOrDefault(Arguments.Verify, false);
            var verbose          = arguments.GetOrDefault(Arguments.Verbose, false);

            var services = new ServiceCollection();

            services.AddSingleton(TelemetryService);
            services.AddSingleton(LoggerFactory);
            services.AddLogging();

            // Prepare the HTTP Client
            services.AddSingleton(p =>
            {
                var httpClient = new HttpClient(new WebRequestHandler
                {
                    AllowPipelining = true
                });

                httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgentUtility.GetUserAgent());
                httpClient.Timeout = TimeSpan.FromMinutes(10);

                return(httpClient);
            });

            // Prepare the catalog reader.
            services.AddSingleton(p =>
            {
                var telemetryService          = p.GetRequiredService <ITelemetryService>();
                var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(telemetryService, verbose);

                return(new CollectorHttpClient(httpMessageHandlerFactory()));
            });

            services.AddTransient(p =>
            {
                var collectorHttpClient = p.GetRequiredService <CollectorHttpClient>();
                var telemetryService    = p.GetRequiredService <ITelemetryService>();

                return(new CatalogIndexReader(new Uri(source), collectorHttpClient, telemetryService));
            });

            // Prepare the Azure Blob Storage container.
            services.AddSingleton(p =>
            {
                var credentials = new StorageCredentials(storageAccount, storageKey);
                var account     = new CloudStorageAccount(credentials, useHttps: true);

                return(account
                       .CreateCloudBlobClient()
                       .GetContainerReference(storageContainer));
            });

            // Prepare the handler that will run on each catalog entry.
            if (verify)
            {
                Logger.LogInformation("Validating that all packages have the proper Content MD5 hash...");

                services.AddTransient <IPackagesContainerHandler, ValidatePackageHashHandler>();
            }
            else
            {
                Logger.LogInformation("Ensuring all packages have a Content MD5 hash...");

                services.AddTransient <IPackagesContainerHandler, FixPackageHashHandler>();
            }

            services.AddTransient <PackagesContainerCatalogProcessor>();

            _serviceProvider = services.BuildServiceProvider();
        }