protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var gallery = arguments.GetOrThrow <string>(Arguments.Gallery); var index = arguments.GetOrThrow <string>(Arguments.Index); var packageBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress); var source = arguments.GetOrThrow <string>(Arguments.Source); var requireSignature = arguments.GetOrDefault(Arguments.RequireSignature, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose); var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments); var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); Logger.LogInformation( "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} endpoints: {Endpoints}", gallery, index, monitoringStorageFactory, auditingStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name))); _packageValidator = new PackageValidatorFactory(LoggerFactory) .Create(gallery, index, packageBaseAddress, auditingStorageFactory, endpointInputs, messageHandlerFactory, requireSignature, verbose); _queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version); _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger <LoggerMonitoringNotificationService>()); _regResource = Repository.Factory.GetCoreV3(index).GetResource <RegistrationResourceV3>(cancellationToken); _client = new CollectorHttpClient(messageHandlerFactory()); }
private static CatalogStorageFactory CreateStorageFactoryImpl(IDictionary <string, string> arguments, IDictionary <string, string> argumentNameMap, bool verbose, bool compressed) { Uri storageBaseAddress = null; var storageBaseAddressStr = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageBaseAddress]); if (!string.IsNullOrEmpty(storageBaseAddressStr)) { storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/"; storageBaseAddress = new Uri(storageBaseAddressStr); } var storageType = arguments.GetOrThrow <string>(Arguments.StorageType); if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase)) { var storagePath = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StoragePath]); if (storageBaseAddress != null) { return(new CatalogFileStorageFactory(storageBaseAddress, storagePath, verbose)); } TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]); return(null); } if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) { var storageAccountName = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageAccountName]); var storageKeyValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageKeyValue]); var storageContainer = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageContainer]); var storagePath = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StoragePath]); var storageSuffix = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageSuffix]); var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds])); var storageServerTimeout = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageServerTimeoutInSeconds])); var storageUseServerSideCopy = arguments.GetOrDefault <bool>(argumentNameMap[Arguments.StorageUseServerSideCopy]); var credentials = new StorageCredentials(storageAccountName, storageKeyValue); var account = string.IsNullOrEmpty(storageSuffix) ? new CloudStorageAccount(credentials, useHttps: true) : new CloudStorageAccount(credentials, storageSuffix, useHttps: true); return(new CatalogAzureStorageFactory( account, storageContainer, storageOperationMaxExecutionTime, storageServerTimeout, storagePath, storageBaseAddress, storageUseServerSideCopy, compressed, verbose)); } throw new ArgumentException($"Unrecognized storageType \"{storageType}\""); }
private static ISecretInjector GetSecretInjector(IDictionary<string, string> arguments) { ISecretReader secretReader; var vaultName = arguments.GetOrDefault<string>(Arguments.VaultName); if (string.IsNullOrEmpty(vaultName)) { secretReader = new EmptySecretReader(); } else { var clientId = arguments.GetOrThrow<string>(Arguments.ClientId); var certificateThumbprint = arguments.GetOrThrow<string>(Arguments.CertificateThumbprint); var storeName = arguments.GetOrDefault(Arguments.StoreName, StoreName.My); var storeLocation = arguments.GetOrDefault(Arguments.StoreLocation, StoreLocation.LocalMachine); var shouldValidateCert = arguments.GetOrDefault(Arguments.ValidateCertificate, false); var keyVaultConfig = new KeyVaultConfiguration(vaultName, clientId, certificateThumbprint, storeName, storeLocation, shouldValidateCert); secretReader = new CachingSecretReader(new KeyVaultReader(keyVaultConfig), arguments.GetOrDefault(Arguments.RefreshIntervalSec, CachingSecretReader.DefaultRefreshIntervalSec)); } return new SecretInjector(secretReader); }
private static ISecretInjector GetSecretInjector(IDictionary <string, string> arguments) { ISecretReader secretReader; var vaultName = arguments.GetOrDefault <string>(Arguments.VaultName); if (string.IsNullOrEmpty(vaultName)) { secretReader = new EmptySecretReader(); } else { var clientId = arguments.GetOrThrow <string>(Arguments.ClientId); var certificateThumbprint = arguments.GetOrThrow <string>(Arguments.CertificateThumbprint); var storeName = arguments.GetOrDefault(Arguments.StoreName, StoreName.My); var storeLocation = arguments.GetOrDefault(Arguments.StoreLocation, StoreLocation.LocalMachine); var shouldValidateCert = arguments.GetOrDefault(Arguments.ValidateCertificate, true); var keyVaultCertificate = CertificateUtility.FindCertificateByThumbprint(storeName, storeLocation, certificateThumbprint, shouldValidateCert); var keyVaultConfig = new KeyVaultConfiguration(vaultName, clientId, keyVaultCertificate); secretReader = new CachingSecretReader(new KeyVaultReader(keyVaultConfig), arguments.GetOrDefault(Arguments.RefreshIntervalSec, CachingSecretReader.DefaultRefreshIntervalSec)); } return(new SecretInjector(secretReader)); }
private void InitPrepare(IDictionary <string, string> arguments) { _outputFolder = arguments.GetOrThrow <string>(Arguments.OutputFolder); _catalogIndex = arguments.GetOrThrow <string>(Arguments.Source); _templateFile = arguments.GetOrThrow <string>(Arguments.TemplateFile); _batchSize = arguments.GetOrThrow <string>(Arguments.BatchSize); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var gallery = arguments.GetOrThrow <string>(Arguments.Gallery); var index = arguments.GetOrThrow <string>(Arguments.Index); var source = arguments.GetOrThrow <string>(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var endpointInputs = CommandHelpers.GetEndpointFactoryInputs(arguments); var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); var queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version); Logger.LogInformation( "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} endpoints: {Endpoints}", gallery, index, monitoringStorageFactory, string.Join(", ", endpointInputs.Select(e => e.Name))); var context = _collectorFactory.Create( queue, source, monitoringStorageFactory, endpointInputs, TelemetryService, messageHandlerFactory); _collector = context.Collector; _front = context.Front; _back = context.Back; }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { _connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString); _source = arguments.GetOrThrow <string>(Arguments.Source); _path = arguments.GetOrThrow <string>(Arguments.Path); _catalogIndexUrl = new Uri(_source); }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { _gallery = arguments.GetOrThrow<string>(Arguments.Gallery); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _id = arguments.GetOrThrow<string>(Arguments.Id); _version = arguments.GetOrDefault<string>(Arguments.Version); _storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose); }
public static Lucene.Net.Store.Directory GetLuceneDirectoryImpl( IDictionary <string, string> arguments, IDictionary <string, string> argumentNameMap, out string destination, bool required = true) { destination = null; try { var luceneDirectoryType = arguments.GetOrThrow <string>(argumentNameMap[Arguments.DirectoryType]); if (luceneDirectoryType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase)) { var lucenePath = arguments.GetOrThrow <string>(argumentNameMap[Arguments.Path]); var directoryInfo = new DirectoryInfo(lucenePath); destination = lucenePath; if (directoryInfo.Exists) { return(new SimpleFSDirectory(directoryInfo)); } directoryInfo.Create(); directoryInfo.Refresh(); return(new SimpleFSDirectory(directoryInfo)); } if (luceneDirectoryType.Equals(Arguments.AzureStorageType, StringComparison.InvariantCultureIgnoreCase)) { var luceneStorageAccountName = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageAccountName]); var luceneStorageKeyValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageKeyValue]); var luceneStorageContainer = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageContainer]); var credentials = new StorageCredentials(luceneStorageAccountName, luceneStorageKeyValue); var account = new CloudStorageAccount(credentials, useHttps: true); destination = luceneStorageContainer; return(new AzureDirectory(account, luceneStorageContainer)); } Trace.TraceError("Unrecognized Lucene Directory Type \"{0}\"", luceneDirectoryType); return(null); } catch (ArgumentException) { if (required) { throw; } return(null); } }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow <string>(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var contentBaseAddress = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress); var galleryBaseAddress = arguments.GetOrDefault <string>(Arguments.GalleryBaseAddress); var isContentFlatContainer = arguments.GetOrDefault <bool>(Arguments.ContentIsFlatContainer); var allIconsInFlatContainer = arguments.GetOrDefault <bool>(Arguments.AllIconsInFlatContainer); var maxConcurrentBatches = MaxConcurrentBatches(arguments.GetOrDefault <int>(Arguments.MaxConcurrentBatches)); // The term "legacy" here refers to the registration hives that do not contain any SemVer 2.0.0 packages. // In production, this is two registration hives: // 1) the first hive released, which is not gzipped and does not have SemVer 2.0.0 packages // 2) the secondary hive released, which is gzipped but does not have SemVer 2.0.0 packages var storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, verbose); Logger.LogInformation( "CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactories.LegacyStorageFactory); if (isContentFlatContainer) { var flatContainerCursorUriString = arguments.GetOrThrow <string>(Arguments.CursorUri); var flatContainerName = arguments.GetOrThrow <string>(Arguments.FlatContainerName); RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName); // In case that the flat container is used as the packages' source the registration needs to wait for the flatcontainer cursor _back = new HttpReadCursor(new Uri(flatContainerCursorUriString)); } else { RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); _back = MemoryCursor.CreateMax(); } _collector = new RegistrationCollector( new Uri(source), storageFactories.LegacyStorageFactory, storageFactories.SemVer2StorageFactory, contentBaseAddress == null ? null : new Uri(contentBaseAddress), galleryBaseAddress == null ? null : new Uri(galleryBaseAddress), allIconsInFlatContainer, TelemetryService, Logger, CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), maxConcurrentBatches: maxConcurrentBatches); var cursorStorage = storageFactories.LegacyStorageFactory.Create(); _front = new DurableCursor(cursorStorage.ResolveUri("cursor.json"), cursorStorage, MemoryCursor.MinValue); storageFactories.SemVer2StorageFactory?.Create(); _destination = storageFactories.LegacyStorageFactory.DestinationAddress; TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination?.AbsoluteUri; }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { _gallery = arguments.GetOrThrow <string>(Arguments.Gallery); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _id = arguments.GetOrThrow <string>(Arguments.Id); _version = arguments.GetOrDefault <string>(Arguments.Version); var storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose); _storage = storageFactory.Create(); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { PrintLightning(); // Hard code against Azure storage. arguments[Arguments.StorageType] = Arguments.AzureStorageType; // Configure the package path provider. var useFlatContainerAsPackageContent = arguments.GetOrDefault <bool>(Arguments.ContentIsFlatContainer, false); if (!useFlatContainerAsPackageContent) { RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); } else { var flatContainerName = arguments.GetOrThrow <string>(Arguments.FlatContainerName); RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerName); } _command = arguments.GetOrThrow <string>(Arguments.Command); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _log = _verbose ? Console.Out : new StringWriter(); _contentBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress); _galleryBaseAddress = arguments.GetOrThrow <string>(Arguments.GalleryBaseAddress); _storageFactories = CommandHelpers.CreateRegistrationStorageFactories(arguments, _verbose); _shouldIncludeSemVer2ForLegacyStorageFactory = RegistrationCollector.GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory); _postProcessGraphForLegacyStorageFactory = RegistrationCollector.GetPostProcessGraphForLegacyStorageFactory(_storageFactories.SemVer2StorageFactory); _forceIconsFromFlatContainer = arguments.GetOrDefault <bool>(Arguments.AllIconsInFlatContainer); _driver = arguments.GetOrDefault(Arguments.Driver, GraphDriver).ToLowerInvariant(); // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments // used by "prepare" should be used when executing "strike". _arguments = arguments; if (_driver != GraphDriver && _driver != JsonDriver) { throw new NotSupportedException($"The lightning driver '{_driver}' is not supported."); } switch (_command.ToLowerInvariant()) { case "charge": case "prepare": InitPrepare(arguments); break; case "strike": InitStrike(arguments); break; default: throw new NotSupportedException($"The lightning command '{_command}' is not supported."); } }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { ServicePointManager.DefaultConnectionLimit = DegreeOfParallelism; var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var packageStorageBase = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress); var failCacheTime = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1)); var auxStorageFactory = CreateAuxStorageFactory(arguments, verbose); var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose); var packageStorage = new AzureStorage( storageBaseUri: new Uri(packageStorageBase), maxExecutionTime: TimeSpan.FromMinutes(15), serverTimeout: TimeSpan.FromMinutes(10), useServerSideCopy: true, compressContent: false, verbose: true, throttle: null); var source = arguments.GetOrThrow <string>(Arguments.Source); var iconProcessor = new IconProcessor(TelemetryService, LoggerFactory.CreateLogger <IconProcessor>()); var httpHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); var httpMessageHandler = httpHandlerFactory(); var httpClient = new HttpClient(httpMessageHandler); var simpleHttpClient = new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()); var catalogClient = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger <CatalogClient>()); var httpResponseProvider = new HttpClientWrapper(httpClient); var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger <ExternalIconContentProvider>()); var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger <IconCopyResultCache>()); var leafProcessor = new CatalogLeafDataProcessor( packageStorage, iconProcessor, externalIconProvider, iconCopyResultCache, TelemetryService, LoggerFactory.CreateLogger <CatalogLeafDataProcessor>()); _collector = new IconsCollector( new Uri(source), TelemetryService, targetStorageFactory, catalogClient, leafProcessor, iconCopyResultCache, auxStorageFactory, CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), LoggerFactory.CreateLogger <IconsCollector>()); var cursorStorage = auxStorageFactory.Create(); _front = new DurableCursor(cursorStorage.ResolveUri("c2icursor.json"), cursorStorage, DateTime.MinValue.ToUniversalTime()); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var verbose = arguments.GetOrDefault(Arguments.Verbose, false); _maxRequeueQueueSize = arguments.GetOrDefault(Arguments.MaxRequeueQueueSize, DefaultMaxQueueSize); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); _packageValidatorContextQueue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version); Logger.LogInformation( "CONFIG storage: {Storage}", monitoringStorageFactory); _monitoringCursor = ValidationFactory.GetFront(monitoringStorageFactory); _galleryCursor = CreateCursor(monitoringStorageFactory, GalleryCursorFileName); _deletedCursor = CreateCursor(monitoringStorageFactory, DeletedCursorFileName); var connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString); var galleryDbConnection = new AzureSqlConnectionFactory( connectionString, SecretInjector, LoggerFactory.CreateLogger <AzureSqlConnectionFactory>()); var packageContentUriBuilder = new PackageContentUriBuilder( arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat)); var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); _galleryDatabaseQueryService = new GalleryDatabaseQueryService( galleryDbConnection, packageContentUriBuilder, TelemetryService, timeoutInSeconds); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( "Auditing", arguments, verbose, new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); _auditingStorage = auditingStorageFactory.Create(); var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); _client = new CollectorHttpClient(messageHandlerFactory()); }
public static EndpointConfiguration GetEndpointConfiguration(IDictionary <string, string> arguments) { var registrationCursorUri = arguments.GetOrThrow <Uri>(Arguments.RegistrationCursorUri); var flatContainerCursorUri = arguments.GetOrThrow <Uri>(Arguments.FlatContainerCursorUri); var instanceNameToSearchBaseUri = GetSuffixToUri(arguments, Arguments.SearchBaseUriPrefix); var instanceNameToSearchCursorUri = GetSuffixToUri(arguments, Arguments.SearchCursorUriPrefix); var instanceNameToSearchConfig = new Dictionary <string, SearchEndpointConfiguration>(); foreach (var pair in instanceNameToSearchBaseUri) { var instanceName = pair.Key; // Find all cursors with an instance name starting with the search base URI instance name. We do this // because there may be multiple potential cursors representing the state of a search service. var matchingCursors = instanceNameToSearchCursorUri.Keys.Where(x => x.StartsWith(instanceName)).ToList(); if (!matchingCursors.Any()) { throw new ArgumentException( $"The -{Arguments.SearchBaseUriPrefix}{instanceName} argument does not have any matching " + $"-{Arguments.SearchCursorUriPrefix}{instanceName}* arguments."); } instanceNameToSearchConfig[instanceName] = new SearchEndpointConfiguration( matchingCursors.Select(x => instanceNameToSearchCursorUri[x]).ToList(), pair.Value); foreach (var key in matchingCursors) { instanceNameToSearchCursorUri.Remove(key); } } // See if there are any search cursor URI arguments left over and error out. Better to fail than to ignore // an argument that the user expected to be relevant. if (instanceNameToSearchCursorUri.Any()) { throw new ArgumentException( $"There are -{Arguments.SearchCursorUriPrefix}* arguments without matching " + $"-{Arguments.SearchBaseUriPrefix}* arguments. The unmatched suffixes were: {string.Join(", ", instanceNameToSearchCursorUri.Keys)}"); } return(new EndpointConfiguration( registrationCursorUri, flatContainerCursorUri, instanceNameToSearchConfig)); }
public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key, string messageWithFormat) { V value = dictionary.GetOrThrow(key, messageWithFormat); dictionary.Remove(key); return(value); }
public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key, Func <K, Exception> exception) { V value = dictionary.GetOrThrow(key, exception); dictionary.Remove(key); return(value); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow <string>(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments); var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); var queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version); Logger.LogInformation( "CONFIG storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); _enqueuer = ValidationFactory.CreatePackageValidatorContextEnqueuer( queue, source, monitoringStorageFactory, endpointConfiguration, TelemetryService, messageHandlerFactory, LoggerFactory); }
public static void AssertAzureStorage(IDictionary <string, string> arguments) { if (arguments.GetOrThrow <string>(Arguments.StorageType) != Arguments.AzureStorageType) { throw new ArgumentException("Only Azure storage is supported!"); } }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { PrintLightning(); // Hard code against Azure storage. arguments[Arguments.StorageType] = Arguments.AzureStorageType; _command = arguments.GetOrThrow <string>(Arguments.Command); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _log = _verbose ? Console.Out : new StringWriter(); // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments // used by "prepare" should be used when executing "strike". _arguments = arguments; switch (_command.ToLowerInvariant()) { case "charge": case "prepare": InitPrepare(arguments); break; case "strike": InitStrike(arguments); break; default: throw new NotSupportedException($"The lightning command '{_command}' is not supported."); } }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { _directory = CommandHelpers.GetLuceneDirectory(arguments); _source = arguments.GetOrThrow<string>(Arguments.Source); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _registration = arguments.GetOrDefault<string>(Arguments.Registration); if (_registration == null) { Logger.LogInformation("Lucene index will be created up to the end of the catalog (alternatively if you provide a registration it will not pass that)"); } _catalogBaseAddress = arguments.GetOrDefault<string>(Arguments.CatalogBaseAddress); if (_catalogBaseAddress == null) { Logger.LogInformation("No catalogBaseAddress was specified so the Lucene index will NOT contain the storage paths"); } _storageBaseAddress = arguments.GetOrDefault<string>(Arguments.StorageBaseAddress); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" registration: \"{Registration}\"" + " catalogBaseAddress: \"{CatalogBaseAddress}\" storageBaseAddress: \"{StorageBaseAddress}\"", _source, _registration ?? "(null)", _catalogBaseAddress ?? "(null)", _storageBaseAddress ?? "(null)"); _handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(_verbose, _catalogBaseAddress, _storageBaseAddress); }
public static V Extract <K, V>(this IDictionary <K, V> dictionary, K key) where K : object { V value = dictionary.GetOrThrow(key); dictionary.Remove(key); return(value); }
public static IStorageQueue <T> CreateStorageQueue <T>(IDictionary <string, string> arguments, int version) { var storageType = arguments.GetOrThrow <string>(Arguments.StorageType); if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) { var storageAccountName = arguments.GetOrThrow <string>(Arguments.StorageAccountName); var storageQueueName = arguments.GetOrDefault <string>(Arguments.StorageQueueName); var account = GetCloudStorageAccount(storageAccountName, endpointSuffix: null, arguments, ArgumentNames); return(new StorageQueue <T>(new AzureStorageQueue(account, storageQueueName), new JsonMessageSerializer <T>(JsonSerializerUtility.SerializerSettings), version)); } else { throw new NotImplementedException("Only Azure storage queues are supported!"); } }
private static Dictionary <string, Uri> GetSuffixToUri(IDictionary <string, string> arguments, string prefix) { var suffixToUri = new Dictionary <string, Uri>(); foreach (var key in arguments.Keys.Where(x => x.StartsWith(prefix))) { var suffix = key.Substring(prefix.Length); suffixToUri[suffix] = arguments.GetOrThrow <Uri>(key); } return(suffixToUri); }
private static CloudStorageAccount GetCloudStorageAccount(string storageAccountName, string endpointSuffix, IDictionary <string, string> arguments, IDictionary <string, string> argumentNameMap) { var storageKeyValue = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageKeyValue]); if (string.IsNullOrEmpty(storageKeyValue)) { var storageSasValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageSasValue]); var credentialSas = new StorageCredentials(storageSasValue); return(new CloudStorageAccount(credentialSas, storageAccountName, endpointSuffix, useHttps: true)); } var credentialKey = new StorageCredentials(storageAccountName, storageKeyValue); return(new CloudStorageAccount(credentialKey, endpointSuffix, useHttps: true)); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { _directory = CommandHelpers.GetLuceneDirectory(arguments, out var destination); _source = arguments.GetOrThrow <string>(Arguments.Source); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _registration = arguments.GetOrDefault <string>(Arguments.Registration); if (_registration == null) { Logger.LogInformation("Lucene index will be created up to the end of the catalog (alternatively if you provide a registration it will not pass that)"); } _catalogBaseAddress = arguments.GetOrDefault <string>(Arguments.CatalogBaseAddress); if (_catalogBaseAddress == null) { Logger.LogInformation("No catalogBaseAddress was specified so the Lucene index will NOT contain the storage paths"); } _storageBaseAddress = arguments.GetOrDefault <string>(Arguments.StorageBaseAddress); var commitTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.CommitTimeoutInSeconds); if (commitTimeoutInSeconds.HasValue) { _commitTimeout = TimeSpan.FromSeconds(commitTimeoutInSeconds.Value); } else { _commitTimeout = null; } Logger.LogInformation("CONFIG source: \"{ConfigSource}\" registration: \"{Registration}\"" + " catalogBaseAddress: \"{CatalogBaseAddress}\" storageBaseAddress: \"{StorageBaseAddress}\" commitTimeout: \"{CommmitTimeout}\"", _source, _registration ?? "(null)", _catalogBaseAddress ?? "(null)", _storageBaseAddress ?? "(null)", _commitTimeout?.ToString() ?? "(null)"); _handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory( TelemetryService, _verbose, _catalogBaseAddress, _storageBaseAddress); _destination = destination; TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination; }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow <string>(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var contentBaseAddress = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress); var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var httpClientTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.HttpClientTimeoutInSeconds); var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null; StorageFactory preferredPackageSourceStorageFactory = null; IAzureStorage preferredPackageSourceStorage = null; var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false); if (preferAlternatePackageSourceStorage) { preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose); preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage; } Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", source, storageFactory, preferredPackageSourceStorageFactory); Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout); MaxDegreeOfParallelism = 256; _collector = new DnxCatalogCollector( new Uri(source), storageFactory, preferredPackageSourceStorage, contentBaseAddress == null ? null : new Uri(contentBaseAddress), TelemetryService, Logger, MaxDegreeOfParallelism, httpClient => new CatalogClient(new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()), LoggerFactory.CreateLogger <CatalogClient>()), CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), httpClientTimeout); var storage = storageFactory.Create(); _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); _back = MemoryCursor.CreateMax(); _destination = storageFactory.BaseAddress; TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri; }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow<string>(Arguments.Source); var unlistShouldDelete = arguments.GetOrDefault(Arguments.UnlistShouldDelete, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var contentBaseAddress = arguments.GetOrDefault<string>(Arguments.ContentBaseAddress); StorageFactory storageFactoryToUse; var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var compressedStorageFactory = CommandHelpers.CreateCompressedStorageFactory(arguments, verbose); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory); RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider(); if (compressedStorageFactory != null) { var secondaryStorageBaseUrlRewriter = new SecondaryStorageBaseUrlRewriter(new List<KeyValuePair<string, string>> { // always rewrite storage root url in seconary new KeyValuePair<string, string>(storageFactory.BaseAddress.ToString(), compressedStorageFactory.BaseAddress.ToString()) }); var aggregateStorageFactory = new AggregateStorageFactory( storageFactory, new[] { compressedStorageFactory }, secondaryStorageBaseUrlRewriter.Rewrite); storageFactoryToUse = aggregateStorageFactory; } else { storageFactoryToUse = storageFactory; } _collector = new RegistrationCollector(new Uri(source), storageFactoryToUse, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress) }; var storage = storageFactoryToUse.Create(); _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); _back = MemoryCursor.CreateMax(); }
public static IEnumerable <EndpointFactory.Input> GetEndpointFactoryInputs(IDictionary <string, string> arguments) { return(arguments.GetOrThrow <string>(Arguments.EndpointsToTest).Split(';').Select(e => { var endpointParts = e.Split('|'); if (endpointParts.Count() < 2) { throw new ArgumentException( $"\"{e}\" is not a valid endpoint! Each endpoint must follow this format: " + $"<endpoint-to-test-name>|<endpoint-to-test-cursor>."); } return new EndpointFactory.Input(endpointParts[0], new Uri(endpointParts[1])); })); }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { Gallery = arguments.GetOrThrow<string>(Arguments.Gallery); Verbose = arguments.GetOrDefault(Arguments.Verbose, false); StartDate = arguments.GetOrDefault(Arguments.StartDate, DateTimeMinValueUtc); var catalogStorageFactory = CommandHelpers.CreateStorageFactory(arguments, Verbose); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", Gallery, catalogStorageFactory); CatalogStorage = catalogStorageFactory.Create(); AuditingStorage = auditingStorageFactory.Create(); Top = 20; Timeout = TimeSpan.FromSeconds(300); }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow<string>(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var contentBaseAddress = arguments.GetOrDefault<string>(Arguments.ContentBaseAddress); var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\"", source, storageFactory); _collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress) }; var storage = storageFactory.Create(); _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); _back = MemoryCursor.CreateMax(); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { Gallery = arguments.GetOrThrow <string>(Arguments.Gallery); Verbose = arguments.GetOrDefault(Arguments.Verbose, false); StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc); SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false); StorageFactory preferredPackageSourceStorageFactory = null; var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false); if (preferAlternatePackageSourceStorage) { preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, Verbose); } var catalogStorageFactory = CommandHelpers.CreateStorageFactory(arguments, Verbose); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", Gallery, catalogStorageFactory, preferredPackageSourceStorageFactory); CatalogStorage = catalogStorageFactory.Create(); AuditingStorage = auditingStorageFactory.Create(); if (preferAlternatePackageSourceStorage) { PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create(); } Destination = catalogStorageFactory.BaseAddress; TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri; Top = 20; Timeout = TimeSpan.FromSeconds(300); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { Verbose = arguments.GetOrDefault(Arguments.Verbose, false); StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc); Top = arguments.GetOrDefault(Arguments.CursorSize, 20); SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false); StorageFactory preferredPackageSourceStorageFactory = null; var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false); if (preferAlternatePackageSourceStorage) { preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( "PreferredPackageSourceStorage", arguments, Verbose, new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); } var catalogStorageFactory = CommandHelpers.CreateStorageFactory( arguments, Verbose, new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( "Auditing", arguments, Verbose, new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", GalleryDbConnection, catalogStorageFactory, preferredPackageSourceStorageFactory); CatalogStorage = catalogStorageFactory.Create(); AuditingStorage = auditingStorageFactory.Create(); if (preferAlternatePackageSourceStorage) { PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create(); } Destination = catalogStorageFactory.BaseAddress; TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri; // Setup gallery database access PackageContentUriBuilder = new PackageContentUriBuilder( arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat)); var connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString); GalleryDbConnection = new AzureSqlConnectionFactory( connectionString, SecretInjector, LoggerFactory.CreateLogger <AzureSqlConnectionFactory>()); var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); Timeout = TimeSpan.FromSeconds(timeoutInSeconds); GalleryDatabaseQueryService = new GalleryDatabaseQueryService( GalleryDbConnection, PackageContentUriBuilder, TelemetryService, timeoutInSeconds); }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { PrintLightning(); _command = arguments.GetOrThrow<string>(Arguments.Command); _verbose = arguments.GetOrDefault(Arguments.Verbose, false); _log = _verbose ? Console.Out : new StringWriter(); switch (_command.ToLowerInvariant()) { case "charge": case "prepare": InitPrepare(arguments); break; case "strike": InitStrike(arguments); break; default: throw new ArgumentNullException(); } _contentBaseAddress = arguments.GetOrThrow<string>(Arguments.ContentBaseAddress); _storageAccount = arguments.GetOrThrow<string>(Arguments.StorageAccount); _storageContainer = arguments.GetOrThrow<string>(Arguments.StorageContainer); _storageBaseAddress = arguments.GetOrThrow<string>(Arguments.StorageBaseAddress); _compress = arguments.GetOrDefault(Arguments.Compress, false); }
private void InitStrike(IDictionary <string, string> arguments) { _indexFile = arguments.GetOrThrow <string>(Arguments.IndexFile); _cursorFile = arguments.GetOrThrow <string>(Arguments.CursorFile); }
private void InitPrepare(IDictionary<string, string> arguments) { _outputFolder = arguments.GetOrThrow<string>(Arguments.OutputFolder); _catalogIndex = arguments.GetOrThrow<string>(Arguments.CatalogIndex); _templateFile = arguments.GetOrThrow<string>(Arguments.TemplateFile); _batchSize = arguments.GetOrThrow<string>(Arguments.BatchSize); }
private void InitStrike(IDictionary<string, string> arguments) { _indexFile = arguments.GetOrThrow<string>(Arguments.IndexFile); _cursorFile = arguments.GetOrThrow<string>(Arguments.CursorFile); }
private static StorageFactory CreateStorageFactoryImpl(IDictionary<string, string> arguments, IDictionary<string, string> argumentNameMap, bool verbose) { Uri storageBaseAddress = null; var storageBaseAddressStr = arguments.GetOrDefault<string>(argumentNameMap[Arguments.StorageBaseAddress]); if (!string.IsNullOrEmpty(storageBaseAddressStr)) { storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/"; storageBaseAddress = new Uri(storageBaseAddressStr); } var storageType = arguments.GetOrThrow<string>(Arguments.StorageType); if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase)) { var storagePath = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StoragePath]); if (storageBaseAddress != null) { return new FileStorageFactory(storageBaseAddress, storagePath) {Verbose = verbose}; } TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]); return null; } if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) { var storageAccountName = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageAccountName]); var storageKeyValue = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageKeyValue]); var storageContainer = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageContainer]); var storagePath = arguments.GetOrDefault<string>(argumentNameMap[Arguments.StoragePath]); var credentials = new StorageCredentials(storageAccountName, storageKeyValue); var account = new CloudStorageAccount(credentials, true); return new AzureStorageFactory(account, storageContainer, storagePath, storageBaseAddress) { Verbose = verbose }; } throw new ArgumentException($"Unrecognized storageType \"{storageType}\""); }
protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken) { _connectionString = arguments.GetOrThrow<string>(Arguments.ConnectionString); _path = arguments.GetOrThrow<string>(Arguments.Path); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var gallery = arguments.GetOrThrow <string>(Arguments.Gallery); var index = arguments.GetOrThrow <string>(Arguments.Index); var packageBaseAddress = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress); var source = arguments.GetOrThrow <string>(Arguments.Source); var requireRepositorySignature = arguments.GetOrDefault(Arguments.RequireRepositorySignature, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); var sqlTimeout = TimeSpan.FromSeconds(timeoutInSeconds); var connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString); var galleryDbConnection = new AzureSqlConnectionFactory( connectionString, SecretInjector, LoggerFactory.CreateLogger <AzureSqlConnectionFactory>()); var packageContentUriBuilder = new PackageContentUriBuilder( arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat)); var galleryDatabase = new GalleryDatabaseQueryService( galleryDbConnection, packageContentUriBuilder, TelemetryService, timeoutInSeconds); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose); var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments); var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); Logger.LogInformation( "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", gallery, index, monitoringStorageFactory, auditingStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); var validatorConfig = new ValidatorConfiguration( packageBaseAddress, requireRepositorySignature); _packageValidator = ValidationFactory.CreatePackageValidator( gallery, index, auditingStorageFactory, validatorConfig, endpointConfiguration, messageHandlerFactory, galleryDatabase, LoggerFactory); _queue = CommandHelpers.CreateStorageQueue <PackageValidatorContext>(arguments, PackageValidatorContext.Version); _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger <LoggerMonitoringNotificationService>()); _client = new CollectorHttpClient(messageHandlerFactory()); _queueLoopDuration = TimeSpan.FromHours( arguments.GetOrDefault( Arguments.QueueLoopDurationHours, DefaultQueueLoopDurationHours)); _queueDelay = TimeSpan.FromSeconds( arguments.GetOrDefault( Arguments.QueueDelaySeconds, DefaultQueueDelaySeconds)); _workerCount = arguments.GetOrDefault(Arguments.WorkerCount, DefaultWorkerCount); SetUserAgentString(); }
public static Lucene.Net.Store.Directory GetLuceneDirectoryImpl(IDictionary<string, string> arguments, IDictionary<string, string> argumentNameMap, bool required = true) { try { var luceneDirectoryType = arguments.GetOrThrow<string>(argumentNameMap[Arguments.DirectoryType]); if (luceneDirectoryType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase)) { var lucenePath = arguments.GetOrThrow<string>(argumentNameMap[Arguments.Path]); var directoryInfo = new DirectoryInfo(lucenePath); if (directoryInfo.Exists) { return new SimpleFSDirectory(directoryInfo); } directoryInfo.Create(); directoryInfo.Refresh(); return new SimpleFSDirectory(directoryInfo); } if (luceneDirectoryType.Equals(Arguments.AzureStorageType, StringComparison.InvariantCultureIgnoreCase)) { var luceneStorageAccountName = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageAccountName]); var luceneStorageKeyValue = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageKeyValue]); var luceneStorageContainer = arguments.GetOrThrow<string>(argumentNameMap[Arguments.StorageContainer]); var credentials = new StorageCredentials(luceneStorageAccountName, luceneStorageKeyValue); var account = new CloudStorageAccount(credentials, true); return new AzureDirectory(account, luceneStorageContainer); } Trace.TraceError("Unrecognized Lucene Directory Type \"{0}\"", luceneDirectoryType); return null; } catch (ArgumentException) { if (required) { throw; } return null; } }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { var source = arguments.GetOrThrow <string>(Arguments.Source); var storageAccount = arguments.GetOrThrow <string>(Arguments.StorageAccountName); var storageKey = arguments.GetOrThrow <string>(Arguments.StorageKeyValue); var storageContainer = arguments.GetOrThrow <string>(Arguments.StorageContainer); var verify = arguments.GetOrDefault(Arguments.Verify, false); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var services = new ServiceCollection(); services.AddSingleton(TelemetryService); services.AddSingleton(LoggerFactory); services.AddLogging(); // Prepare the HTTP Client services.AddSingleton(p => { var httpClient = new HttpClient(new WebRequestHandler { AllowPipelining = true }); httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgentUtility.GetUserAgent()); httpClient.Timeout = TimeSpan.FromMinutes(10); return(httpClient); }); // Prepare the catalog reader. services.AddSingleton(p => { var telemetryService = p.GetRequiredService <ITelemetryService>(); var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(telemetryService, verbose); return(new CollectorHttpClient(httpMessageHandlerFactory())); }); services.AddTransient(p => { var collectorHttpClient = p.GetRequiredService <CollectorHttpClient>(); var telemetryService = p.GetRequiredService <ITelemetryService>(); return(new CatalogIndexReader(new Uri(source), collectorHttpClient, telemetryService)); }); // Prepare the Azure Blob Storage container. services.AddSingleton(p => { var credentials = new StorageCredentials(storageAccount, storageKey); var account = new CloudStorageAccount(credentials, useHttps: true); return(account .CreateCloudBlobClient() .GetContainerReference(storageContainer)); }); // Prepare the handler that will run on each catalog entry. if (verify) { Logger.LogInformation("Validating that all packages have the proper Content MD5 hash..."); services.AddTransient <IPackagesContainerHandler, ValidatePackageHashHandler>(); } else { Logger.LogInformation("Ensuring all packages have a Content MD5 hash..."); services.AddTransient <IPackagesContainerHandler, FixPackageHashHandler>(); } services.AddTransient <PackagesContainerCatalogProcessor>(); _serviceProvider = services.BuildServiceProvider(); }