public void InitializeJobConfiguration(IServiceProvider serviceProvider) { _configuration = serviceProvider.GetRequiredService <IOptionsSnapshot <JobConfiguration> >().Value; _executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds; _maxBlobsToProcess = _configuration.MaxBlobsToProcess ?? DefaultMaxBlobsToProcess; var logHeader = _configuration.LogHeader ?? throw new ArgumentNullException(nameof(_configuration.LogHeader)); var logHeaderDelimiter = _configuration.LogHeaderDelimiter ?? throw new ArgumentNullException(nameof(_configuration.LogHeaderDelimiter)); _logHeaderMetadata = new LogHeaderMetadata(logHeader, logHeaderDelimiter); _blobPrefix = _configuration.BlobPrefix; var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService <ILogger <AzureBlobLeaseManager> >()); var source = new AzureStatsLogSource( ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / _maxBlobsToProcess, blobLeaseManager, serviceProvider.GetRequiredService <ILogger <AzureStatsLogSource> >()); var dest = new AzureStatsLogDestination( ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService <ILogger <AzureStatsLogDestination> >()); var sanitizers = new List <ISanitizer> { new ClientIPSanitizer(_logHeaderMetadata) }; _processor = new Processor(source, dest, _maxBlobsToProcess, sanitizers, serviceProvider.GetRequiredService <ILogger <Processor> >()); }
/// <summary> /// .ctor for the AzureStatsLogSource /// </summary> /// <param name="connectionString">The connection string for the Azure account.</param> /// <param name="containerName">The container name.</param> public AzureStatsLogSource(string connectionString, string containerName, int azureServerTimeoutInSeconds) { _azureAccount = CloudStorageAccount.Parse(connectionString); _blobClient = _azureAccount.CreateCloudBlobClient(); _container = _blobClient.GetContainerReference(containerName); _blobRequestOptions = new BlobRequestOptions(); _blobRequestOptions.ServerTimeout = TimeSpan.FromSeconds(azureServerTimeoutInSeconds); _blobLeaseManager = new AzureBlobLeaseManager(); _deadletterContainerName = $"{containerName}-deadletter"; _archiveContainerName = $"{containerName}-archive"; }
public async Task when_the_leader_aborts_another_node_can_become_leader() { var settings = new BlobSettings(CloudStorageAccount.DevelopmentStorageAccount, "integration", "LeaderAbortingCreatesNewLeader"); var tokenSource = new CancellationTokenSource(); var firstLeader = new AzureBlobLeaseManager(settings, TimeSpan.FromSeconds(15)); var secondLeader = new AzureBlobLeaseManager(settings, TimeSpan.FromSeconds(15)); Assert.True(await firstLeader.AcquireLease(new LeaseOptions(nameof(firstLeader)), tokenSource.Token)); Assert.False(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), tokenSource.Token)); await firstLeader.ReleaseLease(new LeaseReleaseOptions(nameof(firstLeader))); Assert.True(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), tokenSource.Token)); }
/// <summary> /// .ctor for the AzureStatsLogSource /// </summary> /// <param name="connectionString">The connection string for the Azure account.</param> /// <param name="containerName">The container name.</param> public AzureStatsLogSource(CloudStorageAccount storageAccount, string containerName, int azureServerTimeoutInSeconds, AzureBlobLeaseManager blobLeaseManager, ILogger <AzureStatsLogSource> logger) { _azureAccount = storageAccount; _blobClient = _azureAccount.CreateCloudBlobClient(); _container = _blobClient.GetContainerReference(containerName); _blobRequestOptions = new BlobRequestOptions(); _blobRequestOptions.ServerTimeout = TimeSpan.FromSeconds(azureServerTimeoutInSeconds); _blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager)); _deadletterContainerName = $"{containerName}-deadletter"; _archiveContainerName = $"{containerName}-archive"; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); }
public async Task when_the_leader_doesnt_renew_the_lease_another_node_can_become_leader() { var settings = new BlobSettings(CloudStorageAccount.DevelopmentStorageAccount, "integration", "LeaderRenewing"); var leaseLength = TimeSpan.FromSeconds(15); var leaseExpiryWaitTime = TimeSpan.FromSeconds(5); var firstLeader = new AzureBlobLeaseManager(settings, leaseLength); await firstLeader.AcquireLease(new LeaseOptions(nameof(firstLeader)), CancellationToken.None); await Task.Delay(leaseLength + leaseExpiryWaitTime); var secondLeader = new AzureBlobLeaseManager(settings, leaseLength); Assert.True(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), CancellationToken.None)); await secondLeader.ReleaseLease(new LeaseReleaseOptions(nameof(secondLeader))); }
public void InitializeJobConfiguration(IServiceProvider serviceProvider) { _configuration = serviceProvider.GetRequiredService <IOptionsSnapshot <CollectAzureChinaCdnLogsConfiguration> >().Value; _executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds; var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService <ILogger <AzureBlobLeaseManager> >()); var source = new AzureStatsLogSource( ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess, blobLeaseManager, serviceProvider.GetRequiredService <ILogger <AzureStatsLogSource> >()); var dest = new AzureStatsLogDestination( ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService <ILogger <AzureStatsLogDestination> >()); _chinaCollector = new ChinaStatsCollector(source, dest, serviceProvider.GetRequiredService <ILogger <ChinaStatsCollector> >()); }