public void InitializeJobConfiguration(IServiceProvider serviceProvider)
{
_configuration = serviceProvider.GetRequiredService <IOptionsSnapshot <JobConfiguration> >().Value;
_executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds;
_maxBlobsToProcess = _configuration.MaxBlobsToProcess ?? DefaultMaxBlobsToProcess;
var logHeader = _configuration.LogHeader ?? throw new ArgumentNullException(nameof(_configuration.LogHeader));
var logHeaderDelimiter = _configuration.LogHeaderDelimiter ?? throw new ArgumentNullException(nameof(_configuration.LogHeaderDelimiter));
_logHeaderMetadata = new LogHeaderMetadata(logHeader, logHeaderDelimiter);
_blobPrefix = _configuration.BlobPrefix;
var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService <ILogger <AzureBlobLeaseManager> >());
var source = new AzureStatsLogSource(
ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource),
_configuration.AzureContainerNameSource,
_executionTimeoutInSeconds / _maxBlobsToProcess,
blobLeaseManager,
serviceProvider.GetRequiredService <ILogger <AzureStatsLogSource> >());
var dest = new AzureStatsLogDestination(
ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination),
_configuration.AzureContainerNameDestination,
serviceProvider.GetRequiredService <ILogger <AzureStatsLogDestination> >());
var sanitizers = new List <ISanitizer> {
new ClientIPSanitizer(_logHeaderMetadata)
};
_processor = new Processor(source, dest, _maxBlobsToProcess, sanitizers, serviceProvider.GetRequiredService <ILogger <Processor> >());
}
/// <summary>
/// .ctor for the AzureStatsLogSource
/// </summary>
/// <param name="connectionString">The connection string for the Azure account.</param>
/// <param name="containerName">The container name.</param>
public AzureStatsLogSource(string connectionString, string containerName, int azureServerTimeoutInSeconds)
{
_azureAccount = CloudStorageAccount.Parse(connectionString);
_blobClient = _azureAccount.CreateCloudBlobClient();
_container = _blobClient.GetContainerReference(containerName);
_blobRequestOptions = new BlobRequestOptions();
_blobRequestOptions.ServerTimeout = TimeSpan.FromSeconds(azureServerTimeoutInSeconds);
_blobLeaseManager = new AzureBlobLeaseManager();
_deadletterContainerName = $"{containerName}-deadletter";
_archiveContainerName = $"{containerName}-archive";
}
public async Task when_the_leader_aborts_another_node_can_become_leader()
{
var settings = new BlobSettings(CloudStorageAccount.DevelopmentStorageAccount, "integration", "LeaderAbortingCreatesNewLeader");
var tokenSource = new CancellationTokenSource();
var firstLeader = new AzureBlobLeaseManager(settings, TimeSpan.FromSeconds(15));
var secondLeader = new AzureBlobLeaseManager(settings, TimeSpan.FromSeconds(15));
Assert.True(await firstLeader.AcquireLease(new LeaseOptions(nameof(firstLeader)), tokenSource.Token));
Assert.False(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), tokenSource.Token));
await firstLeader.ReleaseLease(new LeaseReleaseOptions(nameof(firstLeader)));
Assert.True(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), tokenSource.Token));
}
/// <summary>
/// .ctor for the AzureStatsLogSource
/// </summary>
/// <param name="connectionString">The connection string for the Azure account.</param>
/// <param name="containerName">The container name.</param>
public AzureStatsLogSource(CloudStorageAccount storageAccount,
string containerName,
int azureServerTimeoutInSeconds,
AzureBlobLeaseManager blobLeaseManager,
ILogger <AzureStatsLogSource> logger)
{
_azureAccount = storageAccount;
_blobClient = _azureAccount.CreateCloudBlobClient();
_container = _blobClient.GetContainerReference(containerName);
_blobRequestOptions = new BlobRequestOptions();
_blobRequestOptions.ServerTimeout = TimeSpan.FromSeconds(azureServerTimeoutInSeconds);
_blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager));
_deadletterContainerName = $"{containerName}-deadletter";
_archiveContainerName = $"{containerName}-archive";
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task when_the_leader_doesnt_renew_the_lease_another_node_can_become_leader()
{
var settings = new BlobSettings(CloudStorageAccount.DevelopmentStorageAccount, "integration", "LeaderRenewing");
var leaseLength = TimeSpan.FromSeconds(15);
var leaseExpiryWaitTime = TimeSpan.FromSeconds(5);
var firstLeader = new AzureBlobLeaseManager(settings, leaseLength);
await firstLeader.AcquireLease(new LeaseOptions(nameof(firstLeader)), CancellationToken.None);
await Task.Delay(leaseLength + leaseExpiryWaitTime);
var secondLeader = new AzureBlobLeaseManager(settings, leaseLength);
Assert.True(await secondLeader.AcquireLease(new LeaseOptions(nameof(secondLeader)), CancellationToken.None));
await secondLeader.ReleaseLease(new LeaseReleaseOptions(nameof(secondLeader)));
}
public void InitializeJobConfiguration(IServiceProvider serviceProvider)
{
_configuration = serviceProvider.GetRequiredService <IOptionsSnapshot <CollectAzureChinaCdnLogsConfiguration> >().Value;
_executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds;
var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService <ILogger <AzureBlobLeaseManager> >());
var source = new AzureStatsLogSource(
ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource),
_configuration.AzureContainerNameSource,
_executionTimeoutInSeconds / MaxFilesToProcess,
blobLeaseManager,
serviceProvider.GetRequiredService <ILogger <AzureStatsLogSource> >());
var dest = new AzureStatsLogDestination(
ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination),
_configuration.AzureContainerNameDestination,
serviceProvider.GetRequiredService <ILogger <AzureStatsLogDestination> >());
_chinaCollector = new ChinaStatsCollector(source, dest, serviceProvider.GetRequiredService <ILogger <ChinaStatsCollector> >());
}
