internal AzureBlobUploader( FabricEvents.ExtensionsEvents traceSource, string logSourceId, string etwLogDirName, string workFolder, StorageAccountFactory storageAccountFactory, string containerName, string fabricNodeInstanceName, string deploymentId, AzureBlobPerformance perfHelper, AccessCondition uploadStreamAccessCondition, AccessCondition uploadFileAccessCondition) : this( traceSource, logSourceId, etwLogDirName, workFolder, storageAccountFactory, string.Empty, false, containerName, string.Empty, fabricNodeInstanceName, deploymentId, perfHelper, uploadStreamAccessCondition, uploadFileAccessCondition) { }
internal AzureFileTrimmer( string sourceFolderName, string workFolder, StorageAccountFactory storageAccountFactory, string containerName, TimeSpan blobDeletionAge, string fabricNodeInstanceName, string deploymentId, bool azureInterfaceAvailable) { // Initialization this.stopping = false; this.sourceFolderName = sourceFolderName; this.localMap = Path.Combine(workFolder, AzureBlobUploader.LocalMapFolder); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.logSourceId = string.Concat(TraceType, "-", Guid.NewGuid().ToString("P")); this.storageAccountFactory = storageAccountFactory; this.containerName = containerName; this.fabricNodeInstanceName = fabricNodeInstanceName; this.blobDeletionAge = blobDeletionAge; this.deploymentId = deploymentId; this.azureInterfaceAvailable = azureInterfaceAvailable; this.localMapTrimmingHelper = new LocalMapTrimmingHelper(sourceFolderName); this.folderTrimmer = new FolderTrimmer( this.traceSource, this.logSourceId, blobDeletionAge); this.perfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Deletion and query are currently synchronous, so the concurrency // count is 1. this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobQuery, 1); this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobDeletion, 1); // Create a timer to delete old logs string timerId = string.Concat( this.logSourceId, OldLogDeletionTimerIdSuffix); var oldLogDeletionInterval = (this.blobDeletionAge < TimeSpan.FromDays(1)) ? AzureConstants.OldLogDeletionIntervalForTest : AzureConstants.OldLogDeletionInterval; this.oldLogDeletionTimer = new DcaTimer( timerId, this.DeleteOldLogsHandler, oldLogDeletionInterval); this.oldLogDeletionTimer.Start(); }
internal AzureTableTrimmer( FabricEvents.ExtensionsEvents traceSource, string logSourceId, StorageAccountFactory storageAccountFactory, string tableName, TimeSpan entityDeletionAge, QueryCreationMethod queryCreationMethod, AzureTablePerformance perfHelper) { // Initialization this.stopping = false; this.traceSource = traceSource; this.logSourceId = logSourceId; this.storageAccountFactory = storageAccountFactory; this.tableName = tableName; this.entityDeletionAge = entityDeletionAge; this.perfHelper = perfHelper; this.queryCreationMethod = queryCreationMethod; // Deletion and query are currently synchronous, so the concurrency // count is 1. this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.TableQuery, 1); this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.TableDeletion, 1); // Create a timer to delete old logs string timerId = String.Concat( this.logSourceId, OldLogDeletionTimerIdSuffix); this.oldLogDeletionTimer = new DcaTimer( timerId, this.DeleteOldLogsHandler, (this.entityDeletionAge < TimeSpan.FromDays(1)) ? AzureConstants.OldLogDeletionIntervalForTest : AzureConstants.OldLogDeletionInterval); this.oldLogDeletionTimer.Start(); }
internal AzureBlobUploader( FabricEvents.ExtensionsEvents traceSource, string logSourceId, string etwLogDirName, string workFolder, StorageAccountFactory storageAccountFactory, string etlFileName, bool isActiveEtl, string containerName, string fabricNodeId, string fabricNodeInstanceName, string deploymentId, AzureBlobPerformance perfHelper, AccessCondition uploadStreamAccessCondition, AccessCondition uploadFileAccessCondition) { this.stopping = false; this.traceSource = traceSource; this.logSourceId = logSourceId; this.etwLogDirName = etwLogDirName; this.storageAccountFactory = storageAccountFactory; this.containerName = containerName; this.etlFileName = etlFileName; this.isActiveEtl = isActiveEtl; this.fabricNodeId = fabricNodeId; this.directoryName = AzureUtility.IsAzureInterfaceAvailable() ? string.Join( "/", string.IsNullOrEmpty(deploymentId) ? AzureUtility.DeploymentId : deploymentId, AzureUtility.RoleName, AzureUtility.RoleInstanceId) : fabricNodeInstanceName; this.localMap = Path.Combine(workFolder, LocalMapFolder); this.uploadStreamAccessCondition = uploadStreamAccessCondition; this.uploadFileAccessCondition = uploadFileAccessCondition; this.perfHelper = perfHelper; // Blob copy is done one at a time, so the concurrency count is 1. this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobCopy, 1); this.streamWriter = new StreamWriter(new MemoryStream()); this.lastEventIndexProcessed.Set(DateTime.MinValue, -1); // Create the container at the destination try { Utility.PerformWithRetries( this.CreateContainer, (object)null, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); } catch (Exception e) { var message = string.Format( "Error creating container {0}, account {1}.", this.containerName, this.storageAccountFactory.Connection.AccountName); this.traceSource.WriteExceptionAsError( this.logSourceId, e, message); throw new InvalidOperationException(message, e); } }
internal AzureFileUploader( FabricEvents.ExtensionsEvents traceSource, string logSourceId, string folderName, string destinationPath, string workFolder, StorageAccountFactory storageAccountFactory, string containerName, TimeSpan uploadIntervalMinutes, TimeSpan fileSyncIntervalInMinutes, TimeSpan blobDeletionAgeMinutes, string fabricNodeInstanceName, string deploymentId) : base( traceSource, logSourceId, folderName, destinationPath, workFolder, uploadIntervalMinutes, fileSyncIntervalInMinutes) { // Initialization this.storageAccountFactory = storageAccountFactory; this.containerName = containerName; this.directoryName = AzureUtility.IsAzureInterfaceAvailable() ? string.Join( "/", string.IsNullOrEmpty(deploymentId) ? AzureUtility.DeploymentId : deploymentId, AzureUtility.RoleName, AzureUtility.RoleInstanceId) : fabricNodeInstanceName; this.fabricNodeInstanceName = fabricNodeInstanceName; this.perfHelper = new AzureBlobPerformance(this.TraceSource, this.LogSourceId); // Blob copy is done one at a time, so the concurrency count is 1. this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobCopy, 1); // Create the container at the destination try { Utility.PerformWithRetries( this.CreateContainer, (object)null, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler)); } catch (Exception e) { var message = string.Format( "Error creating container {0}, account {1}.", this.containerName, this.storageAccountFactory.Connection.AccountName); this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, message); throw new InvalidOperationException(message, e); } // Check if a trimmer already exists to delete old files from this destination. lock (Trimmers) { if (Trimmers.ContainsKey(destinationPath)) { // Trimmer already exists. Increment its reference count. TrimmerInfo trimmerInfo = Trimmers[destinationPath]; trimmerInfo.RefCount++; } else { // Trimmer does not exist. Create it. AzureFileTrimmer trimmer = new AzureFileTrimmer( folderName, LocalMapFolderPath, storageAccountFactory, containerName, blobDeletionAgeMinutes, fabricNodeInstanceName, deploymentId, AzureUtility.IsAzureInterfaceAvailable()); TrimmerInfo trimmerInfo = new TrimmerInfo { RefCount = 1, Trimmer = trimmer }; Trimmers[destinationPath] = trimmerInfo; } } }