public void OnProcessingPeriodStart(string traceFileName, bool isActiveEtl, string traceFileSubFolder) { // Create a new azure blob uploader var blobUploader = new AzureBlobUploader( this.traceSource, this.logSourceId, this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, traceFileName, isActiveEtl, this.blobUploadSettings.EtwTraceContainerName, this.initParam.FabricNodeId, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, this.streamUploadPerfHelper, this.uploadStreamAccessCondition, null); this.streamUploadPerfHelper.AzureBlobUploadPassBegin(); // GetMaxIndexAlreadyProcessed is invoked before the perf helper pass begins. So we save the bytes read // during the call to GetMaxIndexAlreadyProcessed and then record it here. this.streamUploadPerfHelper.BytesRead(this.bytesReadGetMaxIndexAlreadyProcessed); this.blobUploader = blobUploader; this.traceSource.WriteInfo( this.logSourceId, "Filtered traces from {0} will be written to the memory stream in the azure blob uploader.", traceFileName); }
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam) { this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); this.configReader = new AzureBlobEtwConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Read blob-specific settings this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { return; } // Create the destination key var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobEtwUploader, accountName, this.blobUploadSettings.EtwTraceContainerName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, this.destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create a sub-directory for the blob uploader this.workFolder = this.CreateBlobUploaderWorkSubDirectory(); if (string.IsNullOrEmpty(this.workFolder)) { const string Message = "Failed to create work folder for the blob uploader."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from ETL files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, initParam.FabricNodeId, this.etwLogDirName, true, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.blobUploadSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); // Create the helper object that syncs local files to blob storage. // Local files will be created when upload of compressed memory stream to blob storage fails. this.fileBlobUploader = new AzureBlobUploader( this.traceSource, this.logSourceId, this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, this.fileUploadPerfHelper, null, this.uploadFileAccessCondition); // Create a timer to schedule the upload of local files to blob storage string timerId = string.Concat( this.logSourceId, FileUploadTimerIdSuffix); this.fileUploadTimer = new DcaTimer( timerId, this.UploadFilesToDestinationBlob, this.blobUploadSettings.FileSyncInterval); this.fileUploadTimer.Start(); // Initialize trimmer this.trimmer = new AzureFileTrimmer( this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.blobUploadSettings.BlobDeletionAge, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, AzureUtility.IsAzureInterfaceAvailable()); this.traceSource.WriteInfo( this.logSourceId, "Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}", this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.EtwTraceContainerName, this.etwLogDirName); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for Azure blob uploader: {0}", this.blobUploadSettings.Filter); }