public AzureBlobFolderUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new AzureBlobConfigReader( new ConfigReader(initParam.ApplicationInstanceId), this.initParam.SectionName, this.traceSource, this.logSourceId); // Read blob-specific settings from settings.xml this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { // Upload to Azure blob storage is not enabled, so return immediately return; } var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobFolderUploader, accountName, this.blobUploadSettings.ContainerName); }
private BlobUploadSettings GetSettings() { // Check for values in settings.xml var blobUploadSettings = new BlobUploadSettings(); blobUploadSettings.Enabled = this.configReader.GetEnabled(); if (!blobUploadSettings.Enabled) { this.traceSource.WriteInfo( this.logSourceId, "Blob storage upload not enabled"); return(blobUploadSettings); } blobUploadSettings.StorageAccountFactory = this.configReader.GetStorageAccountFactory(); if (blobUploadSettings.StorageAccountFactory == null) { blobUploadSettings.Enabled = false; return(blobUploadSettings); } blobUploadSettings.LttTraceContainerName = this.configReader.GetEtwTraceContainerName(); blobUploadSettings.UploadInterval = this.configReader.GetUploadInterval(); blobUploadSettings.FileSyncInterval = this.configReader.GetFileSyncInterval(); blobUploadSettings.BlobDeletionAge = this.configReader.GetDataDeletionAge(); blobUploadSettings.Filter = this.configReader.GetFilters(); blobUploadSettings.DeploymentId = this.configReader.GetDeploymentId(); // Write settings to log this.traceSource.WriteInfo( this.logSourceId, "Blob storage upload enabled: ETW trace container: {0}, Upload interval (minutes): {1}, Blob deletion age: {2}", blobUploadSettings.LttTraceContainerName, blobUploadSettings.UploadInterval, blobUploadSettings.BlobDeletionAge); return(blobUploadSettings); }
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam) { this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); this.configReader = new AzureBlobEtwConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Read blob-specific settings this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { return; } // Create the destination key var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobEtwUploader, accountName, this.blobUploadSettings.EtwTraceContainerName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, this.destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create a sub-directory for the blob uploader this.workFolder = this.CreateBlobUploaderWorkSubDirectory(); if (string.IsNullOrEmpty(this.workFolder)) { const string Message = "Failed to create work folder for the blob uploader."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from ETL files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, initParam.FabricNodeId, this.etwLogDirName, true, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.blobUploadSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); // Create the helper object that syncs local files to blob storage. // Local files will be created when upload of compressed memory stream to blob storage fails. this.fileBlobUploader = new AzureBlobUploader( this.traceSource, this.logSourceId, this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, this.fileUploadPerfHelper, null, this.uploadFileAccessCondition); // Create a timer to schedule the upload of local files to blob storage string timerId = string.Concat( this.logSourceId, FileUploadTimerIdSuffix); this.fileUploadTimer = new DcaTimer( timerId, this.UploadFilesToDestinationBlob, this.blobUploadSettings.FileSyncInterval); this.fileUploadTimer.Start(); // Initialize trimmer this.trimmer = new AzureFileTrimmer( this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.blobUploadSettings.BlobDeletionAge, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, AzureUtility.IsAzureInterfaceAvailable()); this.traceSource.WriteInfo( this.logSourceId, "Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}", this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.EtwTraceContainerName, this.etwLogDirName); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for Azure blob uploader: {0}", this.blobUploadSettings.Filter); }
public AzureBlobCsvUploader(ConsumerInitializationParameters initParam) { // Initialization this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new AzureBlobConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.diskSpaceManager = initParam.DiskSpaceManager; this.disposed = false; // Read blob-specific settings from settings.xml this.blobUploadSettings = this.GetSettings(); if (this.configReader.IsReadingFromApplicationManifest) { // Check if we can use an existing upload worker object UploadWorkerKey key = new UploadWorkerKey() { // Destination path is an concatenation of storage account name and container name DestinationPath = string.Concat( this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, ";", // This separator cannot occur in account name or container name this.blobUploadSettings.LttTraceContainerName), ApplicationType = this.configReader.GetApplicationType(), }; lock (UploadWorkers) { UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key)); if (null != workerInfo) { // Existing upload worker object is available. Increment its // reference count this.traceSource.WriteInfo( this.logSourceId, "Existing upload worker object for application type {0}, Azure storage account {1} and container {2} is available and will be used.", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); (workerInfo.RefCount)++; workerInfo.UploadWorker.UpdateSettings(this.blobUploadSettings); this.uploadWorker = workerInfo.UploadWorker; } else { // Create a new upload worker object this.traceSource.WriteInfo( this.logSourceId, "Creating upload worker object for application type {0}, Azure storage account {1} and container {2} ...", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters() { TraceSource = this.traceSource, FabricNodeId = initParam.FabricNodeId, FabricNodeInstanceName = initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = initParam.LogDirectory, WorkDirectory = initParam.WorkDirectory, UploaderInstanceId = key.ApplicationType, ParentWorkFolderName = key.ApplicationType, Settings = this.blobUploadSettings }; CsvUploadWorker newWorker; try { newWorker = new CsvUploadWorker(param, this.diskSpaceManager); } catch (Exception) { this.traceSource.WriteError( this.logSourceId, "Failed to create upload worker object for application type {0}, Azure storage account {1} and container {2}.", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); throw; } workerInfo = new UploadWorkerInfo() { Key = key, RefCount = 1, UploadWorker = newWorker }; UploadWorkers.Add(workerInfo); this.uploadWorker = workerInfo.UploadWorker; } } } else { // Create a new upload worker object CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters() { TraceSource = this.traceSource, FabricNodeId = initParam.FabricNodeId, FabricNodeInstanceName = initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = initParam.LogDirectory, WorkDirectory = initParam.WorkDirectory, UploaderInstanceId = this.logSourceId, ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths, Settings = this.blobUploadSettings }; this.uploadWorker = new CsvUploadWorker(param, this.diskSpaceManager); } }