private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestUtility.TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false); var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true); var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, fastDcaInMemoryConsumer); var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, slowDcaInMemoryConsumer); var initParam = new ProducerInitializationParameters { ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter } }; return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, initParam)); }
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam) { this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); this.configReader = new AzureBlobEtwConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Read blob-specific settings this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { return; } // Create the destination key var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobEtwUploader, accountName, this.blobUploadSettings.EtwTraceContainerName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, this.destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create a sub-directory for the blob uploader this.workFolder = this.CreateBlobUploaderWorkSubDirectory(); if (string.IsNullOrEmpty(this.workFolder)) { const string Message = "Failed to create work folder for the blob uploader."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from ETL files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, initParam.FabricNodeId, this.etwLogDirName, true, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.blobUploadSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); // Create the helper object that syncs local files to blob storage. // Local files will be created when upload of compressed memory stream to blob storage fails. this.fileBlobUploader = new AzureBlobUploader( this.traceSource, this.logSourceId, this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, this.fileUploadPerfHelper, null, this.uploadFileAccessCondition); // Create a timer to schedule the upload of local files to blob storage string timerId = string.Concat( this.logSourceId, FileUploadTimerIdSuffix); this.fileUploadTimer = new DcaTimer( timerId, this.UploadFilesToDestinationBlob, this.blobUploadSettings.FileSyncInterval); this.fileUploadTimer.Start(); // Initialize trimmer this.trimmer = new AzureFileTrimmer( this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.blobUploadSettings.BlobDeletionAge, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, AzureUtility.IsAzureInterfaceAvailable()); this.traceSource.WriteInfo( this.logSourceId, "Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}", this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.EtwTraceContainerName, this.etwLogDirName); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for Azure blob uploader: {0}", this.blobUploadSettings.Filter); }
public MdsFileProducer(ConsumerInitializationParameters initParam) { // Initialization this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); this.tableWriters = new Dictionary <string, MdsTableWriter>(); this.mdsFileProducerPerf = new MdsFileProducerPerformance(); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, MdsFileProducerConstants.MethodExecutionInitialRetryIntervalMs, MdsFileProducerConstants.MethodExecutionMaxRetryCount, MdsFileProducerConstants.MethodExecutionMaxRetryIntervalMs); // Read MDS-specific settings from settings.xml this.GetSettings(); if (false == this.mdsFileProducerSettings.Enabled) { // MDS tables creation is not enabled, so return immediately return; } string destinationKey = Path.Combine( this.mdsFileProducerSettings.DirectoryName, this.mdsFileProducerSettings.TableName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from the ETL // files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, this.initParam.FabricNodeId, this.etwLogDirName, false, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.mdsFileProducerSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); this.traceSource.WriteInfo( this.logSourceId, "MDS file producer is configured. Table path: {0}, Table name: {1}, Table priority: {2}, Disk quota (MB): {3}, Data deletion age (minutes): {4}", this.mdsFileProducerSettings.DirectoryName, this.mdsFileProducerSettings.TableName, this.mdsFileProducerSettings.TablePriority, this.mdsFileProducerSettings.DiskQuotaInMB, this.mdsFileProducerSettings.DataDeletionAge); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for MDS file producer: {0}", this.mdsFileProducerSettings.Filter); }