internal EtlProcessor( bool loadWinFabManifests, bool dynamicWinFabManifestLoad, IEnumerable <string> customManifestPaths, string applicationType, string markerFileDirectory, WinFabricEtlType winFabricEtlType, FabricEvents.ExtensionsEvents traceSource, string logSourceId, EtlPerformance perfHelper, IEnumerable <IEtlFileSink> sinks, IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters, IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders, IEnumerable <Guid> appEtwGuids, ITraceFileEventReaderFactory traceFileEventReaderFactory) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.processingWinFabEvents = loadWinFabManifests; this.applicationType = applicationType; this.markerFileDirectory = markerFileDirectory; this.windowsFabricEtlType = winFabricEtlType; this.perfHelper = perfHelper; this.sinks = sinks.ToList().AsReadOnly(); this.etlToCsvFileWriters = etlToCsvFileWriters.ToList().AsReadOnly(); this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly(); this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Load the custom manifests (if any) into the manifest cache this.LoadManifests(customManifestPaths); if (loadWinFabManifests) { if (dynamicWinFabManifestLoad) { // We are going to be loading manifests dynamically based on the version // in the ETL file. Initialize the manifest manager object to manage this // for us. this.winFabManifestMgr = this.InitializeWinFabManifestManager(); } else { // We'll load the default manifests now and just use them throughout this.LoadDefaultManifests(); } } // Provide the sinks a reference to the manifest cache. this.OnEtwManifestCacheAvailable(); }
internal EtlInMemoryProducer( DiskSpaceManager diskSpaceManager, IEtlInMemoryProducerConfigReaderFactory configReaderFactory, ITraceFileEventReaderFactory traceFileEventReaderFactory, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.diskSpaceManager = diskSpaceManager; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Initialization this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.consumerSinks = initParam.ConsumerSinks; // Read settings var configReader = configReaderFactory.CreateEtlInMemoryProducerConfigReader(this.traceSource, this.logSourceId); this.etlInMemoryProducerSettings = configReader.GetSettings(); // ETL in-memory file processing is not enabled or we are not processing // winfab etl files, so return immediately if (false == this.etlInMemoryProducerSettings.Enabled || false == this.etlInMemoryProducerSettings.ProcessingWinFabEtlFiles) { return; } // Create a new worker object var newWorkerParam = new EtlInMemoryProducerWorker.EtlInMemoryProducerWorkerParameters() { TraceSource = this.traceSource, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, EtlInMemoryProducer = this, LatestSettings = this.etlInMemoryProducerSettings }; var newWorker = new EtlInMemoryProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); this.producerWorker = newWorker; }
internal EtlProcessor( bool loadWinFabManifests, bool dynamicWinFabManifestLoad, string markerFileDirectory, WinFabricEtlType winFabricEtlType, FabricEvents.ExtensionsEvents traceSource, string logSourceId, EtlInMemoryPerformance perfHelper, IEnumerable <IEtlInMemorySink> sinks, IEnumerable <EtlToInMemoryBufferWriter> etlToInMemoryBufferWriters, ITraceFileEventReaderFactory traceFileEventReaderFactory) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.processingWinFabEvents = loadWinFabManifests; this.markerFileDirectory = markerFileDirectory; this.windowsFabricEtlType = winFabricEtlType; this.sinks = sinks.ToList().AsReadOnly(); this.etlToInMemoryBufferWriters = etlToInMemoryBufferWriters.ToList().AsReadOnly(); this.traceFileEventReaderFactory = traceFileEventReaderFactory; this.perfHelper = perfHelper; if (loadWinFabManifests) { if (dynamicWinFabManifestLoad) { // We are going to be loading manifests dynamically based on the version // in the ETL file. Initialize the manifest manager object to manage this // for us. this.winFabManifestMgr = this.InitializeWinFabManifestManager(); } else { // We'll load the default manifests now and just use them throughout this.LoadDefaultManifests(); } } // Provide the sinks a reference to the manifest cache. this.sinks.ForEach(sink => sink.SetEtwManifestCache(this.etwManifestCache)); }
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName)) .Returns("true"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName)) .Returns(logDirectory); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName)) .Returns(TableName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName)) .Returns(MdsFileProducerConstants.DefaultTablePriority); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName)) .Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName)) .Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName)) .Returns(LogFilter); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the MDS consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); var mdsFileProducer = new MdsFileProducer(consumerInitParam); var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam)); }
internal EtlInMemoryProducerWorker( EtlInMemoryProducerWorkerParameters initParam, DiskSpaceManager diskSpaceManager, ITraceFileEventReaderFactory traceFileEventReaderFactory) { this.logSourceId = initParam.ProducerInstanceId; this.traceSource = initParam.TraceSource; this.cancellationTokenSource = new CancellationTokenSource(); this.perfHelper = new EtlInMemoryPerformance(this.traceSource); this.diskSpaceManager = diskSpaceManager; // Initialize the settings this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam); if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType) { // If we're processing the default ETL files, we should keep track of // whether or not we're on the FMM node. This information is used by // some other plugin types. Utility.LastFmmEventTimestamp = DateTime.MinValue; } // Initialize the sink list this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly(); this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly(); this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this)); // Figure out where the ETL files are located this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory( this.etlInMemoryProducerWorkerSettings.EtlPath, initParam.LogDirectory, this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType); this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory( this.etlInMemoryProducerWorkerSettings.EtlPath, initParam.LogDirectory, this.traceDirectory, this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType); this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders( this.etlInMemoryProducerWorkerSettings.EtlPath, this.etlInMemoryProducerWorkerSettings.EtlFilePatterns, this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType, message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly(); if (0 == this.providers.Count) { // No ETL files to read, so return immediately this.traceSource.WriteWarning( this.logSourceId, "No ETL files have been specified for processing."); } this.checkpointManager = new CheckpointManager( this.etlInMemoryProducerWorkerSettings.EtlPath, initParam.LogDirectory, this.traceDirectory, this.traceSource, this.logSourceId); this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval; if (this.etlReadInterval > TimeSpan.Zero) { // Create the directory that contains the marker files. this.CreateDirectoriesForEtlProcessing(); // We need to collect bootstrap traces this.bootstrapTraceProcessor = new BootstrapTraceProcessor( this.traceDirectory, this.markerFileDirectory, this.etlToInMemoryBufferWriters, this.etlInMemoryProducerWorkerSettings.EtlReadInterval, this.traceSource, this.logSourceId); this.bootstrapTraceProcessor.Start(); // Create the ETL processor this.etlProcessor = new EtlProcessor( true, this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(), this.markerFileDirectory, this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType, this.traceSource, this.logSourceId, this.perfHelper, this.sinks, this.etlToInMemoryBufferWriters, traceFileEventReaderFactory); // Create a periodic timer to read ETL files var timerId = string.Concat( this.logSourceId, EtlReadTimerIdSuffix); this.etlReadTimer = new DcaTimer( timerId, state => this.EtlReadCallback(this.cancellationTokenSource.Token), this.etlReadInterval); this.etlReadTimer.Start(); // Figure out how much processing time is available to each provider. this.ComputePerProviderEtlProcessingTimeSeconds(); } // Disk manager set up for traces foreach (var provider in this.providers) { var capturedProvider = provider; this.diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern), f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes)); } // Disk manager set up for marker files this.diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(), f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes)); }
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestUtility.TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false); var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true); var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, fastDcaInMemoryConsumer); var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, slowDcaInMemoryConsumer); var initParam = new ProducerInitializationParameters { ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter } }; return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, initParam)); }
internal EtlProducerWorker( EtlProducerWorkerParameters initParam, DiskSpaceManager diskSpaceManager, ITraceFileEventReaderFactory traceFileEventReaderFactory) { this.logSourceId = initParam.ProducerInstanceId; this.traceSource = initParam.TraceSource; this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest; this.perfHelper = new EtlPerformance(this.traceSource); this.cancellationTokenSource = new CancellationTokenSource(); this.diskSpaceManager = diskSpaceManager; // Initialize the settings this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam); if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType) { // If we're processing the default ETL files, we should keep track of // whether or not we're on the FMM node. This information is used by // some other plugin types. Utility.LastFmmEventTimestamp = DateTime.MinValue; } // Initialize the sink list this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks( initParam.EtlProducers, message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly(); this.etlToCsvFileWriters = EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly(); this.bufferedEtwEventProviders = EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly(); // Figure out where the ETL files are located this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory( initParam.IsReadingFromApplicationManifest, this.etlProducerSettings.EtlPath, initParam.LogDirectory, this.etlProducerSettings.WindowsFabricEtlType); this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory( initParam.IsReadingFromApplicationManifest, this.etlProducerSettings.EtlPath, initParam.LogDirectory, this.etlProducerSettings.WindowsFabricEtlType, this.traceDirectory, initParam.ProducerInstanceId); if (initParam.IsReadingFromApplicationManifest) { lock (InternalMarkerFileDirectoriesForApps) { InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory); } } this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders( initParam.IsReadingFromApplicationManifest, this.etlProducerSettings.EtlPath, this.etlProducerSettings.WindowsFabricEtlType, this.etlProducerSettings.EtlFilePatterns, message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly(); if (0 == this.providers.Count) { // No ETL files to read, so return immediately this.traceSource.WriteWarning( this.logSourceId, "No ETL files have been specified for processing."); } this.checkpointManager = new CheckpointManager( initParam.IsReadingFromApplicationManifest, this.etlProducerSettings.EtlPath, initParam.LogDirectory, this.traceDirectory, initParam.ProducerInstanceId, this.traceSource, this.logSourceId); if (false == initParam.IsReadingFromApplicationManifest) { // Ensure that no other instance of EtlProducerWorker is processing the // same ETL files var patternsAdded = new List <string>(); var isUnique = VerifyEtlFilesUniqueToCurrentInstance( this.traceDirectory, this.providers, patternsAdded, message => this.traceSource.WriteError(this.logSourceId, message)); this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly(); if (!isUnique) { throw new InvalidOperationException( string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory)); } } this.etlReadInterval = this.etlProducerSettings.EtlReadInterval; if (this.etlReadInterval > TimeSpan.Zero) { // Create the directory that contains the marker files. this.CreateDirectoriesForEtlProcessing(); if (false == initParam.IsReadingFromApplicationManifest) { // We need to collect bootstrap traces this.bootstrapTraceProcessor = new BootstrapTraceProcessor( this.traceDirectory, this.markerFileDirectory, this.etlToCsvFileWriters, this.etlProducerSettings.EtlReadInterval, this.traceSource, this.logSourceId); this.bootstrapTraceProcessor.Start(); } // Create the ETL processor this.etlProcessor = new EtlProcessor( false == initParam.IsReadingFromApplicationManifest, this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(), this.etlProducerSettings.CustomManifestPaths, initParam.ApplicationType, this.markerFileDirectory, this.etlProducerSettings.WindowsFabricEtlType, this.traceSource, this.logSourceId, this.perfHelper, this.sinks, this.etlToCsvFileWriters, this.bufferedEtwEventProviders, this.etlProducerSettings.AppEtwGuids, traceFileEventReaderFactory); // Create a periodic timer to read ETL files var timerId = string.Concat( this.logSourceId, EtlReadTimerIdSuffix); this.etlReadTimer = new DcaTimer( timerId, state => this.EtlReadCallback(this.cancellationTokenSource.Token), this.etlReadInterval); this.etlReadTimer.Start(); // If there is a huge backlog of ETL files to process, we limit the // amount of time that we spend on processing ETL files in each // pass. Figure out how much processing time is available to each // provider. this.ComputePerProviderEtlProcessingTimeSeconds(); } foreach (var provider in this.providers) { var capturedProvider = provider; if (Directory.Exists(this.traceDirectory)) { diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern), f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes)); } } if (Directory.Exists(this.markerFileDirectory)) { diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(), f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes)); } }
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory, out EtlInMemoryProducer etlInMemoryProducer, out string containerName, out StorageAccountFactory storageAccountFactory, AccessCondition uploadStreamAccessCondition = null, AccessCondition uploadFileAccessCondition = null) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName)) .Returns("true"); containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName)) .Returns(containerName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName)) .Returns("0.25"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName)) .Returns("1"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName)) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName)) .Returns("*.*:4"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId)) .Returns(AzureConstants.DefaultDeploymentId); var accountKey = GetTestStorageAccountKey(); var isEncrypted = false; var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey); configStore .Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted)) .Returns(storageConnectionString); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the Azure Blob Uploader consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); // Save storage connection for clean up var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType); storageAccountFactory = azureUtility.GetStorageAccountFactory( new ConfigReader(consumerInitParam.ApplicationInstanceId), TestConfigSectionName, AzureConstants.ConnectionStringParamName); var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition); var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer etlInMemoryProducer = new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam); }
internal EtlProducer( DiskSpaceManager diskSpaceManager, IEtlProducerConfigReaderFactory configReaderFactory, ITraceFileEventReaderFactory traceFileEventReaderFactory, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.diskSpaceManager = diskSpaceManager; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Initialization this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.serviceConfigSections = new List <string>(); this.logDirectory = initParam.LogDirectory; this.consumerSinks = initParam.ConsumerSinks; // Read instance-specific settings from the settings file var configReader = configReaderFactory.CreateEtlProducerConfigReader(this.traceSource, this.logSourceId); this.etlProducerSettings = configReader.GetSettings(); if (false == this.etlProducerSettings.Enabled) { // ETL file processing is not enabled, so return immediately return; } if (!this.etlProducerSettings.ProcessingWinFabEtlFiles) { // If we are collecting ETW events on behalf of an app, then we will // need information from the <ETW> section of the service manifest. this.serviceConfigSections.Add(ServiceConfig.EtwElement); // Check if we can use an existing worker object string applicationType = this.etlProducerSettings.ApplicationType; lock (ProducerWorkers) { EtlProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(w => w.ApplicationType.Equals( applicationType, StringComparison.Ordinal)); if (null != workerInfo) { // Existing worker object is available. this.traceSource.WriteInfo( this.logSourceId, "Existing ETL producer worker object for application type {0} is available. Restarting the worker object now.", applicationType); // Restart the worker object workerInfo.ProducerWorker.Dispose(); workerInfo.ProducerWorker = null; List <EtlProducer> etlProducers = new List <EtlProducer>(workerInfo.EtlProducers) { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = applicationType, LogDirectory = initParam.LogDirectory, ProducerInstanceId = applicationType, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); workerInfo.EtlProducers.Add(this); workerInfo.ProducerWorker = newWorker; } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to restart ETL producer worker object for application type {0}.", applicationType); } } else { // Create a new worker object this.traceSource.WriteInfo( this.logSourceId, "Creating ETL producer worker object for application type {0} ...", applicationType); List <EtlProducer> etlProducers = new List <EtlProducer> { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = applicationType, LogDirectory = initParam.LogDirectory, ProducerInstanceId = applicationType, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); workerInfo = new EtlProducerWorkerInfo() { ApplicationType = applicationType, EtlProducers = new List <EtlProducer>(), ProducerWorker = newWorker }; workerInfo.EtlProducers.Add(this); ProducerWorkers.Add(workerInfo); } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to create ETL producer worker object for application type {0}.", applicationType); } } } } else { // Create a new worker object List <EtlProducer> etlProducers = new List <EtlProducer> { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = string.Empty, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); this.producerWorker = newWorker; } catch (InvalidOperationException) { } } }
private static EtlProducer CreateEtlProducerForTest( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlProducerSettings = new EtlProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestUtility.TestEtlFilePatterns, new List <string>(), new Dictionary <string, List <ServiceEtwManifestInfo> >(), true, new[] { Guid.Parse(EtwProviderGuid) }, null); var cr = TestUtility.MockRepository.Create <IEtlProducerConfigReader>(); cr.Setup(c => c.GetSettings()).Returns(etlProducerSettings); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyOldDataDeletionIntervalInSeconds")) .Returns("3600"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "ApplicationLogsFormatVersion")) .Returns((string)null); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyDtrDeletionAgeInMinutes")) .Returns("60"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlProducerConfigReaderFactory>(); mockConfigReaderFactory .Setup(f => f.CreateEtlProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(cr.Object); var mockTraceSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var csvWriter = new EtlToCsvFileWriter( mockTraceSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, mockDiskSpaceManager.Object); var initParam = new ProducerInitializationParameters { ConsumerSinks = new[] { csvWriter } }; return(new EtlProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceSourceFactory.Object, initParam)); }