public LttProducer( DiskSpaceManager diskSpaceManager, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.AppInstanceId = initParam.ApplicationInstanceId; this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.serviceConfigSections = new List <string>(); this.consumerSinks = initParam.ConsumerSinks; // Read the timer config value from dca section var configReader = new ConfigReader(initParam.ApplicationInstanceId); long lttReadIntervalMinutes = configReader.GetUnencryptedConfigValue( initParam.SectionName, LttProducerConstants.LttReadIntervalParamName, LttProducerConstants.DefaultLttReadIntervalMinutes); if (initParam.ApplicationInstanceId == Utility.WindowsFabricApplicationInstanceId) { this.CreateWindowsFabricLttProducerWorkerInfo(initParam, lttReadIntervalMinutes); } else { this.CreateAppProducerWorkerInfo(initParam, lttReadIntervalMinutes); } }
internal IDcaProducer CreateProducer(string producerInstance, ProducerInitializationParameters initParam, string producerType) { // Create the producer object IDcaProducer producer; #if DotNetCoreClrLinux if (producerType.Equals(StandardPluginTypes.LttProducer)) { producer = new LttProducer( this.diskSpaceManager, new TraceEventSourceFactory(), initParam); } #else if (producerType.Equals(StandardPluginTypes.EtlFileProducer)) { var configReaderFactory = new EtlProducerConfigReader.EtlProducerConfigReaderFactory( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName); producer = new EtlProducer( this.diskSpaceManager, configReaderFactory, new TraceFileEventReaderFactory(), new TraceEventSourceFactory(), initParam); } else if (producerType.Equals(StandardPluginTypes.EtlInMemoryProducer)) { var configReaderFactory = new EtlInMemoryProducerConfigReader.EtlInMemoryProducerConfigReaderFactory( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName); producer = new EtlInMemoryProducer( this.diskSpaceManager, configReaderFactory, new TraceFileEventReaderFactory(), new TraceEventSourceFactory(), initParam); } #if !DotNetCoreClrIOT else if (producerType.Equals(StandardPluginTypes.PerfCounterConfigReader)) { producer = new PerfCounterConfigReader(initParam); } #endif #endif else { Debug.Assert( producerType.Equals(StandardPluginTypes.FolderProducer), "No other producer types exist."); producer = new FolderProducer(this.diskSpaceManager, initParam); } Utility.TraceSource.WriteInfo( TraceType, "Created producer {0}.", producerInstance); return(producer); }
public PerfCounterConfigReader(ProducerInitializationParameters initializationParameters) { // Initialization this.initParam = initializationParameters; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(this.initParam.ApplicationInstanceId); // Read instance-specific settings from settings.xml this.GetSettings(); if (false == this.configReaderSettings.Enabled) { // Producer is not enabled, so return immediately return; } // Read the performance counter configuration from settings.xml this.GetPerformanceCounterConfiguration(); if (false == this.perfCounterConfig.IsEnabled) { // Performance counter collection is not enabled, so return immediately return; } if (null == initializationParameters.ConsumerSinks) { return; } foreach (object sinkAsObject in initializationParameters.ConsumerSinks) { IPerformanceCounterConfigurationSink perfCounterConfigSink = null; try { perfCounterConfigSink = (IPerformanceCounterConfigurationSink)sinkAsObject; } catch (InvalidCastException e) { this.traceSource.WriteError( this.logSourceId, "Exception occured while casting a sink object of type {0} to interface IPerformanceCounterConfigurationSink. Exception information: {1}.", sinkAsObject.GetType(), e); } if (null == perfCounterConfigSink) { continue; } perfCounterConfigSink.RegisterPerformanceCounterConfiguration( this.perfCounterConfig.SamplingInterval, this.perfCounterConfig.CounterPaths); } }
private LttProducerWorkerInfo CreateWorkerInfo(ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { // Create a new worker object this.traceSource.WriteInfo( this.logSourceId, $"Creating Ltt producer worker object for {initParam.ApplicationInstanceId}"); LttProducerWorkerInfo workerInfo = new LttProducerWorkerInfo(); this.AddLttProducerWorkerToWorkerInfo(workerInfo, initParam, lttReadIntervalMinutes); return(workerInfo); }
private void CreateWindowsFabricLttProducerWorkerInfo( ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { lock (ProducerWorkerLock) { if (LttProducer.ProducerWorkerInfo == null) { LttProducer.ProducerWorkerInfo = this.CreateWorkerInfo(initParam, lttReadIntervalMinutes); } else { this.UpdateWorkerInfo(LttProducer.ProducerWorkerInfo, initParam, lttReadIntervalMinutes); } } }
internal EtlInMemoryProducer( DiskSpaceManager diskSpaceManager, IEtlInMemoryProducerConfigReaderFactory configReaderFactory, ITraceFileEventReaderFactory traceFileEventReaderFactory, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.diskSpaceManager = diskSpaceManager; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Initialization this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.consumerSinks = initParam.ConsumerSinks; // Read settings var configReader = configReaderFactory.CreateEtlInMemoryProducerConfigReader(this.traceSource, this.logSourceId); this.etlInMemoryProducerSettings = configReader.GetSettings(); // ETL in-memory file processing is not enabled or we are not processing // winfab etl files, so return immediately if (false == this.etlInMemoryProducerSettings.Enabled || false == this.etlInMemoryProducerSettings.ProcessingWinFabEtlFiles) { return; } // Create a new worker object var newWorkerParam = new EtlInMemoryProducerWorker.EtlInMemoryProducerWorkerParameters() { TraceSource = this.traceSource, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, EtlInMemoryProducer = this, LatestSettings = this.etlInMemoryProducerSettings }; var newWorker = new EtlInMemoryProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); this.producerWorker = newWorker; }
private void CreateAppProducerWorkerInfo( ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { lock (AppProducerWorkersLock) { if (!LttProducer.AppProducerWorkersInfos.ContainsKey(initParam.ApplicationInstanceId)) { LttProducerWorkerInfo workerInfo = CreateWorkerInfo(initParam, lttReadIntervalMinutes); LttProducer.AppProducerWorkersInfos.Add(initParam.ApplicationInstanceId, workerInfo); } else { this.UpdateWorkerInfo( LttProducer.AppProducerWorkersInfos[initParam.ApplicationInstanceId], initParam, lttReadIntervalMinutes); } } }
private void UpdateWorkerInfo(LttProducerWorkerInfo workerInfo, ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { // Existing worker object is available. this.traceSource.WriteInfo( this.logSourceId, $"Existing Ltt producer worker object. Restarting the worker object now for {initParam.ApplicationInstanceId}"); // Save the old value for comparision long oldLttReadIntervalMinutes = workerInfo.ProducerWorker.LttReadIntervalMinutes; // Restart the worker object workerInfo.ProducerWorker.Dispose(); workerInfo.ProducerWorker = null; // Keep the smaller value intact // as this worker handles both producers Ltt trace conversion and table events if (oldLttReadIntervalMinutes < lttReadIntervalMinutes) { lttReadIntervalMinutes = oldLttReadIntervalMinutes; } this.AddLttProducerWorkerToWorkerInfo(workerInfo, initParam, lttReadIntervalMinutes); }
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName)) .Returns("true"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName)) .Returns(logDirectory); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName)) .Returns(TableName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName)) .Returns(MdsFileProducerConstants.DefaultTablePriority); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName)) .Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName)) .Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName)) .Returns(LogFilter); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the MDS consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); var mdsFileProducer = new MdsFileProducer(consumerInitParam); var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam)); }
private static EtlProducer CreateEtlProducerForTest( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlProducerSettings = new EtlProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestUtility.TestEtlFilePatterns, new List <string>(), new Dictionary <string, List <ServiceEtwManifestInfo> >(), true, new[] { Guid.Parse(EtwProviderGuid) }, null); var cr = TestUtility.MockRepository.Create <IEtlProducerConfigReader>(); cr.Setup(c => c.GetSettings()).Returns(etlProducerSettings); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyOldDataDeletionIntervalInSeconds")) .Returns("3600"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "ApplicationLogsFormatVersion")) .Returns((string)null); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyDtrDeletionAgeInMinutes")) .Returns("60"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlProducerConfigReaderFactory>(); mockConfigReaderFactory .Setup(f => f.CreateEtlProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(cr.Object); var mockTraceSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var csvWriter = new EtlToCsvFileWriter( mockTraceSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, mockDiskSpaceManager.Object); var initParam = new ProducerInitializationParameters { ConsumerSinks = new[] { csvWriter } }; return(new EtlProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceSourceFactory.Object, initParam)); }
private static Dictionary <string, IDcaProducer> CreateProducers( ProducerFactory producerFactory, IDictionary <string, List <object> > producerConsumerMap, DCASettings settings, string applicationInstanceId, IList <string> errorEvents) { // Initialize producer instance list var producers = new Dictionary <string, IDcaProducer>(); Debug.Assert(null != producerConsumerMap, "Map of producers to consumers must be initialized."); foreach (string producerInstance in settings.ProducerInstances.Keys) { // Get the producer instance information DCASettings.ProducerInstanceInfo producerInstanceInfo = settings.ProducerInstances[producerInstance]; // Prepare the producer initialization parameters ProducerInitializationParameters initParam = new ProducerInitializationParameters(); initParam.ApplicationInstanceId = applicationInstanceId; initParam.SectionName = producerInstanceInfo.SectionName; initParam.LogDirectory = Utility.LogDirectory; if (ContainerEnvironment.IsContainerApplication(applicationInstanceId)) { if (producerInstanceInfo.TypeName != StandardPluginTypes.EtlFileProducer) { continue; } initParam.LogDirectory = ContainerEnvironment.GetContainerLogFolder(applicationInstanceId); } initParam.WorkDirectory = Utility.DcaWorkFolder; if (producerConsumerMap.ContainsKey(producerInstance)) { initParam.ConsumerSinks = producerConsumerMap[producerInstance]; producerConsumerMap.Remove(producerInstance); } else { initParam.ConsumerSinks = null; } // Create producer instance try { var producerInterface = producerFactory.CreateProducer( producerInstance, initParam, producerInstanceInfo.TypeName); // Add the producer to the producer list producers[producerInstance] = producerInterface; } catch (Exception e) { // Initialize the producer Utility.TraceSource.WriteError( TraceType, "Failed to create producer {0}. {1}", producerInstance, e); var message = string.Format( StringResources.DCAError_UnhandledPluginExceptionHealthDescription, producerInstanceInfo.SectionName, e.Message); errorEvents.Add(message); } } return(producers); }
private LttProducerWorker.LttProducerWorkerParameters CreateLttProducerWorkerParameters(LttProducerWorkerInfo workerInfo, ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { List <LttProducer> lttProducers = new List <LttProducer>(workerInfo.LttProducers) { this }; return(new LttProducerWorker.LttProducerWorkerParameters() { TraceSource = this.traceSource, LogDirectory = initParam.LogDirectory, WorkDirectory = initParam.WorkDirectory, ProducerInstanceId = this.logSourceId, LttProducers = lttProducers, LatestSettings = initParam, LttReadIntervalMinutes = lttReadIntervalMinutes }); }
private void AddLttProducerWorkerToWorkerInfo(LttProducerWorkerInfo workerInfo, ProducerInitializationParameters initParam, long lttReadIntervalMinutes) { LttProducerWorker.LttProducerWorkerParameters parameters = CreateLttProducerWorkerParameters(workerInfo, initParam, lttReadIntervalMinutes); try { LttProducerWorker newWorker = new LttProducerWorker(parameters); workerInfo.LttProducers.Add(this); workerInfo.ProducerWorker = newWorker; } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to restart Ltt producer worker object."); } }
public LttProducer( DiskSpaceManager diskSpaceManager, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.serviceConfigSections = new List <string>(); this.consumerSinks = initParam.ConsumerSinks; // Read the timer config value from dca section var configReader = new ConfigReader(initParam.ApplicationInstanceId); long newLttReadIntervalMinutes = configReader.GetUnencryptedConfigValue( initParam.SectionName, LttProducerConstants.LttReadIntervalParamName, LttProducerConstants.DefaultLttReadIntervalMinutes); lock (ProducerWorkers) { LttProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(); if (null != workerInfo) { // Existing worker object is available. this.traceSource.WriteInfo( this.logSourceId, "Existing Ltt producer worker object. Restarting the worker object now."); // Save the old value for comparision long oldLttReadIntervalMinutes = workerInfo.ProducerWorker.LttReadIntervalMinutes; // Restart the worker object workerInfo.ProducerWorker.Dispose(); workerInfo.ProducerWorker = null; // Keep the smaller value intact // as this worker handles both producers Ltt trace conversion and table events if (oldLttReadIntervalMinutes < newLttReadIntervalMinutes) { newLttReadIntervalMinutes = oldLttReadIntervalMinutes; } List <LttProducer> LttProducers = new List <LttProducer>(workerInfo.LttProducers) { this }; LttProducerWorker.LttProducerWorkerParameters newWorkerParam = new LttProducerWorker.LttProducerWorkerParameters() { TraceSource = this.traceSource, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, LttProducers = LttProducers, LatestSettings = initParam, LttReadIntervalMinutes = newLttReadIntervalMinutes }; try { LttProducerWorker newWorker = new LttProducerWorker(newWorkerParam); workerInfo.LttProducers.Add(this); workerInfo.ProducerWorker = newWorker; } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to restart Ltt producer worker object."); } } else { // Create a new worker object this.traceSource.WriteInfo( this.logSourceId, "Creating Ltt producer worker object ..."); List <LttProducer> LttProducers = new List <LttProducer> { this }; LttProducerWorker.LttProducerWorkerParameters newWorkerParam = new LttProducerWorker.LttProducerWorkerParameters() { TraceSource = this.traceSource, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, LttProducers = LttProducers, LatestSettings = initParam, LttReadIntervalMinutes = newLttReadIntervalMinutes }; try { LttProducerWorker newWorker = new LttProducerWorker(newWorkerParam); workerInfo = new LttProducerWorkerInfo() { LttProducers = new List <LttProducer>(), ProducerWorker = newWorker }; workerInfo.LttProducers.Add(this); ProducerWorkers.Add(workerInfo); } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to create Ltt producer worker object."); } } } }
public FolderProducer(DiskSpaceManager diskSpaceManager, ProducerInitializationParameters initializationParameters) { this.diskSpaceManager = diskSpaceManager; // Initialization this.initParam = initializationParameters; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(this.initParam.ApplicationInstanceId); this.additionalAppConfigSections = new List <string>(); this.serviceConfigSections = new List <string>(); // Read instance-specific settings from settings.xml this.GetSettings(); if (false == this.folderProducerSettings.Enabled) { // Producer is not enabled, so return immediately return; } if (this.configReader.IsReadingFromApplicationManifest && FolderProducerType.WindowsFabricCrashDumps == this.folderProducerSettings.Type) { this.serviceConfigSections.Add(ServiceConfig.ExeHostElement); } var additionalFoldersToTrim = new List <string>(); #if !DotNetCoreClr if (FolderProducerType.WindowsFabricPerformanceCounters == this.folderProducerSettings.Type) { // We will need information from the <PerformanceCounterLocalStore> section of the // service manifest. this.additionalAppConfigSections.Add(PerformanceCounterCommon.PerformanceCounterSectionName); // The performance counter binary files cannot be read while the OS is still // writing to them. Therefore, we make the files available to the consumer only // when the OS has finished writing to them. Hence we need a special processor // for these files. List <string> additionalPerfCounterFoldersToTrim; bool perfCounterCollectionEnabled; // There should be only one path in the path list for performance counters string perfCounterPath = this.folderProducerSettings.Paths[0]; this.perfCounterFolderProcessor = PerfCounterFolderProcessor.Create( this.traceSource, this.logSourceId, this.configReader, this.initParam.LogDirectory, perfCounterPath, out perfCounterCollectionEnabled, out additionalPerfCounterFoldersToTrim); if (null == this.perfCounterFolderProcessor) { return; } if (false == perfCounterCollectionEnabled) { return; } if (null != additionalPerfCounterFoldersToTrim) { additionalFoldersToTrim.AddRange(additionalPerfCounterFoldersToTrim); } } #endif if (null != this.initParam.ConsumerSinks) { foreach (object sinkAsObject in this.initParam.ConsumerSinks) { IFolderSink folderSink = null; try { folderSink = (IFolderSink)sinkAsObject; } catch (InvalidCastException e) { this.traceSource.WriteError( this.logSourceId, "Exception occured while casting a sink object of type {0} to interface IFolderSink. Exception information: {1}.", sinkAsObject.GetType(), e); } if (null == folderSink) { continue; } folderSink.RegisterFolders(this.folderProducerSettings.Paths); } } #if DotNetCoreClrLinux if (FolderProducerType.WindowsFabricCrashDumps == this.folderProducerSettings.Type) { const int filePermissions = Helpers.LINUX_USER_READ | Helpers.LINUX_USER_WRITE | Helpers.LINUX_USER_EXECUTE | Helpers.LINUX_GROUP_READ | Helpers.LINUX_GROUP_WRITE | Helpers.LINUX_GROUP_EXECUTE | Helpers.LINUX_OTHER_READ | Helpers.LINUX_OTHER_WRITE | Helpers.LINUX_OTHER_EXECUTE; Helpers.UpdateFilePermission(this.folderProducerSettings.Paths[0], filePermissions); using (FileStream fs = new FileStream("/proc/sys/kernel/core_pattern", FileMode.Open)) { using (StreamWriter sw = new StreamWriter(fs)) { sw.Write(Path.Combine(this.folderProducerSettings.Paths[0], "%e.%p.dmp")); } } } #endif var foldersToTrim = this.folderProducerSettings.Paths.Concat(additionalFoldersToTrim).ToArray(); if (this.IsDiskSpaceManagementEnabled()) { foreach (string folderPath in foldersToTrim) { // Figure out the timestamp before which all files will be deleted this.diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(folderPath).EnumerateFiles("*", SearchOption.AllDirectories), f => f.LastWriteTimeUtc < DateTime.UtcNow - MinimumFileRetentionTime, // isSafeToDelete f => (!Utility.IgnoreUploadFileList.Exists(x => x.Equals(f)) && f.LastWriteTimeUtc >= DateTime.UtcNow - this.folderProducerSettings.DataDeletionAge)); // shouldBeRetained } } }
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestUtility.TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false); var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true); var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, fastDcaInMemoryConsumer); var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( mockTraceEventSourceFactory.Object, LogSourceId, TestNodeId, TestUtility.GetActualOutputFolderPath(logDirectory), true, slowDcaInMemoryConsumer); var initParam = new ProducerInitializationParameters { ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter } }; return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, initParam)); }
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory, out EtlInMemoryProducer etlInMemoryProducer, out string containerName, out StorageAccountFactory storageAccountFactory, AccessCondition uploadStreamAccessCondition = null, AccessCondition uploadFileAccessCondition = null) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName)) .Returns("true"); containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName)) .Returns(containerName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName)) .Returns("0.25"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName)) .Returns("1"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName)) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName)) .Returns("*.*:4"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId)) .Returns(AzureConstants.DefaultDeploymentId); var accountKey = GetTestStorageAccountKey(); var isEncrypted = false; var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey); configStore .Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted)) .Returns(storageConnectionString); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the Azure Blob Uploader consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); // Save storage connection for clean up var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType); storageAccountFactory = azureUtility.GetStorageAccountFactory( new ConfigReader(consumerInitParam.ApplicationInstanceId), TestConfigSectionName, AzureConstants.ConnectionStringParamName); var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition); var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer etlInMemoryProducer = new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam); }
internal EtlProducer( DiskSpaceManager diskSpaceManager, IEtlProducerConfigReaderFactory configReaderFactory, ITraceFileEventReaderFactory traceFileEventReaderFactory, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.diskSpaceManager = diskSpaceManager; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Initialization this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.serviceConfigSections = new List <string>(); this.logDirectory = initParam.LogDirectory; this.consumerSinks = initParam.ConsumerSinks; // Read instance-specific settings from the settings file var configReader = configReaderFactory.CreateEtlProducerConfigReader(this.traceSource, this.logSourceId); this.etlProducerSettings = configReader.GetSettings(); if (false == this.etlProducerSettings.Enabled) { // ETL file processing is not enabled, so return immediately return; } if (!this.etlProducerSettings.ProcessingWinFabEtlFiles) { // If we are collecting ETW events on behalf of an app, then we will // need information from the <ETW> section of the service manifest. this.serviceConfigSections.Add(ServiceConfig.EtwElement); // Check if we can use an existing worker object string applicationType = this.etlProducerSettings.ApplicationType; lock (ProducerWorkers) { EtlProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(w => w.ApplicationType.Equals( applicationType, StringComparison.Ordinal)); if (null != workerInfo) { // Existing worker object is available. this.traceSource.WriteInfo( this.logSourceId, "Existing ETL producer worker object for application type {0} is available. Restarting the worker object now.", applicationType); // Restart the worker object workerInfo.ProducerWorker.Dispose(); workerInfo.ProducerWorker = null; List <EtlProducer> etlProducers = new List <EtlProducer>(workerInfo.EtlProducers) { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = applicationType, LogDirectory = initParam.LogDirectory, ProducerInstanceId = applicationType, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); workerInfo.EtlProducers.Add(this); workerInfo.ProducerWorker = newWorker; } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to restart ETL producer worker object for application type {0}.", applicationType); } } else { // Create a new worker object this.traceSource.WriteInfo( this.logSourceId, "Creating ETL producer worker object for application type {0} ...", applicationType); List <EtlProducer> etlProducers = new List <EtlProducer> { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = applicationType, LogDirectory = initParam.LogDirectory, ProducerInstanceId = applicationType, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); workerInfo = new EtlProducerWorkerInfo() { ApplicationType = applicationType, EtlProducers = new List <EtlProducer>(), ProducerWorker = newWorker }; workerInfo.EtlProducers.Add(this); ProducerWorkers.Add(workerInfo); } catch (InvalidOperationException) { this.traceSource.WriteError( this.logSourceId, "Failed to create ETL producer worker object for application type {0}.", applicationType); } } } } else { // Create a new worker object List <EtlProducer> etlProducers = new List <EtlProducer> { this }; EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters() { TraceSource = this.traceSource, IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles, ApplicationType = string.Empty, LogDirectory = initParam.LogDirectory, ProducerInstanceId = this.logSourceId, EtlProducers = etlProducers, LatestSettings = this.etlProducerSettings }; try { EtlProducerWorker newWorker = new EtlProducerWorker( newWorkerParam, this.diskSpaceManager, this.traceFileEventReaderFactory); this.producerWorker = newWorker; } catch (InvalidOperationException) { } } }