Ejemplo n.º 1
0
        internal EtlProcessor(
            bool loadWinFabManifests,
            bool dynamicWinFabManifestLoad,
            IEnumerable <string> customManifestPaths,
            string applicationType,
            string markerFileDirectory,
            WinFabricEtlType winFabricEtlType,
            FabricEvents.ExtensionsEvents traceSource,
            string logSourceId,
            EtlPerformance perfHelper,
            IEnumerable <IEtlFileSink> sinks,
            IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters,
            IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders,
            IEnumerable <Guid> appEtwGuids,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.traceSource            = traceSource;
            this.logSourceId            = logSourceId;
            this.processingWinFabEvents = loadWinFabManifests;
            this.applicationType        = applicationType;
            this.markerFileDirectory    = markerFileDirectory;
            this.windowsFabricEtlType   = winFabricEtlType;
            this.perfHelper             = perfHelper;
            this.sinks = sinks.ToList().AsReadOnly();
            this.etlToCsvFileWriters       = etlToCsvFileWriters.ToList().AsReadOnly();
            this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly();
            this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null;

            this.traceFileEventReaderFactory = traceFileEventReaderFactory;

            // Load the custom manifests (if any) into the manifest cache
            this.LoadManifests(customManifestPaths);

            if (loadWinFabManifests)
            {
                if (dynamicWinFabManifestLoad)
                {
                    // We are going to be loading manifests dynamically based on the version
                    // in the ETL file. Initialize the manifest manager object to manage this
                    // for us.
                    this.winFabManifestMgr = this.InitializeWinFabManifestManager();
                }
                else
                {
                    // We'll load the default manifests now and just use them throughout
                    this.LoadDefaultManifests();
                }
            }

            // Provide the sinks a reference to the manifest cache.
            this.OnEtwManifestCacheAvailable();
        }
Ejemplo n.º 2
0
        internal EtlInMemoryProducer(
            DiskSpaceManager diskSpaceManager,
            IEtlInMemoryProducerConfigReaderFactory configReaderFactory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory,
            ITraceEventSourceFactory traceEventSourceFactory,
            ProducerInitializationParameters initParam)
        {
            this.diskSpaceManager            = diskSpaceManager;
            this.traceFileEventReaderFactory = traceFileEventReaderFactory;

            // Initialization
            this.traceSource   = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
            this.logSourceId   = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
            this.consumerSinks = initParam.ConsumerSinks;

            // Read settings
            var configReader = configReaderFactory.CreateEtlInMemoryProducerConfigReader(this.traceSource, this.logSourceId);

            this.etlInMemoryProducerSettings = configReader.GetSettings();

            // ETL in-memory file processing is not enabled or we are not processing
            // winfab etl files, so return immediately
            if (false == this.etlInMemoryProducerSettings.Enabled || false == this.etlInMemoryProducerSettings.ProcessingWinFabEtlFiles)
            {
                return;
            }

            // Create a new worker object
            var newWorkerParam = new EtlInMemoryProducerWorker.EtlInMemoryProducerWorkerParameters()
            {
                TraceSource         = this.traceSource,
                LogDirectory        = initParam.LogDirectory,
                ProducerInstanceId  = this.logSourceId,
                EtlInMemoryProducer = this,
                LatestSettings      = this.etlInMemoryProducerSettings
            };

            var newWorker = new EtlInMemoryProducerWorker(
                newWorkerParam,
                this.diskSpaceManager,
                this.traceFileEventReaderFactory);

            this.producerWorker = newWorker;
        }
Ejemplo n.º 3
0
        internal EtlProcessor(
            bool loadWinFabManifests,
            bool dynamicWinFabManifestLoad,
            string markerFileDirectory,
            WinFabricEtlType winFabricEtlType,
            FabricEvents.ExtensionsEvents traceSource,
            string logSourceId,
            EtlInMemoryPerformance perfHelper,
            IEnumerable <IEtlInMemorySink> sinks,
            IEnumerable <EtlToInMemoryBufferWriter> etlToInMemoryBufferWriters,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.traceSource            = traceSource;
            this.logSourceId            = logSourceId;
            this.processingWinFabEvents = loadWinFabManifests;
            this.markerFileDirectory    = markerFileDirectory;
            this.windowsFabricEtlType   = winFabricEtlType;
            this.sinks = sinks.ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters  = etlToInMemoryBufferWriters.ToList().AsReadOnly();
            this.traceFileEventReaderFactory = traceFileEventReaderFactory;
            this.perfHelper = perfHelper;

            if (loadWinFabManifests)
            {
                if (dynamicWinFabManifestLoad)
                {
                    // We are going to be loading manifests dynamically based on the version
                    // in the ETL file. Initialize the manifest manager object to manage this
                    // for us.
                    this.winFabManifestMgr = this.InitializeWinFabManifestManager();
                }
                else
                {
                    // We'll load the default manifests now and just use them throughout
                    this.LoadDefaultManifests();
                }
            }

            // Provide the sinks a reference to the manifest cache.
            this.sinks.ForEach(sink => sink.SetEtwManifestCache(this.etwManifestCache));
        }
Ejemplo n.º 4
0
        private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            var mockDiskSpaceManager        = TestUtility.MockRepository.Create <DiskSpaceManager>();
            var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
                true,
                TimeSpan.FromSeconds(1),
                TimeSpan.FromDays(3000),
                WinFabricEtlType.DefaultEtl,
                logDirectory,
                TestEtlFilePatterns,
                true);

            var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();

            configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);

            var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();

            mockTraceEventSourceFactory
            .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
            .Returns(new ErrorAndWarningFreeTraceEventSource());

            var configStore = TestUtility.MockRepository.Create <IConfigStore>();

            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
            .Returns("100");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
            .Returns("0");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName))
            .Returns("true");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName))
            .Returns(logDirectory);
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName))
            .Returns(TableName);
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName))
            .Returns(MdsFileProducerConstants.DefaultTablePriority);
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName))
            .Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString());
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName))
            .Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString());
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName))
            .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName))
            .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName))
            .Returns(LogFilter);

            Utility.InitializeConfigStore(configStore.Object);

            var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();

            mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
            .Returns(configReader.Object);

            // Create the MDS consumer
            ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
            var consumerInitParam = new ConsumerInitializationParameters(
                Utility.WindowsFabricApplicationInstanceId,
                TestConfigSectionName,
                TestFabricNodeId,
                TestFabricNodeName,
                Utility.LogDirectory,
                Utility.DcaWorkFolder,
                new DiskSpaceManager());

            var mdsFileProducer           = new MdsFileProducer(consumerInitParam);
            var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink();

            var producerInitParam = new ProducerInitializationParameters
            {
                ConsumerSinks = new[] { etlToInMemoryBufferWriter }
            };

            // Create the in-memory producer
            return(new EtlInMemoryProducer(
                       mockDiskSpaceManager.Object,
                       mockConfigReaderFactory.Object,
                       traceFileEventReaderFactory,
                       mockTraceEventSourceFactory.Object,
                       producerInitParam));
        }
        internal EtlInMemoryProducerWorker(
            EtlInMemoryProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId             = initParam.ProducerInstanceId;
            this.traceSource             = initParam.TraceSource;
            this.cancellationTokenSource = new CancellationTokenSource();
            this.perfHelper       = new EtlInMemoryPerformance(this.traceSource);
            this.diskSpaceManager = diskSpaceManager;

            // Initialize the settings
            this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));

            // Figure out where the ETL files are located
            this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.traceSource,
                this.logSourceId);

            this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                // We need to collect bootstrap traces
                this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                    this.traceDirectory,
                    this.markerFileDirectory,
                    this.etlToInMemoryBufferWriters,
                    this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
                    this.traceSource,
                    this.logSourceId);
                this.bootstrapTraceProcessor.Start();

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    true,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.markerFileDirectory,
                    this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToInMemoryBufferWriters,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // Figure out how much processing time is available to each provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            // Disk manager set up for traces
            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                this.diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                    f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }

            // Disk manager set up for marker files
            this.diskSpaceManager.RegisterFolder(
                this.logSourceId,
                () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
        }
Ejemplo n.º 6
0
        private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest(
            string logDirectory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            var mockDiskSpaceManager        = TestUtility.MockRepository.Create <DiskSpaceManager>();
            var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
                true,
                TimeSpan.FromSeconds(1),
                TimeSpan.FromDays(3000),
                WinFabricEtlType.DefaultEtl,
                logDirectory,
                TestUtility.TestEtlFilePatterns,
                true);

            var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();

            configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);

            var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();

            mockTraceEventSourceFactory
            .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
            .Returns(new ErrorAndWarningFreeTraceEventSource());

            var configStore = TestUtility.MockRepository.Create <IConfigStore>();

            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
            .Returns("100");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
            .Returns("0");
            Utility.InitializeConfigStore(configStore.Object);

            var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();

            mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
            .Returns(configReader.Object);

            var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false);
            var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true);

            var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                mockTraceEventSourceFactory.Object,
                LogSourceId,
                TestNodeId,
                TestUtility.GetActualOutputFolderPath(logDirectory),
                true,
                fastDcaInMemoryConsumer);

            var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                mockTraceEventSourceFactory.Object,
                LogSourceId,
                TestNodeId,
                TestUtility.GetActualOutputFolderPath(logDirectory),
                true,
                slowDcaInMemoryConsumer);

            var initParam = new ProducerInitializationParameters
            {
                ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter }
            };

            return(new EtlInMemoryProducer(
                       mockDiskSpaceManager.Object,
                       mockConfigReaderFactory.Object,
                       traceFileEventReaderFactory,
                       mockTraceEventSourceFactory.Object,
                       initParam));
        }
Ejemplo n.º 7
0
        internal EtlProducerWorker(
            EtlProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId = initParam.ProducerInstanceId;
            this.traceSource = initParam.TraceSource;
            this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
            this.perfHelper = new EtlPerformance(this.traceSource);
            this.cancellationTokenSource = new CancellationTokenSource();
            this.diskSpaceManager        = diskSpaceManager;

            // Initialize the settings
            this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
                initParam.EtlProducers,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
            this.etlToCsvFileWriters =
                EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
            this.bufferedEtwEventProviders =
                EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();

            // Figure out where the ETL files are located
            this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.traceDirectory,
                initParam.ProducerInstanceId);

            if (initParam.IsReadingFromApplicationManifest)
            {
                lock (InternalMarkerFileDirectoriesForApps)
                {
                    InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
                }
            }

            this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.etlProducerSettings.EtlFilePatterns,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                initParam.ProducerInstanceId,
                this.traceSource,
                this.logSourceId);

            if (false == initParam.IsReadingFromApplicationManifest)
            {
                // Ensure that no other instance of EtlProducerWorker is processing the
                // same ETL files
                var patternsAdded = new List <string>();
                var isUnique      = VerifyEtlFilesUniqueToCurrentInstance(
                    this.traceDirectory,
                    this.providers,
                    patternsAdded,
                    message => this.traceSource.WriteError(this.logSourceId, message));
                this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();

                if (!isUnique)
                {
                    throw new InvalidOperationException(
                              string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
                }
            }

            this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                if (false == initParam.IsReadingFromApplicationManifest)
                {
                    // We need to collect bootstrap traces
                    this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                        this.traceDirectory,
                        this.markerFileDirectory,
                        this.etlToCsvFileWriters,
                        this.etlProducerSettings.EtlReadInterval,
                        this.traceSource,
                        this.logSourceId);
                    this.bootstrapTraceProcessor.Start();
                }

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    false == initParam.IsReadingFromApplicationManifest,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.etlProducerSettings.CustomManifestPaths,
                    initParam.ApplicationType,
                    this.markerFileDirectory,
                    this.etlProducerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToCsvFileWriters,
                    this.bufferedEtwEventProviders,
                    this.etlProducerSettings.AppEtwGuids,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // If there is a huge backlog of ETL files to process, we limit the
                // amount of time that we spend on processing ETL files in each
                // pass. Figure out how much processing time is available to each
                // provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                if (Directory.Exists(this.traceDirectory))
                {
                    diskSpaceManager.RegisterFolder(
                        this.logSourceId,
                        () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                        f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                        f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
                }
            }

            if (Directory.Exists(this.markerFileDirectory))
            {
                diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                    f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }
        }
        private static void CreateEtlInMemoryProducerWithAzureBlobConsumer(
            string logDirectory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory,
            out EtlInMemoryProducer etlInMemoryProducer,
            out string containerName,
            out StorageAccountFactory storageAccountFactory,
            AccessCondition uploadStreamAccessCondition = null,
            AccessCondition uploadFileAccessCondition   = null)
        {
            var mockDiskSpaceManager        = TestUtility.MockRepository.Create <DiskSpaceManager>();
            var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
                true,
                TimeSpan.FromSeconds(1),
                TimeSpan.FromDays(3000),
                WinFabricEtlType.DefaultEtl,
                logDirectory,
                TestEtlFilePatterns,
                true);

            var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();

            configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);

            var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();

            mockTraceEventSourceFactory
            .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
            .Returns(new ErrorAndWarningFreeTraceEventSource());

            var configStore = TestUtility.MockRepository.Create <IConfigStore>();

            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
            .Returns("100");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
            .Returns("0");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName))
            .Returns("true");

            containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid());
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName))
            .Returns(containerName);

            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName))
            .Returns("0.25");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName))
            .Returns("1");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName))
            .Returns("0");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName))
            .Returns("*.*:4");
            configStore
            .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId))
            .Returns(AzureConstants.DefaultDeploymentId);

            var accountKey              = GetTestStorageAccountKey();
            var isEncrypted             = false;
            var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey);

            configStore
            .Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted))
            .Returns(storageConnectionString);

            Utility.InitializeConfigStore(configStore.Object);

            var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();

            mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
            .Returns(configReader.Object);

            // Create the Azure Blob Uploader consumer
            ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
            var consumerInitParam = new ConsumerInitializationParameters(
                Utility.WindowsFabricApplicationInstanceId,
                TestConfigSectionName,
                TestFabricNodeId,
                TestFabricNodeName,
                Utility.LogDirectory,
                Utility.DcaWorkFolder,
                new DiskSpaceManager());

            // Save storage connection for clean up
            var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType);

            storageAccountFactory = azureUtility.GetStorageAccountFactory(
                new ConfigReader(consumerInitParam.ApplicationInstanceId),
                TestConfigSectionName,
                AzureConstants.ConnectionStringParamName);

            var azureBlobConsumer         = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition);
            var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink();

            var producerInitParam = new ProducerInitializationParameters
            {
                ConsumerSinks = new[] { etlToInMemoryBufferWriter }
            };

            // Create the in-memory producer
            etlInMemoryProducer = new EtlInMemoryProducer(
                mockDiskSpaceManager.Object,
                mockConfigReaderFactory.Object,
                traceFileEventReaderFactory,
                mockTraceEventSourceFactory.Object,
                producerInitParam);
        }
Ejemplo n.º 9
0
        internal EtlProducer(
            DiskSpaceManager diskSpaceManager,
            IEtlProducerConfigReaderFactory configReaderFactory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory,
            ITraceEventSourceFactory traceEventSourceFactory,
            ProducerInitializationParameters initParam)
        {
            this.diskSpaceManager            = diskSpaceManager;
            this.traceFileEventReaderFactory = traceFileEventReaderFactory;

            // Initialization
            this.logSourceId           = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
            this.traceSource           = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
            this.serviceConfigSections = new List <string>();
            this.logDirectory          = initParam.LogDirectory;
            this.consumerSinks         = initParam.ConsumerSinks;

            // Read instance-specific settings from the settings file
            var configReader = configReaderFactory.CreateEtlProducerConfigReader(this.traceSource, this.logSourceId);

            this.etlProducerSettings = configReader.GetSettings();
            if (false == this.etlProducerSettings.Enabled)
            {
                // ETL file processing is not enabled, so return immediately
                return;
            }

            if (!this.etlProducerSettings.ProcessingWinFabEtlFiles)
            {
                // If we are collecting ETW events on behalf of an app, then we will
                // need information from the <ETW> section of the service manifest.
                this.serviceConfigSections.Add(ServiceConfig.EtwElement);

                // Check if we can use an existing worker object
                string applicationType = this.etlProducerSettings.ApplicationType;
                lock (ProducerWorkers)
                {
                    EtlProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(w => w.ApplicationType.Equals(
                                                                                          applicationType,
                                                                                          StringComparison.Ordinal));
                    if (null != workerInfo)
                    {
                        // Existing worker object is available.
                        this.traceSource.WriteInfo(
                            this.logSourceId,
                            "Existing ETL producer worker object for application type {0} is available. Restarting the worker object now.",
                            applicationType);

                        // Restart the worker object
                        workerInfo.ProducerWorker.Dispose();
                        workerInfo.ProducerWorker = null;

                        List <EtlProducer> etlProducers = new List <EtlProducer>(workerInfo.EtlProducers)
                        {
                            this
                        };
                        EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
                        {
                            TraceSource = this.traceSource,
                            IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
                            ApplicationType    = applicationType,
                            LogDirectory       = initParam.LogDirectory,
                            ProducerInstanceId = applicationType,
                            EtlProducers       = etlProducers,
                            LatestSettings     = this.etlProducerSettings
                        };
                        try
                        {
                            EtlProducerWorker newWorker = new EtlProducerWorker(
                                newWorkerParam,
                                this.diskSpaceManager,
                                this.traceFileEventReaderFactory);
                            workerInfo.EtlProducers.Add(this);
                            workerInfo.ProducerWorker = newWorker;
                        }
                        catch (InvalidOperationException)
                        {
                            this.traceSource.WriteError(
                                this.logSourceId,
                                "Failed to restart ETL producer worker object for application type {0}.",
                                applicationType);
                        }
                    }
                    else
                    {
                        // Create a new worker object
                        this.traceSource.WriteInfo(
                            this.logSourceId,
                            "Creating ETL producer worker object for application type {0} ...",
                            applicationType);

                        List <EtlProducer> etlProducers = new List <EtlProducer> {
                            this
                        };
                        EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
                        {
                            TraceSource = this.traceSource,
                            IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
                            ApplicationType    = applicationType,
                            LogDirectory       = initParam.LogDirectory,
                            ProducerInstanceId = applicationType,
                            EtlProducers       = etlProducers,
                            LatestSettings     = this.etlProducerSettings
                        };
                        try
                        {
                            EtlProducerWorker newWorker = new EtlProducerWorker(
                                newWorkerParam,
                                this.diskSpaceManager,
                                this.traceFileEventReaderFactory);
                            workerInfo = new EtlProducerWorkerInfo()
                            {
                                ApplicationType = applicationType,
                                EtlProducers    = new List <EtlProducer>(),
                                ProducerWorker  = newWorker
                            };
                            workerInfo.EtlProducers.Add(this);

                            ProducerWorkers.Add(workerInfo);
                        }
                        catch (InvalidOperationException)
                        {
                            this.traceSource.WriteError(
                                this.logSourceId,
                                "Failed to create ETL producer worker object for application type {0}.",
                                applicationType);
                        }
                    }
                }
            }
            else
            {
                // Create a new  worker object
                List <EtlProducer> etlProducers = new List <EtlProducer> {
                    this
                };
                EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
                {
                    TraceSource = this.traceSource,
                    IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
                    ApplicationType    = string.Empty,
                    LogDirectory       = initParam.LogDirectory,
                    ProducerInstanceId = this.logSourceId,
                    EtlProducers       = etlProducers,
                    LatestSettings     = this.etlProducerSettings
                };
                try
                {
                    EtlProducerWorker newWorker = new EtlProducerWorker(
                        newWorkerParam,
                        this.diskSpaceManager,
                        this.traceFileEventReaderFactory);
                    this.producerWorker = newWorker;
                }
                catch (InvalidOperationException)
                {
                }
            }
        }
Ejemplo n.º 10
0
        private static EtlProducer CreateEtlProducerForTest(
            string logDirectory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
            var etlProducerSettings  = new EtlProducerSettings(
                true,
                TimeSpan.FromSeconds(1),
                TimeSpan.FromDays(3000),
                WinFabricEtlType.DefaultEtl,
                logDirectory,
                TestUtility.TestEtlFilePatterns,
                new List <string>(),
                new Dictionary <string, List <ServiceEtwManifestInfo> >(),
                true,
                new[] { Guid.Parse(EtwProviderGuid) },
                null);

            var cr = TestUtility.MockRepository.Create <IEtlProducerConfigReader>();

            cr.Setup(c => c.GetSettings()).Returns(etlProducerSettings);

            var configStore = TestUtility.MockRepository.Create <IConfigStore>();

            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyOldDataDeletionIntervalInSeconds"))
            .Returns("3600");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "ApplicationLogsFormatVersion"))
            .Returns((string)null);
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyDtrDeletionAgeInMinutes"))
            .Returns("60");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
            .Returns("100");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
            .Returns("0");
            Utility.InitializeConfigStore(configStore.Object);

            var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlProducerConfigReaderFactory>();

            mockConfigReaderFactory
            .Setup(f => f.CreateEtlProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
            .Returns(cr.Object);

            var mockTraceSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();

            mockTraceSourceFactory
            .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
            .Returns(new ErrorAndWarningFreeTraceEventSource());

            var csvWriter = new EtlToCsvFileWriter(
                mockTraceSourceFactory.Object,
                LogSourceId,
                TestNodeId,
                TestUtility.GetActualOutputFolderPath(logDirectory),
                true,
                mockDiskSpaceManager.Object);
            var initParam = new ProducerInitializationParameters
            {
                ConsumerSinks = new[] { csvWriter }
            };

            return(new EtlProducer(
                       mockDiskSpaceManager.Object,
                       mockConfigReaderFactory.Object,
                       traceFileEventReaderFactory,
                       mockTraceSourceFactory.Object,
                       initParam));
        }