internal EtlInMemoryProducerWorker(
            EtlInMemoryProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId             = initParam.ProducerInstanceId;
            this.traceSource             = initParam.TraceSource;
            this.cancellationTokenSource = new CancellationTokenSource();
            this.perfHelper       = new EtlInMemoryPerformance(this.traceSource);
            this.diskSpaceManager = diskSpaceManager;

            // Initialize the settings
            this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));

            // Figure out where the ETL files are located
            this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.traceSource,
                this.logSourceId);

            this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                // We need to collect bootstrap traces
                this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                    this.traceDirectory,
                    this.markerFileDirectory,
                    this.etlToInMemoryBufferWriters,
                    this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
                    this.traceSource,
                    this.logSourceId);
                this.bootstrapTraceProcessor.Start();

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    true,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.markerFileDirectory,
                    this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToInMemoryBufferWriters,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // Figure out how much processing time is available to each provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            // Disk manager set up for traces
            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                this.diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                    f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }

            // Disk manager set up for marker files
            this.diskSpaceManager.RegisterFolder(
                this.logSourceId,
                () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
        }
Пример #2
0
        internal EtlProducerWorker(
            EtlProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId = initParam.ProducerInstanceId;
            this.traceSource = initParam.TraceSource;
            this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
            this.perfHelper = new EtlPerformance(this.traceSource);
            this.cancellationTokenSource = new CancellationTokenSource();
            this.diskSpaceManager        = diskSpaceManager;

            // Initialize the settings
            this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
                initParam.EtlProducers,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
            this.etlToCsvFileWriters =
                EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
            this.bufferedEtwEventProviders =
                EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();

            // Figure out where the ETL files are located
            this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.traceDirectory,
                initParam.ProducerInstanceId);

            if (initParam.IsReadingFromApplicationManifest)
            {
                lock (InternalMarkerFileDirectoriesForApps)
                {
                    InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
                }
            }

            this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.etlProducerSettings.EtlFilePatterns,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                initParam.ProducerInstanceId,
                this.traceSource,
                this.logSourceId);

            if (false == initParam.IsReadingFromApplicationManifest)
            {
                // Ensure that no other instance of EtlProducerWorker is processing the
                // same ETL files
                var patternsAdded = new List <string>();
                var isUnique      = VerifyEtlFilesUniqueToCurrentInstance(
                    this.traceDirectory,
                    this.providers,
                    patternsAdded,
                    message => this.traceSource.WriteError(this.logSourceId, message));
                this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();

                if (!isUnique)
                {
                    throw new InvalidOperationException(
                              string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
                }
            }

            this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                if (false == initParam.IsReadingFromApplicationManifest)
                {
                    // We need to collect bootstrap traces
                    this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                        this.traceDirectory,
                        this.markerFileDirectory,
                        this.etlToCsvFileWriters,
                        this.etlProducerSettings.EtlReadInterval,
                        this.traceSource,
                        this.logSourceId);
                    this.bootstrapTraceProcessor.Start();
                }

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    false == initParam.IsReadingFromApplicationManifest,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.etlProducerSettings.CustomManifestPaths,
                    initParam.ApplicationType,
                    this.markerFileDirectory,
                    this.etlProducerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToCsvFileWriters,
                    this.bufferedEtwEventProviders,
                    this.etlProducerSettings.AppEtwGuids,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // If there is a huge backlog of ETL files to process, we limit the
                // amount of time that we spend on processing ETL files in each
                // pass. Figure out how much processing time is available to each
                // provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                if (Directory.Exists(this.traceDirectory))
                {
                    diskSpaceManager.RegisterFolder(
                        this.logSourceId,
                        () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                        f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                        f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
                }
            }

            if (Directory.Exists(this.markerFileDirectory))
            {
                diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                    f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }
        }