Beispiel #1
0
        internal EtlProcessor(
            bool loadWinFabManifests,
            bool dynamicWinFabManifestLoad,
            string markerFileDirectory,
            WinFabricEtlType winFabricEtlType,
            FabricEvents.ExtensionsEvents traceSource,
            string logSourceId,
            EtlInMemoryPerformance perfHelper,
            IEnumerable <IEtlInMemorySink> sinks,
            IEnumerable <EtlToInMemoryBufferWriter> etlToInMemoryBufferWriters,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.traceSource            = traceSource;
            this.logSourceId            = logSourceId;
            this.processingWinFabEvents = loadWinFabManifests;
            this.markerFileDirectory    = markerFileDirectory;
            this.windowsFabricEtlType   = winFabricEtlType;
            this.sinks = sinks.ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters  = etlToInMemoryBufferWriters.ToList().AsReadOnly();
            this.traceFileEventReaderFactory = traceFileEventReaderFactory;
            this.perfHelper = perfHelper;

            if (loadWinFabManifests)
            {
                if (dynamicWinFabManifestLoad)
                {
                    // We are going to be loading manifests dynamically based on the version
                    // in the ETL file. Initialize the manifest manager object to manage this
                    // for us.
                    this.winFabManifestMgr = this.InitializeWinFabManifestManager();
                }
                else
                {
                    // We'll load the default manifests now and just use them throughout
                    this.LoadDefaultManifests();
                }
            }

            // Provide the sinks a reference to the manifest cache.
            this.sinks.ForEach(sink => sink.SetEtwManifestCache(this.etwManifestCache));
        }
        internal EtlInMemoryProducerWorker(
            EtlInMemoryProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId             = initParam.ProducerInstanceId;
            this.traceSource             = initParam.TraceSource;
            this.cancellationTokenSource = new CancellationTokenSource();
            this.perfHelper       = new EtlInMemoryPerformance(this.traceSource);
            this.diskSpaceManager = diskSpaceManager;

            // Initialize the settings
            this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
            this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));

            // Figure out where the ETL files are located
            this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);

            this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
                this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                this.etlInMemoryProducerWorkerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                this.traceSource,
                this.logSourceId);

            this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                // We need to collect bootstrap traces
                this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                    this.traceDirectory,
                    this.markerFileDirectory,
                    this.etlToInMemoryBufferWriters,
                    this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
                    this.traceSource,
                    this.logSourceId);
                this.bootstrapTraceProcessor.Start();

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    true,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.markerFileDirectory,
                    this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToInMemoryBufferWriters,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // Figure out how much processing time is available to each provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            // Disk manager set up for traces
            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                this.diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                    f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }

            // Disk manager set up for marker files
            this.diskSpaceManager.RegisterFolder(
                this.logSourceId,
                () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
        }