コード例 #1
0
ファイル: EtlProcessor.cs プロジェクト: zmyer/service-fabric
        internal EtlProcessor(
            bool loadWinFabManifests,
            bool dynamicWinFabManifestLoad,
            IEnumerable <string> customManifestPaths,
            string applicationType,
            string markerFileDirectory,
            WinFabricEtlType winFabricEtlType,
            FabricEvents.ExtensionsEvents traceSource,
            string logSourceId,
            EtlPerformance perfHelper,
            IEnumerable <IEtlFileSink> sinks,
            IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters,
            IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders,
            IEnumerable <Guid> appEtwGuids,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.traceSource            = traceSource;
            this.logSourceId            = logSourceId;
            this.processingWinFabEvents = loadWinFabManifests;
            this.applicationType        = applicationType;
            this.markerFileDirectory    = markerFileDirectory;
            this.windowsFabricEtlType   = winFabricEtlType;
            this.perfHelper             = perfHelper;
            this.sinks = sinks.ToList().AsReadOnly();
            this.etlToCsvFileWriters       = etlToCsvFileWriters.ToList().AsReadOnly();
            this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly();
            this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null;

            this.traceFileEventReaderFactory = traceFileEventReaderFactory;

            // Load the custom manifests (if any) into the manifest cache
            this.LoadManifests(customManifestPaths);

            if (loadWinFabManifests)
            {
                if (dynamicWinFabManifestLoad)
                {
                    // We are going to be loading manifests dynamically based on the version
                    // in the ETL file. Initialize the manifest manager object to manage this
                    // for us.
                    this.winFabManifestMgr = this.InitializeWinFabManifestManager();
                }
                else
                {
                    // We'll load the default manifests now and just use them throughout
                    this.LoadDefaultManifests();
                }
            }

            // Provide the sinks a reference to the manifest cache.
            this.OnEtwManifestCacheAvailable();
        }
コード例 #2
0
        internal EtlProducerWorker(
            EtlProducerWorkerParameters initParam,
            DiskSpaceManager diskSpaceManager,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            this.logSourceId = initParam.ProducerInstanceId;
            this.traceSource = initParam.TraceSource;
            this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
            this.perfHelper = new EtlPerformance(this.traceSource);
            this.cancellationTokenSource = new CancellationTokenSource();
            this.diskSpaceManager        = diskSpaceManager;

            // Initialize the settings
            this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
            if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
            {
                // If we're processing the default ETL files, we should keep track of
                // whether or not we're on the FMM node. This information is used by
                // some other plugin types.
                Utility.LastFmmEventTimestamp = DateTime.MinValue;
            }

            // Initialize the sink list
            this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
                initParam.EtlProducers,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
            this.etlToCsvFileWriters =
                EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
            this.bufferedEtwEventProviders =
                EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();

            // Figure out where the ETL files are located
            this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType);

            this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.traceDirectory,
                initParam.ProducerInstanceId);

            if (initParam.IsReadingFromApplicationManifest)
            {
                lock (InternalMarkerFileDirectoriesForApps)
                {
                    InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
                }
            }

            this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                this.etlProducerSettings.WindowsFabricEtlType,
                this.etlProducerSettings.EtlFilePatterns,
                message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();

            if (0 == this.providers.Count)
            {
                // No ETL files to read, so return immediately
                this.traceSource.WriteWarning(
                    this.logSourceId,
                    "No ETL files have been specified for processing.");
            }

            this.checkpointManager = new CheckpointManager(
                initParam.IsReadingFromApplicationManifest,
                this.etlProducerSettings.EtlPath,
                initParam.LogDirectory,
                this.traceDirectory,
                initParam.ProducerInstanceId,
                this.traceSource,
                this.logSourceId);

            if (false == initParam.IsReadingFromApplicationManifest)
            {
                // Ensure that no other instance of EtlProducerWorker is processing the
                // same ETL files
                var patternsAdded = new List <string>();
                var isUnique      = VerifyEtlFilesUniqueToCurrentInstance(
                    this.traceDirectory,
                    this.providers,
                    patternsAdded,
                    message => this.traceSource.WriteError(this.logSourceId, message));
                this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();

                if (!isUnique)
                {
                    throw new InvalidOperationException(
                              string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
                }
            }

            this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
            if (this.etlReadInterval > TimeSpan.Zero)
            {
                // Create the directory that contains the marker files.
                this.CreateDirectoriesForEtlProcessing();

                if (false == initParam.IsReadingFromApplicationManifest)
                {
                    // We need to collect bootstrap traces
                    this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
                        this.traceDirectory,
                        this.markerFileDirectory,
                        this.etlToCsvFileWriters,
                        this.etlProducerSettings.EtlReadInterval,
                        this.traceSource,
                        this.logSourceId);
                    this.bootstrapTraceProcessor.Start();
                }

                // Create the ETL processor
                this.etlProcessor = new EtlProcessor(
                    false == initParam.IsReadingFromApplicationManifest,
                    this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
                    this.etlProducerSettings.CustomManifestPaths,
                    initParam.ApplicationType,
                    this.markerFileDirectory,
                    this.etlProducerSettings.WindowsFabricEtlType,
                    this.traceSource,
                    this.logSourceId,
                    this.perfHelper,
                    this.sinks,
                    this.etlToCsvFileWriters,
                    this.bufferedEtwEventProviders,
                    this.etlProducerSettings.AppEtwGuids,
                    traceFileEventReaderFactory);

                // Create a periodic timer to read ETL files
                var timerId = string.Concat(
                    this.logSourceId,
                    EtlReadTimerIdSuffix);
                this.etlReadTimer = new DcaTimer(
                    timerId,
                    state => this.EtlReadCallback(this.cancellationTokenSource.Token),
                    this.etlReadInterval);
                this.etlReadTimer.Start();

                // If there is a huge backlog of ETL files to process, we limit the
                // amount of time that we spend on processing ETL files in each
                // pass. Figure out how much processing time is available to each
                // provider.
                this.ComputePerProviderEtlProcessingTimeSeconds();
            }

            foreach (var provider in this.providers)
            {
                var capturedProvider = provider;
                if (Directory.Exists(this.traceDirectory))
                {
                    diskSpaceManager.RegisterFolder(
                        this.logSourceId,
                        () => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
                        f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
                        f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
                }
            }

            if (Directory.Exists(this.markerFileDirectory))
            {
                diskSpaceManager.RegisterFolder(
                    this.logSourceId,
                    () => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
                    f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
                    f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
            }
        }