internal EtlProcessor(
bool loadWinFabManifests,
bool dynamicWinFabManifestLoad,
IEnumerable <string> customManifestPaths,
string applicationType,
string markerFileDirectory,
WinFabricEtlType winFabricEtlType,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
EtlPerformance perfHelper,
IEnumerable <IEtlFileSink> sinks,
IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters,
IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders,
IEnumerable <Guid> appEtwGuids,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.processingWinFabEvents = loadWinFabManifests;
this.applicationType = applicationType;
this.markerFileDirectory = markerFileDirectory;
this.windowsFabricEtlType = winFabricEtlType;
this.perfHelper = perfHelper;
this.sinks = sinks.ToList().AsReadOnly();
this.etlToCsvFileWriters = etlToCsvFileWriters.ToList().AsReadOnly();
this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly();
this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Load the custom manifests (if any) into the manifest cache
this.LoadManifests(customManifestPaths);
if (loadWinFabManifests)
{
if (dynamicWinFabManifestLoad)
{
// We are going to be loading manifests dynamically based on the version
// in the ETL file. Initialize the manifest manager object to manage this
// for us.
this.winFabManifestMgr = this.InitializeWinFabManifestManager();
}
else
{
// We'll load the default manifests now and just use them throughout
this.LoadDefaultManifests();
}
}
// Provide the sinks a reference to the manifest cache.
this.OnEtwManifestCacheAvailable();
}
internal EtlProducerWorker(
EtlProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
this.perfHelper = new EtlPerformance(this.traceSource);
this.cancellationTokenSource = new CancellationTokenSource();
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
initParam.EtlProducers,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
this.etlToCsvFileWriters =
EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
this.bufferedEtwEventProviders =
EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();
// Figure out where the ETL files are located
this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceDirectory,
initParam.ProducerInstanceId);
if (initParam.IsReadingFromApplicationManifest)
{
lock (InternalMarkerFileDirectoriesForApps)
{
InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
}
}
this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
this.etlProducerSettings.WindowsFabricEtlType,
this.etlProducerSettings.EtlFilePatterns,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
initParam.ProducerInstanceId,
this.traceSource,
this.logSourceId);
if (false == initParam.IsReadingFromApplicationManifest)
{
// Ensure that no other instance of EtlProducerWorker is processing the
// same ETL files
var patternsAdded = new List <string>();
var isUnique = VerifyEtlFilesUniqueToCurrentInstance(
this.traceDirectory,
this.providers,
patternsAdded,
message => this.traceSource.WriteError(this.logSourceId, message));
this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();
if (!isUnique)
{
throw new InvalidOperationException(
string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
}
}
this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
if (false == initParam.IsReadingFromApplicationManifest)
{
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToCsvFileWriters,
this.etlProducerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
}
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
false == initParam.IsReadingFromApplicationManifest,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.etlProducerSettings.CustomManifestPaths,
initParam.ApplicationType,
this.markerFileDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToCsvFileWriters,
this.bufferedEtwEventProviders,
this.etlProducerSettings.AppEtwGuids,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// If there is a huge backlog of ETL files to process, we limit the
// amount of time that we spend on processing ETL files in each
// pass. Figure out how much processing time is available to each
// provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
foreach (var provider in this.providers)
{
var capturedProvider = provider;
if (Directory.Exists(this.traceDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}
if (Directory.Exists(this.markerFileDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}