public LttProducer(
DiskSpaceManager diskSpaceManager,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.AppInstanceId = initParam.ApplicationInstanceId;
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.serviceConfigSections = new List <string>();
this.consumerSinks = initParam.ConsumerSinks;
// Read the timer config value from dca section
var configReader = new ConfigReader(initParam.ApplicationInstanceId);
long lttReadIntervalMinutes = configReader.GetUnencryptedConfigValue(
initParam.SectionName,
LttProducerConstants.LttReadIntervalParamName,
LttProducerConstants.DefaultLttReadIntervalMinutes);
if (initParam.ApplicationInstanceId == Utility.WindowsFabricApplicationInstanceId)
{
this.CreateWindowsFabricLttProducerWorkerInfo(initParam, lttReadIntervalMinutes);
}
else
{
this.CreateAppProducerWorkerInfo(initParam, lttReadIntervalMinutes);
}
}
internal BufferedEtwEventProvider(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string traceBufferingFolder,
TimeSpan uploadInterval,
TimeSpan eventDeletionAgeIn,
IBufferedEtwEventSink eventSink)
: base(traceEventSourceFactory, logSourceId)
{
this.eventSink = eventSink;
this.bookmark = null;
this.perfHelper = new BufferedEtwEventPerformance(this.TraceSource, this.LogSourceId);
this.eventDeletionAge = eventDeletionAgeIn;
// Compute how much time we spend delivering buffered ETW events to
// the consumer in each pass. We define this to be roughly 1/4th of
// the interval between passes. This is an arbitrary choice.
this.eventDeliveryPassLength = TimeSpan.FromMilliseconds(uploadInterval.TotalMilliseconds / 4);
// Create the buffered event directory, in case it doesn't already exist.
this.etwEventCache = traceBufferingFolder;
FabricDirectory.CreateDirectory(this.etwEventCache);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing buffered events: {0}",
this.etwEventCache);
// Create a timer to schedule the delivery of ETW events to the consumer
string timerId = string.Concat(
this.LogSourceId,
EventDeliveryTimerIdSuffix);
this.eventDeliveryTimer = new DcaTimer(
timerId,
this.DeliverEventsToConsumer,
uploadInterval);
this.eventDeliveryTimer.Start();
// Create a timer to delete old logs
timerId = string.Concat(
this.LogSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval =
(this.eventDeletionAge < TimeSpan.FromDays(1))
? EtlConsumerConstants.OldLogDeletionIntervalForTest
: EtlConsumerConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
protected EventProcessor(ITraceEventSourceFactory traceEventSourceFactory, string logSourceId)
{
this.TraceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.LogSourceId = logSourceId;
this.Stopping = false;
this.Filter = null;
this.FilterLock = new object();
this.EtlProducerLock = new object();
// Initialize pending events
this.pendingEtwEvents.EventList = new Queue <DecodedEventWrapper>();
this.ResetPendingEvents();
}
// Creates a helper object that decodes ETW events and writes the decoded
// events to text files
public static IEtlToCsvFileWriter CreateEtlToCsvFileWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwCsvFolder,
bool disableDtrCompression)
{
MdsUploaderTest.EtwCsvFolder = etwCsvFolder;
return((IEtlToCsvFileWriter) new EtlToCsvFileWriter(
traceEventSourceFactory,
logSourceId,
fabricNodeId,
etwCsvFolder,
disableDtrCompression));
}
internal EtlToInMemoryBufferWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwLogDirName,
bool compressionEnabled,
IDcaInMemoryConsumer inMemoryConsumer)
: base(traceEventSourceFactory, logSourceId)
{
this.fabricNodeId = fabricNodeId;
this.etwLogDirName = etwLogDirName;
this.compressionEnabled = compressionEnabled;
this.inMemoryConsumer = inMemoryConsumer;
#if !DotNetCoreClr
this.perfHelper = new EtlToInMemoryBufferPerformance(this.TraceSource, this.LogSourceId);
#endif
}
internal EtlToCsvFileWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwCsvFolder,
bool dtrCompressionDisabled,
DiskSpaceManager diskSpaceManager)
: this(
traceEventSourceFactory,
logSourceId,
fabricNodeId,
etwCsvFolder,
dtrCompressionDisabled,
diskSpaceManager,
new EtlToCsvFileWriterConfigReader())
{
}
internal EtlInMemoryProducer(
DiskSpaceManager diskSpaceManager,
IEtlInMemoryProducerConfigReaderFactory configReaderFactory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.diskSpaceManager = diskSpaceManager;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Initialization
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.consumerSinks = initParam.ConsumerSinks;
// Read settings
var configReader = configReaderFactory.CreateEtlInMemoryProducerConfigReader(this.traceSource, this.logSourceId);
this.etlInMemoryProducerSettings = configReader.GetSettings();
// ETL in-memory file processing is not enabled or we are not processing
// winfab etl files, so return immediately
if (false == this.etlInMemoryProducerSettings.Enabled || false == this.etlInMemoryProducerSettings.ProcessingWinFabEtlFiles)
{
return;
}
// Create a new worker object
var newWorkerParam = new EtlInMemoryProducerWorker.EtlInMemoryProducerWorkerParameters()
{
TraceSource = this.traceSource,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
EtlInMemoryProducer = this,
LatestSettings = this.etlInMemoryProducerSettings
};
var newWorker = new EtlInMemoryProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
this.producerWorker = newWorker;
}
internal EtlProducer(
DiskSpaceManager diskSpaceManager,
IEtlProducerConfigReaderFactory configReaderFactory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.diskSpaceManager = diskSpaceManager;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Initialization
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.serviceConfigSections = new List <string>();
this.logDirectory = initParam.LogDirectory;
this.consumerSinks = initParam.ConsumerSinks;
// Read instance-specific settings from the settings file
var configReader = configReaderFactory.CreateEtlProducerConfigReader(this.traceSource, this.logSourceId);
this.etlProducerSettings = configReader.GetSettings();
if (false == this.etlProducerSettings.Enabled)
{
// ETL file processing is not enabled, so return immediately
return;
}
if (!this.etlProducerSettings.ProcessingWinFabEtlFiles)
{
// If we are collecting ETW events on behalf of an app, then we will
// need information from the <ETW> section of the service manifest.
this.serviceConfigSections.Add(ServiceConfig.EtwElement);
// Check if we can use an existing worker object
string applicationType = this.etlProducerSettings.ApplicationType;
lock (ProducerWorkers)
{
EtlProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(w => w.ApplicationType.Equals(
applicationType,
StringComparison.Ordinal));
if (null != workerInfo)
{
// Existing worker object is available.
this.traceSource.WriteInfo(
this.logSourceId,
"Existing ETL producer worker object for application type {0} is available. Restarting the worker object now.",
applicationType);
// Restart the worker object
workerInfo.ProducerWorker.Dispose();
workerInfo.ProducerWorker = null;
List <EtlProducer> etlProducers = new List <EtlProducer>(workerInfo.EtlProducers)
{
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = applicationType,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = applicationType,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
workerInfo.EtlProducers.Add(this);
workerInfo.ProducerWorker = newWorker;
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to restart ETL producer worker object for application type {0}.",
applicationType);
}
}
else
{
// Create a new worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating ETL producer worker object for application type {0} ...",
applicationType);
List <EtlProducer> etlProducers = new List <EtlProducer> {
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = applicationType,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = applicationType,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
workerInfo = new EtlProducerWorkerInfo()
{
ApplicationType = applicationType,
EtlProducers = new List <EtlProducer>(),
ProducerWorker = newWorker
};
workerInfo.EtlProducers.Add(this);
ProducerWorkers.Add(workerInfo);
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create ETL producer worker object for application type {0}.",
applicationType);
}
}
}
}
else
{
// Create a new worker object
List <EtlProducer> etlProducers = new List <EtlProducer> {
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = string.Empty,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
this.producerWorker = newWorker;
}
catch (InvalidOperationException)
{
}
}
}
internal EtlToCsvFileWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwCsvFolder,
bool dtrCompressionDisabled,
DiskSpaceManager diskSpaceManager,
IEtlToCsvFileWriterConfigReader configReader)
: base(traceEventSourceFactory, logSourceId)
{
this.organizeWindowsFabricTracesByType = true;
this.fabricNodeId = fabricNodeId;
this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled;
this.diskSpaceManager = diskSpaceManager;
this.configReader = configReader;
#if !DotNetCoreClr
this.perfHelper = new EtlToCsvPerformance(this.TraceSource, this.LogSourceId);
#endif
// Create the directory that containing filtered traces, in case it
// doesn't already exist
this.filteredTraceDirName = etwCsvFolder;
FabricDirectory.CreateDirectory(this.filteredTraceDirName);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing filtered ETW traces: {0}",
this.filteredTraceDirName);
// Create a timer to delete old logs
// Figure out the retention time for the CSV files
// Time after which the CSV file on disk becomes a candidate for deletion
// Read this value from config every time. Do not cache it. That's how
// we pick up the latest value when an update happens.
//
// From the above files, get the ones whose corresponding ETL files
// have already been fully processed. We should delete only the files
// whose corresponding ETL files have been fully processed. All other
// files should be kept around because their file name gives us the
// bookmark up to which we have processed events.
var deletionAge = configReader.GetDtrDeletionAge();
diskSpaceManager.RegisterFolder(
logSourceId,
() =>
{
// Get the filtered ETW trace files that are old enough to be deleted
var dirInfo = new DirectoryInfo(this.filteredTraceDirName);
return(dirInfo.EnumerateFiles(EtlConsumerConstants.FilteredEtwTraceSearchPattern, SearchOption.AllDirectories));
},
f => f.LastWriteTimeUtc < DateTime.UtcNow - deletionAge, // we don't have any indication whether work is done currently, use timer to estimate
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge,
f =>
{
DateTime lastEventTimeStamp;
GetLastEventTimestamp(f.Name, out lastEventTimeStamp);
this.lastDeletedDtrName = f.Name;
this.lastDeletedDtrEventTime = lastEventTimeStamp;
try
{
FabricFile.Delete(f.FullName);
return(true);
}
catch (Exception)
{
return(false);
}
});
diskSpaceManager.RetentionPassCompleted += () =>
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The last dtr file deleted during disk space manager pass {0} with events up til {1}.",
this.lastDeletedDtrName,
this.lastDeletedDtrEventTime);
};
diskSpaceManager.RegisterFolder(
logSourceId,
() => new DirectoryInfo(this.filteredTraceDirName)
.EnumerateFiles(EtlConsumerConstants.BootstrapTraceSearchPattern, SearchOption.AllDirectories),
f => true,
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge);
}
public LttProducer(
DiskSpaceManager diskSpaceManager,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.serviceConfigSections = new List <string>();
this.consumerSinks = initParam.ConsumerSinks;
// Read the timer config value from dca section
var configReader = new ConfigReader(initParam.ApplicationInstanceId);
long newLttReadIntervalMinutes = configReader.GetUnencryptedConfigValue(
initParam.SectionName,
LttProducerConstants.LttReadIntervalParamName,
LttProducerConstants.DefaultLttReadIntervalMinutes);
lock (ProducerWorkers)
{
LttProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault();
if (null != workerInfo)
{
// Existing worker object is available.
this.traceSource.WriteInfo(
this.logSourceId,
"Existing Ltt producer worker object. Restarting the worker object now.");
// Save the old value for comparision
long oldLttReadIntervalMinutes = workerInfo.ProducerWorker.LttReadIntervalMinutes;
// Restart the worker object
workerInfo.ProducerWorker.Dispose();
workerInfo.ProducerWorker = null;
// Keep the smaller value intact
// as this worker handles both producers Ltt trace conversion and table events
if (oldLttReadIntervalMinutes < newLttReadIntervalMinutes)
{
newLttReadIntervalMinutes = oldLttReadIntervalMinutes;
}
List <LttProducer> LttProducers = new List <LttProducer>(workerInfo.LttProducers)
{
this
};
LttProducerWorker.LttProducerWorkerParameters newWorkerParam = new LttProducerWorker.LttProducerWorkerParameters()
{
TraceSource = this.traceSource,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
LttProducers = LttProducers,
LatestSettings = initParam,
LttReadIntervalMinutes = newLttReadIntervalMinutes
};
try
{
LttProducerWorker newWorker = new LttProducerWorker(newWorkerParam);
workerInfo.LttProducers.Add(this);
workerInfo.ProducerWorker = newWorker;
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to restart Ltt producer worker object.");
}
}
else
{
// Create a new worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating Ltt producer worker object ...");
List <LttProducer> LttProducers = new List <LttProducer> {
this
};
LttProducerWorker.LttProducerWorkerParameters newWorkerParam = new LttProducerWorker.LttProducerWorkerParameters()
{
TraceSource = this.traceSource,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
LttProducers = LttProducers,
LatestSettings = initParam,
LttReadIntervalMinutes = newLttReadIntervalMinutes
};
try
{
LttProducerWorker newWorker = new LttProducerWorker(newWorkerParam);
workerInfo = new LttProducerWorkerInfo()
{
LttProducers = new List <LttProducer>(),
ProducerWorker = newWorker
};
workerInfo.LttProducers.Add(this);
ProducerWorkers.Add(workerInfo);
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create Ltt producer worker object.");
}
}
}
}
