internal EtlProcessor(
bool loadWinFabManifests,
bool dynamicWinFabManifestLoad,
IEnumerable <string> customManifestPaths,
string applicationType,
string markerFileDirectory,
WinFabricEtlType winFabricEtlType,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
EtlPerformance perfHelper,
IEnumerable <IEtlFileSink> sinks,
IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters,
IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders,
IEnumerable <Guid> appEtwGuids,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.processingWinFabEvents = loadWinFabManifests;
this.applicationType = applicationType;
this.markerFileDirectory = markerFileDirectory;
this.windowsFabricEtlType = winFabricEtlType;
this.perfHelper = perfHelper;
this.sinks = sinks.ToList().AsReadOnly();
this.etlToCsvFileWriters = etlToCsvFileWriters.ToList().AsReadOnly();
this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly();
this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Load the custom manifests (if any) into the manifest cache
this.LoadManifests(customManifestPaths);
if (loadWinFabManifests)
{
if (dynamicWinFabManifestLoad)
{
// We are going to be loading manifests dynamically based on the version
// in the ETL file. Initialize the manifest manager object to manage this
// for us.
this.winFabManifestMgr = this.InitializeWinFabManifestManager();
}
else
{
// We'll load the default manifests now and just use them throughout
this.LoadDefaultManifests();
}
}
// Provide the sinks a reference to the manifest cache.
this.OnEtwManifestCacheAvailable();
}
internal EtlInMemoryProducer(
DiskSpaceManager diskSpaceManager,
IEtlInMemoryProducerConfigReaderFactory configReaderFactory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.diskSpaceManager = diskSpaceManager;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Initialization
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.consumerSinks = initParam.ConsumerSinks;
// Read settings
var configReader = configReaderFactory.CreateEtlInMemoryProducerConfigReader(this.traceSource, this.logSourceId);
this.etlInMemoryProducerSettings = configReader.GetSettings();
// ETL in-memory file processing is not enabled or we are not processing
// winfab etl files, so return immediately
if (false == this.etlInMemoryProducerSettings.Enabled || false == this.etlInMemoryProducerSettings.ProcessingWinFabEtlFiles)
{
return;
}
// Create a new worker object
var newWorkerParam = new EtlInMemoryProducerWorker.EtlInMemoryProducerWorkerParameters()
{
TraceSource = this.traceSource,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
EtlInMemoryProducer = this,
LatestSettings = this.etlInMemoryProducerSettings
};
var newWorker = new EtlInMemoryProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
this.producerWorker = newWorker;
}
internal EtlProcessor(
bool loadWinFabManifests,
bool dynamicWinFabManifestLoad,
string markerFileDirectory,
WinFabricEtlType winFabricEtlType,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
EtlInMemoryPerformance perfHelper,
IEnumerable <IEtlInMemorySink> sinks,
IEnumerable <EtlToInMemoryBufferWriter> etlToInMemoryBufferWriters,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.processingWinFabEvents = loadWinFabManifests;
this.markerFileDirectory = markerFileDirectory;
this.windowsFabricEtlType = winFabricEtlType;
this.sinks = sinks.ToList().AsReadOnly();
this.etlToInMemoryBufferWriters = etlToInMemoryBufferWriters.ToList().AsReadOnly();
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
this.perfHelper = perfHelper;
if (loadWinFabManifests)
{
if (dynamicWinFabManifestLoad)
{
// We are going to be loading manifests dynamically based on the version
// in the ETL file. Initialize the manifest manager object to manage this
// for us.
this.winFabManifestMgr = this.InitializeWinFabManifestManager();
}
else
{
// We'll load the default manifests now and just use them throughout
this.LoadDefaultManifests();
}
}
// Provide the sinks a reference to the manifest cache.
this.sinks.ForEach(sink => sink.SetEtwManifestCache(this.etwManifestCache));
}
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName))
.Returns("true");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName))
.Returns(logDirectory);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName))
.Returns(TableName);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName))
.Returns(MdsFileProducerConstants.DefaultTablePriority);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName))
.Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName))
.Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName))
.Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName))
.Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName))
.Returns(LogFilter);
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
// Create the MDS consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var consumerInitParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
var mdsFileProducer = new MdsFileProducer(consumerInitParam);
var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink();
var producerInitParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { etlToInMemoryBufferWriter }
};
// Create the in-memory producer
return(new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
producerInitParam));
}
internal EtlInMemoryProducerWorker(
EtlInMemoryProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.cancellationTokenSource = new CancellationTokenSource();
this.perfHelper = new EtlInMemoryPerformance(this.traceSource);
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));
// Figure out where the ETL files are located
this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
this.etlInMemoryProducerWorkerSettings.EtlPath,
this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.traceSource,
this.logSourceId);
this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToInMemoryBufferWriters,
this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
true,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.markerFileDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToInMemoryBufferWriters,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// Figure out how much processing time is available to each provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
// Disk manager set up for traces
foreach (var provider in this.providers)
{
var capturedProvider = provider;
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
// Disk manager set up for marker files
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestUtility.TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false);
var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true);
var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
mockTraceEventSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
fastDcaInMemoryConsumer);
var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
mockTraceEventSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
slowDcaInMemoryConsumer);
var initParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter }
};
return(new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
initParam));
}
internal EtlProducerWorker(
EtlProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
this.perfHelper = new EtlPerformance(this.traceSource);
this.cancellationTokenSource = new CancellationTokenSource();
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
initParam.EtlProducers,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
this.etlToCsvFileWriters =
EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
this.bufferedEtwEventProviders =
EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();
// Figure out where the ETL files are located
this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceDirectory,
initParam.ProducerInstanceId);
if (initParam.IsReadingFromApplicationManifest)
{
lock (InternalMarkerFileDirectoriesForApps)
{
InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
}
}
this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
this.etlProducerSettings.WindowsFabricEtlType,
this.etlProducerSettings.EtlFilePatterns,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
initParam.ProducerInstanceId,
this.traceSource,
this.logSourceId);
if (false == initParam.IsReadingFromApplicationManifest)
{
// Ensure that no other instance of EtlProducerWorker is processing the
// same ETL files
var patternsAdded = new List <string>();
var isUnique = VerifyEtlFilesUniqueToCurrentInstance(
this.traceDirectory,
this.providers,
patternsAdded,
message => this.traceSource.WriteError(this.logSourceId, message));
this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();
if (!isUnique)
{
throw new InvalidOperationException(
string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
}
}
this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
if (false == initParam.IsReadingFromApplicationManifest)
{
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToCsvFileWriters,
this.etlProducerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
}
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
false == initParam.IsReadingFromApplicationManifest,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.etlProducerSettings.CustomManifestPaths,
initParam.ApplicationType,
this.markerFileDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToCsvFileWriters,
this.bufferedEtwEventProviders,
this.etlProducerSettings.AppEtwGuids,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// If there is a huge backlog of ETL files to process, we limit the
// amount of time that we spend on processing ETL files in each
// pass. Figure out how much processing time is available to each
// provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
foreach (var provider in this.providers)
{
var capturedProvider = provider;
if (Directory.Exists(this.traceDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}
if (Directory.Exists(this.markerFileDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
out EtlInMemoryProducer etlInMemoryProducer,
out string containerName,
out StorageAccountFactory storageAccountFactory,
AccessCondition uploadStreamAccessCondition = null,
AccessCondition uploadFileAccessCondition = null)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName))
.Returns("true");
containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName))
.Returns(containerName);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName))
.Returns("0.25");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName))
.Returns("1");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName))
.Returns("*.*:4");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId))
.Returns(AzureConstants.DefaultDeploymentId);
var accountKey = GetTestStorageAccountKey();
var isEncrypted = false;
var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey);
configStore
.Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted))
.Returns(storageConnectionString);
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
// Create the Azure Blob Uploader consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var consumerInitParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
// Save storage connection for clean up
var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType);
storageAccountFactory = azureUtility.GetStorageAccountFactory(
new ConfigReader(consumerInitParam.ApplicationInstanceId),
TestConfigSectionName,
AzureConstants.ConnectionStringParamName);
var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition);
var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink();
var producerInitParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { etlToInMemoryBufferWriter }
};
// Create the in-memory producer
etlInMemoryProducer = new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
producerInitParam);
}
internal EtlProducer(
DiskSpaceManager diskSpaceManager,
IEtlProducerConfigReaderFactory configReaderFactory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
ITraceEventSourceFactory traceEventSourceFactory,
ProducerInitializationParameters initParam)
{
this.diskSpaceManager = diskSpaceManager;
this.traceFileEventReaderFactory = traceFileEventReaderFactory;
// Initialization
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA);
this.serviceConfigSections = new List <string>();
this.logDirectory = initParam.LogDirectory;
this.consumerSinks = initParam.ConsumerSinks;
// Read instance-specific settings from the settings file
var configReader = configReaderFactory.CreateEtlProducerConfigReader(this.traceSource, this.logSourceId);
this.etlProducerSettings = configReader.GetSettings();
if (false == this.etlProducerSettings.Enabled)
{
// ETL file processing is not enabled, so return immediately
return;
}
if (!this.etlProducerSettings.ProcessingWinFabEtlFiles)
{
// If we are collecting ETW events on behalf of an app, then we will
// need information from the <ETW> section of the service manifest.
this.serviceConfigSections.Add(ServiceConfig.EtwElement);
// Check if we can use an existing worker object
string applicationType = this.etlProducerSettings.ApplicationType;
lock (ProducerWorkers)
{
EtlProducerWorkerInfo workerInfo = ProducerWorkers.FirstOrDefault(w => w.ApplicationType.Equals(
applicationType,
StringComparison.Ordinal));
if (null != workerInfo)
{
// Existing worker object is available.
this.traceSource.WriteInfo(
this.logSourceId,
"Existing ETL producer worker object for application type {0} is available. Restarting the worker object now.",
applicationType);
// Restart the worker object
workerInfo.ProducerWorker.Dispose();
workerInfo.ProducerWorker = null;
List <EtlProducer> etlProducers = new List <EtlProducer>(workerInfo.EtlProducers)
{
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = applicationType,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = applicationType,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
workerInfo.EtlProducers.Add(this);
workerInfo.ProducerWorker = newWorker;
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to restart ETL producer worker object for application type {0}.",
applicationType);
}
}
else
{
// Create a new worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating ETL producer worker object for application type {0} ...",
applicationType);
List <EtlProducer> etlProducers = new List <EtlProducer> {
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = applicationType,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = applicationType,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
workerInfo = new EtlProducerWorkerInfo()
{
ApplicationType = applicationType,
EtlProducers = new List <EtlProducer>(),
ProducerWorker = newWorker
};
workerInfo.EtlProducers.Add(this);
ProducerWorkers.Add(workerInfo);
}
catch (InvalidOperationException)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create ETL producer worker object for application type {0}.",
applicationType);
}
}
}
}
else
{
// Create a new worker object
List <EtlProducer> etlProducers = new List <EtlProducer> {
this
};
EtlProducerWorker.EtlProducerWorkerParameters newWorkerParam = new EtlProducerWorker.EtlProducerWorkerParameters()
{
TraceSource = this.traceSource,
IsReadingFromApplicationManifest = !this.etlProducerSettings.ProcessingWinFabEtlFiles,
ApplicationType = string.Empty,
LogDirectory = initParam.LogDirectory,
ProducerInstanceId = this.logSourceId,
EtlProducers = etlProducers,
LatestSettings = this.etlProducerSettings
};
try
{
EtlProducerWorker newWorker = new EtlProducerWorker(
newWorkerParam,
this.diskSpaceManager,
this.traceFileEventReaderFactory);
this.producerWorker = newWorker;
}
catch (InvalidOperationException)
{
}
}
}
private static EtlProducer CreateEtlProducerForTest(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlProducerSettings = new EtlProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestUtility.TestEtlFilePatterns,
new List <string>(),
new Dictionary <string, List <ServiceEtwManifestInfo> >(),
true,
new[] { Guid.Parse(EtwProviderGuid) },
null);
var cr = TestUtility.MockRepository.Create <IEtlProducerConfigReader>();
cr.Setup(c => c.GetSettings()).Returns(etlProducerSettings);
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyOldDataDeletionIntervalInSeconds"))
.Returns("3600");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "ApplicationLogsFormatVersion"))
.Returns((string)null);
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyDtrDeletionAgeInMinutes"))
.Returns("60");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlProducerConfigReaderFactory>();
mockConfigReaderFactory
.Setup(f => f.CreateEtlProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(cr.Object);
var mockTraceSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var csvWriter = new EtlToCsvFileWriter(
mockTraceSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
mockDiskSpaceManager.Object);
var initParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { csvWriter }
};
return(new EtlProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceSourceFactory.Object,
initParam));
}
