private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestUtility.TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false);
var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true);
var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
mockTraceEventSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
fastDcaInMemoryConsumer);
var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
mockTraceEventSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
slowDcaInMemoryConsumer);
var initParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter }
};
return(new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
initParam));
}
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam)
{
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
this.configReader = new AzureBlobEtwConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Read blob-specific settings
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
return;
}
// Create the destination key
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
this.destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create a sub-directory for the blob uploader
this.workFolder = this.CreateBlobUploaderWorkSubDirectory();
if (string.IsNullOrEmpty(this.workFolder))
{
const string Message = "Failed to create work folder for the blob uploader.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from ETL files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
initParam.FabricNodeId,
this.etwLogDirName,
true,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
// Create the helper object that syncs local files to blob storage.
// Local files will be created when upload of compressed memory stream to blob storage fails.
this.fileBlobUploader = new AzureBlobUploader(
this.traceSource,
this.logSourceId,
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
this.fileUploadPerfHelper,
null,
this.uploadFileAccessCondition);
// Create a timer to schedule the upload of local files to blob storage
string timerId = string.Concat(
this.logSourceId,
FileUploadTimerIdSuffix);
this.fileUploadTimer = new DcaTimer(
timerId,
this.UploadFilesToDestinationBlob,
this.blobUploadSettings.FileSyncInterval);
this.fileUploadTimer.Start();
// Initialize trimmer
this.trimmer = new AzureFileTrimmer(
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
AzureUtility.IsAzureInterfaceAvailable());
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwLogDirName);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for Azure blob uploader: {0}",
this.blobUploadSettings.Filter);
}
public MdsFileProducer(ConsumerInitializationParameters initParam)
{
// Initialization
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new ConfigReader(initParam.ApplicationInstanceId);
this.tableWriters = new Dictionary <string, MdsTableWriter>();
this.mdsFileProducerPerf = new MdsFileProducerPerformance();
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
MdsFileProducerConstants.MethodExecutionInitialRetryIntervalMs,
MdsFileProducerConstants.MethodExecutionMaxRetryCount,
MdsFileProducerConstants.MethodExecutionMaxRetryIntervalMs);
// Read MDS-specific settings from settings.xml
this.GetSettings();
if (false == this.mdsFileProducerSettings.Enabled)
{
// MDS tables creation is not enabled, so return immediately
return;
}
string destinationKey = Path.Combine(
this.mdsFileProducerSettings.DirectoryName,
this.mdsFileProducerSettings.TableName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from the ETL
// files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
this.initParam.FabricNodeId,
this.etwLogDirName,
false,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.mdsFileProducerSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
this.traceSource.WriteInfo(
this.logSourceId,
"MDS file producer is configured. Table path: {0}, Table name: {1}, Table priority: {2}, Disk quota (MB): {3}, Data deletion age (minutes): {4}",
this.mdsFileProducerSettings.DirectoryName,
this.mdsFileProducerSettings.TableName,
this.mdsFileProducerSettings.TablePriority,
this.mdsFileProducerSettings.DiskQuotaInMB,
this.mdsFileProducerSettings.DataDeletionAge);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for MDS file producer: {0}",
this.mdsFileProducerSettings.Filter);
}