internal EtwCsvUploadWorker(EtwCsvUploadWorkerParameters param, FabricEvents.ExtensionsEvents traceSource)
{
// Initialization
Action onError = () =>
{
if (null != this.etlToCsvWriter)
{
this.etlToCsvWriter.Dispose();
}
};
this.logSourceId = param.UploaderInstanceId;
this.traceSource = traceSource;
this.fileUploadSettings = param.Settings;
this.etwCsvFolder = string.Empty;
this.initParam = param;
if (false == this.fileUploadSettings.DestinationIsLocalAppFolder)
{
// Append fabric node instance name to destination path, so that
// each node uploads to a different subfolder.
this.destinationPathForNode = Path.Combine(
this.fileUploadSettings.DestinationPath,
param.FabricNodeInstanceName);
}
else
{
this.destinationPathForNode = this.fileUploadSettings.DestinationPath;
}
// Create a sub-directory for ourselves under the log directory
this.etwCsvFolder = this.GetEtwCsvSubDirectory();
if (!string.IsNullOrWhiteSpace(this.etwCsvFolder))
{
// Create the helper object that writes events delivered from the ETL
// files into CSV files.
this.etlToCsvWriter = new EtlToCsvFileWriter(
new TraceEventSourceFactory(),
this.logSourceId,
param.FabricNodeId,
this.etwCsvFolder,
false,
param.DiskSpaceManager,
new EtlToCsvFileWriterConfigReader());
}
// Set the event filter
this.etlToCsvWriter.SetEtwEventFilter(
this.fileUploadSettings.Filter,
param.IsReadingFromApplicationManifest ? string.Empty : WinFabDefaultFilter.StringRepresentation,
param.IsReadingFromApplicationManifest ? string.Empty : WinFabSummaryFilter.StringRepresentation,
!param.IsReadingFromApplicationManifest);
// Create a sub-directory for the uploader under the log directory
if (!this.GetUploaderWorkSubDirectory())
{
onError();
throw new InvalidOperationException();
}
try
{
this.uploader = new FileShareUploader(
this.traceSource,
this.logSourceId,
param.IsReadingFromApplicationManifest,
this.etwCsvFolder,
this.destinationPathForNode,
this.fileUploadSettings.AccessInfo,
this.workFolder,
this.fileUploadSettings.UploadInterval,
this.fileUploadSettings.FileSyncInterval,
this.fileUploadSettings.FileDeletionAgeMinutes,
this.fileUploadSettings.DestinationIsLocalAppFolder,
param.FabricNodeId);
this.uploader.Start();
}
catch (Exception)
{
onError();
throw;
}
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to file share is configured. Destination: {0}, Local trace Path: {1}.",
this.fileUploadSettings.DestinationPath,
this.etwCsvFolder);
}
public EtwCsvUploadWorker(EtwCsvUploadWorkerParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = this.initParam.UploaderInstanceId;
this.traceSource = this.initParam.TraceSource;
this.blobUploadSettings = this.initParam.Settings;
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwCsvUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// Create a sub-directory for ourselves under the log directory
bool success = this.GetEtwCsvSubDirectory();
// Create the helper object that writes events delivered from the ETL
// files into CSV files.
if (success)
{
this.etlToCsvWriter = new EtlToCsvFileWriter(
new TraceEventSourceFactory(),
this.logSourceId,
this.initParam.FabricNodeId,
this.etwCsvFolder,
false,
initParam.DiskSpaceManager);
// Set the event filter
this.etlToCsvWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
this.initParam.IsReadingFromApplicationManifest ? string.Empty : WinFabDefaultFilter.StringRepresentation,
this.initParam.IsReadingFromApplicationManifest ? string.Empty : WinFabSummaryFilter.StringRepresentation,
!this.initParam.IsReadingFromApplicationManifest);
// Create a sub-directory for the uploader under the log directory
success = this.GetUploaderWorkSubDirectory();
}
if (success)
{
// Create and initialize the uploader
//
// NOTE: By specifying 'true' for the 'filterDeletionByNodeId' parameter,
// we only delete those blobs that were uploaded by the current node. We
// identify this via the Fabric node ID that the ETL-to-CSV writer prefixed
// to the file name before uploading. This is done so that all nodes don't
// wastefully try to delete all blobs.
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.EtwTraceContainerName);
this.uploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
this.etwCsvFolder,
destinationPath,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
this.uploader.Start();
}
catch (Exception e)
{
throw new InvalidOperationException("AzureFileUploader could not be constructed.", e);
}
}
if (success)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace Path: {2}, Upload interval (minutes): {3}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwCsvFolder,
this.blobUploadSettings.UploadInterval.TotalMinutes);
}
else
{
if (null != this.etlToCsvWriter)
{
this.etlToCsvWriter.Dispose();
this.etlToCsvWriter = null;
}
}
if (!success)
{
throw new InvalidOperationException();
}
}
public void TestCsvNotDeletedOnDiskFull()
{
var mockTraceSourceFactory = new Mock <ITraceEventSourceFactory>(MockBehavior.Strict);
mockTraceSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var mockConfigReader = new Mock <IEtlToCsvFileWriterConfigReader>(MockBehavior.Strict);
mockConfigReader
.Setup(cr => cr.IsDtrCompressionDisabledGlobally())
.Returns(false);
mockConfigReader
.Setup(cr => cr.GetDtrDeletionAge())
.Returns(TimeSpan.FromMinutes(1));
var mockEtlProducer = new Mock <IEtlProducer>(MockBehavior.Strict);
mockEtlProducer
.Setup(ep => ep.IsProcessingWindowsFabricEtlFiles())
.Returns(true);
mockEtlProducer
.Setup(ep => ep.HasEtlFileBeenFullyProcessed(EtlToCsvFileWriter.GetEtlFileNameFromTraceFileName(this.testCsvFileNames[0])))
.Returns(true);
mockEtlProducer
.Setup(ep => ep.HasEtlFileBeenFullyProcessed(EtlToCsvFileWriter.GetEtlFileNameFromTraceFileName(this.testCsvFileNames[1])))
.Returns(true);
mockEtlProducer
.Setup(ep => ep.HasEtlFileBeenFullyProcessed(EtlToCsvFileWriter.GetEtlFileNameFromTraceFileName(this.testCsvFileNames[2])))
.Returns(false);
var diskSpaceManager = new DiskSpaceManager(() => 1 << 30, () => 0, () => 80, TimeSpan.FromMilliseconds(5));
var passCompletedEvent = new ManualResetEvent(false);
var fileDeletedEvent = new ManualResetEvent(false);
var testSetupEvent = new ManualResetEvent(false);
diskSpaceManager.GetAvailableSpace = d =>
{
testSetupEvent.WaitOne();
passCompletedEvent.Reset();
fileDeletedEvent.Set();
return(1 << 30); // 1GB
};
diskSpaceManager.RetentionPassCompleted += () => { passCompletedEvent.Set(); };
var writer = new EtlToCsvFileWriter(
mockTraceSourceFactory.Object,
TestLogSourceId,
TestNodeId,
TestCsvFolder,
false,
diskSpaceManager,
mockConfigReader.Object);
writer.SetEtlProducer(mockEtlProducer.Object);
File.WriteAllText(Path.Combine(TestCsvFolder, this.testCsvFileNames[0]), TestContents);
File.SetLastWriteTimeUtc(Path.Combine(TestCsvFolder, this.testCsvFileNames[0]), DateTime.UtcNow - TimeSpan.FromHours(2));
File.WriteAllText(Path.Combine(TestCsvFolder, this.testCsvFileNames[1]), TestContents);
File.WriteAllText(Path.Combine(TestCsvFolder, this.testCsvFileNames[2]), TestContents);
testSetupEvent.Set();
Assert.IsTrue(fileDeletedEvent.WaitOne(TestTimeout), "File delete should happen within timeout.");
Assert.IsTrue(passCompletedEvent.WaitOne(TestTimeout), "A retention pass should happen within timeout.");
// Wait for a second completion to ensure both local and global policies get applied.
passCompletedEvent.Reset();
Assert.IsTrue(passCompletedEvent.WaitOne(TestTimeout), "A retention pass should happen within timeout.");
Assert.IsFalse(File.Exists(Path.Combine(TestCsvFolder, this.testCsvFileNames[0])), "File should be deleted by local policy.");
Assert.IsTrue(File.Exists(Path.Combine(TestCsvFolder, this.testCsvFileNames[1])), "File is not old enough to be deleted.");
Assert.IsTrue(File.Exists(Path.Combine(TestCsvFolder, this.testCsvFileNames[2])), "File is not old enough to be deleted.");
}
private static EtlProducer CreateEtlProducerForTest(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlProducerSettings = new EtlProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestUtility.TestEtlFilePatterns,
new List <string>(),
new Dictionary <string, List <ServiceEtwManifestInfo> >(),
true,
new[] { Guid.Parse(EtwProviderGuid) },
null);
var cr = TestUtility.MockRepository.Create <IEtlProducerConfigReader>();
cr.Setup(c => c.GetSettings()).Returns(etlProducerSettings);
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyOldDataDeletionIntervalInSeconds"))
.Returns("3600");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "ApplicationLogsFormatVersion"))
.Returns((string)null);
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "TestOnlyDtrDeletionAgeInMinutes"))
.Returns("60");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlProducerConfigReaderFactory>();
mockConfigReaderFactory
.Setup(f => f.CreateEtlProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(cr.Object);
var mockTraceSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var csvWriter = new EtlToCsvFileWriter(
mockTraceSourceFactory.Object,
LogSourceId,
TestNodeId,
TestUtility.GetActualOutputFolderPath(logDirectory),
true,
mockDiskSpaceManager.Object);
var initParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { csvWriter }
};
return(new EtlProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceSourceFactory.Object,
initParam));
}