internal CsvToUploadFolderWriter(
string logSourceId,
string fabricNodeId,
string csvFolder,
string sourceFolder,
DiskSpaceManager diskSpaceManager,
bool dtrCompressionDisabled,
IEtlToCsvFileWriterConfigReader configReader)
{
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = logSourceId;
this.organizeWindowsFabricTracesByType = true;
this.fabricNodeId = fabricNodeId;
this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled;
this.configReader = configReader;
this.disposed = false;
this.stopping = false;
this.compressCsvFiles = FileCompressor.CompressionEnabled &&
(false == this.dtrCompressionDisabledByConsumer) &&
(false == this.configReader.IsDtrCompressionDisabledGlobally());
this.diskSpaceManager = diskSpaceManager;
try
{
string currentAssemblyLocation = typeof(CsvToUploadFolderWriter).GetTypeInfo().Assembly.Location;
string versionFile = Path.Combine(Path.GetDirectoryName(currentAssemblyLocation), ClusterVersionFile);
this.fabricVersion = File.ReadAllText(versionFile);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Could not find the version of current service fabric code");
throw;
}
// Create the directory that containing filtered traces, in case it
// doesn't already exist
this.csvFolder = csvFolder;
FabricDirectory.CreateDirectory(this.csvFolder);
CreateWindowsFabricTraceSubFolders();
this.sourceFolder = sourceFolder;
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing trace files: {0} Directory containing dtr traces: {1}",
this.sourceFolder,
this.csvFolder);
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.csvFolder).EnumerateFiles("*.dtr*", SearchOption.AllDirectories),
null,
f => f.LastWriteTimeUtc > (DateTime.UtcNow - configReader.GetDtrDeletionAge()));
string timerIdMove = string.Concat(
this.logSourceId,
moveFilesTimerIdSuffix);
this.csvMoveFilesTimer = new DcaTimer(
timerIdMove,
this.CsvMoveFilesHandler,
1 * 60 * 1000);
this.csvMoveFilesTimer.Start();
}
internal EtlToCsvFileWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwCsvFolder,
bool dtrCompressionDisabled,
DiskSpaceManager diskSpaceManager,
IEtlToCsvFileWriterConfigReader configReader)
: base(traceEventSourceFactory, logSourceId)
{
this.organizeWindowsFabricTracesByType = true;
this.fabricNodeId = fabricNodeId;
this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled;
this.diskSpaceManager = diskSpaceManager;
this.configReader = configReader;
#if !DotNetCoreClr
this.perfHelper = new EtlToCsvPerformance(this.TraceSource, this.LogSourceId);
#endif
// Create the directory that containing filtered traces, in case it
// doesn't already exist
this.filteredTraceDirName = etwCsvFolder;
FabricDirectory.CreateDirectory(this.filteredTraceDirName);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing filtered ETW traces: {0}",
this.filteredTraceDirName);
// Create a timer to delete old logs
// Figure out the retention time for the CSV files
// Time after which the CSV file on disk becomes a candidate for deletion
// Read this value from config every time. Do not cache it. That's how
// we pick up the latest value when an update happens.
//
// From the above files, get the ones whose corresponding ETL files
// have already been fully processed. We should delete only the files
// whose corresponding ETL files have been fully processed. All other
// files should be kept around because their file name gives us the
// bookmark up to which we have processed events.
var deletionAge = configReader.GetDtrDeletionAge();
diskSpaceManager.RegisterFolder(
logSourceId,
() =>
{
// Get the filtered ETW trace files that are old enough to be deleted
var dirInfo = new DirectoryInfo(this.filteredTraceDirName);
return(dirInfo.EnumerateFiles(EtlConsumerConstants.FilteredEtwTraceSearchPattern, SearchOption.AllDirectories));
},
f => f.LastWriteTimeUtc < DateTime.UtcNow - deletionAge, // we don't have any indication whether work is done currently, use timer to estimate
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge,
f =>
{
DateTime lastEventTimeStamp;
GetLastEventTimestamp(f.Name, out lastEventTimeStamp);
this.lastDeletedDtrName = f.Name;
this.lastDeletedDtrEventTime = lastEventTimeStamp;
try
{
FabricFile.Delete(f.FullName);
return(true);
}
catch (Exception)
{
return(false);
}
});
diskSpaceManager.RetentionPassCompleted += () =>
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The last dtr file deleted during disk space manager pass {0} with events up til {1}.",
this.lastDeletedDtrName,
this.lastDeletedDtrEventTime);
};
diskSpaceManager.RegisterFolder(
logSourceId,
() => new DirectoryInfo(this.filteredTraceDirName)
.EnumerateFiles(EtlConsumerConstants.BootstrapTraceSearchPattern, SearchOption.AllDirectories),
f => true,
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge);
}
