public bool OnBootstrapTraceFileAvailable(string fileName)
{
// Copy the bootstrap trace file to the ETW CSV directory
//
// Add the fabric node ID as a prefix, so that we can later associate
// the traces with the Fabric node ID.
string bootstrapTraceFileNameWithoutPath = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}",
this.fabricNodeId,
Path.GetFileName(fileName));
string bootstrapTraceDestinationFileName = Path.Combine(
this.filteredTraceDirName,
this.organizeWindowsFabricTracesByType ? BootstrapTracesFolderName : string.Empty,
bootstrapTraceFileNameWithoutPath);
try
{
InternalFileSink.CopyFile(fileName, bootstrapTraceDestinationFileName, true, false);
// logging bytes read and written
var fileInfo = new FileInfo(fileName);
#if !DotNetCoreClr
this.perfHelper.BytesRead(fileInfo.Length);
this.perfHelper.BytesWritten(fileInfo.Length);
#endif
this.TraceSource.WriteNoise(
this.LogSourceId,
"Copied bootstrap trace file {0} to the directory {1}.",
fileName,
this.filteredTraceDirName);
return(true);
}
catch (Exception e)
{
// Failed to copy bootstrap trace file. Just skip this file and continue.
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to copy file. Source: {0}, destination: {1}.",
fileName,
bootstrapTraceDestinationFileName);
return(false);
}
}
internal void OnEtlFileReadStart(string fileName, bool isActiveEtl, string currentBookmark)
{
// Reset the last event index
this.lastEventIndex.Set(DateTime.MinValue, -1);
// Get the index up to which we have already processed events from
// this file.
this.maxIndexAlreadyProcessed = this.GetMaxIndexAlreadyProcessed(
fileName);
// Create a new file sink.
this.fileSink = new InternalFileSink(
this.TraceSource,
this.LogSourceId);
this.fileSink.Initialize();
this.TraceSource.WriteInfo(
this.LogSourceId,
"Filtered traces from {0} will be written to {1}.",
fileName,
this.fileSink.TempFileName);
}
private EventIndex GetMaxIndexAlreadyProcessed(string etlFileName)
{
// From the ETW CSV directory, retrieve the names of all CSV files
// that we generated from this ETL file
string traceFilePattern = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}_*.{2}*",
this.fabricNodeId,
Path.GetFileNameWithoutExtension(etlFileName),
EtlConsumerConstants.FilteredEtwTraceFileExtension);
string subFolder = this.GetTraceFileSubFolder(etlFileName);
string folderToSearch = Path.Combine(this.filteredTraceDirName, subFolder);
EventIndex index = new EventIndex();
string[] traceFiles;
try
{
InternalFileSink.GetFilesMatchingPattern(
folderToSearch,
traceFilePattern,
out traceFiles);
}
catch (Exception e)
{
this.TraceSource.WriteError(
this.LogSourceId,
"Failed to retrieve files matching pattern {0} in directory {1}. This may cause events to be duplicated in multiple files. {2}",
traceFilePattern,
this.filteredTraceDirName,
e);
// We'll assume that no events from this file have been processed yet.
// This may end up being an incorrect assumption, in which case some
// events can end up duplicated in multiple files.
index.Set(DateTime.MinValue, -1);
return(index);
}
if (traceFiles.Length == 0)
{
// No events from this file have been processed yet
index.Set(DateTime.MinValue, -1);
return(index);
}
// Sort the files based on file name and get the name of the newest
// file
Array.Sort(traceFiles);
string newestFile = traceFiles[traceFiles.Length - 1];
// From the name of the newest file, extract the maximum index already
// processed
string newestFileWithoutExtension = Path.GetFileNameWithoutExtension(newestFile);
if (DtrRegEx.IsMatch(newestFileWithoutExtension))
{
newestFileWithoutExtension = Path.GetFileNameWithoutExtension(newestFileWithoutExtension);
}
string[] fileNameParts = newestFileWithoutExtension.Split('_');
Debug.Assert(fileNameParts.Length >= 4, "All file name schemas have at least 4 '_' seperated parts.");
long timestampTicks = long.Parse(
fileNameParts[fileNameParts.Length - 2],
CultureInfo.InvariantCulture);
DateTime timestamp = new DateTime(timestampTicks);
int timestampDifferentiator = int.Parse(
fileNameParts[fileNameParts.Length - 1],
CultureInfo.InvariantCulture);
index.Set(timestamp, timestampDifferentiator);
return(index);
}
private void CopyTraceFileForUpload(string fileName, bool isActiveEtl)
{
// If the temporary trace file does not contain events and if the ETL
// file that is being processed is an active ETL file, then don't
// copy the temporary file to the ETW CSV directory.
//
// In contrast, temporary trace files corresponding to inactive ETL
// files are copied even if they don't have events in them (i.e. a
// zero-byte file is copied) because it makes it easier to identify
// gaps in traces if the DCA happens to fall behind on trace processing.
// For details, please see comments later in this function.
if (0 == this.fileSink.WriteStatistics.EventsWritten && isActiveEtl)
{
return;
}
// Build the destination name for the filtered trace file
//
// !!! WARNING !!!
// The trace viewer tool parses the file names of the filtered trace files.
// Changing the file name format might require a change to trace viewer as well.
// !!! WARNING !!!
//
// If the ETL file is an active ETL file, the trace file name is of the form:
// <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<TimestampDifferentiatorOfLastEventProcessed>.dtr
// For containers the file structure is:
// <FabricNodeID>_<ContainerName>_<etlFileName>_<TimestampOfLastEventProcessed>_<TimestampDifferentiatorOfLastEventProcessed>.dtr
//
// If the ETL file is an inactive ETL file, the trace file name is of the form:
// <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<Int32.MaxValue>.dtr
// For containers the file structure is:
// <FabricNodeID>_<ContainerName>_<etlFileName>_<TimestampOfLastEventProcessed>_<Int32.MaxValue>.dtr
//
// Using Int32.MaxValue as a component of the trace file name makes
// it easy to identify gaps in the filtered traces if the DCA is
// falling behind on trace processing. Recall that an inactive ETL
// file is always processed fully. Only active ETL files are processed
// in chunks. Therefore, the presence of Int32.MaxValue indicates that
// the corresponding ETL file is inactive and has been fully processed
// by the DCA. Thus, gaps **within** an ETL file (i.e. unprocessed
// chunks within the file) can be identified by the absence of a file
// containing Int32.MaxValue in its name.
//
// It is also worth noting that ETL file names are sequentially
// numbered, which helps in identifying gaps **between** ETL files
// (i.e. ETL files that were not processed at all). And the use of
// Int32.MaxValue is an enhancement that enables us to identify gaps
// within an ETL file. Using these two concepts, we can look at a set
// of filtered trace files and determine whether they are complete.
// And if not complete, we can also identify where all the gaps are.
string differentiator = string.Format(
CultureInfo.InvariantCulture,
"{0:D10}",
isActiveEtl ? this.lastEventIndex.TimestampDifferentiator : int.MaxValue);
string traceFileNamePrefix = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}_{2:D20}_{3}.",
this.fabricNodeId,
Path.GetFileNameWithoutExtension(fileName),
this.lastEventIndex.Timestamp.Ticks,
differentiator);
var applicationInstanceId = ContainerEnvironment.GetContainerApplicationInstanceId(this.LogSourceId);
if (ContainerEnvironment.IsContainerApplication(applicationInstanceId))
{
// Note that the a hash of the applicationInstanceId is being used to reduce file name length in around 70 characters
// This is done to workaround PathTooLong exception in FileUploaderBase.cs since we don't have an interop for FileSystemWatcher
// and .NET 4.5 used does not support long paths yet.
traceFileNamePrefix = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1:X8}_{2}_{3:D20}_{4}.",
this.fabricNodeId,
Path.GetFileName(applicationInstanceId).GetHashCode(),
Path.GetFileNameWithoutExtension(fileName),
this.lastEventIndex.Timestamp.Ticks,
differentiator);
}
string traceFileNameWithoutPath = string.Concat(
traceFileNamePrefix,
EtlConsumerConstants.FilteredEtwTraceFileExtension);
string compressedTraceFileNameWithoutPath = string.Concat(
traceFileNamePrefix,
EtlConsumerConstants.FilteredEtwTraceFileExtension,
EtlConsumerConstants.CompressedFilteredEtwTraceFileExtension);
string subFolder = this.GetTraceFileSubFolder(fileName);
string traceFileDestinationPath = Path.Combine(
this.filteredTraceDirName,
subFolder);
string traceFileDestinationName = Path.Combine(
traceFileDestinationPath,
this.compressCsvFiles ? compressedTraceFileNameWithoutPath : traceFileNameWithoutPath);
string alternateTraceFileDestinationName = Path.Combine(
traceFileDestinationPath,
this.compressCsvFiles ? traceFileNameWithoutPath : compressedTraceFileNameWithoutPath);
// If a file with the same name already exists at the destination,
// then don't copy the file over. This is because if the file already
// existed at the destination, then the file that are about to copy
// over must be a zero-byte file because we always ignore events that
// we have already processed. Therefore, we don't want to overwrite
// a file that contains events with a zero-byte file.
if (InternalFileSink.FileExists(traceFileDestinationName) ||
InternalFileSink.FileExists(alternateTraceFileDestinationName))
{
Debug.Assert(0 == this.fileSink.WriteStatistics.EventsWritten, "The temporary trace file must be a zero-byte file.");
// Also, the ETL file must be an inactive ETL file because if it
// had been an active ETL file and the temporary trace file was
// empty, then we would have already returned from this method
// due to a check made at the beginning of this method.
Debug.Assert(false == isActiveEtl, "File must be inactive.");
return;
}
// Copy the file
try
{
InternalFileSink.CopyFile(this.fileSink.TempFileName, traceFileDestinationName, false, this.compressCsvFiles);
// logging bytes read and written
var fileInfo = new FileInfo(this.fileSink.TempFileName);
#if !DotNetCoreClr
this.perfHelper.BytesRead(fileInfo.Length);
this.perfHelper.BytesWritten(fileInfo.Length);
#endif
this.TraceSource.WriteInfo(
this.LogSourceId,
"Filtered traces from {0} are ready. They have been moved from {1} to {2}.",
fileName,
this.fileSink.TempFileName,
traceFileDestinationName);
}
catch (Exception e)
{
// Log an error and move on
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to copy file. Source: {0}, destination: {1}.",
this.fileSink.TempFileName,
traceFileDestinationName);
}
}
