internal CsvToUploadFolderWriter( string logSourceId, string fabricNodeId, string csvFolder, string sourceFolder, DiskSpaceManager diskSpaceManager, bool dtrCompressionDisabled, IEtlToCsvFileWriterConfigReader configReader) { this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.logSourceId = logSourceId; this.organizeWindowsFabricTracesByType = true; this.fabricNodeId = fabricNodeId; this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled; this.configReader = configReader; this.disposed = false; this.stopping = false; this.compressCsvFiles = FileCompressor.CompressionEnabled && (false == this.dtrCompressionDisabledByConsumer) && (false == this.configReader.IsDtrCompressionDisabledGlobally()); this.diskSpaceManager = diskSpaceManager; try { string currentAssemblyLocation = typeof(CsvToUploadFolderWriter).GetTypeInfo().Assembly.Location; string versionFile = Path.Combine(Path.GetDirectoryName(currentAssemblyLocation), ClusterVersionFile); this.fabricVersion = File.ReadAllText(versionFile); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Could not find the version of current service fabric code"); throw; } // Create the directory that containing filtered traces, in case it // doesn't already exist this.csvFolder = csvFolder; FabricDirectory.CreateDirectory(this.csvFolder); CreateWindowsFabricTraceSubFolders(); this.sourceFolder = sourceFolder; this.traceSource.WriteInfo( this.logSourceId, "Directory containing trace files: {0} Directory containing dtr traces: {1}", this.sourceFolder, this.csvFolder); diskSpaceManager.RegisterFolder( this.logSourceId, () => new DirectoryInfo(this.csvFolder).EnumerateFiles("*.dtr*", SearchOption.AllDirectories), null, f => f.LastWriteTimeUtc > (DateTime.UtcNow - configReader.GetDtrDeletionAge())); string timerIdMove = string.Concat( this.logSourceId, moveFilesTimerIdSuffix); this.csvMoveFilesTimer = new DcaTimer( timerIdMove, this.CsvMoveFilesHandler, 1 * 60 * 1000); this.csvMoveFilesTimer.Start(); }
internal EtlToCsvFileWriter( ITraceEventSourceFactory traceEventSourceFactory, string logSourceId, string fabricNodeId, string etwCsvFolder, bool dtrCompressionDisabled, DiskSpaceManager diskSpaceManager, IEtlToCsvFileWriterConfigReader configReader) : base(traceEventSourceFactory, logSourceId) { this.organizeWindowsFabricTracesByType = true; this.fabricNodeId = fabricNodeId; this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled; this.diskSpaceManager = diskSpaceManager; this.configReader = configReader; #if !DotNetCoreClr this.perfHelper = new EtlToCsvPerformance(this.TraceSource, this.LogSourceId); #endif // Create the directory that containing filtered traces, in case it // doesn't already exist this.filteredTraceDirName = etwCsvFolder; FabricDirectory.CreateDirectory(this.filteredTraceDirName); this.TraceSource.WriteInfo( this.LogSourceId, "Directory containing filtered ETW traces: {0}", this.filteredTraceDirName); // Create a timer to delete old logs // Figure out the retention time for the CSV files // Time after which the CSV file on disk becomes a candidate for deletion // Read this value from config every time. Do not cache it. That's how // we pick up the latest value when an update happens. // // From the above files, get the ones whose corresponding ETL files // have already been fully processed. We should delete only the files // whose corresponding ETL files have been fully processed. All other // files should be kept around because their file name gives us the // bookmark up to which we have processed events. var deletionAge = configReader.GetDtrDeletionAge(); diskSpaceManager.RegisterFolder( logSourceId, () => { // Get the filtered ETW trace files that are old enough to be deleted var dirInfo = new DirectoryInfo(this.filteredTraceDirName); return(dirInfo.EnumerateFiles(EtlConsumerConstants.FilteredEtwTraceSearchPattern, SearchOption.AllDirectories)); }, f => f.LastWriteTimeUtc < DateTime.UtcNow - deletionAge, // we don't have any indication whether work is done currently, use timer to estimate f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge, f => { DateTime lastEventTimeStamp; GetLastEventTimestamp(f.Name, out lastEventTimeStamp); this.lastDeletedDtrName = f.Name; this.lastDeletedDtrEventTime = lastEventTimeStamp; try { FabricFile.Delete(f.FullName); return(true); } catch (Exception) { return(false); } }); diskSpaceManager.RetentionPassCompleted += () => { this.TraceSource.WriteInfo( this.LogSourceId, "The last dtr file deleted during disk space manager pass {0} with events up til {1}.", this.lastDeletedDtrName, this.lastDeletedDtrEventTime); }; diskSpaceManager.RegisterFolder( logSourceId, () => new DirectoryInfo(this.filteredTraceDirName) .EnumerateFiles(EtlConsumerConstants.BootstrapTraceSearchPattern, SearchOption.AllDirectories), f => true, f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge); }