internal LttProducerWorker(LttProducerWorkerParameters initParam)
{
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(initParam.LatestSettings.ApplicationInstanceId, "_", initParam.LatestSettings.SectionName);
this.traceSource.WriteInfo(
this.logSourceId,
"LttProducer: ApplicationInstanceId:{0} SectionName:{1} LogDirectory:{2} WorkDirectory:{0}",
initParam.LatestSettings.ApplicationInstanceId,
initParam.LatestSettings.SectionName,
initParam.LogDirectory,
initParam.LatestSettings.WorkDirectory);
this.applicationInstanceId = initParam.LatestSettings.ApplicationInstanceId;
this.lttngTracesDirectory = Path.Combine(initParam.LogDirectory, LttProducerConstants.LttSubDirectoryUnderLogDirectory);
this.dtrTracesDirectory = Path.Combine(initParam.LogDirectory, LttProducerConstants.DtrSubDirectoryUnderLogDirectory);
this.LttReadIntervalMinutes = initParam.LttReadIntervalMinutes;
// splitting sinks into their two different supported types
var sinks = InitializeSinks(initParam.LttProducers,
new List <string>()
{
this.dtrTracesDirectory
},
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
this.sinksIEtlFile = (sinks.Where(s => s is IEtlFileSink)).Select(s => s as IEtlFileSink).ToList().AsReadOnly();
this.sinksICsvFile = (sinks.Where(s => s is ICsvFileSink)).Select(s => s as ICsvFileSink).ToList().AsReadOnly();
this.traceSource.WriteInfo(
this.logSourceId,
"Total number of sinks: {0}, IEtlFileSinks: {1}, ICsvFileSinks: {2}",
sinks.Count,
this.sinksIEtlFile.Count,
this.sinksICsvFile.Count);
this.lastReadTimestampUnixEpochNanoSec = this.InitializeLastReadTimestamp();
this.lttngTraceProcessor = new LttTraceProcessor(
this.traceSource,
this.logSourceId,
this.dtrTracesDirectory,
this.sinksIEtlFile,
this.sinksICsvFile);
// getting active service fabric lttng trace session
this.activeLttTraceSessionOutputPath = this.GetServiceFabricSessionOutputPath();
// try to process existing traces left from previous session
this.ProcessPreviousLttSessionTraces(this.activeLttTraceSessionOutputPath);
string timerId = string.Concat(initParam.LatestSettings.ApplicationInstanceId,
LttProducerConstants.LttTimerPrefix);
this.LttReadTimer = new DcaTimer(
timerId,
this.LttReadCallback,
this.LttReadIntervalMinutes * 60 * 1000);
this.LttReadTimer.Start();
}
internal BufferedEtwEventProvider(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string traceBufferingFolder,
TimeSpan uploadInterval,
TimeSpan eventDeletionAgeIn,
IBufferedEtwEventSink eventSink)
: base(traceEventSourceFactory, logSourceId)
{
this.eventSink = eventSink;
this.bookmark = null;
this.perfHelper = new BufferedEtwEventPerformance(this.TraceSource, this.LogSourceId);
this.eventDeletionAge = eventDeletionAgeIn;
// Compute how much time we spend delivering buffered ETW events to
// the consumer in each pass. We define this to be roughly 1/4th of
// the interval between passes. This is an arbitrary choice.
this.eventDeliveryPassLength = TimeSpan.FromMilliseconds(uploadInterval.TotalMilliseconds / 4);
// Create the buffered event directory, in case it doesn't already exist.
this.etwEventCache = traceBufferingFolder;
FabricDirectory.CreateDirectory(this.etwEventCache);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing buffered events: {0}",
this.etwEventCache);
// Create a timer to schedule the delivery of ETW events to the consumer
string timerId = string.Concat(
this.LogSourceId,
EventDeliveryTimerIdSuffix);
this.eventDeliveryTimer = new DcaTimer(
timerId,
this.DeliverEventsToConsumer,
uploadInterval);
this.eventDeliveryTimer.Start();
// Create a timer to delete old logs
timerId = string.Concat(
this.LogSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval =
(this.eventDeletionAge < TimeSpan.FromDays(1))
? EtlConsumerConstants.OldLogDeletionIntervalForTest
: EtlConsumerConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
internal AzureFileTrimmer(
string sourceFolderName,
string workFolder,
StorageAccountFactory storageAccountFactory,
string containerName,
TimeSpan blobDeletionAge,
string fabricNodeInstanceName,
string deploymentId,
bool azureInterfaceAvailable)
{
// Initialization
this.stopping = false;
this.sourceFolderName = sourceFolderName;
this.localMap = Path.Combine(workFolder, AzureBlobUploader.LocalMapFolder);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(TraceType, "-", Guid.NewGuid().ToString("P"));
this.storageAccountFactory = storageAccountFactory;
this.containerName = containerName;
this.fabricNodeInstanceName = fabricNodeInstanceName;
this.blobDeletionAge = blobDeletionAge;
this.deploymentId = deploymentId;
this.azureInterfaceAvailable = azureInterfaceAvailable;
this.localMapTrimmingHelper = new LocalMapTrimmingHelper(sourceFolderName);
this.folderTrimmer = new FolderTrimmer(
this.traceSource,
this.logSourceId,
blobDeletionAge);
this.perfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Deletion and query are currently synchronous, so the concurrency
// count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobQuery,
1);
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobDeletion,
1);
// Create a timer to delete old logs
string timerId = string.Concat(
this.logSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval =
(this.blobDeletionAge < TimeSpan.FromDays(1))
? AzureConstants.OldLogDeletionIntervalForTest
: AzureConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
internal ContainerManager(AppInstanceManager appInstanceManager, ContainerEnvironment containerEnvironment)
{
this.appInstanceManager = appInstanceManager;
this.containerEnvironment = containerEnvironment;
Utility.TraceSource.WriteInfo(
TraceType,
"Watching directory for creation of new container trace folders {0}",
this.containerEnvironment.ContainerLogRootDirectory);
// Watcher needs to be created before startup listing of folders
// Otherwise there is the risk a file will be created between the two and it will be missed.
// WriteLock is created so prevent race condition with file watchers
this.processingMarkersRWLock.EnterWriteLock();
try
{
if (Directory.Exists(this.containerEnvironment.ContainerLogRootDirectory))
{
this.CreateContainerFolderWacher();
foreach (var directory in Directory.EnumerateDirectories(this.containerEnvironment.ContainerLogRootDirectory))
{
this.ContainerLogFolderProcessingStartup(directory);
}
}
else
{
Utility.TraceSource.WriteError(
TraceType,
"Not monitoring directory for creation of new container trace folders since directory doesn't exist {0}",
this.containerEnvironment.ContainerLogRootDirectory);
}
}
finally
{
this.processingMarkersRWLock.ExitWriteLock();
}
// Arm the cleanup timer.
this.cleanupTimer = new DcaTimer(
ContainerManager.TraceType,
new TimerCallback(this.BackgroundCleanup),
(long)this.containerEnvironment.ContainerFolderCleanupTimerInterval.TotalMilliseconds);
this.cleanupTimer.Start((long)this.containerEnvironment.ContainerFolderCleanupTimerInterval.TotalMilliseconds);
}
internal ConfigUpdateHandler()
{
this.ignoreConfigurationUpdates = true;
this.currentConfigChangeReactionTime = TimeSpan.FromDays(1);
this.windowsFabricApplicationCreated = new ManualResetEvent(false);
this.changedSections = new HashSet <string>();
this.changeDeliveryLock = new object();
// Create the timer that will handle the configuration change after the
// reaction time has elapsed. Using a reaction time helps in collapsing
// multiple changes that might occur close together in time. This avoids
// repeated updating of the data collector in response to each of those
// changes.
this.changeDeliveryTimer = new DcaTimer(
TimerId,
this.DeliverChangedSections,
DefaultConfigChangeReactionTime);
}
public BootstrapTraceProcessor(
string traceDirectory,
string markerFileDirectory,
IEnumerable <ITraceOutputWriter> traceOutputWriters,
TimeSpan scanInterval,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId)
{
this.traceDirectory = traceDirectory;
this.markerFileDirectory = markerFileDirectory;
this.traceOutputWriters = traceOutputWriters.ToList().AsReadOnly();
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.timer = new DcaTimer(
string.Format("{0}_bootstrap", logSourceId),
this.DoScan,
scanInterval);
}
internal MdsFileProducerPerformance()
{
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.perfCountersList = new List <PerformanceCounter>();
bool success = this.SetupPerformanceCounters();
if (success)
{
// Timer for tracing performance counters.
string timerId = string.Concat(
TraceTypePerformance,
MdsCountersReadTimerIdSuffix);
this.mdsCountersReadTimer = new DcaTimer(
timerId,
this.ReadCounters,
TimeSpan.FromSeconds(ReadCountersTimerInterval));
}
}
internal EtlToCsvPerformance(FabricEvents.ExtensionsEvents traceSource, string logSourceId)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.perfCountersList = new List <PerformanceCounter>();
bool success = this.SetupPerformanceCounters();
if (success)
{
// Timer for tracing performance counters.
string timerId = string.Concat(
this.logSourceId,
EtlToCsvCountersReadTimerIdSuffix);
this.etlToCsvCountersReadTimer = new DcaTimer(
timerId,
this.ReadCounters,
TimeSpan.FromSeconds(ReadCountersTimerInterval));
}
}
internal AzureTableTrimmer(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
StorageAccountFactory storageAccountFactory,
string tableName,
TimeSpan entityDeletionAge,
QueryCreationMethod queryCreationMethod,
AzureTablePerformance perfHelper)
{
// Initialization
this.stopping = false;
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.storageAccountFactory = storageAccountFactory;
this.tableName = tableName;
this.entityDeletionAge = entityDeletionAge;
this.perfHelper = perfHelper;
this.queryCreationMethod = queryCreationMethod;
// Deletion and query are currently synchronous, so the concurrency
// count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.TableQuery,
1);
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.TableDeletion,
1);
// Create a timer to delete old logs
string timerId = String.Concat(
this.logSourceId,
OldLogDeletionTimerIdSuffix);
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
(this.entityDeletionAge < TimeSpan.FromDays(1)) ?
AzureConstants.OldLogDeletionIntervalForTest :
AzureConstants.OldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
internal FileShareTrimmer(
string folderName,
string destinationPath,
FolderTrimmer.DestinationOperationPerformer destinationOperationPerformer,
string localMap,
TimeSpan fileDeletionAge)
{
// Initialization
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(TraceType, "-", Guid.NewGuid().ToString("P"));
this.destinationPath = destinationPath;
this.destinationOperationPerformer = destinationOperationPerformer;
this.localMap = localMap;
this.localMapTrimmingHelper = new LocalMapTrimmingHelper(folderName);
this.folderTrimmer = new FolderTrimmer(
this.traceSource,
this.logSourceId,
fileDeletionAge);
this.perfHelper = new FileSharePerformance(this.traceSource, this.logSourceId);
// File deletion is done one at a time, so the
// concurrency count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.FileShareDeletion,
1);
// Create a timer to delete old logs
string timerId = string.Concat(
this.logSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval = (fileDeletionAge < TimeSpan.FromDays(1)) ?
FileShareUploaderConstants.OldLogDeletionIntervalForTest :
FileShareUploaderConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldFilesHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
public ContainerManager(AppInstanceManager appInstanceManager)
{
Utility.TraceSource.WriteInfo(TraceType, "Watching directory for creation of new container trace folders {0}", ContainerEnvironment.ContainerLogRootDirectory);
this.appInstanceManager = appInstanceManager;
if (Directory.Exists(ContainerEnvironment.ContainerLogRootDirectory))
{
foreach (var directory in Directory.EnumerateDirectories(ContainerEnvironment.ContainerLogRootDirectory))
{
var subDirectory = Path.GetFileName(directory);
var appInstanceId = Path.Combine(ContainerEnvironment.ContainersRootFolderName, subDirectory);
Utility.TraceSource.WriteInfo(TraceType, "Reading container traces from {0}", appInstanceId);
if (!File.Exists(Path.Combine(directory, ContainerEnvironment.ContainerDeletionMarkerFileName)))
{
this.appInstanceManager.CreateApplicationInstance(appInstanceId, null);
}
else
{
this.applicationInstancesToDelete.Add(appInstanceId);
}
}
}
else
{
Directory.CreateDirectory(ContainerEnvironment.ContainerLogRootDirectory);
}
// Arm FSW to watch for new folders.
this.watcher = new FileSystemWatcher(ContainerEnvironment.ContainerLogRootDirectory);
this.watcher.Created += this.OnFolderCreated;
this.watcher.EnableRaisingEvents = true;
// Arm the cleanup timer.
this.cleanupTimer = new DcaTimer(
ContainerManager.TraceType,
new TimerCallback(this.BackgroundCleanup),
(long)TimeSpan.FromMinutes(10.0).TotalMilliseconds);
this.cleanupTimer.Start();
}
private bool Initialize(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
ConfigReader configReader,
string logDirectory,
string outputFolderPath)
{
// Initialization
this.stopping = false;
this.logSourceId = logSourceId;
this.traceSource = traceSource;
this.configReader = configReader;
this.logDirectory = logDirectory;
this.outputFolderPath = outputFolderPath;
this.archiveFolderIsUnderBinaryFolder = false;
// Read performance counter collection settings from settings.xml
this.GetPerformanceCounterConfiguration();
if (false == this.perfCounterCollectionEnabled)
{
// Performance counter collection is not enabled, so return immediately
return(true);
}
// Create a timer to process performance counter binary files
string timerId = string.Concat(
this.logSourceId,
PerfCounterFolderProcessingTimerIdSuffix);
this.perfCounterFolderProcessingTimer = new DcaTimer(
timerId,
this.ProcessPerfCounterFolder,
DefaultPerfCounterFolderProcessingInterval);
this.perfCounterFolderProcessingTimer.Start();
return(true);
}
internal AppInstanceEtlFileDataReader(
string directory,
ServicePackageTableBackup tableBackup,
ServicePackageTableManager.AddOrUpdateHandler addOrUpdateServicePackageHandler,
ServicePackageTableManager.RemoveHandler removeServicePackageHandler)
: base(addOrUpdateServicePackageHandler, removeServicePackageHandler)
{
this.etlDirectory = directory;
this.tableBackup = tableBackup;
this.currentEtlReadIntervalMillisec = TimeSpan.MaxValue;
this.currentEtlFileFlushInterval = TimeSpan.MaxValue;
this.currentAppEtwTracesDeletionAgeMinutes = TimeSpan.MaxValue;
// Start reading events from the ETL file.
this.etlReadTimer = new DcaTimer(
AppInstanceEtlReadTimerId,
this.Read,
this.GetEtlReadInterval());
this.etlReadTimer.Start();
// Create a timer to periodically delete old ETL files
var oldDataDeletionInterval = TimeSpan.FromSeconds(Utility.GetUnencryptedConfigValue(
ConfigReader.DiagnosticsSectionName,
TestOnlyAppDataDeletionIntervalSecondsParamName,
(int)DefaultOldDataDeletionInterval.TotalSeconds));
Utility.TraceSource.WriteInfo(
TraceType,
"Interval at which we check for old data to delete: {0} seconds.",
oldDataDeletionInterval);
this.oldDataDeletionTimer = new DcaTimer(
OldDataDeletionTimerId,
this.DeleteOldEtlFiles,
oldDataDeletionInterval);
this.oldDataDeletionTimer.Start();
}
internal EtlInMemoryProducerWorker(
EtlInMemoryProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.cancellationTokenSource = new CancellationTokenSource();
this.perfHelper = new EtlInMemoryPerformance(this.traceSource);
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));
// Figure out where the ETL files are located
this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
this.etlInMemoryProducerWorkerSettings.EtlPath,
this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.traceSource,
this.logSourceId);
this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToInMemoryBufferWriters,
this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
true,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.markerFileDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToInMemoryBufferWriters,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// Figure out how much processing time is available to each provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
// Disk manager set up for traces
foreach (var provider in this.providers)
{
var capturedProvider = provider;
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
// Disk manager set up for marker files
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam)
{
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
this.configReader = new AzureBlobEtwConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Read blob-specific settings
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
return;
}
// Create the destination key
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
this.destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create a sub-directory for the blob uploader
this.workFolder = this.CreateBlobUploaderWorkSubDirectory();
if (string.IsNullOrEmpty(this.workFolder))
{
const string Message = "Failed to create work folder for the blob uploader.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from ETL files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
initParam.FabricNodeId,
this.etwLogDirName,
true,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
// Create the helper object that syncs local files to blob storage.
// Local files will be created when upload of compressed memory stream to blob storage fails.
this.fileBlobUploader = new AzureBlobUploader(
this.traceSource,
this.logSourceId,
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
this.fileUploadPerfHelper,
null,
this.uploadFileAccessCondition);
// Create a timer to schedule the upload of local files to blob storage
string timerId = string.Concat(
this.logSourceId,
FileUploadTimerIdSuffix);
this.fileUploadTimer = new DcaTimer(
timerId,
this.UploadFilesToDestinationBlob,
this.blobUploadSettings.FileSyncInterval);
this.fileUploadTimer.Start();
// Initialize trimmer
this.trimmer = new AzureFileTrimmer(
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
AzureUtility.IsAzureInterfaceAvailable());
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwLogDirName);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for Azure blob uploader: {0}",
this.blobUploadSettings.Filter);
}
internal ServicePackageTableManager(AppInstanceManager appInstanceMgr)
{
this.applicationInstanceManager = appInstanceMgr;
this.servicePackageTable = new ServicePackageTable();
// Get the interval at which we need to read ETL files
int servicePackageNotificationIntervalSeconds = Utility.GetUnencryptedConfigValue(
HostingSectionName,
ServicePackageNotificationIntervalInSecondsParamName,
DefaultServicePackageNotificationIntervalInSeconds);
this.maxServicePackageInactiveTimeSeconds = servicePackageNotificationIntervalSeconds * MaxServicePackageInactiveTimeMultiplier;
Utility.TraceSource.WriteInfo(
TraceType,
"Interval at which we check for inactive service packages: {0} seconds.",
servicePackageNotificationIntervalSeconds);
Utility.TraceSource.WriteInfo(
TraceType,
"Maximum time for which a service package can remain inactive before being deleted: {0} seconds.",
this.maxServicePackageInactiveTimeSeconds);
// Create the folder where the backup files are stored
string tableBackupDirectory = Path.Combine(
Utility.LogDirectory,
AppInstanceDataDirName,
AppInstanceTableDirName);
FabricDirectory.CreateDirectory(tableBackupDirectory);
// Initialize the application activation table
AppActivationTable.Initialize(tableBackupDirectory);
// Retrieve the file where we last saved the service package table
this.tableBackup = new ServicePackageTableBackup(
tableBackupDirectory,
this.AddOrUpdateServicePackage);
// Initialize the service package table from the file that we last saved.
if (false == this.tableBackup.Read())
{
Utility.TraceSource.WriteError(
TraceType,
"Unable to initialize service package table from backup file on disk.");
}
// Initialize the timestamp up to which ETW events can be read
Utility.ApplicationEtwTracesEndTime = this.tableBackup.LatestBackupTime.Timestamp;
// Compute the directory containing ETL files containing information about
// application instances
string etlFileDirectory = Path.Combine(
Utility.LogDirectory,
AppInstanceDataDirName,
AppInstanceDataEtlDirName);
FabricDirectory.CreateDirectory(etlFileDirectory);
#if !DotNetCoreClrLinux
// Create the object that reads events from ETL files.
this.etlFileReader = new AppInstanceEtlFileDataReader(
etlFileDirectory,
this.tableBackup,
this.AddOrUpdateServicePackage,
this.RemoveServicePackage);
#endif
long inactiveServicePackageScanIntervalMillisec = ((long)servicePackageNotificationIntervalSeconds) * 1000;
this.inactiveServicePackageScanTimer = new DcaTimer(
InactiveServicePackageScanTimerId,
this.MarkInactiveServicePackagesForDeletion,
inactiveServicePackageScanIntervalMillisec);
this.inactiveServicePackageScanTimer.Start();
}
internal CsvToUploadFolderWriter(
string logSourceId,
string fabricNodeId,
string csvFolder,
string sourceFolder,
DiskSpaceManager diskSpaceManager,
bool dtrCompressionDisabled,
IEtlToCsvFileWriterConfigReader configReader)
{
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = logSourceId;
this.organizeWindowsFabricTracesByType = true;
this.fabricNodeId = fabricNodeId;
this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled;
this.configReader = configReader;
this.disposed = false;
this.stopping = false;
this.compressCsvFiles = FileCompressor.CompressionEnabled &&
(false == this.dtrCompressionDisabledByConsumer) &&
(false == this.configReader.IsDtrCompressionDisabledGlobally());
this.diskSpaceManager = diskSpaceManager;
try
{
string currentAssemblyLocation = typeof(CsvToUploadFolderWriter).GetTypeInfo().Assembly.Location;
string versionFile = Path.Combine(Path.GetDirectoryName(currentAssemblyLocation), ClusterVersionFile);
this.fabricVersion = File.ReadAllText(versionFile);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Could not find the version of current service fabric code");
throw;
}
// Create the directory that containing filtered traces, in case it
// doesn't already exist
this.csvFolder = csvFolder;
FabricDirectory.CreateDirectory(this.csvFolder);
CreateWindowsFabricTraceSubFolders();
this.sourceFolder = sourceFolder;
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing trace files: {0} Directory containing dtr traces: {1}",
this.sourceFolder,
this.csvFolder);
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.csvFolder).EnumerateFiles("*.dtr*", SearchOption.AllDirectories),
null,
f => f.LastWriteTimeUtc > (DateTime.UtcNow - configReader.GetDtrDeletionAge()));
string timerIdMove = string.Concat(
this.logSourceId,
moveFilesTimerIdSuffix);
this.csvMoveFilesTimer = new DcaTimer(
timerIdMove,
this.CsvMoveFilesHandler,
1 * 60 * 1000);
this.csvMoveFilesTimer.Start();
}
internal EtlProducerWorker(
EtlProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.isReadingFromApplicationManifest = initParam.IsReadingFromApplicationManifest;
this.perfHelper = new EtlPerformance(this.traceSource);
this.cancellationTokenSource = new CancellationTokenSource();
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlProducerSettings = EtlProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlProducerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = EtlProducerWorkerSettingsHelper.InitializeSinks(
initParam.EtlProducers,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
this.etlToCsvFileWriters =
EtlProducerWorkerSettingsHelper.InitializeFileWriters(this.sinks, this).AsReadOnly();
this.bufferedEtwEventProviders =
EtlProducerWorkerSettingsHelper.InitializeBufferedEtwEventProviders(this.sinks, this).AsReadOnly();
// Figure out where the ETL files are located
this.traceDirectory = EtlProducerWorkerSettingsHelper.InitializeTraceDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceDirectory,
initParam.ProducerInstanceId);
if (initParam.IsReadingFromApplicationManifest)
{
lock (InternalMarkerFileDirectoriesForApps)
{
InternalMarkerFileDirectoriesForApps.Add(this.markerFileDirectory);
}
}
this.providers = EtlProducerWorkerSettingsHelper.InitializeProviders(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
this.etlProducerSettings.WindowsFabricEtlType,
this.etlProducerSettings.EtlFilePatterns,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
initParam.IsReadingFromApplicationManifest,
this.etlProducerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
initParam.ProducerInstanceId,
this.traceSource,
this.logSourceId);
if (false == initParam.IsReadingFromApplicationManifest)
{
// Ensure that no other instance of EtlProducerWorker is processing the
// same ETL files
var patternsAdded = new List <string>();
var isUnique = VerifyEtlFilesUniqueToCurrentInstance(
this.traceDirectory,
this.providers,
patternsAdded,
message => this.traceSource.WriteError(this.logSourceId, message));
this.patternsAddedToKnownEtlFileSet = patternsAdded.AsReadOnly();
if (!isUnique)
{
throw new InvalidOperationException(
string.Format("{0} is already being monitored for files matching one of the file patterns.", this.traceDirectory));
}
}
this.etlReadInterval = this.etlProducerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
if (false == initParam.IsReadingFromApplicationManifest)
{
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToCsvFileWriters,
this.etlProducerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
}
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
false == initParam.IsReadingFromApplicationManifest,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.etlProducerSettings.CustomManifestPaths,
initParam.ApplicationType,
this.markerFileDirectory,
this.etlProducerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToCsvFileWriters,
this.bufferedEtwEventProviders,
this.etlProducerSettings.AppEtwGuids,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// If there is a huge backlog of ETL files to process, we limit the
// amount of time that we spend on processing ETL files in each
// pass. Figure out how much processing time is available to each
// provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
foreach (var provider in this.providers)
{
var capturedProvider = provider;
if (Directory.Exists(this.traceDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}
if (Directory.Exists(this.markerFileDirectory))
{
diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
}
protected FileUploaderBase(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
string folderName,
string destinationId,
string workFolder,
TimeSpan uploadInterval,
TimeSpan fileSyncInterval)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.folderName = folderName;
this.destinationId = destinationId;
this.localMap = Path.Combine(workFolder, LocalMapFolder);
this.localMapFolderPath = this.localMap;
this.uploadingWinFabCrashDumps = folderName.StartsWith(
Utility.CrashDumpsDirectory,
StringComparison.OrdinalIgnoreCase);
// Create the local map folder
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricDirectory.CreateDirectory(this.localMap);
});
}
catch (Exception e)
{
throw new IOException(
string.Format("Unable to create directory {0}.", this.localMap),
e);
}
this.perfHelper = new FileUploaderBasePerformance(this.TraceSource, this.LogSourceId);
this.filesToCopy = new HashSet <string>();
// When we are first initialized, we always need a file sync because
// we don't know what local files may have been modified during the
// time that we were not running.
this.fileSyncNeeded = true;
// Configure the file watcher to watch the folder of interest
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricDirectory.CreateDirectory(this.folderName);
this.StartFileWatcher();
});
}
catch (Exception e)
{
throw new IOException(
string.Format("Unable to create directory {0}.", this.folderName),
e);
}
// Create a timer to schedule the upload of files to the destination
string timerId = string.Concat(
this.LogSourceId,
FileUploadTimerIdSuffix);
this.fileUploadTimer = new DcaTimer(
timerId,
this.UploadToFileShare,
uploadInterval,
true);
// Create a timer to check whether files on the local node need to
// be sync'ed with the files on the destination.
timerId = string.Concat(
this.LogSourceId,
FileSyncTimerIdSuffix);
this.fileSyncTimer = new DcaTimer(
timerId,
this.CheckForFilesToBeSynced,
fileSyncInterval);
}
