private void InitializeCommon(FabricEvents.ExtensionsEvents traceSource, string logSourceId, string fileFullPath, int fileFormatVersion) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.fileFullPath = fileFullPath; this.fileFormatVersion = fileFormatVersion; }
internal AzureBlobUploader( FabricEvents.ExtensionsEvents traceSource, string logSourceId, string etwLogDirName, string workFolder, StorageAccountFactory storageAccountFactory, string containerName, string fabricNodeInstanceName, string deploymentId, AzureBlobPerformance perfHelper, AccessCondition uploadStreamAccessCondition, AccessCondition uploadFileAccessCondition) : this( traceSource, logSourceId, etwLogDirName, workFolder, storageAccountFactory, string.Empty, false, containerName, string.Empty, fabricNodeInstanceName, deploymentId, perfHelper, uploadStreamAccessCondition, uploadFileAccessCondition) { }
public CheckpointManager( bool isReadingFromApplicationManifest, string etlPath, string logDirectory, string traceDirectory, string producerInstanceId, FabricEvents.ExtensionsEvents traceSource, string logSourceId) { this.traceSource = traceSource; this.logSourceId = logSourceId; // Directory where the producer saves the file containing information // about the point upto which we last read the ETL files this.lastEtlFileReadFileDirectoryPath = InitializeLastEtlFileReadFilePath( isReadingFromApplicationManifest, etlPath, logDirectory, traceDirectory, producerInstanceId); // If the directory containing the last ETL read information is // different from the log directory, then create it now. if (false == this.lastEtlFileReadFileDirectoryPath.Equals( logDirectory)) { FabricDirectory.CreateDirectory(this.lastEtlFileReadFileDirectoryPath); this.traceSource.WriteInfo( this.logSourceId, "Directory containing last ETL read info: {0}", this.lastEtlFileReadFileDirectoryPath); } }
internal LttProducerWorker(LttProducerWorkerParameters initParam) { this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.logSourceId = string.Concat(initParam.LatestSettings.ApplicationInstanceId, "_", initParam.LatestSettings.SectionName); this.traceSource.WriteInfo( this.logSourceId, "LttProducer: ApplicationInstanceId:{0} SectionName:{1} LogDirectory:{2} WorkDirectory:{0}", initParam.LatestSettings.ApplicationInstanceId, initParam.LatestSettings.SectionName, initParam.LogDirectory, initParam.LatestSettings.WorkDirectory); this.applicationInstanceId = initParam.LatestSettings.ApplicationInstanceId; this.lttngTracesDirectory = Path.Combine(initParam.LogDirectory, LttProducerConstants.LttSubDirectoryUnderLogDirectory); this.dtrTracesDirectory = Path.Combine(initParam.LogDirectory, LttProducerConstants.DtrSubDirectoryUnderLogDirectory); this.LttReadIntervalMinutes = initParam.LttReadIntervalMinutes; // splitting sinks into their two different supported types var sinks = InitializeSinks(initParam.LttProducers, new List <string>() { this.dtrTracesDirectory }, message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly(); this.sinksIEtlFile = (sinks.Where(s => s is IEtlFileSink)).Select(s => s as IEtlFileSink).ToList().AsReadOnly(); this.sinksICsvFile = (sinks.Where(s => s is ICsvFileSink)).Select(s => s as ICsvFileSink).ToList().AsReadOnly(); this.traceSource.WriteInfo( this.logSourceId, "Total number of sinks: {0}, IEtlFileSinks: {1}, ICsvFileSinks: {2}", sinks.Count, this.sinksIEtlFile.Count, this.sinksICsvFile.Count); this.lastReadTimestampUnixEpochNanoSec = this.InitializeLastReadTimestamp(); this.lttngTraceProcessor = new LttTraceProcessor( this.traceSource, this.logSourceId, this.dtrTracesDirectory, this.sinksIEtlFile, this.sinksICsvFile); // getting active service fabric lttng trace session this.activeLttTraceSessionOutputPath = this.GetServiceFabricSessionOutputPath(); // try to process existing traces left from previous session this.ProcessPreviousLttSessionTraces(this.activeLttTraceSessionOutputPath); string timerId = string.Concat(initParam.LatestSettings.ApplicationInstanceId, LttProducerConstants.LttTimerPrefix); this.LttReadTimer = new DcaTimer( timerId, this.LttReadCallback, this.LttReadIntervalMinutes * 60 * 1000); this.LttReadTimer.Start(); }
private static bool GetDestinationPathFromUri( FabricEvents.ExtensionsEvents traceSource, string logSourceId, string sectionName, string destinationPathAsUri, out string destinationPath) { destinationPath = string.Empty; try { Uri uri = new Uri(destinationPathAsUri); destinationPath = uri.LocalPath; } catch (UriFormatException e) { traceSource.WriteError( logSourceId, "The value '{0}' specified as the file share destination in section {1} could not be parsed as a URI. Exception information: {2}.", destinationPathAsUri, sectionName, e); return(false); } return(true); }
public AzureBlobFolderUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new AzureBlobConfigReader( new ConfigReader(initParam.ApplicationInstanceId), this.initParam.SectionName, this.traceSource, this.logSourceId); // Read blob-specific settings from settings.xml this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { // Upload to Azure blob storage is not enabled, so return immediately return; } var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobFolderUploader, accountName, this.blobUploadSettings.ContainerName); }
public LastEndTimeFileParameters( FabricEvents.ExtensionsEvents traceSource, string logSourceId) { this.traceSource = traceSource; this.logSourceId = logSourceId; }
public DsmsStorageAccountFactory(StorageConnection connection) : base(connection) { this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); var sourceLocation = connection.DsmsSourceLocation; this.traceSource.WriteInfo(TraceType, "Invoking ClientInitialize and creating DsmsStorageCredentials instance for source location: {0} and EndPointSuffix: {1}", sourceLocation, connection.DsmsEndpointSuffix); try { if (connection.DsmsAutopilotServiceName == null) { CredentialManagementClient.Instance.ClientInitialize(); } else { CredentialManagementClient.Instance.ClientInitializeForAp(connection.DsmsAutopilotServiceName); } this.dsmsCredentials = DsmsStorageCredentials.GetStorageCredentials(sourceLocation); } catch (Exception ex) { this.traceSource.WriteExceptionAsError(TraceType, ex); throw; } }
public LttProducer( DiskSpaceManager diskSpaceManager, ITraceEventSourceFactory traceEventSourceFactory, ProducerInitializationParameters initParam) { this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.AppInstanceId = initParam.ApplicationInstanceId; this.traceSource = traceEventSourceFactory.CreateTraceEventSource(FabricEvents.Tasks.FabricDCA); this.serviceConfigSections = new List <string>(); this.consumerSinks = initParam.ConsumerSinks; // Read the timer config value from dca section var configReader = new ConfigReader(initParam.ApplicationInstanceId); long lttReadIntervalMinutes = configReader.GetUnencryptedConfigValue( initParam.SectionName, LttProducerConstants.LttReadIntervalParamName, LttProducerConstants.DefaultLttReadIntervalMinutes); if (initParam.ApplicationInstanceId == Utility.WindowsFabricApplicationInstanceId) { this.CreateWindowsFabricLttProducerWorkerInfo(initParam, lttReadIntervalMinutes); } else { this.CreateAppProducerWorkerInfo(initParam, lttReadIntervalMinutes); } }
public CheckpointManager( IEnumerable <EtlInMemoryProducerWorker.ProviderInfo> providers, string etlPath, string logDirectory, string traceDirectory, FabricEvents.ExtensionsEvents traceSource, string logSourceId) { this.traceSource = traceSource; this.logSourceId = logSourceId; // Directory where the producer saves the file containing information // about the point upto which we last read the ETL files var lastEtlFileReadFilePath = InitializeLastEtlFileReadFilePath( etlPath, logDirectory, traceDirectory); // If the directory containing the last ETL read information is // different from the log directory, then create it now. if (false == lastEtlFileReadFilePath.Equals( logDirectory)) { FabricDirectory.CreateDirectory(lastEtlFileReadFilePath); this.traceSource.WriteInfo( this.logSourceId, "Directory containing last ETL read info: {0}", lastEtlFileReadFilePath); } this.SetProviderCheckpointFileName(providers, lastEtlFileReadFilePath); }
static FileImageStore() { // Try reading config (this may require access to tracing which is why we setup something up above TraceConfig.InitializeFromConfigStore(); // Create the trace source based on the config FileImageStore.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.ImageStoreClient); }
internal MockActorManager(ActorService actorService) { this.actorService = actorService; this.diagnosticsManager = new MockDiagnosticsManager(actorService); this.eventManager = new MockActorEventManager(actorService.ActorTypeInformation); this.remindersByActorId = new ConcurrentDictionary <ActorId, ConcurrentDictionary <string, ActorReminder> >(); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.ActorFramework); }
static AppTrace() { // Try reading config TraceConfig.InitializeFromConfigStore(); // Create the trace source AppTrace.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.SystemFabric); }
static ImageBuilderExe() { TraceConfig.InitializeFromConfigStore(); ImageBuilderExe.TraceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.ImageBuilder); #if !DotNetCoreClr ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; #endif }
internal FolderTrimmer( FabricEvents.ExtensionsEvents traceSource, string logSourceId, TimeSpan fileDeletionAge) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.fileDeletionAge = fileDeletionAge; }
public PerfCounterConfigReader(ProducerInitializationParameters initializationParameters) { // Initialization this.initParam = initializationParameters; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(this.initParam.ApplicationInstanceId); // Read instance-specific settings from settings.xml this.GetSettings(); if (false == this.configReaderSettings.Enabled) { // Producer is not enabled, so return immediately return; } // Read the performance counter configuration from settings.xml this.GetPerformanceCounterConfiguration(); if (false == this.perfCounterConfig.IsEnabled) { // Performance counter collection is not enabled, so return immediately return; } if (null == initializationParameters.ConsumerSinks) { return; } foreach (object sinkAsObject in initializationParameters.ConsumerSinks) { IPerformanceCounterConfigurationSink perfCounterConfigSink = null; try { perfCounterConfigSink = (IPerformanceCounterConfigurationSink)sinkAsObject; } catch (InvalidCastException e) { this.traceSource.WriteError( this.logSourceId, "Exception occured while casting a sink object of type {0} to interface IPerformanceCounterConfigurationSink. Exception information: {1}.", sinkAsObject.GetType(), e); } if (null == perfCounterConfigSink) { continue; } perfCounterConfigSink.RegisterPerformanceCounterConfiguration( this.perfCounterConfig.SamplingInterval, this.perfCounterConfig.CounterPaths); } }
internal InMemorySink(FabricEvents.ExtensionsEvents traceSource, string logSourceId, string traceFileSubFolder, Action <DecodedEventWrapper, string> consumerProcessTraceEventAction) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.inMemorySink = new BlockingCollection <DecodedEventWrapper>(BoundedCapacity); this.consumerProcessTraceEventAction = consumerProcessTraceEventAction; this.consumerProcessingCompleted = new ManualResetEvent(false); this.traceFileSubFolder = traceFileSubFolder; this.CreateAndRunTraceEventConsumerTask(); }
public IEtlProducerConfigReader CreateEtlProducerConfigReader( FabricEvents.ExtensionsEvents traceSource, string logSourceId) { return(new EtlProducerConfigReader( this.configReader, this.sectionName, traceSource, logSourceId)); }
static DeployerTrace() { #if DotNetCoreClrLinux // TODO - Following code will be removed once fully transitioned to structured traces in Linux // this is used to read the configuration which tells whether to use structured traces in Linux. TraceConfig.InitializeFromConfigStore(); #endif traceSource = FabricEvents.ExtensionsEvents.GetEventSource(FabricEvents.Tasks.FabricDeployer); TraceConfig.SetDefaultLevel(TraceSinkType.Console, EventLevel.Error); TraceConfig.SetDefaultLevel(TraceSinkType.ETW, EventLevel.Informational); }
public FileShareFolderUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); // Read file-share-specific settings from settings.xml this.GetSettings(); }
public CsvFileToSinkProcessor( FileInfo fileInfo, ReadOnlyCollection <ICsvFileSink> sinks, string logSourceId, FabricEvents.ExtensionsEvents TraceSource) { this.fileInfo = fileInfo; this.sinks = sinks; this.logSourceId = logSourceId; this.traceSource = TraceSource; }
static ImageStoreClient() { // Try reading config TraceConfig.InitializeFromConfigStore(); // Create the trace source based on the config ImageStoreClient.TraceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.ImageStoreClient); #if !DotNetCoreClr ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; #endif }
public EtlProducerConfigReader( ConfigReader configReader, string sectionName, FabricEvents.ExtensionsEvents traceSource, string logSourceId) { this.configReader = configReader; this.sectionName = sectionName; this.traceSource = traceSource; this.logSourceId = logSourceId; }
internal AzureFileTrimmer( string sourceFolderName, string workFolder, StorageAccountFactory storageAccountFactory, string containerName, TimeSpan blobDeletionAge, string fabricNodeInstanceName, string deploymentId, bool azureInterfaceAvailable) { // Initialization this.stopping = false; this.sourceFolderName = sourceFolderName; this.localMap = Path.Combine(workFolder, AzureBlobUploader.LocalMapFolder); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.logSourceId = string.Concat(TraceType, "-", Guid.NewGuid().ToString("P")); this.storageAccountFactory = storageAccountFactory; this.containerName = containerName; this.fabricNodeInstanceName = fabricNodeInstanceName; this.blobDeletionAge = blobDeletionAge; this.deploymentId = deploymentId; this.azureInterfaceAvailable = azureInterfaceAvailable; this.localMapTrimmingHelper = new LocalMapTrimmingHelper(sourceFolderName); this.folderTrimmer = new FolderTrimmer( this.traceSource, this.logSourceId, blobDeletionAge); this.perfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Deletion and query are currently synchronous, so the concurrency // count is 1. this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobQuery, 1); this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.BlobDeletion, 1); // Create a timer to delete old logs string timerId = string.Concat( this.logSourceId, OldLogDeletionTimerIdSuffix); var oldLogDeletionInterval = (this.blobDeletionAge < TimeSpan.FromDays(1)) ? AzureConstants.OldLogDeletionIntervalForTest : AzureConstants.OldLogDeletionInterval; this.oldLogDeletionTimer = new DcaTimer( timerId, this.DeleteOldLogsHandler, oldLogDeletionInterval); this.oldLogDeletionTimer.Start(); }
public AzureBlobEtwConfigReader( IConfigReader configReader, string sectionName, FabricEvents.ExtensionsEvents traceSource, string logSourceId) { this.configReader = configReader; this.sectionName = sectionName; this.traceSource = traceSource; this.logSourceId = logSourceId; this.azureUtility = new AzureUtility(traceSource, logSourceId); }
internal EtlProcessor( bool loadWinFabManifests, bool dynamicWinFabManifestLoad, IEnumerable <string> customManifestPaths, string applicationType, string markerFileDirectory, WinFabricEtlType winFabricEtlType, FabricEvents.ExtensionsEvents traceSource, string logSourceId, EtlPerformance perfHelper, IEnumerable <IEtlFileSink> sinks, IEnumerable <EtlToCsvFileWriter> etlToCsvFileWriters, IEnumerable <BufferedEtwEventProvider> bufferedEtwEventProviders, IEnumerable <Guid> appEtwGuids, ITraceFileEventReaderFactory traceFileEventReaderFactory) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.processingWinFabEvents = loadWinFabManifests; this.applicationType = applicationType; this.markerFileDirectory = markerFileDirectory; this.windowsFabricEtlType = winFabricEtlType; this.perfHelper = perfHelper; this.sinks = sinks.ToList().AsReadOnly(); this.etlToCsvFileWriters = etlToCsvFileWriters.ToList().AsReadOnly(); this.bufferedEtwEventProviders = bufferedEtwEventProviders.ToList().AsReadOnly(); this.appEtwGuids = appEtwGuids != null?appEtwGuids.ToList().AsReadOnly() : null; this.traceFileEventReaderFactory = traceFileEventReaderFactory; // Load the custom manifests (if any) into the manifest cache this.LoadManifests(customManifestPaths); if (loadWinFabManifests) { if (dynamicWinFabManifestLoad) { // We are going to be loading manifests dynamically based on the version // in the ETL file. Initialize the manifest manager object to manage this // for us. this.winFabManifestMgr = this.InitializeWinFabManifestManager(); } else { // We'll load the default manifests now and just use them throughout this.LoadDefaultManifests(); } } // Provide the sinks a reference to the manifest cache. this.OnEtwManifestCacheAvailable(); }
public SyslogConsumer(ConsumerInitializationParameters initParam) { this.logSourceId = String.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.PopulateSettings(initParam); if (this.isEnabled) { this.traceSource.WriteInfo(this.logSourceId, "SyslogConsumer is Enabled"); this.sysLogWriter = new LocalSysLogger(this.logIdentity, this.syslogFacility); } }
public static PerfCounterFolderProcessor Create( FabricEvents.ExtensionsEvents traceSource, string logSourceId, ConfigReader configReader, string logDirectory, string outputFolderPath, out bool isEnabled, out List <string> additionalFoldersToTrim) { additionalFoldersToTrim = null; isEnabled = false; // Create a new instance of the folder processor PerfCounterFolderProcessor folderProcessor = new PerfCounterFolderProcessor(); // PerfCounterFolderProcessor is a singleton, so make sure there are no other instances. object original = Interlocked.CompareExchange(ref folderProcessorSingleton, folderProcessor, null); if (null != original) { traceSource.WriteError( logSourceId, "Cannot have more than one producer of type {0}, whose {1} value is {2}.", StandardPluginTypes.FolderProducer, FolderProducerValidator.FolderTypeParamName, FolderProducerValidator.ServiceFabricPerformanceCounters); return(null); } // Initialize the folder processor if (false == folderProcessor.Initialize(traceSource, logSourceId, configReader, logDirectory, outputFolderPath)) { return(null); } isEnabled = folderProcessor.perfCounterCollectionEnabled; if (isEnabled) { // In addition to the output folder, which is already trimmed by default, we also need // the perf counter binary folder and the perf counter binary archive folder to be trimmed. additionalFoldersToTrim = new List <string> { folderProcessor.perfCounterBinaryFolder }; if (false == folderProcessor.archiveFolderIsUnderBinaryFolder) { additionalFoldersToTrim.Add(folderProcessor.perfCounterBinaryArchiveFolder); } } return(folderProcessor); }
internal static ITelemetryWriter CreateTelemetryWriter( ConfigReader configReader, string clusterId, string nodeName, TelemetryCollection.LogErrorDelegate logErrorDelegate) { #if !DotNetCoreClrLinux bool createFileShareWriter = false; #endif FabricEvents.ExtensionsEvents traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); traceSource.WriteInfo("TelemetryWriterFactory", "Entering TelemetryWriterFactory"); #if !DotNetCoreClrLinux string fabricLogsShare; bool isFabricLogsLocalFolder; if (FileShareCommon.GetDestinationPath( traceSource, string.Concat(ConfigReader.FileShareWinFabSectionName), configReader, ConfigReader.FileShareWinFabSectionName, out isFabricLogsLocalFolder, out fabricLogsShare)) { traceSource.WriteInfo("TelemetryWriterFactory", "isFabricLogsLocalFolder: {0}, fabricLogsShare: {1}", isFabricLogsLocalFolder, fabricLogsShare); string customizedGoalStateFileUrl = configReader.GetUnencryptedConfigValue( ConfigReader.UpgradeOrchestrationServiceConfigSectionName, ConfigReader.GoalStateFileUrlParamName, string.Empty); if (customizedGoalStateFileUrl != string.Empty) { createFileShareWriter = true; } } if (createFileShareWriter) { string fileShareDirectory = Path.Combine(Path.GetDirectoryName(fabricLogsShare), "fabrictelemetries-" + clusterId, nodeName); traceSource.WriteInfo("TelemetryWriterFactory", "return fileShare {0}", fileShareDirectory); return(new FileShareTelemetryWriter(fileShareDirectory, logErrorDelegate)); } else #endif { traceSource.WriteInfo("TelemetryWriterFactory", "return appinsight"); return(new AppInsightsTelemetryWriter()); } }
public ConsumerProgressManager( FabricEvents.ExtensionsEvents traceSource, string logSourceId, int methodExecutionInitialRetryIntervalMs, int methodExecutionMaxRetryCount, int methodExecutionMaxRetryIntervalMs) { this.traceSource = traceSource; this.logSourceId = logSourceId; this.methodExecutionInitialRetryIntervalMs = methodExecutionInitialRetryIntervalMs; this.methodExecutionMaxRetryCount = methodExecutionMaxRetryCount; this.methodExecutionMaxRetryIntervalMs = methodExecutionMaxRetryIntervalMs; this.bookmarkFolders = new Dictionary <string, string>(); this.bookmarkLastEventReadPosition = new Dictionary <string, long>(); }