public AzureBlobFolderUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new AzureBlobConfigReader( new ConfigReader(initParam.ApplicationInstanceId), this.initParam.SectionName, this.traceSource, this.logSourceId); // Read blob-specific settings from settings.xml this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { // Upload to Azure blob storage is not enabled, so return immediately return; } var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobFolderUploader, accountName, this.blobUploadSettings.ContainerName); }
private void PopulateSettings(ConsumerInitializationParameters initParam) { var configReader = new ConfigReader(initParam.ApplicationInstanceId); this.isEnabled = configReader.GetUnencryptedConfigValue( initParam.SectionName, SyslogConstant.EnabledParamName, DefaultEnabledStateValue); this.logIdentity = configReader.GetUnencryptedConfigValue( initParam.SectionName, SyslogConstant.IdentityParamName, DefaultLogIdentity); var facilityString = configReader.GetUnencryptedConfigValue( initParam.SectionName, SyslogConstant.FacilityParamName, DefaultLogFacility.ToString()); // It is bound to succeed since the param is already validated to be the right enum type in validator. this.syslogFacility = (SyslogFacility)Enum.Parse(typeof(SyslogFacility), facilityString, true); this.traceSource.WriteInfo( this.logSourceId, string.Format("SyslogConsumer Populated with these settings. Enabled: {0}, Identity: {1}, Facility: {2}", this.isEnabled, this.logIdentity, this.syslogFacility)); }
public FileShareFolderUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); // Read file-share-specific settings from settings.xml this.GetSettings(); }
public SyslogConsumer(ConsumerInitializationParameters initParam) { this.logSourceId = String.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.PopulateSettings(initParam); if (this.isEnabled) { this.traceSource.WriteInfo(this.logSourceId, "SyslogConsumer is Enabled"); this.sysLogWriter = new LocalSysLogger(this.logIdentity, this.syslogFacility); } }
private bool CreateMdsUploader(string sectionName, out MdsEtwEventUploader uploader) { // Once Uploader is constructed DiskSpaceManager is no longer needed. using (var dsm = new DiskSpaceManager()) { ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var initParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, sectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, dsm); uploader = new MdsEtwEventUploader(initParam); EtwCsvFolder = uploader.EtwCsvFolder; return(true); } }
public void TestMethodSetup() { // Initialization bool result = StartDtrRead.Reset() && DtrReadCompleted.Reset() && EndOfTest.Reset(); Verify.IsTrue(result, "Successfully reset all events at the start of the test"); // Parse the configuration file this.testStartTime = DateTime.UtcNow; ParseConfig(testStartTime); // Create the log and work directories Utility.LogDirectory = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), LogFolderName); Utility.TraceSource.WriteInfo( TraceType, "Log folder: {0}", Utility.LogDirectory); Utility.InitializeWorkDirectory(); // Create the deleted items folder this.deletedItemsFolder = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), DeletedItemsFolderName); FabricDirectory.CreateDirectory(deletedItemsFolder); Utility.TraceSource.WriteInfo( TraceType, "Deleted items folder: {0}", this.deletedItemsFolder); // Create the MDS upload consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var initParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); this.uploader = new MdsEtwEventUploader(initParam); EtwCsvFolder = this.uploader.EtwCsvFolder; }
private AzureTableQueryableEventUploader CreateAndInitializeUploader() { const string TestFabricNodeInstanceName = "test"; const string TestLogDirectory = "Logs"; const string TestWorkDirectory = "Work"; // Once Uploader is constructed DiskSpaceManager is no longer needed. using (var testDiskSpaceManager = new DiskSpaceManager()) { ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); ConsumerInitializationParameters initParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeInstanceName, TestLogDirectory, TestWorkDirectory, testDiskSpaceManager); return(new AzureTableQueryableEventUploader(initParam)); } }
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam) { this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); this.configReader = new AzureBlobEtwConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId); // Read blob-specific settings this.blobUploadSettings = this.GetSettings(); if (false == this.blobUploadSettings.Enabled) { return; } // Create the destination key var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName; this.destinationKey = string.Join( "_", StandardPluginTypes.AzureBlobEtwUploader, accountName, this.blobUploadSettings.EtwTraceContainerName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, this.destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create a sub-directory for the blob uploader this.workFolder = this.CreateBlobUploaderWorkSubDirectory(); if (string.IsNullOrEmpty(this.workFolder)) { const string Message = "Failed to create work folder for the blob uploader."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from ETL files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, initParam.FabricNodeId, this.etwLogDirName, true, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.blobUploadSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); // Create the helper object that syncs local files to blob storage. // Local files will be created when upload of compressed memory stream to blob storage fails. this.fileBlobUploader = new AzureBlobUploader( this.traceSource, this.logSourceId, this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, this.fileUploadPerfHelper, null, this.uploadFileAccessCondition); // Create a timer to schedule the upload of local files to blob storage string timerId = string.Concat( this.logSourceId, FileUploadTimerIdSuffix); this.fileUploadTimer = new DcaTimer( timerId, this.UploadFilesToDestinationBlob, this.blobUploadSettings.FileSyncInterval); this.fileUploadTimer.Start(); // Initialize trimmer this.trimmer = new AzureFileTrimmer( this.etwLogDirName, this.workFolder, this.blobUploadSettings.StorageAccountFactory, this.blobUploadSettings.EtwTraceContainerName, this.blobUploadSettings.BlobDeletionAge, this.initParam.FabricNodeInstanceName, this.blobUploadSettings.DeploymentId, AzureUtility.IsAzureInterfaceAvailable()); this.traceSource.WriteInfo( this.logSourceId, "Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}", this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.EtwTraceContainerName, this.etwLogDirName); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for Azure blob uploader: {0}", this.blobUploadSettings.Filter); }
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam, AccessCondition uploadStreamAccessCondition, AccessCondition uploadFileAccessCondition) : this(initParam) { this.uploadStreamAccessCondition = uploadStreamAccessCondition; this.uploadFileAccessCondition = uploadFileAccessCondition; }
public FileShareEtwCsvUploader(ConsumerInitializationParameters initParam) { // Initialization this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); // Read file-share-specific settings from settings.xml this.GetSettings(); if (false == this.fileUploadSettings.Enabled) { // Upload to file share is not enabled, so return immediately return; } if (this.configReader.IsReadingFromApplicationManifest) { // Check if we can use an existing upload worker object UploadWorkerKey key = new UploadWorkerKey() { DestinationPath = this.fileUploadSettings.DestinationPath, ApplicationType = this.configReader.GetApplicationType(), }; lock (UploadWorkers) { UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key)); if (null != workerInfo) { // Existing upload worker object is available. Increment its // reference count this.traceSource.WriteInfo( this.logSourceId, "Existing upload worker object for application type {0} and destination {1} is available and will be used.", key.ApplicationType, this.fileUploadSettings.DestinationPath); workerInfo.RefCount++; workerInfo.UploadWorker.UpdateSettings(this.fileUploadSettings); this.uploadWorker = workerInfo.UploadWorker; } else { // Create a new upload worker object this.traceSource.WriteInfo( this.logSourceId, "Creating upload worker object for application type {0} and destination {1} ...", key.ApplicationType, this.fileUploadSettings.DestinationPath); EtwCsvUploadWorker.EtwCsvUploadWorkerParameters param = new EtwCsvUploadWorker.EtwCsvUploadWorkerParameters() { FabricNodeId = this.initParam.FabricNodeId, FabricNodeInstanceName = this.initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = this.initParam.LogDirectory, WorkDirectory = this.initParam.WorkDirectory, UploaderInstanceId = key.ApplicationType, ParentWorkFolderName = key.ApplicationType, Settings = this.fileUploadSettings, DiskSpaceManager = this.initParam.DiskSpaceManager }; try { EtwCsvUploadWorker newWorker = new EtwCsvUploadWorker(param, this.traceSource); workerInfo = new UploadWorkerInfo { Key = key, RefCount = 1, UploadWorker = newWorker }; UploadWorkers.Add(workerInfo); this.uploadWorker = workerInfo.UploadWorker; } catch (InvalidOperationException e) { var message = string.Format( "Failed to create upload worker object for application type {0} and destination {1}.", key.ApplicationType, this.fileUploadSettings.DestinationPath); this.traceSource.WriteError( this.logSourceId, message); throw new InvalidOperationException(message, e); } } } } else { // Create a new upload worker object var param = new EtwCsvUploadWorker.EtwCsvUploadWorkerParameters { FabricNodeId = this.initParam.FabricNodeId, FabricNodeInstanceName = this.initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = this.initParam.LogDirectory, WorkDirectory = this.initParam.WorkDirectory, UploaderInstanceId = this.logSourceId, ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths, Settings = this.fileUploadSettings, DiskSpaceManager = initParam.DiskSpaceManager }; this.uploadWorker = new EtwCsvUploadWorker(param, this.traceSource); } }
public MdsFileProducer(ConsumerInitializationParameters initParam) { // Initialization this.stopping = false; this.initParam = initParam; this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); this.tableWriters = new Dictionary <string, MdsTableWriter>(); this.mdsFileProducerPerf = new MdsFileProducerPerformance(); this.progressManager = new ConsumerProgressManager( this.traceSource, this.logSourceId, MdsFileProducerConstants.MethodExecutionInitialRetryIntervalMs, MdsFileProducerConstants.MethodExecutionMaxRetryCount, MdsFileProducerConstants.MethodExecutionMaxRetryIntervalMs); // Read MDS-specific settings from settings.xml this.GetSettings(); if (false == this.mdsFileProducerSettings.Enabled) { // MDS tables creation is not enabled, so return immediately return; } string destinationKey = Path.Combine( this.mdsFileProducerSettings.DirectoryName, this.mdsFileProducerSettings.TableName); // initialize bookmark folders and files var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles( this.initParam.WorkDirectory, destinationKey); if (false == initializeBookmarkFoldersAndFilesSuccess) { const string Message = "Failed to initialize bookmark folders and files."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create etw log directory this.etwLogDirName = this.CreateEtwLogDirectory(); if (string.IsNullOrEmpty(this.etwLogDirName)) { const string Message = "Failed to create etw log directory."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Create the helper object that writes events delivered from the ETL // files into an in-memory buffer. this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter( new TraceEventSourceFactory(), this.logSourceId, this.initParam.FabricNodeId, this.etwLogDirName, false, this); // Set the event filter this.etlToInMemoryBufferWriter.SetEtwEventFilter( this.mdsFileProducerSettings.Filter, WinFabDefaultFilter.StringRepresentation, WinFabSummaryFilter.StringRepresentation, true); this.traceSource.WriteInfo( this.logSourceId, "MDS file producer is configured. Table path: {0}, Table name: {1}, Table priority: {2}, Disk quota (MB): {3}, Data deletion age (minutes): {4}", this.mdsFileProducerSettings.DirectoryName, this.mdsFileProducerSettings.TableName, this.mdsFileProducerSettings.TablePriority, this.mdsFileProducerSettings.DiskQuotaInMB, this.mdsFileProducerSettings.DataDeletionAge); this.traceSource.WriteInfo( this.logSourceId, "Windows Fabric event filters for MDS file producer: {0}", this.mdsFileProducerSettings.Filter); }
public AzureTableEtwEventUploader(ConsumerInitializationParameters initParam) { // Initialization this.stopping = false; this.initParam = initParam; this.logSourceId = String.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new ConfigReader(initParam.ApplicationInstanceId); this.azureUtility = new AzureUtility(this.traceSource, this.logSourceId); this.perfHelper = new AzureTablePerformance(this.traceSource, this.logSourceId); // Make sure that the Azure interfaces are available if (false == AzureUtility.IsAzureInterfaceAvailable()) { const string Message = "Due to unavailability of Azure interfaces, ETW traces will not be uploaded to Azure table storage."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } this.azureNodeInstanceId = AzureUtility.RoleInstanceId; // Read table-specific settings from settings.xml GetSettings(); if (false == this.tableUploadSettings.Enabled) { // Upload to Azure table storage is not enabled, so return immediately return; } // Create a sub-directory for ourselves under the log directory string bufferedEventFolder = GetBufferedEventSubDirectory(); if (String.IsNullOrEmpty(bufferedEventFolder)) { throw new InvalidOperationException("Unable to get buffered event subdirectory."); } // Create the helper object that buffers events delivered from the ETL // files into CSV files on disk. this.bufferedEventProvider = new BufferedEtwEventProvider( new TraceEventSourceFactory(), this.logSourceId, bufferedEventFolder, this.tableUploadSettings.UploadIntervalMinutes, this.tableUploadSettings.EntityDeletionAge, this); if (null == this.bufferedEventProvider) { const string Message = "Failed to create buffered event provider helper object."; this.traceSource.WriteError( this.logSourceId, Message); throw new InvalidOperationException(Message); } // Set the filter for Windows Fabric events this.bufferedEventProvider.SetEtwEventFilter( this.tableUploadSettings.Filter, defaultTableFilter, WinFabSummaryFilter.StringRepresentation, true); // Initialize the batch upload concurrency count Debug.Assert(this.tableUploadSettings.BatchUploadConcurrencyCount <= AzureConstants.MaxBatchConcurrencyCount); if (this.tableUploadSettings.BatchUploadConcurrencyCount <= AzureConstants.MaxBatchConcurrencyCount) { this.batchConcurrencyCount = this.tableUploadSettings.BatchUploadConcurrencyCount; } else { this.traceSource.WriteError( this.logSourceId, "{0} is an invalid value for table batch concurrency count. The maximum supported value is {1} and that value will be used instead.", this.tableUploadSettings.BatchUploadConcurrencyCount, AzureConstants.MaxBatchConcurrencyCount); this.batchConcurrencyCount = AzureConstants.MaxBatchConcurrencyCount; } this.perfHelper.ExternalOperationInitialize( ExternalOperationTime.ExternalOperationType.TableUpload, this.batchConcurrencyCount); // Create the table try { CreateTable(); } catch (Exception e) { const string Message = "Due to an error in table creation ETW traces will not be uploaded to Azure table storage."; this.traceSource.WriteExceptionAsError( this.logSourceId, e, Message); throw new InvalidOperationException(Message, e); } this.traceSource.WriteInfo( this.logSourceId, "Created table for uploading ETW traces. Storage account: {0}, Table name: {1}", this.tableUploadSettings.StorageAccountFactory.Connection.AccountName, this.tableUploadSettings.TableName); // Initialize old log deletion this.trimmer = new AzureTableTrimmer( this.traceSource, this.logSourceId, this.tableUploadSettings.StorageAccountFactory, this.tableUploadSettings.TableName, this.tableUploadSettings.EntityDeletionAge, this.CreateDeletionQuery, this.perfHelper); }
private static Dictionary <string, IDcaConsumer> CreateConsumers( ConsumerFactory consumerFactory, IDictionary <string, List <object> > producerConsumerMap, DCASettings settings, DiskSpaceManager diskSpaceManager, string applicationInstanceId, IList <string> errorEvents) { // Initialize consumer instance list var consumers = new Dictionary <string, IDcaConsumer>(); foreach (string consumerInstance in settings.ConsumerInstances.Keys) { // Get the consumer instance information DCASettings.ConsumerInstanceInfo consumerInstanceInfo = settings.ConsumerInstances[consumerInstance]; // Prepare the consumer initialization parameters var initParam = new ConsumerInitializationParameters( applicationInstanceId, consumerInstanceInfo.SectionName, Utility.FabricNodeId, Utility.FabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, diskSpaceManager); // if the application is a container move to container log folder. if (ContainerEnvironment.IsContainerApplication(applicationInstanceId)) { initParam = new ConsumerInitializationParameters( applicationInstanceId, consumerInstanceInfo.SectionName, Utility.FabricNodeId, Utility.FabricNodeName, ContainerEnvironment.GetContainerLogFolder(applicationInstanceId), Utility.DcaWorkFolder, diskSpaceManager); } // Create consumer instance IDcaConsumer consumerInterface; try { consumerInterface = consumerFactory.CreateConsumer( consumerInstance, initParam, consumerInstanceInfo.TypeInfo.AssemblyName, consumerInstanceInfo.TypeInfo.TypeName); } catch (Exception e) { // We should continue trying to create other consumers. errorEvents.Add(e.Message); continue; } // Get the consumer's data sink object sink = consumerInterface.GetDataSink(); if (null == sink) { // The consumer does not wish to provide a data sink. // One situation this might happen is if the consumer has been // disabled. This is not an error, so just move on to the next // consumer. continue; } // Add the data sink to the corresponding producer's consumer sink list string producerInstance = consumerInstanceInfo.ProducerInstance; Debug.Assert(false == string.IsNullOrEmpty(producerInstance), "Consumers must be tied to a producer"); if (false == producerConsumerMap.ContainsKey(producerInstance)) { producerConsumerMap[producerInstance] = new List <object>(); } producerConsumerMap[producerInstance].Add(sink); // Add the consumer to the consumer list consumers[consumerInstance] = consumerInterface; } return(consumers); }
internal IDcaConsumer CreateConsumer( string consumerInstance, ConsumerInitializationParameters initParam, string moduleName, string typeName) { // Load the consumer assembly Action <string> reportError = message => { Utility.TraceSource.WriteError(TraceType, message); }; Assembly assembly; try { assembly = Assembly.Load(new AssemblyName(moduleName)); } catch (Exception e) { var message = string.Format( "Exception occured while loading assembly {0} for creating consumer {1}. The consumer type might not be available in the runtime version you are using.", moduleName, consumerInstance); reportError(string.Format("{0} Exception information: {1}.", message, e)); throw new InvalidOperationException(message, e); } // Get the type of the consumer object from the assembly Type type; try { type = assembly.GetType(typeName); if (null == type) { throw new TypeLoadException(string.Format("Could not load type {0} from assembly {1}.", typeName, assembly.FullName)); } } catch (Exception e) { var message = string.Format( "Exception occured while retrieving type {0} in assembly {1} for creating consumer {2}. The consumer type might not be available in the runtime version you are using.", typeName, moduleName, consumerInstance); reportError(string.Format("{0} Exception information: {1}.", message, e)); throw new InvalidOperationException(message, e); } // Create the consumer object object consumerObject; try { #if DotNetCoreClr consumerObject = Activator.CreateInstance(type, new object[] { initParam }); #else consumerObject = Activator.CreateInstance( type, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, null, new object[] { initParam }, null); #endif } catch (Exception e) { var message = string.Format( "Exception occured while creating an object of type {0} (assembly {1}) for creating consumer {2}.", typeName, moduleName, consumerInstance); reportError(string.Format("{0} Exception information: {1}.", message, e)); throw new InvalidOperationException(message, e); } IDcaConsumer consumer; try { consumer = (IDcaConsumer)consumerObject; } catch (InvalidCastException e) { var message = string.Format( "Exception occured while casting an object of type {0} (assembly {1}) to interface IDcaConsumer while creating consumer {2}", typeName, moduleName, consumerInstance); reportError(string.Format("{0} Exception information: {1}.", message, e)); throw new InvalidOperationException(message, e); } Utility.TraceSource.WriteInfo( TraceType, "Created consumer {0}.", consumerInstance); return(consumer); }
public AzureBlobCsvUploader(ConsumerInitializationParameters initParam) { // Initialization this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName); this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA); this.configReader = new AzureBlobConfigReader( new ConfigReader(initParam.ApplicationInstanceId), initParam.SectionName, this.traceSource, this.logSourceId); this.diskSpaceManager = initParam.DiskSpaceManager; this.disposed = false; // Read blob-specific settings from settings.xml this.blobUploadSettings = this.GetSettings(); if (this.configReader.IsReadingFromApplicationManifest) { // Check if we can use an existing upload worker object UploadWorkerKey key = new UploadWorkerKey() { // Destination path is an concatenation of storage account name and container name DestinationPath = string.Concat( this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ? AzureConstants.DevelopmentStorageConnectionString : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, ";", // This separator cannot occur in account name or container name this.blobUploadSettings.LttTraceContainerName), ApplicationType = this.configReader.GetApplicationType(), }; lock (UploadWorkers) { UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key)); if (null != workerInfo) { // Existing upload worker object is available. Increment its // reference count this.traceSource.WriteInfo( this.logSourceId, "Existing upload worker object for application type {0}, Azure storage account {1} and container {2} is available and will be used.", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); (workerInfo.RefCount)++; workerInfo.UploadWorker.UpdateSettings(this.blobUploadSettings); this.uploadWorker = workerInfo.UploadWorker; } else { // Create a new upload worker object this.traceSource.WriteInfo( this.logSourceId, "Creating upload worker object for application type {0}, Azure storage account {1} and container {2} ...", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters() { TraceSource = this.traceSource, FabricNodeId = initParam.FabricNodeId, FabricNodeInstanceName = initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = initParam.LogDirectory, WorkDirectory = initParam.WorkDirectory, UploaderInstanceId = key.ApplicationType, ParentWorkFolderName = key.ApplicationType, Settings = this.blobUploadSettings }; CsvUploadWorker newWorker; try { newWorker = new CsvUploadWorker(param, this.diskSpaceManager); } catch (Exception) { this.traceSource.WriteError( this.logSourceId, "Failed to create upload worker object for application type {0}, Azure storage account {1} and container {2}.", key.ApplicationType, this.blobUploadSettings.StorageAccountFactory.Connection.AccountName, this.blobUploadSettings.LttTraceContainerName); throw; } workerInfo = new UploadWorkerInfo() { Key = key, RefCount = 1, UploadWorker = newWorker }; UploadWorkers.Add(workerInfo); this.uploadWorker = workerInfo.UploadWorker; } } } else { // Create a new upload worker object CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters() { TraceSource = this.traceSource, FabricNodeId = initParam.FabricNodeId, FabricNodeInstanceName = initParam.FabricNodeInstanceName, IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest, LogDirectory = initParam.LogDirectory, WorkDirectory = initParam.WorkDirectory, UploaderInstanceId = this.logSourceId, ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths, Settings = this.blobUploadSettings }; this.uploadWorker = new CsvUploadWorker(param, this.diskSpaceManager); } }
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName)) .Returns("true"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName)) .Returns(logDirectory); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName)) .Returns(TableName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName)) .Returns(MdsFileProducerConstants.DefaultTablePriority); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName)) .Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName)) .Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName)) .Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName)) .Returns(LogFilter); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the MDS consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); var mdsFileProducer = new MdsFileProducer(consumerInitParam); var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer return(new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam)); }
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer( string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory, out EtlInMemoryProducer etlInMemoryProducer, out string containerName, out StorageAccountFactory storageAccountFactory, AccessCondition uploadStreamAccessCondition = null, AccessCondition uploadFileAccessCondition = null) { var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>(); var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings( true, TimeSpan.FromSeconds(1), TimeSpan.FromDays(3000), WinFabricEtlType.DefaultEtl, logDirectory, TestEtlFilePatterns, true); var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>(); configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings); var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>(); mockTraceEventSourceFactory .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>())) .Returns(new ErrorAndWarningFreeTraceEventSource()); var configStore = TestUtility.MockRepository.Create <IConfigStore>(); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB")) .Returns("100"); configStore .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB")) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName)) .Returns("true"); containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid()); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName)) .Returns(containerName); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName)) .Returns("0.25"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName)) .Returns("1"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName)) .Returns("0"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName)) .Returns("*.*:4"); configStore .Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId)) .Returns(AzureConstants.DefaultDeploymentId); var accountKey = GetTestStorageAccountKey(); var isEncrypted = false; var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey); configStore .Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted)) .Returns(storageConnectionString); Utility.InitializeConfigStore(configStore.Object); var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>(); mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>())) .Returns(configReader.Object); // Create the Azure Blob Uploader consumer ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null); var consumerInitParam = new ConsumerInitializationParameters( Utility.WindowsFabricApplicationInstanceId, TestConfigSectionName, TestFabricNodeId, TestFabricNodeName, Utility.LogDirectory, Utility.DcaWorkFolder, new DiskSpaceManager()); // Save storage connection for clean up var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType); storageAccountFactory = azureUtility.GetStorageAccountFactory( new ConfigReader(consumerInitParam.ApplicationInstanceId), TestConfigSectionName, AzureConstants.ConnectionStringParamName); var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition); var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink(); var producerInitParam = new ProducerInitializationParameters { ConsumerSinks = new[] { etlToInMemoryBufferWriter } }; // Create the in-memory producer etlInMemoryProducer = new EtlInMemoryProducer( mockDiskSpaceManager.Object, mockConfigReaderFactory.Object, traceFileEventReaderFactory, mockTraceEventSourceFactory.Object, producerInitParam); }
public AzureTableQueryableEventUploader(ConsumerInitializationParameters initParam) : base(initParam) { }
public AzureTableOperationalEventUploader(ConsumerInitializationParameters initParam) : base(initParam) { }