Exemplo n.º 1
0
        private static EtlInMemoryProducer CreateEtlInMemoryProducerWithFastAndSlowConsumerForTest(
            string logDirectory,
            ITraceFileEventReaderFactory traceFileEventReaderFactory)
        {
            var mockDiskSpaceManager        = TestUtility.MockRepository.Create <DiskSpaceManager>();
            var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
                true,
                TimeSpan.FromSeconds(1),
                TimeSpan.FromDays(3000),
                WinFabricEtlType.DefaultEtl,
                logDirectory,
                TestUtility.TestEtlFilePatterns,
                true);

            var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();

            configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);

            var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();

            mockTraceEventSourceFactory
            .Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
            .Returns(new ErrorAndWarningFreeTraceEventSource());

            var configStore = TestUtility.MockRepository.Create <IConfigStore>();

            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
            .Returns("100");
            configStore
            .Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
            .Returns("0");
            Utility.InitializeConfigStore(configStore.Object);

            var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();

            mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
            .Returns(configReader.Object);

            var fastDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, false);
            var slowDcaInMemoryConsumer = CreateEtlToInMemoryBufferWriter(logDirectory, true);

            var fastEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                mockTraceEventSourceFactory.Object,
                LogSourceId,
                TestNodeId,
                TestUtility.GetActualOutputFolderPath(logDirectory),
                true,
                fastDcaInMemoryConsumer);

            var slowEtlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                mockTraceEventSourceFactory.Object,
                LogSourceId,
                TestNodeId,
                TestUtility.GetActualOutputFolderPath(logDirectory),
                true,
                slowDcaInMemoryConsumer);

            var initParam = new ProducerInitializationParameters
            {
                ConsumerSinks = new[] { fastEtlToInMemoryBufferWriter, slowEtlToInMemoryBufferWriter }
            };

            return(new EtlInMemoryProducer(
                       mockDiskSpaceManager.Object,
                       mockConfigReaderFactory.Object,
                       traceFileEventReaderFactory,
                       mockTraceEventSourceFactory.Object,
                       initParam));
        }
Exemplo n.º 2
0
        public AzureBlobEtwUploader(ConsumerInitializationParameters initParam)
        {
            this.stopping        = false;
            this.initParam       = initParam;
            this.logSourceId     = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
            this.traceSource     = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
            this.progressManager = new ConsumerProgressManager(
                this.traceSource,
                this.logSourceId,
                AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
                AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
                AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);

            this.configReader = new AzureBlobEtwConfigReader(
                new ConfigReader(initParam.ApplicationInstanceId),
                initParam.SectionName,
                this.traceSource,
                this.logSourceId);

            this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
            this.fileUploadPerfHelper   = new AzureBlobPerformance(this.traceSource, this.logSourceId);

            // Read blob-specific settings
            this.blobUploadSettings = this.GetSettings();
            if (false == this.blobUploadSettings.Enabled)
            {
                return;
            }

            // Create the destination key
            var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
               ? AzureConstants.DevelopmentStorageConnectionString
               : this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;

            this.destinationKey = string.Join(
                "_",
                StandardPluginTypes.AzureBlobEtwUploader,
                accountName,
                this.blobUploadSettings.EtwTraceContainerName);

            // initialize bookmark folders and files
            var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
                this.initParam.WorkDirectory,
                this.destinationKey);

            if (false == initializeBookmarkFoldersAndFilesSuccess)
            {
                const string Message = "Failed to initialize bookmark folders and files.";
                this.traceSource.WriteError(
                    this.logSourceId,
                    Message);
                throw new InvalidOperationException(Message);
            }

            // Create etw log directory
            this.etwLogDirName = this.CreateEtwLogDirectory();
            if (string.IsNullOrEmpty(this.etwLogDirName))
            {
                const string Message = "Failed to create etw log directory.";
                this.traceSource.WriteError(
                    this.logSourceId,
                    Message);
                throw new InvalidOperationException(Message);
            }

            // Create a sub-directory for the blob uploader
            this.workFolder = this.CreateBlobUploaderWorkSubDirectory();
            if (string.IsNullOrEmpty(this.workFolder))
            {
                const string Message = "Failed to create work folder for the blob uploader.";
                this.traceSource.WriteError(
                    this.logSourceId,
                    Message);
                throw new InvalidOperationException(Message);
            }

            // Create the helper object that writes events delivered from ETL files into an in-memory buffer.
            this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                new TraceEventSourceFactory(),
                this.logSourceId,
                initParam.FabricNodeId,
                this.etwLogDirName,
                true,
                this);

            // Set the event filter
            this.etlToInMemoryBufferWriter.SetEtwEventFilter(
                this.blobUploadSettings.Filter,
                WinFabDefaultFilter.StringRepresentation,
                WinFabSummaryFilter.StringRepresentation,
                true);

            // Create the helper object that syncs local files to blob storage.
            // Local files will be created when upload of compressed memory stream to blob storage fails.
            this.fileBlobUploader = new AzureBlobUploader(
                this.traceSource,
                this.logSourceId,
                this.etwLogDirName,
                this.workFolder,
                this.blobUploadSettings.StorageAccountFactory,
                this.blobUploadSettings.EtwTraceContainerName,
                this.initParam.FabricNodeInstanceName,
                this.blobUploadSettings.DeploymentId,
                this.fileUploadPerfHelper,
                null,
                this.uploadFileAccessCondition);

            // Create a timer to schedule the upload of local files to blob storage
            string timerId = string.Concat(
                this.logSourceId,
                FileUploadTimerIdSuffix);

            this.fileUploadTimer = new DcaTimer(
                timerId,
                this.UploadFilesToDestinationBlob,
                this.blobUploadSettings.FileSyncInterval);

            this.fileUploadTimer.Start();

            // Initialize trimmer
            this.trimmer = new AzureFileTrimmer(
                this.etwLogDirName,
                this.workFolder,
                this.blobUploadSettings.StorageAccountFactory,
                this.blobUploadSettings.EtwTraceContainerName,
                this.blobUploadSettings.BlobDeletionAge,
                this.initParam.FabricNodeInstanceName,
                this.blobUploadSettings.DeploymentId,
                AzureUtility.IsAzureInterfaceAvailable());

            this.traceSource.WriteInfo(
                this.logSourceId,
                "Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}",
                this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
                this.blobUploadSettings.EtwTraceContainerName,
                this.etwLogDirName);

            this.traceSource.WriteInfo(
                this.logSourceId,
                "Windows Fabric event filters for Azure blob uploader: {0}",
                this.blobUploadSettings.Filter);
        }
Exemplo n.º 3
0
        public MdsFileProducer(ConsumerInitializationParameters initParam)
        {
            // Initialization
            this.stopping            = false;
            this.initParam           = initParam;
            this.logSourceId         = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
            this.traceSource         = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
            this.configReader        = new ConfigReader(initParam.ApplicationInstanceId);
            this.tableWriters        = new Dictionary <string, MdsTableWriter>();
            this.mdsFileProducerPerf = new MdsFileProducerPerformance();
            this.progressManager     = new ConsumerProgressManager(
                this.traceSource,
                this.logSourceId,
                MdsFileProducerConstants.MethodExecutionInitialRetryIntervalMs,
                MdsFileProducerConstants.MethodExecutionMaxRetryCount,
                MdsFileProducerConstants.MethodExecutionMaxRetryIntervalMs);

            // Read MDS-specific settings from settings.xml
            this.GetSettings();
            if (false == this.mdsFileProducerSettings.Enabled)
            {
                // MDS tables creation is not enabled, so return immediately
                return;
            }

            string destinationKey = Path.Combine(
                this.mdsFileProducerSettings.DirectoryName,
                this.mdsFileProducerSettings.TableName);

            // initialize bookmark folders and files
            var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
                this.initParam.WorkDirectory,
                destinationKey);

            if (false == initializeBookmarkFoldersAndFilesSuccess)
            {
                const string Message = "Failed to initialize bookmark folders and files.";
                this.traceSource.WriteError(
                    this.logSourceId,
                    Message);
                throw new InvalidOperationException(Message);
            }

            // Create etw log directory
            this.etwLogDirName = this.CreateEtwLogDirectory();
            if (string.IsNullOrEmpty(this.etwLogDirName))
            {
                const string Message = "Failed to create etw log directory.";
                this.traceSource.WriteError(
                    this.logSourceId,
                    Message);
                throw new InvalidOperationException(Message);
            }

            // Create the helper object that writes events delivered from the ETL
            // files into an in-memory buffer.
            this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
                new TraceEventSourceFactory(),
                this.logSourceId,
                this.initParam.FabricNodeId,
                this.etwLogDirName,
                false,
                this);

            // Set the event filter
            this.etlToInMemoryBufferWriter.SetEtwEventFilter(
                this.mdsFileProducerSettings.Filter,
                WinFabDefaultFilter.StringRepresentation,
                WinFabSummaryFilter.StringRepresentation,
                true);

            this.traceSource.WriteInfo(
                this.logSourceId,
                "MDS file producer is configured. Table path: {0}, Table name: {1}, Table priority: {2}, Disk quota (MB): {3}, Data deletion age (minutes): {4}",
                this.mdsFileProducerSettings.DirectoryName,
                this.mdsFileProducerSettings.TableName,
                this.mdsFileProducerSettings.TablePriority,
                this.mdsFileProducerSettings.DiskQuotaInMB,
                this.mdsFileProducerSettings.DataDeletionAge);

            this.traceSource.WriteInfo(
                this.logSourceId,
                "Windows Fabric event filters for MDS file producer: {0}",
                this.mdsFileProducerSettings.Filter);
        }