public AzureBlobFolderUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new AzureBlobConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
this.initParam.SectionName,
this.traceSource,
this.logSourceId);
// Read blob-specific settings from settings.xml
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
// Upload to Azure blob storage is not enabled, so return immediately
return;
}
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobFolderUploader,
accountName,
this.blobUploadSettings.ContainerName);
}
private void PopulateSettings(ConsumerInitializationParameters initParam)
{
var configReader = new ConfigReader(initParam.ApplicationInstanceId);
this.isEnabled = configReader.GetUnencryptedConfigValue(
initParam.SectionName,
SyslogConstant.EnabledParamName,
DefaultEnabledStateValue);
this.logIdentity = configReader.GetUnencryptedConfigValue(
initParam.SectionName,
SyslogConstant.IdentityParamName,
DefaultLogIdentity);
var facilityString = configReader.GetUnencryptedConfigValue(
initParam.SectionName,
SyslogConstant.FacilityParamName,
DefaultLogFacility.ToString());
// It is bound to succeed since the param is already validated to be the right enum type in validator.
this.syslogFacility = (SyslogFacility)Enum.Parse(typeof(SyslogFacility), facilityString, true);
this.traceSource.WriteInfo(
this.logSourceId,
string.Format("SyslogConsumer Populated with these settings. Enabled: {0}, Identity: {1}, Facility: {2}", this.isEnabled, this.logIdentity, this.syslogFacility));
}
public FileShareFolderUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new ConfigReader(initParam.ApplicationInstanceId);
// Read file-share-specific settings from settings.xml
this.GetSettings();
}
public SyslogConsumer(ConsumerInitializationParameters initParam)
{
this.logSourceId = String.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.PopulateSettings(initParam);
if (this.isEnabled)
{
this.traceSource.WriteInfo(this.logSourceId, "SyslogConsumer is Enabled");
this.sysLogWriter = new LocalSysLogger(this.logIdentity, this.syslogFacility);
}
}
private bool CreateMdsUploader(string sectionName, out MdsEtwEventUploader uploader)
{
// Once Uploader is constructed DiskSpaceManager is no longer needed.
using (var dsm = new DiskSpaceManager())
{
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var initParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
sectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
dsm);
uploader = new MdsEtwEventUploader(initParam);
EtwCsvFolder = uploader.EtwCsvFolder;
return(true);
}
}
public void TestMethodSetup()
{
// Initialization
bool result = StartDtrRead.Reset() && DtrReadCompleted.Reset() && EndOfTest.Reset();
Verify.IsTrue(result, "Successfully reset all events at the start of the test");
// Parse the configuration file
this.testStartTime = DateTime.UtcNow;
ParseConfig(testStartTime);
// Create the log and work directories
Utility.LogDirectory = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), LogFolderName);
Utility.TraceSource.WriteInfo(
TraceType,
"Log folder: {0}",
Utility.LogDirectory);
Utility.InitializeWorkDirectory();
// Create the deleted items folder
this.deletedItemsFolder = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), DeletedItemsFolderName);
FabricDirectory.CreateDirectory(deletedItemsFolder);
Utility.TraceSource.WriteInfo(
TraceType,
"Deleted items folder: {0}",
this.deletedItemsFolder);
// Create the MDS upload consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var initParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
this.uploader = new MdsEtwEventUploader(initParam);
EtwCsvFolder = this.uploader.EtwCsvFolder;
}
private AzureTableQueryableEventUploader CreateAndInitializeUploader()
{
const string TestFabricNodeInstanceName = "test";
const string TestLogDirectory = "Logs";
const string TestWorkDirectory = "Work";
// Once Uploader is constructed DiskSpaceManager is no longer needed.
using (var testDiskSpaceManager = new DiskSpaceManager())
{
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
ConsumerInitializationParameters initParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeInstanceName,
TestLogDirectory,
TestWorkDirectory,
testDiskSpaceManager);
return(new AzureTableQueryableEventUploader(initParam));
}
}
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam)
{
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
this.configReader = new AzureBlobEtwConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Read blob-specific settings
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
return;
}
// Create the destination key
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
this.destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create a sub-directory for the blob uploader
this.workFolder = this.CreateBlobUploaderWorkSubDirectory();
if (string.IsNullOrEmpty(this.workFolder))
{
const string Message = "Failed to create work folder for the blob uploader.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from ETL files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
initParam.FabricNodeId,
this.etwLogDirName,
true,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
// Create the helper object that syncs local files to blob storage.
// Local files will be created when upload of compressed memory stream to blob storage fails.
this.fileBlobUploader = new AzureBlobUploader(
this.traceSource,
this.logSourceId,
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
this.fileUploadPerfHelper,
null,
this.uploadFileAccessCondition);
// Create a timer to schedule the upload of local files to blob storage
string timerId = string.Concat(
this.logSourceId,
FileUploadTimerIdSuffix);
this.fileUploadTimer = new DcaTimer(
timerId,
this.UploadFilesToDestinationBlob,
this.blobUploadSettings.FileSyncInterval);
this.fileUploadTimer.Start();
// Initialize trimmer
this.trimmer = new AzureFileTrimmer(
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
AzureUtility.IsAzureInterfaceAvailable());
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwLogDirName);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for Azure blob uploader: {0}",
this.blobUploadSettings.Filter);
}
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam, AccessCondition uploadStreamAccessCondition, AccessCondition uploadFileAccessCondition)
: this(initParam)
{
this.uploadStreamAccessCondition = uploadStreamAccessCondition;
this.uploadFileAccessCondition = uploadFileAccessCondition;
}
public FileShareEtwCsvUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new ConfigReader(initParam.ApplicationInstanceId);
// Read file-share-specific settings from settings.xml
this.GetSettings();
if (false == this.fileUploadSettings.Enabled)
{
// Upload to file share is not enabled, so return immediately
return;
}
if (this.configReader.IsReadingFromApplicationManifest)
{
// Check if we can use an existing upload worker object
UploadWorkerKey key = new UploadWorkerKey()
{
DestinationPath = this.fileUploadSettings.DestinationPath,
ApplicationType = this.configReader.GetApplicationType(),
};
lock (UploadWorkers)
{
UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key));
if (null != workerInfo)
{
// Existing upload worker object is available. Increment its
// reference count
this.traceSource.WriteInfo(
this.logSourceId,
"Existing upload worker object for application type {0} and destination {1} is available and will be used.",
key.ApplicationType,
this.fileUploadSettings.DestinationPath);
workerInfo.RefCount++;
workerInfo.UploadWorker.UpdateSettings(this.fileUploadSettings);
this.uploadWorker = workerInfo.UploadWorker;
}
else
{
// Create a new upload worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating upload worker object for application type {0} and destination {1} ...",
key.ApplicationType,
this.fileUploadSettings.DestinationPath);
EtwCsvUploadWorker.EtwCsvUploadWorkerParameters param = new EtwCsvUploadWorker.EtwCsvUploadWorkerParameters()
{
FabricNodeId = this.initParam.FabricNodeId,
FabricNodeInstanceName = this.initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = this.initParam.LogDirectory,
WorkDirectory = this.initParam.WorkDirectory,
UploaderInstanceId = key.ApplicationType,
ParentWorkFolderName = key.ApplicationType,
Settings = this.fileUploadSettings,
DiskSpaceManager = this.initParam.DiskSpaceManager
};
try
{
EtwCsvUploadWorker newWorker = new EtwCsvUploadWorker(param, this.traceSource);
workerInfo = new UploadWorkerInfo
{
Key = key,
RefCount = 1,
UploadWorker = newWorker
};
UploadWorkers.Add(workerInfo);
this.uploadWorker = workerInfo.UploadWorker;
}
catch (InvalidOperationException e)
{
var message = string.Format(
"Failed to create upload worker object for application type {0} and destination {1}.",
key.ApplicationType,
this.fileUploadSettings.DestinationPath);
this.traceSource.WriteError(
this.logSourceId,
message);
throw new InvalidOperationException(message, e);
}
}
}
}
else
{
// Create a new upload worker object
var param = new EtwCsvUploadWorker.EtwCsvUploadWorkerParameters
{
FabricNodeId = this.initParam.FabricNodeId,
FabricNodeInstanceName = this.initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = this.initParam.LogDirectory,
WorkDirectory = this.initParam.WorkDirectory,
UploaderInstanceId = this.logSourceId,
ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths,
Settings = this.fileUploadSettings,
DiskSpaceManager = initParam.DiskSpaceManager
};
this.uploadWorker = new EtwCsvUploadWorker(param, this.traceSource);
}
}
public MdsFileProducer(ConsumerInitializationParameters initParam)
{
// Initialization
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new ConfigReader(initParam.ApplicationInstanceId);
this.tableWriters = new Dictionary <string, MdsTableWriter>();
this.mdsFileProducerPerf = new MdsFileProducerPerformance();
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
MdsFileProducerConstants.MethodExecutionInitialRetryIntervalMs,
MdsFileProducerConstants.MethodExecutionMaxRetryCount,
MdsFileProducerConstants.MethodExecutionMaxRetryIntervalMs);
// Read MDS-specific settings from settings.xml
this.GetSettings();
if (false == this.mdsFileProducerSettings.Enabled)
{
// MDS tables creation is not enabled, so return immediately
return;
}
string destinationKey = Path.Combine(
this.mdsFileProducerSettings.DirectoryName,
this.mdsFileProducerSettings.TableName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from the ETL
// files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
this.initParam.FabricNodeId,
this.etwLogDirName,
false,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.mdsFileProducerSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
this.traceSource.WriteInfo(
this.logSourceId,
"MDS file producer is configured. Table path: {0}, Table name: {1}, Table priority: {2}, Disk quota (MB): {3}, Data deletion age (minutes): {4}",
this.mdsFileProducerSettings.DirectoryName,
this.mdsFileProducerSettings.TableName,
this.mdsFileProducerSettings.TablePriority,
this.mdsFileProducerSettings.DiskQuotaInMB,
this.mdsFileProducerSettings.DataDeletionAge);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for MDS file producer: {0}",
this.mdsFileProducerSettings.Filter);
}
public AzureTableEtwEventUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.stopping = false;
this.initParam = initParam;
this.logSourceId = String.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new ConfigReader(initParam.ApplicationInstanceId);
this.azureUtility = new AzureUtility(this.traceSource, this.logSourceId);
this.perfHelper = new AzureTablePerformance(this.traceSource, this.logSourceId);
// Make sure that the Azure interfaces are available
if (false == AzureUtility.IsAzureInterfaceAvailable())
{
const string Message = "Due to unavailability of Azure interfaces, ETW traces will not be uploaded to Azure table storage.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
this.azureNodeInstanceId = AzureUtility.RoleInstanceId;
// Read table-specific settings from settings.xml
GetSettings();
if (false == this.tableUploadSettings.Enabled)
{
// Upload to Azure table storage is not enabled, so return immediately
return;
}
// Create a sub-directory for ourselves under the log directory
string bufferedEventFolder = GetBufferedEventSubDirectory();
if (String.IsNullOrEmpty(bufferedEventFolder))
{
throw new InvalidOperationException("Unable to get buffered event subdirectory.");
}
// Create the helper object that buffers events delivered from the ETL
// files into CSV files on disk.
this.bufferedEventProvider = new BufferedEtwEventProvider(
new TraceEventSourceFactory(),
this.logSourceId,
bufferedEventFolder,
this.tableUploadSettings.UploadIntervalMinutes,
this.tableUploadSettings.EntityDeletionAge,
this);
if (null == this.bufferedEventProvider)
{
const string Message = "Failed to create buffered event provider helper object.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Set the filter for Windows Fabric events
this.bufferedEventProvider.SetEtwEventFilter(
this.tableUploadSettings.Filter,
defaultTableFilter,
WinFabSummaryFilter.StringRepresentation,
true);
// Initialize the batch upload concurrency count
Debug.Assert(this.tableUploadSettings.BatchUploadConcurrencyCount <= AzureConstants.MaxBatchConcurrencyCount);
if (this.tableUploadSettings.BatchUploadConcurrencyCount <= AzureConstants.MaxBatchConcurrencyCount)
{
this.batchConcurrencyCount = this.tableUploadSettings.BatchUploadConcurrencyCount;
}
else
{
this.traceSource.WriteError(
this.logSourceId,
"{0} is an invalid value for table batch concurrency count. The maximum supported value is {1} and that value will be used instead.",
this.tableUploadSettings.BatchUploadConcurrencyCount,
AzureConstants.MaxBatchConcurrencyCount);
this.batchConcurrencyCount = AzureConstants.MaxBatchConcurrencyCount;
}
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.TableUpload,
this.batchConcurrencyCount);
// Create the table
try
{
CreateTable();
}
catch (Exception e)
{
const string Message =
"Due to an error in table creation ETW traces will not be uploaded to Azure table storage.";
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
Message);
throw new InvalidOperationException(Message, e);
}
this.traceSource.WriteInfo(
this.logSourceId,
"Created table for uploading ETW traces. Storage account: {0}, Table name: {1}",
this.tableUploadSettings.StorageAccountFactory.Connection.AccountName,
this.tableUploadSettings.TableName);
// Initialize old log deletion
this.trimmer = new AzureTableTrimmer(
this.traceSource,
this.logSourceId,
this.tableUploadSettings.StorageAccountFactory,
this.tableUploadSettings.TableName,
this.tableUploadSettings.EntityDeletionAge,
this.CreateDeletionQuery,
this.perfHelper);
}
private static Dictionary <string, IDcaConsumer> CreateConsumers(
ConsumerFactory consumerFactory,
IDictionary <string, List <object> > producerConsumerMap,
DCASettings settings,
DiskSpaceManager diskSpaceManager,
string applicationInstanceId,
IList <string> errorEvents)
{
// Initialize consumer instance list
var consumers = new Dictionary <string, IDcaConsumer>();
foreach (string consumerInstance in settings.ConsumerInstances.Keys)
{
// Get the consumer instance information
DCASettings.ConsumerInstanceInfo consumerInstanceInfo = settings.ConsumerInstances[consumerInstance];
// Prepare the consumer initialization parameters
var initParam = new ConsumerInitializationParameters(
applicationInstanceId,
consumerInstanceInfo.SectionName,
Utility.FabricNodeId,
Utility.FabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
diskSpaceManager);
// if the application is a container move to container log folder.
if (ContainerEnvironment.IsContainerApplication(applicationInstanceId))
{
initParam = new ConsumerInitializationParameters(
applicationInstanceId,
consumerInstanceInfo.SectionName,
Utility.FabricNodeId,
Utility.FabricNodeName,
ContainerEnvironment.GetContainerLogFolder(applicationInstanceId),
Utility.DcaWorkFolder,
diskSpaceManager);
}
// Create consumer instance
IDcaConsumer consumerInterface;
try
{
consumerInterface = consumerFactory.CreateConsumer(
consumerInstance,
initParam,
consumerInstanceInfo.TypeInfo.AssemblyName,
consumerInstanceInfo.TypeInfo.TypeName);
}
catch (Exception e)
{
// We should continue trying to create other consumers.
errorEvents.Add(e.Message);
continue;
}
// Get the consumer's data sink
object sink = consumerInterface.GetDataSink();
if (null == sink)
{
// The consumer does not wish to provide a data sink.
// One situation this might happen is if the consumer has been
// disabled. This is not an error, so just move on to the next
// consumer.
continue;
}
// Add the data sink to the corresponding producer's consumer sink list
string producerInstance = consumerInstanceInfo.ProducerInstance;
Debug.Assert(false == string.IsNullOrEmpty(producerInstance), "Consumers must be tied to a producer");
if (false == producerConsumerMap.ContainsKey(producerInstance))
{
producerConsumerMap[producerInstance] = new List <object>();
}
producerConsumerMap[producerInstance].Add(sink);
// Add the consumer to the consumer list
consumers[consumerInstance] = consumerInterface;
}
return(consumers);
}
internal IDcaConsumer CreateConsumer(
string consumerInstance,
ConsumerInitializationParameters initParam,
string moduleName,
string typeName)
{
// Load the consumer assembly
Action <string> reportError = message =>
{
Utility.TraceSource.WriteError(TraceType, message);
};
Assembly assembly;
try
{
assembly = Assembly.Load(new AssemblyName(moduleName));
}
catch (Exception e)
{
var message = string.Format(
"Exception occured while loading assembly {0} for creating consumer {1}. The consumer type might not be available in the runtime version you are using.",
moduleName,
consumerInstance);
reportError(string.Format("{0} Exception information: {1}.", message, e));
throw new InvalidOperationException(message, e);
}
// Get the type of the consumer object from the assembly
Type type;
try
{
type = assembly.GetType(typeName);
if (null == type)
{
throw new TypeLoadException(string.Format("Could not load type {0} from assembly {1}.", typeName, assembly.FullName));
}
}
catch (Exception e)
{
var message = string.Format(
"Exception occured while retrieving type {0} in assembly {1} for creating consumer {2}. The consumer type might not be available in the runtime version you are using.",
typeName,
moduleName,
consumerInstance);
reportError(string.Format("{0} Exception information: {1}.", message, e));
throw new InvalidOperationException(message, e);
}
// Create the consumer object
object consumerObject;
try
{
#if DotNetCoreClr
consumerObject = Activator.CreateInstance(type, new object[] { initParam });
#else
consumerObject = Activator.CreateInstance(
type,
BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance,
null,
new object[] { initParam },
null);
#endif
}
catch (Exception e)
{
var message = string.Format(
"Exception occured while creating an object of type {0} (assembly {1}) for creating consumer {2}.",
typeName,
moduleName,
consumerInstance);
reportError(string.Format("{0} Exception information: {1}.", message, e));
throw new InvalidOperationException(message, e);
}
IDcaConsumer consumer;
try
{
consumer = (IDcaConsumer)consumerObject;
}
catch (InvalidCastException e)
{
var message = string.Format(
"Exception occured while casting an object of type {0} (assembly {1}) to interface IDcaConsumer while creating consumer {2}",
typeName,
moduleName,
consumerInstance);
reportError(string.Format("{0} Exception information: {1}.", message, e));
throw new InvalidOperationException(message, e);
}
Utility.TraceSource.WriteInfo(
TraceType,
"Created consumer {0}.",
consumerInstance);
return(consumer);
}
public AzureBlobCsvUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new AzureBlobConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.diskSpaceManager = initParam.DiskSpaceManager;
this.disposed = false;
// Read blob-specific settings from settings.xml
this.blobUploadSettings = this.GetSettings();
if (this.configReader.IsReadingFromApplicationManifest)
{
// Check if we can use an existing upload worker object
UploadWorkerKey key = new UploadWorkerKey()
{
// Destination path is an concatenation of storage account name and container name
DestinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ?
AzureConstants.DevelopmentStorageConnectionString :
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.LttTraceContainerName),
ApplicationType = this.configReader.GetApplicationType(),
};
lock (UploadWorkers)
{
UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key));
if (null != workerInfo)
{
// Existing upload worker object is available. Increment its
// reference count
this.traceSource.WriteInfo(
this.logSourceId,
"Existing upload worker object for application type {0}, Azure storage account {1} and container {2} is available and will be used.",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
(workerInfo.RefCount)++;
workerInfo.UploadWorker.UpdateSettings(this.blobUploadSettings);
this.uploadWorker = workerInfo.UploadWorker;
}
else
{
// Create a new upload worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating upload worker object for application type {0}, Azure storage account {1} and container {2} ...",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters()
{
TraceSource = this.traceSource,
FabricNodeId = initParam.FabricNodeId,
FabricNodeInstanceName = initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = initParam.LogDirectory,
WorkDirectory = initParam.WorkDirectory,
UploaderInstanceId = key.ApplicationType,
ParentWorkFolderName = key.ApplicationType,
Settings = this.blobUploadSettings
};
CsvUploadWorker newWorker;
try
{
newWorker = new CsvUploadWorker(param, this.diskSpaceManager);
}
catch (Exception)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create upload worker object for application type {0}, Azure storage account {1} and container {2}.",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
throw;
}
workerInfo = new UploadWorkerInfo()
{
Key = key,
RefCount = 1,
UploadWorker = newWorker
};
UploadWorkers.Add(workerInfo);
this.uploadWorker = workerInfo.UploadWorker;
}
}
}
else
{
// Create a new upload worker object
CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters()
{
TraceSource = this.traceSource,
FabricNodeId = initParam.FabricNodeId,
FabricNodeInstanceName = initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = initParam.LogDirectory,
WorkDirectory = initParam.WorkDirectory,
UploaderInstanceId = this.logSourceId,
ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths,
Settings = this.blobUploadSettings
};
this.uploadWorker = new CsvUploadWorker(param, this.diskSpaceManager);
}
}
private static EtlInMemoryProducer CreateEtlInMemoryProducerWithMDSConsumer(string logDirectory, ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.EnabledParamName))
.Returns("true");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DirectoryParamName))
.Returns(logDirectory);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TableParamName))
.Returns(TableName);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TablePriorityParamName))
.Returns(MdsFileProducerConstants.DefaultTablePriority);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DiskQuotaParamName))
.Returns(MdsFileProducerConstants.DefaultDiskQuotaInMB.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.BookmarkBatchSizeParamName))
.Returns(MdsFileProducerConstants.DefaultBookmarkBatchSize.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.DataDeletionAgeParamName))
.Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.TestDataDeletionAgeParamName))
.Returns(MdsFileProducerConstants.DefaultDataDeletionAge.TotalDays.ToString());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, MdsFileProducerConstants.LogFilterParamName))
.Returns(LogFilter);
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
// Create the MDS consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var consumerInitParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
var mdsFileProducer = new MdsFileProducer(consumerInitParam);
var etlToInMemoryBufferWriter = mdsFileProducer.GetDataSink();
var producerInitParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { etlToInMemoryBufferWriter }
};
// Create the in-memory producer
return(new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
producerInitParam));
}
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
out EtlInMemoryProducer etlInMemoryProducer,
out string containerName,
out StorageAccountFactory storageAccountFactory,
AccessCondition uploadStreamAccessCondition = null,
AccessCondition uploadFileAccessCondition = null)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName))
.Returns("true");
containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName))
.Returns(containerName);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName))
.Returns("0.25");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName))
.Returns("1");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName))
.Returns("*.*:4");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId))
.Returns(AzureConstants.DefaultDeploymentId);
var accountKey = GetTestStorageAccountKey();
var isEncrypted = false;
var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey);
configStore
.Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted))
.Returns(storageConnectionString);
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
// Create the Azure Blob Uploader consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var consumerInitParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
// Save storage connection for clean up
var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType);
storageAccountFactory = azureUtility.GetStorageAccountFactory(
new ConfigReader(consumerInitParam.ApplicationInstanceId),
TestConfigSectionName,
AzureConstants.ConnectionStringParamName);
var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition);
var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink();
var producerInitParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { etlToInMemoryBufferWriter }
};
// Create the in-memory producer
etlInMemoryProducer = new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
producerInitParam);
}
public AzureTableQueryableEventUploader(ConsumerInitializationParameters initParam)
: base(initParam)
{
}
public AzureTableOperationalEventUploader(ConsumerInitializationParameters initParam)
: base(initParam)
{
}
