public AzureBlobFolderUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = string.Concat(this.initParam.ApplicationInstanceId, "_", this.initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new AzureBlobConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
this.initParam.SectionName,
this.traceSource,
this.logSourceId);
// Read blob-specific settings from settings.xml
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
// Upload to Azure blob storage is not enabled, so return immediately
return;
}
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobFolderUploader,
accountName,
this.blobUploadSettings.ContainerName);
}
private BlobUploadSettings GetSettings()
{
// Check for values in settings.xml
var blobUploadSettings = new BlobUploadSettings();
blobUploadSettings.Enabled = this.configReader.GetEnabled();
if (!blobUploadSettings.Enabled)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Blob storage upload not enabled");
return(blobUploadSettings);
}
blobUploadSettings.StorageAccountFactory = this.configReader.GetStorageAccountFactory();
if (blobUploadSettings.StorageAccountFactory == null)
{
blobUploadSettings.Enabled = false;
return(blobUploadSettings);
}
blobUploadSettings.LttTraceContainerName = this.configReader.GetEtwTraceContainerName();
blobUploadSettings.UploadInterval = this.configReader.GetUploadInterval();
blobUploadSettings.FileSyncInterval = this.configReader.GetFileSyncInterval();
blobUploadSettings.BlobDeletionAge = this.configReader.GetDataDeletionAge();
blobUploadSettings.Filter = this.configReader.GetFilters();
blobUploadSettings.DeploymentId = this.configReader.GetDeploymentId();
// Write settings to log
this.traceSource.WriteInfo(
this.logSourceId,
"Blob storage upload enabled: ETW trace container: {0}, Upload interval (minutes): {1}, Blob deletion age: {2}",
blobUploadSettings.LttTraceContainerName,
blobUploadSettings.UploadInterval,
blobUploadSettings.BlobDeletionAge);
return(blobUploadSettings);
}
public AzureBlobEtwUploader(ConsumerInitializationParameters initParam)
{
this.stopping = false;
this.initParam = initParam;
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.progressManager = new ConsumerProgressManager(
this.traceSource,
this.logSourceId,
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
this.configReader = new AzureBlobEtwConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.streamUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
this.fileUploadPerfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Read blob-specific settings
this.blobUploadSettings = this.GetSettings();
if (false == this.blobUploadSettings.Enabled)
{
return;
}
// Create the destination key
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// initialize bookmark folders and files
var initializeBookmarkFoldersAndFilesSuccess = this.progressManager.InitializeBookmarkFoldersAndFiles(
this.initParam.WorkDirectory,
this.destinationKey);
if (false == initializeBookmarkFoldersAndFilesSuccess)
{
const string Message = "Failed to initialize bookmark folders and files.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create etw log directory
this.etwLogDirName = this.CreateEtwLogDirectory();
if (string.IsNullOrEmpty(this.etwLogDirName))
{
const string Message = "Failed to create etw log directory.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create a sub-directory for the blob uploader
this.workFolder = this.CreateBlobUploaderWorkSubDirectory();
if (string.IsNullOrEmpty(this.workFolder))
{
const string Message = "Failed to create work folder for the blob uploader.";
this.traceSource.WriteError(
this.logSourceId,
Message);
throw new InvalidOperationException(Message);
}
// Create the helper object that writes events delivered from ETL files into an in-memory buffer.
this.etlToInMemoryBufferWriter = new EtlToInMemoryBufferWriter(
new TraceEventSourceFactory(),
this.logSourceId,
initParam.FabricNodeId,
this.etwLogDirName,
true,
this);
// Set the event filter
this.etlToInMemoryBufferWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
WinFabDefaultFilter.StringRepresentation,
WinFabSummaryFilter.StringRepresentation,
true);
// Create the helper object that syncs local files to blob storage.
// Local files will be created when upload of compressed memory stream to blob storage fails.
this.fileBlobUploader = new AzureBlobUploader(
this.traceSource,
this.logSourceId,
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
this.fileUploadPerfHelper,
null,
this.uploadFileAccessCondition);
// Create a timer to schedule the upload of local files to blob storage
string timerId = string.Concat(
this.logSourceId,
FileUploadTimerIdSuffix);
this.fileUploadTimer = new DcaTimer(
timerId,
this.UploadFilesToDestinationBlob,
this.blobUploadSettings.FileSyncInterval);
this.fileUploadTimer.Start();
// Initialize trimmer
this.trimmer = new AzureFileTrimmer(
this.etwLogDirName,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId,
AzureUtility.IsAzureInterfaceAvailable());
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace path: {2}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwLogDirName);
this.traceSource.WriteInfo(
this.logSourceId,
"Windows Fabric event filters for Azure blob uploader: {0}",
this.blobUploadSettings.Filter);
}
public AzureBlobCsvUploader(ConsumerInitializationParameters initParam)
{
// Initialization
this.logSourceId = string.Concat(initParam.ApplicationInstanceId, "_", initParam.SectionName);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.configReader = new AzureBlobConfigReader(
new ConfigReader(initParam.ApplicationInstanceId),
initParam.SectionName,
this.traceSource,
this.logSourceId);
this.diskSpaceManager = initParam.DiskSpaceManager;
this.disposed = false;
// Read blob-specific settings from settings.xml
this.blobUploadSettings = this.GetSettings();
if (this.configReader.IsReadingFromApplicationManifest)
{
// Check if we can use an existing upload worker object
UploadWorkerKey key = new UploadWorkerKey()
{
// Destination path is an concatenation of storage account name and container name
DestinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ?
AzureConstants.DevelopmentStorageConnectionString :
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.LttTraceContainerName),
ApplicationType = this.configReader.GetApplicationType(),
};
lock (UploadWorkers)
{
UploadWorkerInfo workerInfo = UploadWorkers.FirstOrDefault(w => w.Matches(key));
if (null != workerInfo)
{
// Existing upload worker object is available. Increment its
// reference count
this.traceSource.WriteInfo(
this.logSourceId,
"Existing upload worker object for application type {0}, Azure storage account {1} and container {2} is available and will be used.",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
(workerInfo.RefCount)++;
workerInfo.UploadWorker.UpdateSettings(this.blobUploadSettings);
this.uploadWorker = workerInfo.UploadWorker;
}
else
{
// Create a new upload worker object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating upload worker object for application type {0}, Azure storage account {1} and container {2} ...",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters()
{
TraceSource = this.traceSource,
FabricNodeId = initParam.FabricNodeId,
FabricNodeInstanceName = initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = initParam.LogDirectory,
WorkDirectory = initParam.WorkDirectory,
UploaderInstanceId = key.ApplicationType,
ParentWorkFolderName = key.ApplicationType,
Settings = this.blobUploadSettings
};
CsvUploadWorker newWorker;
try
{
newWorker = new CsvUploadWorker(param, this.diskSpaceManager);
}
catch (Exception)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create upload worker object for application type {0}, Azure storage account {1} and container {2}.",
key.ApplicationType,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName);
throw;
}
workerInfo = new UploadWorkerInfo()
{
Key = key,
RefCount = 1,
UploadWorker = newWorker
};
UploadWorkers.Add(workerInfo);
this.uploadWorker = workerInfo.UploadWorker;
}
}
}
else
{
// Create a new upload worker object
CsvUploadWorker.CsvUploadWorkerParameters param = new CsvUploadWorker.CsvUploadWorkerParameters()
{
TraceSource = this.traceSource,
FabricNodeId = initParam.FabricNodeId,
FabricNodeInstanceName = initParam.FabricNodeInstanceName,
IsReadingFromApplicationManifest = this.configReader.IsReadingFromApplicationManifest,
LogDirectory = initParam.LogDirectory,
WorkDirectory = initParam.WorkDirectory,
UploaderInstanceId = this.logSourceId,
ParentWorkFolderName = Utility.ShortWindowsFabricIdForPaths,
Settings = this.blobUploadSettings
};
this.uploadWorker = new CsvUploadWorker(param, this.diskSpaceManager);
}
}
