private void StopUploaderForWinFabFolder(AzureFileUploader uploader)
{
if (null == uploader)
{
return;
}
uploader.Dispose();
}
internal void UpdateSettings(AzureBlobCsvUploader.BlobUploadSettings newSettings)
{
bool updateFilter = ShouldUpdateFilter(newSettings);
bool updateUploader = ShouldUpdateUploader(newSettings);
if (updateFilter || updateUploader)
{
this.blobUploadSettings = newSettings;
}
if (updateUploader)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Due to settings change, the uploader will be stopped and restarted.");
// Stop the uploader
if (null != this.uploader)
{
this.uploader.Dispose();
}
// Restart the upload with the new settings
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.LttTraceContainerName);
this.uploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
this.csvFolder,
destinationPath,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.LttTraceContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
this.uploader.Start();
}
catch (Exception ex)
{
throw new InvalidOperationException("AzureFileUploader could not be constructed.", ex);
}
}
}
private bool CreateUploaderForWinFabFolder(
string source,
string workFolder,
out AzureFileUploader uploader)
{
uploader = null;
// Create and initialize the uploader
//
// NOTE: We do not make any assumptions about the contents of general
// data folders. Hence, as part of old data deletion, we attempt to
// delete all blobs in the corresponding container, without trying to
// filter blobs by Fabric node ID. We do this by specifying 'false'
// for the 'filterDeletionByNodeId' parameter.
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.ContainerName);
AzureFileUploader newUploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
source,
destinationPath,
workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.ContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
newUploader.Start();
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Container: {1}, Local data Path: {2}, Upload interval (minutes): {3}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName,
source,
this.blobUploadSettings.UploadInterval);
uploader = newUploader;
return(true);
}
catch (Exception)
{
return(false);
}
}
private void StopUploaderForApplicationFolder(AzureFileUploader uploader)
{
if (null == uploader)
{
return;
}
lock (AllUploaders)
{
// Drop the reference count on the uploader object
UploaderInfo uploaderInfo = AllUploaders.FirstOrDefault(w => w.Uploader.Equals(uploader));
uploaderInfo.RefCount--;
if (0 == uploaderInfo.RefCount)
{
// Tell the uploader object to stop
this.traceSource.WriteInfo(
this.logSourceId,
"Stopping uploader object for application type {0}, source {1} and destination {2} ...",
uploaderInfo.Key.ApplicationType,
uploaderInfo.Key.SourcePath,
uploaderInfo.Key.DestinationPath);
uploaderInfo.Uploader.Dispose();
AllUploaders.Remove(uploaderInfo);
}
else
{
this.traceSource.WriteInfo(
this.logSourceId,
"Uploader object for application type {0}, source {1}, storage account {2}, container {3} is still in use by other uploaders, so let it continue running",
uploaderInfo.Key.ApplicationType,
uploaderInfo.Key.SourcePath,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName);
}
}
}
private bool CreateUploaderForApplicationFolder(
string uploaderId,
string source,
string workFolder,
out AzureFileUploader uploader)
{
uploader = null;
bool success = true;
AzureFileUploader newUploader;
// Check if we can use an existing uploader object
UploaderKey key = new UploaderKey()
{
SourcePath = source,
// Destination path is an concatenation of storage account name and container name
DestinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ?
AzureConstants.DevelopmentStorageConnectionString :
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.ContainerName),
ApplicationType = this.configReader.GetApplicationType(),
};
lock (AllUploaders)
{
UploaderInfo uploaderInfo = AllUploaders.FirstOrDefault(w => w.Matches(key));
if (null != uploaderInfo)
{
// Existing uploader object is available. Increment its reference count
this.traceSource.WriteInfo(
this.logSourceId,
"Existing uploader object for application type {0}, source {1}, storage account {2}, container {3} is available and will be used.",
key.ApplicationType,
source,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName);
uploaderInfo.RefCount++;
newUploader = uploaderInfo.Uploader;
success = true;
}
else
{
// Create a new uploader object
this.traceSource.WriteInfo(
this.logSourceId,
"Creating uploader object for application type {0}, source {1}, storage account {2}, container {3} ...",
key.ApplicationType,
source,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName);
// Create and initialize the uploader
//
// NOTE: We do not make any assumptions about the contents of general
// data folders. Hence, as part of old data deletion, we attempt to
// delete all files in the corresponding container, without trying to
// filter files by Fabric node ID. We do this by specifying 'false'
// for the 'filterDeletionByNodeId' parameter.
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.ContainerName);
newUploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
source,
destinationPath,
workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.ContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
newUploader.Start();
uploaderInfo = new UploaderInfo()
{
Key = key,
RefCount = 1,
Uploader = newUploader
};
AllUploaders.Add(uploaderInfo);
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to file share is configured. Storage account: {0}, Container: {1}, Local folder path: {2}, Upload interval (minutes): {3}.",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName,
source,
this.blobUploadSettings.UploadInterval);
uploader = newUploader;
}
catch (Exception)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create uploader object for application type {0}, source {1}, storage account {2}, container {3}.",
key.ApplicationType,
source,
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.ContainerName);
}
}
}
return(success);
}
internal CsvUploadWorker(CsvUploadWorkerParameters initParam, DiskSpaceManager diskSpaceManager)
{
// Initialization
this.FlushDataOnDispose = false;
this.initParam = initParam;
this.logSourceId = this.initParam.UploaderInstanceId;
this.traceSource = this.initParam.TraceSource;
this.blobUploadSettings = this.initParam.Settings;
this.azureUtility = new AzureUtility(this.traceSource, this.logSourceId);
this.destinationKey = String.Join(
"_",
new string[]
{
StandardPluginTypes.AzureBlobCsvUploader,
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage ?
AzureConstants.DevelopmentStorageConnectionString :
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName
});
this.disposed = false;
// Create a sub-directory for ourselves under the log directory
bool success = GetCsvSubDirectory();
string sourceDirectory = Path.Combine(initParam.LogDirectory, "Traces");
// Create the helper object that writes events delivered from the LTT
// files into CSV files.
if (success)
{
this.csvToUploadFolderWriter = new CsvToUploadFolderWriter(
this.logSourceId,
this.initParam.FabricNodeId,
this.csvFolder,
sourceDirectory,
diskSpaceManager,
false);
if (null == this.csvToUploadFolderWriter)
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to create CSV to Upload Foler writer helper object.");
success = false;
}
}
if (success)
{
// Create a sub-directory for the uploader under the log directory
success = GetUploaderWorkSubDirectory();
}
if (this.blobUploadSettings.Enabled)
{
// Create and initialize the uploader
//
// NOTE: By specifying 'true' for the 'filterDeletionByNodeId' parameter,
// we only delete those blobs that were uploaded by the current node. We
// identify this via the Fabric node ID that the ETL-to-CSV writer prefixed
// to the file name before uploading. This is done so that all nodes don't
// wastefully try to delete all blobs.
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.LttTraceContainerName);
this.uploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
this.csvFolder,
destinationPath,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.LttTraceContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
this.uploader.Start();
}
catch (Exception ex)
{
throw new InvalidOperationException("AzureFileUploader could not be constructed.", ex);
}
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace Path: {2}," +
"Upload interval (minutes): {3}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.LttTraceContainerName,
this.csvFolder,
this.blobUploadSettings.UploadInterval);
}
else
{
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is disabled (Storage key not available). Only log age management is enabled."
+ "Local trace Path: {0}",
this.csvFolder
);
}
}
public EtwCsvUploadWorker(EtwCsvUploadWorkerParameters initParam)
{
// Initialization
this.initParam = initParam;
this.logSourceId = this.initParam.UploaderInstanceId;
this.traceSource = this.initParam.TraceSource;
this.blobUploadSettings = this.initParam.Settings;
var accountName = this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName;
this.destinationKey = string.Join(
"_",
StandardPluginTypes.AzureBlobEtwCsvUploader,
accountName,
this.blobUploadSettings.EtwTraceContainerName);
// Create a sub-directory for ourselves under the log directory
bool success = this.GetEtwCsvSubDirectory();
// Create the helper object that writes events delivered from the ETL
// files into CSV files.
if (success)
{
this.etlToCsvWriter = new EtlToCsvFileWriter(
new TraceEventSourceFactory(),
this.logSourceId,
this.initParam.FabricNodeId,
this.etwCsvFolder,
false,
initParam.DiskSpaceManager);
// Set the event filter
this.etlToCsvWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
this.initParam.IsReadingFromApplicationManifest ? string.Empty : WinFabDefaultFilter.StringRepresentation,
this.initParam.IsReadingFromApplicationManifest ? string.Empty : WinFabSummaryFilter.StringRepresentation,
!this.initParam.IsReadingFromApplicationManifest);
// Create a sub-directory for the uploader under the log directory
success = this.GetUploaderWorkSubDirectory();
}
if (success)
{
// Create and initialize the uploader
//
// NOTE: By specifying 'true' for the 'filterDeletionByNodeId' parameter,
// we only delete those blobs that were uploaded by the current node. We
// identify this via the Fabric node ID that the ETL-to-CSV writer prefixed
// to the file name before uploading. This is done so that all nodes don't
// wastefully try to delete all blobs.
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.EtwTraceContainerName);
this.uploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
this.etwCsvFolder,
destinationPath,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
this.uploader.Start();
}
catch (Exception e)
{
throw new InvalidOperationException("AzureFileUploader could not be constructed.", e);
}
}
if (success)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Upload to blob storage is configured. Storage account: {0}, Trace container: {1}, Local trace Path: {2}, Upload interval (minutes): {3}",
this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
this.blobUploadSettings.EtwTraceContainerName,
this.etwCsvFolder,
this.blobUploadSettings.UploadInterval.TotalMinutes);
}
else
{
if (null != this.etlToCsvWriter)
{
this.etlToCsvWriter.Dispose();
this.etlToCsvWriter = null;
}
}
if (!success)
{
throw new InvalidOperationException();
}
}
internal void UpdateSettings(AzureBlobEtwCsvUploader.BlobUploadSettings newSettings)
{
bool updateFilter = this.ShouldUpdateFilter(newSettings);
bool updateUploader = this.ShouldUpdateUploader(newSettings);
if (updateFilter || updateUploader)
{
this.blobUploadSettings = newSettings;
}
if (updateFilter)
{
if (null != this.etlToCsvWriter)
{
// Update the filter
Debug.Assert(this.initParam.IsReadingFromApplicationManifest, "Updating the trace filter is only allowed for applications.");
this.etlToCsvWriter.SetEtwEventFilter(
this.blobUploadSettings.Filter,
string.Empty,
string.Empty,
false);
this.traceSource.WriteInfo(
this.logSourceId,
"ETW event filter has been updated due to settings change. New ETW event filter: {0}.",
this.blobUploadSettings.Filter);
}
}
if (updateUploader)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Due to settings change, the uploader will be stopped and restarted.");
// Stop the uploader
if (null != this.uploader)
{
this.uploader.Dispose();
}
// Restart the upload with the new settings
try
{
var destinationPath = string.Concat(
this.blobUploadSettings.StorageAccountFactory.Connection.UseDevelopmentStorage
? AzureConstants.DevelopmentStorageConnectionString
: this.blobUploadSettings.StorageAccountFactory.Connection.AccountName,
";", // This separator cannot occur in account name or container name
this.blobUploadSettings.EtwTraceContainerName);
this.uploader = new AzureFileUploader(
this.traceSource,
this.logSourceId,
this.etwCsvFolder,
destinationPath,
this.workFolder,
this.blobUploadSettings.StorageAccountFactory,
this.blobUploadSettings.EtwTraceContainerName,
this.blobUploadSettings.UploadInterval,
this.blobUploadSettings.FileSyncInterval,
this.blobUploadSettings.BlobDeletionAge,
this.initParam.FabricNodeInstanceName,
this.blobUploadSettings.DeploymentId);
this.uploader.Start();
}
catch (Exception)
{
this.uploader = null;
this.traceSource.WriteError(
this.logSourceId,
"Failed to restart uploader.");
}
}
}