internal AzureBlobUploader(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
string etwLogDirName,
string workFolder,
StorageAccountFactory storageAccountFactory,
string containerName,
string fabricNodeInstanceName,
string deploymentId,
AzureBlobPerformance perfHelper,
AccessCondition uploadStreamAccessCondition,
AccessCondition uploadFileAccessCondition)
: this(
traceSource,
logSourceId,
etwLogDirName,
workFolder,
storageAccountFactory,
string.Empty,
false,
containerName,
string.Empty,
fabricNodeInstanceName,
deploymentId,
perfHelper,
uploadStreamAccessCondition,
uploadFileAccessCondition)
{
}
internal AzureFileTrimmer(
string sourceFolderName,
string workFolder,
StorageAccountFactory storageAccountFactory,
string containerName,
TimeSpan blobDeletionAge,
string fabricNodeInstanceName,
string deploymentId,
bool azureInterfaceAvailable)
{
// Initialization
this.stopping = false;
this.sourceFolderName = sourceFolderName;
this.localMap = Path.Combine(workFolder, AzureBlobUploader.LocalMapFolder);
this.traceSource = new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA);
this.logSourceId = string.Concat(TraceType, "-", Guid.NewGuid().ToString("P"));
this.storageAccountFactory = storageAccountFactory;
this.containerName = containerName;
this.fabricNodeInstanceName = fabricNodeInstanceName;
this.blobDeletionAge = blobDeletionAge;
this.deploymentId = deploymentId;
this.azureInterfaceAvailable = azureInterfaceAvailable;
this.localMapTrimmingHelper = new LocalMapTrimmingHelper(sourceFolderName);
this.folderTrimmer = new FolderTrimmer(
this.traceSource,
this.logSourceId,
blobDeletionAge);
this.perfHelper = new AzureBlobPerformance(this.traceSource, this.logSourceId);
// Deletion and query are currently synchronous, so the concurrency
// count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobQuery,
1);
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobDeletion,
1);
// Create a timer to delete old logs
string timerId = string.Concat(
this.logSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval =
(this.blobDeletionAge < TimeSpan.FromDays(1))
? AzureConstants.OldLogDeletionIntervalForTest
: AzureConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
internal AzureTableTrimmer(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
StorageAccountFactory storageAccountFactory,
string tableName,
TimeSpan entityDeletionAge,
QueryCreationMethod queryCreationMethod,
AzureTablePerformance perfHelper)
{
// Initialization
this.stopping = false;
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.storageAccountFactory = storageAccountFactory;
this.tableName = tableName;
this.entityDeletionAge = entityDeletionAge;
this.perfHelper = perfHelper;
this.queryCreationMethod = queryCreationMethod;
// Deletion and query are currently synchronous, so the concurrency
// count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.TableQuery,
1);
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.TableDeletion,
1);
// Create a timer to delete old logs
string timerId = String.Concat(
this.logSourceId,
OldLogDeletionTimerIdSuffix);
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
(this.entityDeletionAge < TimeSpan.FromDays(1)) ?
AzureConstants.OldLogDeletionIntervalForTest :
AzureConstants.OldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
internal AzureBlobUploader(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
string etwLogDirName,
string workFolder,
StorageAccountFactory storageAccountFactory,
string etlFileName,
bool isActiveEtl,
string containerName,
string fabricNodeId,
string fabricNodeInstanceName,
string deploymentId,
AzureBlobPerformance perfHelper,
AccessCondition uploadStreamAccessCondition,
AccessCondition uploadFileAccessCondition)
{
this.stopping = false;
this.traceSource = traceSource;
this.logSourceId = logSourceId;
this.etwLogDirName = etwLogDirName;
this.storageAccountFactory = storageAccountFactory;
this.containerName = containerName;
this.etlFileName = etlFileName;
this.isActiveEtl = isActiveEtl;
this.fabricNodeId = fabricNodeId;
this.directoryName = AzureUtility.IsAzureInterfaceAvailable()
? string.Join(
"/",
string.IsNullOrEmpty(deploymentId) ? AzureUtility.DeploymentId : deploymentId,
AzureUtility.RoleName,
AzureUtility.RoleInstanceId)
: fabricNodeInstanceName;
this.localMap = Path.Combine(workFolder, LocalMapFolder);
this.uploadStreamAccessCondition = uploadStreamAccessCondition;
this.uploadFileAccessCondition = uploadFileAccessCondition;
this.perfHelper = perfHelper;
// Blob copy is done one at a time, so the concurrency count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobCopy,
1);
this.streamWriter = new StreamWriter(new MemoryStream());
this.lastEventIndexProcessed.Set(DateTime.MinValue, -1);
// Create the container at the destination
try
{
Utility.PerformWithRetries(
this.CreateContainer,
(object)null,
new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
var message = string.Format(
"Error creating container {0}, account {1}.",
this.containerName,
this.storageAccountFactory.Connection.AccountName);
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
message);
throw new InvalidOperationException(message, e);
}
}
internal AzureFileUploader(
FabricEvents.ExtensionsEvents traceSource,
string logSourceId,
string folderName,
string destinationPath,
string workFolder,
StorageAccountFactory storageAccountFactory,
string containerName,
TimeSpan uploadIntervalMinutes,
TimeSpan fileSyncIntervalInMinutes,
TimeSpan blobDeletionAgeMinutes,
string fabricNodeInstanceName,
string deploymentId)
: base(
traceSource,
logSourceId,
folderName,
destinationPath,
workFolder,
uploadIntervalMinutes,
fileSyncIntervalInMinutes)
{
// Initialization
this.storageAccountFactory = storageAccountFactory;
this.containerName = containerName;
this.directoryName = AzureUtility.IsAzureInterfaceAvailable()
? string.Join(
"/",
string.IsNullOrEmpty(deploymentId) ? AzureUtility.DeploymentId : deploymentId,
AzureUtility.RoleName,
AzureUtility.RoleInstanceId)
: fabricNodeInstanceName;
this.fabricNodeInstanceName = fabricNodeInstanceName;
this.perfHelper = new AzureBlobPerformance(this.TraceSource, this.LogSourceId);
// Blob copy is done one at a time, so the concurrency count is 1.
this.perfHelper.ExternalOperationInitialize(
ExternalOperationTime.ExternalOperationType.BlobCopy,
1);
// Create the container at the destination
try
{
Utility.PerformWithRetries(
this.CreateContainer,
(object)null,
new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler));
}
catch (Exception e)
{
var message = string.Format(
"Error creating container {0}, account {1}.",
this.containerName,
this.storageAccountFactory.Connection.AccountName);
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
message);
throw new InvalidOperationException(message, e);
}
// Check if a trimmer already exists to delete old files from this destination.
lock (Trimmers)
{
if (Trimmers.ContainsKey(destinationPath))
{
// Trimmer already exists. Increment its reference count.
TrimmerInfo trimmerInfo = Trimmers[destinationPath];
trimmerInfo.RefCount++;
}
else
{
// Trimmer does not exist. Create it.
AzureFileTrimmer trimmer = new AzureFileTrimmer(
folderName,
LocalMapFolderPath,
storageAccountFactory,
containerName,
blobDeletionAgeMinutes,
fabricNodeInstanceName,
deploymentId,
AzureUtility.IsAzureInterfaceAvailable());
TrimmerInfo trimmerInfo = new TrimmerInfo
{
RefCount = 1,
Trimmer = trimmer
};
Trimmers[destinationPath] = trimmerInfo;
}
}
}
