/// <summary>
/// Post-processing of copying (from XStore to SMB)
/// </summary>
/// <param name="task">The task to end.</param>
private static void EndCopyFromXStoreToSMB(XStoreFileOperationTask task)
{
if (task.IsSucceeded)
{
// Step 3, 4 and 5 for atomic copy
// 3. delete dstFolder.old
// 4. rename dstFolder -> dstFolder.old
// 5. rename dstFolder.new -> dstFolder
// get the original destination path
string newDstRootUri = task.DstUri + NewExtension;
string oldDstRootUri = task.DstUri + OldExtension;
// Step 3, 4 & 5
if (FabricDirectory.Exists(oldDstRootUri))
{
// delete dstFolder.old
FabricDirectory.Delete(oldDstRootUri, recursive: true, deleteReadOnlyFiles: true);
}
if (FabricDirectory.Exists(task.DstUri))
{
// rename dstFolder -> dstFolder.old
RenameFolder(task.DstUri, oldDstRootUri);
}
// rename dstFolder.new -> dstFolder; and clean up
RenameFolder(newDstRootUri, task.DstUri);
if (FabricDirectory.Exists(oldDstRootUri))
{
FabricDirectory.Delete(oldDstRootUri, recursive: true, deleteReadOnlyFiles: true);
}
}
}
/// <summary>
/// Delete a file (blob) from XStore
/// </summary>
/// <param name="blobContainer">The container from which to delete.</param>
/// <param name="task">The task to perform.</param>
private static void DeleteFileFromXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
blob.DeleteIfExists();
#else
blob.DeleteIfExistsAsync().Wait();
#endif
}
/// <summary>
/// Retry the task
/// </summary>
/// <param name="task">The task to be performed.</param>
private void RetryTask(XStoreFileOperationTask task)
{
// retrycount + 1 and enqueue again
if (task.RetryCount + 1 > MaxRetryAttempts)
{
// Enqueue to endqueue
task.IsSucceeded = false;
this.xstoreTaskPool.AddTaskToEndTaskQueue(task);
}
else
{
task.RetryCount = task.RetryCount + 1;
this.xstoreTaskPool.AddTaskToTaskQueue(task);
}
}
/// <summary>
/// Post-processing of copying (from SMB to XStore)
/// </summary>
/// <param name="blobContainer">The blob container from which copy is happening.</param>
/// <param name="task">The task to end.</param>
private static void EndCopyFromSMBToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (task.IsSucceeded)
{
// Generate a manifest on the xstore for the copied folder - only need for atomic copy
// When doing atomic copy we have following assumptions:
// 1) only one writer
// 2) we don't overwrite to the exisiting folder
// 3) later this folder is copied to smb with the exact path, no sub folder or parent folder
// therefore it is enough to only generate a manifest file on the folder only
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.UploadText(DateTime.Now.ToString());
#else
blob.UploadTextAsync(DateTime.Now.ToString()).Wait();
#endif
}
}
/// <summary>
/// Upload a file from SMB to xstore
/// </summary>
/// <param name="blobContainer">
/// Blob container
/// </param>
/// <param name="task">
/// Task object
/// </param>
private void TransferFileFromSMBToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
Int64 size = FabricFile.GetSize(task.SrcUri);
// Create the Blob and upload the file
var blob = blobContainer.GetBlockBlobReference(task.DstUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.UploadFromFile(task.SrcUri, (int)size, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.UploadFromFile(task.SrcUri, (int)size, TimeSpan.MaxValue);
}
}
/// <summary>
/// Worker thread to perform the file operations
/// </summary>
private void ThreadRunImpl()
{
// Initialize xstore blobcontainer and connection in the thread
// This is to prevent we block the dispatcher
// Connect to azure blob storage
string connectionString = this.storeParams.ConnectionString;
var storageAccount = CloudStorageAccount.Parse(connectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.DefaultRequestOptions = new BlobRequestOptions();
blobClient.DefaultRequestOptions.MaximumExecutionTime = new TimeSpan(0, 5, 0);
// For large file copies set up a custom timeout period; and using parallel settings
blobClient.DefaultRequestOptions.ParallelOperationThreadCount = XStoreCommon.ParallelOperationThreadCount;
// Get and create the container
CloudBlobContainer blobContainer = blobClient.GetContainerReference(this.storeParams.Container);
while (WaitHandle.WaitAny(this.syncEvents.EventArray) != 1)
{
XStoreFileOperationTask task = this.xstoreTaskPool.GetTaskFromTaskQueue();
if (null != task)
{
// We have got valid task
if (task.RetryCount > 0)
{
// Sleep random time to prevent all worker threads do the next try at the same time
Random rand = new Random(DateTime.Now.Millisecond + this.workerID);
int timeoutInSeconds = rand.Next(MinRetryInSeconds, MaxRetryInSeconds);
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[RETRY] sleep {0} seconds to work on {1}", timeoutInSeconds, task.SrcUri));
Thread.Sleep(timeoutInSeconds * 1000);
}
if (task.IsFolder &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore &&
task.OpType != XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore)
{
try
{
this.HandleFolder(blobContainer, task);
}
catch (Exception e)
{
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[{0}] [{1}] EXCEPTION = {2}", Thread.CurrentThread.ManagedThreadId, DateTime.Now, e.Message));
if (e is TimeoutException)
{
throw;
}
if (e is StorageException && e.InnerException is TimeoutException)
{
throw new TimeoutException(e.InnerException.Message);
}
if (ExceptionHandler.IsFatalException(e))
{
throw;
}
this.RetryTask(task);
}
finally
{
this.xstoreTaskPool.DecrementTaskCount();
}
}
else
{
try
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
this.TransferFileFromXStoreToSMB(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
this.TransferFileFromSMBToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
this.TransferFileFromXStoreToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore:
DeleteFileFromXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore:
EndCopyFromSMBToXStore(blobContainer, task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB:
EndCopyFromXStoreToSMB(task);
break;
case XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore:
EndCopyFromXStoreToXStore(blobContainer, task);
break;
default:
break;
}
}
catch (Exception e)
{
traceSource.WriteError(
ClassName,
string.Format(CultureInfo.InvariantCulture, "[{0}] [{1}] EXCEPTION = {2}", Thread.CurrentThread.ManagedThreadId, DateTime.Now, e.Message));
string extraTracing = task.GetExtraTracing();
if (!string.IsNullOrEmpty(extraTracing))
{
traceSource.WriteInfo(ClassName, extraTracing);
}
if (e is TimeoutException)
{
throw;
}
if (e is StorageException && e.InnerException is TimeoutException)
{
throw new TimeoutException(e.InnerException.Message);
}
if (ExceptionHandler.IsFatalException(e))
{
throw;
}
this.RetryTask(task);
}
finally
{
this.xstoreTaskPool.DecrementTaskCount();
}
}
}
}
}
/// <summary>
/// Processing folder
/// </summary>
/// <param name="blobContainer">The blob container.</param>
/// <param name="task">The task to be performed.</param>
private void HandleFolder(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
// expand the task and enqueue new tasks
string[] files;
string[] subFolders;
// See if we need to handle the folder
if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && FabricDirectory.Exists(task.DstUri))
{
// We will skip the copy because the final destination exists
return;
}
else if ((task.FileCopyFlag == CopyFlag.AtomicCopy ||
task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists) &&
task.IsRoot)
{
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
task.DstUri = task.DstUri + NewExtension;
// step1: delete dstFolder.new
if (FabricDirectory.Exists(task.DstUri))
{
FabricDirectory.Delete(task.DstUri, recursive: true, deleteReadOnlyFiles: true);
}
}
}
else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
{
// We will skip the copy because the final destination exists
return;
}
else if (task.IsRoot)
{
// if this is the root uri of this operation
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
// remove the manifest of this folder
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null);
#else
blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null, null).Wait();
#endif
}
}
else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
{
// We will skip the copy because the final destination exists
return;
}
else if (task.IsRoot)
{
// if this is the root uri of this operation
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
// remove the manifest of this folder
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper));
#else
blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper), null).Wait();
#endif
}
}
string dstUri = null;
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore:
this.GetFilesAndSubfoldersFromXStore(blobContainer, task.SrcUri, out files, out subFolders, task.TimeoutHelper);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
default:
GetFilesAndSubfoldersFromSMB(task.SrcUri, out files, out subFolders);
break;
}
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
foreach (var file in files)
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
dstUri = ConvertBlobReferenceToSMBPath(file, task.SrcUri, task.DstUri);
if (task.OperationId != null)
{
XStoreCommon.AddDownloadContentEntry(task.OperationId.GetValueOrDefault(), file, dstUri);
}
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
dstUri = ConvertSMBPathToBlobReference(file, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
dstUri = ConvertBlobReferenceToBlobReference(file, task.SrcUri, task.DstUri);
break;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, file, dstUri, false, 0, task.TimeoutHelper, task.OperationId);
newTask.FileCopyFlag = task.FileCopyFlag;
tasks.Enqueue(newTask);
}
foreach (var folder in subFolders)
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
dstUri = ConvertBlobReferenceToSMBPath(folder, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
dstUri = ConvertSMBPathToBlobReference(folder, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
dstUri = ConvertBlobReferenceToBlobReference(folder, task.SrcUri, task.DstUri);
break;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, folder, dstUri, true, 0, task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
tasks.Enqueue(newTask);
}
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
}
/// <summary>
/// Download file from XStore to SMB
/// </summary>
/// <param name="blobContainer">The blob container having the file.</param>
/// <param name="task">The task to be performed.</param>
private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
// download the file
if (task.PartialID >= 0)
{
// This is already a divided work; let's go ahead and download
string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(dstSMBPath))
{
FabricFile.Delete(dstSMBPath, deleteReadonly: true);
}
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
}
}
else
{
// we are going to download a file, which we don't know the size yet
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
blob.FetchAttributes(null, this.defaultRequestOption);
#else
blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
int blobLength = (int)blob.Properties.Length;
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(task.DstUri);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(task.DstUri))
{
FabricFile.Delete(task.DstUri, deleteReadonly: true);
}
if (blobLength < DownloadSizeThreshold)
{
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
}
else
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
}
return;
}
else
{
// For large files we divided the work to couple threads.
int numThreads = Math.Min(
blobLength / ParrallelDownloadSize,
ParrallelDownloadThreadCount);
// Create new tasks to parallel download
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
int offset = 0;
for (int i = 0; i < numThreads; i++)
{
int length;
if (i < numThreads - 1)
{
length = blobLength / numThreads;
}
else
{
length = blobLength - offset;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
newTask.Offset = offset;
newTask.Length = length;
newTask.PartialID = i;
tasks.Enqueue(newTask);
offset += length;
}
// enqueue all divided tasks
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
// Add an EndTask to the endTaskQueue as well; to combine all downloads
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
endTask.PartialID = numThreads;
endTask.Content = task.DstUri;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
}
}
}
/// <summary>
/// Copy a file from xstore to xstore
/// </summary>
/// <param name="blobContainer">
/// Blob container
/// </param>
/// <param name="task">
/// Task object
/// </param>
private void TransferFileFromXStoreToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
var destinationBlob = blobContainer.GetBlockBlobReference(task.DstUri);
var sourceBlob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
sourceBlob.FetchAttributes();
#else
sourceBlob.FetchAttributesAsync().Wait();
#endif
int blobLength = (int)sourceBlob.Properties.Length;
var blobWrapper = new XStoreBlobWrapper(destinationBlob);
blobWrapper.CopyFromXStore(sourceBlob, blobLength, task.TimeoutHelper != null ? task.TimeoutHelper.GetRemainingTime() : TimeSpan.MaxValue);
}
