/// <summary>
/// Post-processing of copying (from XStore to XStore)
/// </summary>
/// <param name="blobContainer">The blob container from which copy is happening.</param>
/// <param name="task">The task to end.</param>
private static void EndCopyFromXStoreToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (task.IsSucceeded)
{
// Generate a manifest on the xstore for the copied folder - only need for atomic copy
// When doing atomic copy we have following assumptions:
// 1) only one writer
// 2) we don't overwrite to the exisiting folder
// 3) later this folder is copied to smb with the exact path, no sub folder or parent folder
// therefore it is enough to only generate a manifest file on the folder only
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.UploadText(DateTime.Now.ToString());
#else
blob.UploadTextAsync(DateTime.Now.ToString()).Wait();
#endif
}
}
/// <summary>
/// Processing folder
/// </summary>
/// <param name="blobContainer">The blob container.</param>
/// <param name="task">The task to be performed.</param>
private void HandleFolder(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
// expand the task and enqueue new tasks
string[] files;
string[] subFolders;
// See if we need to handle the folder
if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && FabricDirectory.Exists(task.DstUri))
{
// We will skip the copy because the final destination exists
return;
}
else if ((task.FileCopyFlag == CopyFlag.AtomicCopy ||
task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists) &&
task.IsRoot)
{
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
task.DstUri = task.DstUri + NewExtension;
// step1: delete dstFolder.new
if (FabricDirectory.Exists(task.DstUri))
{
FabricDirectory.Delete(task.DstUri, recursive: true, deleteReadOnlyFiles: true);
}
}
}
else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
{
// We will skip the copy because the final destination exists
return;
}
else if (task.IsRoot)
{
// if this is the root uri of this operation
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
// remove the manifest of this folder
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null);
#else
blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null, null).Wait();
#endif
}
}
else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore)
{
if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists &&
XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null))
{
// We will skip the copy because the final destination exists
return;
}
else if (task.IsRoot)
{
// if this is the root uri of this operation
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore,
task.SrcUri,
task.DstUri,
true,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
// remove the manifest of this folder
var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri));
#if !DotNetCoreClr
blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper));
#else
blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper), null).Wait();
#endif
}
}
string dstUri = null;
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore:
this.GetFilesAndSubfoldersFromXStore(blobContainer, task.SrcUri, out files, out subFolders, task.TimeoutHelper);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
default:
GetFilesAndSubfoldersFromSMB(task.SrcUri, out files, out subFolders);
break;
}
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
foreach (var file in files)
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
dstUri = ConvertBlobReferenceToSMBPath(file, task.SrcUri, task.DstUri);
if (task.OperationId != null)
{
XStoreCommon.AddDownloadContentEntry(task.OperationId.GetValueOrDefault(), file, dstUri);
}
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
dstUri = ConvertSMBPathToBlobReference(file, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
dstUri = ConvertBlobReferenceToBlobReference(file, task.SrcUri, task.DstUri);
break;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, file, dstUri, false, 0, task.TimeoutHelper, task.OperationId);
newTask.FileCopyFlag = task.FileCopyFlag;
tasks.Enqueue(newTask);
}
foreach (var folder in subFolders)
{
switch (task.OpType)
{
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB:
dstUri = ConvertBlobReferenceToSMBPath(folder, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore:
dstUri = ConvertSMBPathToBlobReference(folder, task.SrcUri, task.DstUri);
break;
case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore:
dstUri = ConvertBlobReferenceToBlobReference(folder, task.SrcUri, task.DstUri);
break;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, folder, dstUri, true, 0, task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
tasks.Enqueue(newTask);
}
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
}