/// <summary> /// Post-processing of copying (from XStore to XStore) /// </summary> /// <param name="blobContainer">The blob container from which copy is happening.</param> /// <param name="task">The task to end.</param> private static void EndCopyFromXStoreToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task) { if (task.IsSucceeded) { // Generate a manifest on the xstore for the copied folder - only need for atomic copy // When doing atomic copy we have following assumptions: // 1) only one writer // 2) we don't overwrite to the exisiting folder // 3) later this folder is copied to smb with the exact path, no sub folder or parent folder // therefore it is enough to only generate a manifest file on the folder only var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri)); #if !DotNetCoreClr blob.UploadText(DateTime.Now.ToString()); #else blob.UploadTextAsync(DateTime.Now.ToString()).Wait(); #endif } }
/// <summary> /// Processing folder /// </summary> /// <param name="blobContainer">The blob container.</param> /// <param name="task">The task to be performed.</param> private void HandleFolder(CloudBlobContainer blobContainer, XStoreFileOperationTask task) { // expand the task and enqueue new tasks string[] files; string[] subFolders; // See if we need to handle the folder if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB) { if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && FabricDirectory.Exists(task.DstUri)) { // We will skip the copy because the final destination exists return; } else if ((task.FileCopyFlag == CopyFlag.AtomicCopy || task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists) && task.IsRoot) { XStoreFileOperationTask endTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToSMB, task.SrcUri, task.DstUri, true, 0, task.TimeoutHelper); endTask.IsSucceeded = true; endTask.FileCopyFlag = task.FileCopyFlag; this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask); task.DstUri = task.DstUri + NewExtension; // step1: delete dstFolder.new if (FabricDirectory.Exists(task.DstUri)) { FabricDirectory.Delete(task.DstUri, recursive: true, deleteReadOnlyFiles: true); } } } else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore) { if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null)) { // We will skip the copy because the final destination exists return; } else if (task.IsRoot) { // if this is the root uri of this operation XStoreFileOperationTask endTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.EndCopyFromSMBToXStore, task.SrcUri, task.DstUri, true, 0, task.TimeoutHelper); endTask.IsSucceeded = true; endTask.FileCopyFlag = task.FileCopyFlag; this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask); // remove the manifest of this folder var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri)); #if !DotNetCoreClr blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null); #else blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null, null).Wait(); #endif } } else if (task.OpType == XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore) { if (task.FileCopyFlag == CopyFlag.AtomicCopySkipIfExists && XStoreCommon.XStoreFolderExists(blobContainer, task.DstUri, false, task.TimeoutHelper != null ? GetRequestOptions(task.TimeoutHelper) : null)) { // We will skip the copy because the final destination exists return; } else if (task.IsRoot) { // if this is the root uri of this operation XStoreFileOperationTask endTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.EndCopyFromXStoreToXStore, task.SrcUri, task.DstUri, true, 0, task.TimeoutHelper); endTask.IsSucceeded = true; endTask.FileCopyFlag = task.FileCopyFlag; this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask); // remove the manifest of this folder var blob = blobContainer.GetBlockBlobReference(XStoreCommon.GetXStoreFolderManifest(task.DstUri)); #if !DotNetCoreClr blob.DeleteIfExists(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper)); #else blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, task.TimeoutHelper == null ? null : GetRequestOptions(task.TimeoutHelper), null).Wait(); #endif } } string dstUri = null; switch (task.OpType) { case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB: case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore: case XStoreFileOperationTask.XStoreTaskType.RemoveFromXStore: this.GetFilesAndSubfoldersFromXStore(blobContainer, task.SrcUri, out files, out subFolders, task.TimeoutHelper); break; case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore: default: GetFilesAndSubfoldersFromSMB(task.SrcUri, out files, out subFolders); break; } Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>(); foreach (var file in files) { switch (task.OpType) { case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB: dstUri = ConvertBlobReferenceToSMBPath(file, task.SrcUri, task.DstUri); if (task.OperationId != null) { XStoreCommon.AddDownloadContentEntry(task.OperationId.GetValueOrDefault(), file, dstUri); } break; case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore: dstUri = ConvertSMBPathToBlobReference(file, task.SrcUri, task.DstUri); break; case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore: dstUri = ConvertBlobReferenceToBlobReference(file, task.SrcUri, task.DstUri); break; } XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, file, dstUri, false, 0, task.TimeoutHelper, task.OperationId); newTask.FileCopyFlag = task.FileCopyFlag; tasks.Enqueue(newTask); } foreach (var folder in subFolders) { switch (task.OpType) { case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB: dstUri = ConvertBlobReferenceToSMBPath(folder, task.SrcUri, task.DstUri); break; case XStoreFileOperationTask.XStoreTaskType.CopyFromSMBToXStore: dstUri = ConvertSMBPathToBlobReference(folder, task.SrcUri, task.DstUri); break; case XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToXStore: dstUri = ConvertBlobReferenceToBlobReference(folder, task.SrcUri, task.DstUri); break; } XStoreFileOperationTask newTask = new XStoreFileOperationTask(task.OpType, folder, dstUri, true, 0, task.TimeoutHelper); newTask.FileCopyFlag = task.FileCopyFlag; tasks.Enqueue(newTask); } this.xstoreTaskPool.AddTaskToTaskQueue(tasks); }