/// <summary>
/// Copy a file from xstore to xstore
/// </summary>
/// <param name="blobContainer">
/// Blob container
/// </param>
/// <param name="task">
/// Task object
/// </param>
private void TransferFileFromXStoreToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
var destinationBlob = blobContainer.GetBlockBlobReference(task.DstUri);
var sourceBlob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
sourceBlob.FetchAttributes();
#else
sourceBlob.FetchAttributesAsync().Wait();
#endif
int blobLength = (int)sourceBlob.Properties.Length;
var blobWrapper = new XStoreBlobWrapper(destinationBlob);
blobWrapper.CopyFromXStore(sourceBlob, blobLength, task.TimeoutHelper != null ? task.TimeoutHelper.GetRemainingTime() : TimeSpan.MaxValue);
}
/// <summary>
/// Upload a file from SMB to xstore
/// </summary>
/// <param name="blobContainer">
/// Blob container
/// </param>
/// <param name="task">
/// Task object
/// </param>
private void TransferFileFromSMBToXStore(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
Int64 size = FabricFile.GetSize(task.SrcUri);
// Create the Blob and upload the file
var blob = blobContainer.GetBlockBlobReference(task.DstUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.UploadFromFile(task.SrcUri, (int)size, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.UploadFromFile(task.SrcUri, (int)size, TimeSpan.MaxValue);
}
}
/// <summary>
/// Download file from XStore to SMB
/// </summary>
/// <param name="blobContainer">The blob container having the file.</param>
/// <param name="task">The task to be performed.</param>
private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
// download the file
if (task.PartialID >= 0)
{
// This is already a divided work; let's go ahead and download
string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(dstSMBPath))
{
FabricFile.Delete(dstSMBPath, deleteReadonly: true);
}
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
}
}
else
{
// we are going to download a file, which we don't know the size yet
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
blob.FetchAttributes(null, this.defaultRequestOption);
#else
blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
int blobLength = (int)blob.Properties.Length;
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(task.DstUri);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(task.DstUri))
{
FabricFile.Delete(task.DstUri, deleteReadonly: true);
}
if (blobLength < DownloadSizeThreshold)
{
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
}
else
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
}
return;
}
else
{
// For large files we divided the work to couple threads.
int numThreads = Math.Min(
blobLength / ParrallelDownloadSize,
ParrallelDownloadThreadCount);
// Create new tasks to parallel download
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
int offset = 0;
for (int i = 0; i < numThreads; i++)
{
int length;
if (i < numThreads - 1)
{
length = blobLength / numThreads;
}
else
{
length = blobLength - offset;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
newTask.Offset = offset;
newTask.Length = length;
newTask.PartialID = i;
tasks.Enqueue(newTask);
offset += length;
}
// enqueue all divided tasks
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
// Add an EndTask to the endTaskQueue as well; to combine all downloads
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
endTask.PartialID = numThreads;
endTask.Content = task.DstUri;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
}
}
}
