public SyncTransferController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException("transferScheduler");
}
if (null == transferJob)
{
throw new ArgumentNullException("transferJob");
}
this.SharedTransferData = new SharedTransferData()
{
TransferJob = this.TransferJob,
AvailableData = new ConcurrentDictionary <long, TransferData>(),
};
if (null == transferJob.CheckPoint)
{
transferJob.CheckPoint = new SingleObjectCheckpoint();
}
reader = this.GetReader(transferJob.Source);
writer = this.GetWriter(transferJob.Destination);
}
/// <summary>
/// Initializes a new instance of the <see cref="BlockBlobServiceSideSyncCopyController"/> class.
/// </summary>
/// <param name="scheduler">Scheduler object which creates this object.</param>
/// <param name="transferJob">Instance of job to start async copy.</param>
/// <param name="userCancellationToken">Token user input to notify about cancellation.</param>
internal BlockBlobServiceSideSyncCopyController(
TransferScheduler scheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(scheduler, transferJob, userCancellationToken)
{
TransferLocation sourceLocation = transferJob.Source;
if (sourceLocation.Type == TransferLocationType.AzureBlob)
{
var blobLocation = sourceLocation as AzureBlobLocation;
this.SourceHandler = new ServiceSideSyncCopySource.BlobSourceHandler(blobLocation, transferJob);
}
else if (sourceLocation.Type == TransferLocationType.AzureFile)
{
this.SourceHandler = new ServiceSideSyncCopySource.FileSourceHandler(sourceLocation as AzureFileLocation, transferJob);
}
else
{
throw new ArgumentException(
Resources.OnlySupportBlobAzureFileSource,
"transferJob");
}
this.destLocation = transferJob.Destination as AzureBlobLocation;
this.destBlockBlob = this.destLocation.Blob as CloudBlockBlob;
this.DestHandler = new ServiceSideSyncCopyDest.BlockBlobDestHandler(this.destLocation, transferJob);
this.hasWork = true;
}
protected RangeBasedServiceSideSyncCopy(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
}
public FileAsyncCopyController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
: base(transferScheduler, transferJob, cancellationToken)
{
if (transferJob.Destination.Type != TransferLocationType.AzureFile)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest.AzureFile"),
"transferJob");
}
if (transferJob.Source.Type != TransferLocationType.SourceUri &&
transferJob.Source.Type != TransferLocationType.AzureBlob &&
transferJob.Source.Type != TransferLocationType.AzureFile)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ProvideExactlyOneOfThreeParameters,
"Source.SourceUri",
"Source.Blob",
"Source.AzureFile"),
"transferJob");
}
this.destLocation = this.TransferJob.Destination as AzureFileLocation;
this.destFile = this.destLocation.AzureFile;
}
public SyncTransferController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException("transferScheduler");
}
if (null == transferJob)
{
throw new ArgumentNullException("transferJob");
}
this.SharedTransferData = new SharedTransferData()
{
TransferJob = this.TransferJob,
AvailableData = new ConcurrentDictionary<long, TransferData>(),
};
if (null == transferJob.CheckPoint)
{
transferJob.CheckPoint = new SingleObjectCheckpoint();
}
reader = this.GetReader(transferJob.Source);
writer = this.GetWriter(transferJob.Destination);
}
internal CloudFileWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.cloudFile = this.TransferJob.Destination.AzureFile;
}
internal CloudFileWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.cloudFile = this.TransferJob.Destination.AzureFile;
}
protected RangeBasedWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.hasWork = true;
}
protected RangeBasedWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.hasWork = true;
}
internal PageBlobWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.pageBlob = this.TransferJob.Destination.Blob as CloudPageBlob;
}
internal PageBlobWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.pageBlob = this.TransferJob.Destination.Blob as CloudPageBlob;
}
public StreamedReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.hasWork = true;
}
public CloudFileReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
:base(scheduler, controller, cancellationToken)
{
this.file = this.SharedTransferData.TransferJob.Source.AzureFile;
Debug.Assert(null != this.file, "Initializing a CloudFileReader, the source location should be a CloudFile instance.");
}
protected TransferReaderWriterBase(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
{
this.Scheduler = scheduler;
this.Controller = controller;
this.CancellationToken = cancellationToken;
}
public PageBlobReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
pageBlob = this.SharedTransferData.TransferJob.Source.Blob as CloudPageBlob;
Debug.Assert(null != this.pageBlob, "Initializing a PageBlobReader, the source location should be a CloudPageBlob instance.");
}
public StreamedWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.hasWork = true;
this.state = State.OpenOutputStream;
}
public StreamedWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.hasWork = true;
this.state = State.OpenOutputStream;
}
public CloudFileReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.file = this.SharedTransferData.TransferJob.Source.AzureFile;
Debug.Assert(null != this.file, "Initializing a CloudFileReader, the source location should be a CloudFile instance.");
}
public StreamedReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.hasWork = true;
}
/// <summary>
/// Initializes a new instance of the <see cref="PageBlobServiceSideSyncCopyController"/> class.
/// </summary>
/// <param name="scheduler">Scheduler object which creates this object.</param>
/// <param name="transferJob">Instance of job to start async copy.</param>
/// <param name="userCancellationToken">Token user input to notify about cancellation.</param>
internal PageBlobServiceSideSyncCopyController(
TransferScheduler scheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(scheduler, transferJob, userCancellationToken)
{
this.destPageBlob = destLocation.Blob as CloudPageBlob;
this.hasWork = true;
}
public PageBlobReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
:base(scheduler, controller, cancellationToken)
{
pageBlob = this.SharedTransferData.TransferJob.Source.Blob as CloudPageBlob;
Debug.Assert(null != this.pageBlob, "Initializing a PageBlobReader, the source location should be a CloudPageBlob instance.");
}
protected TransferReaderWriterBase(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
{
this.Scheduler = scheduler;
this.Controller = controller;
this.CancellationToken = cancellationToken;
}
public RangeBasedReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.Location = this.transferJob.Source;
this.workToken = 1;
}
public StreamedReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.workToken = 1;
this.readLength = 0;
}
public RangeBasedReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.Location = this.transferJob.Source;
this.hasWork = true;
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncCopyController"/> class.
/// </summary>
/// <param name="scheduler">Scheduler object which creates this object.</param>
/// <param name="transferJob">Instance of job to start async copy.</param>
/// <param name="userCancellationToken">Token user input to notify about cancellation.</param>
internal AsyncCopyController(
TransferScheduler scheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(scheduler, transferJob, userCancellationToken)
{
if (null == transferJob.Destination)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest"),
"transferJob");
}
switch(this.TransferJob.Source.Type)
{
case TransferLocationType.AzureBlob:
this.SourceBlob = (this.TransferJob.Source as AzureBlobLocation).Blob;
break;
case TransferLocationType.AzureFile:
this.SourceFile = (this.TransferJob.Source as AzureFileLocation).AzureFile;
break;
case TransferLocationType.SourceUri:
this.SourceUri = (this.TransferJob.Source as UriLocation).Uri;
break;
default:
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ProvideExactlyOneOfThreeParameters,
"Source.SourceUri",
"Source.Blob",
"Source.AzureFile"),
"transferJob");
}
// initialize the status refresh timer
this.statusRefreshTimer = new Timer(
new TimerCallback(
delegate(object timerState)
{
this.hasWork = true;
}));
this.SetInitialStatus();
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncCopyController"/> class.
/// </summary>
/// <param name="scheduler">Scheduler object which creates this object.</param>
/// <param name="transferJob">Instance of job to start async copy.</param>
/// <param name="userCancellationToken">Token user input to notify about cancellation.</param>
internal AsyncCopyController(
TransferScheduler scheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(scheduler, transferJob, userCancellationToken)
{
if (null == transferJob.Destination)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest"),
"transferJob");
}
switch (this.TransferJob.Source.Type)
{
case TransferLocationType.AzureBlob:
this.SourceBlob = (this.TransferJob.Source as AzureBlobLocation).Blob;
break;
case TransferLocationType.AzureFile:
this.SourceFile = (this.TransferJob.Source as AzureFileLocation).AzureFile;
break;
case TransferLocationType.SourceUri:
this.SourceUri = (this.TransferJob.Source as UriLocation).Uri;
break;
default:
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ProvideExactlyOneOfThreeParameters,
"Source.SourceUri",
"Source.Blob",
"Source.AzureFile"),
"transferJob");
}
// initialize the status refresh timer
this.statusRefreshTimer = new Timer(
new TimerCallback(
delegate(object timerState)
{
this.hasWork = true;
#if DOTNET5_4
}), null, -1, Timeout.Infinite);
#else
}));
public BlockBlobWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.destLocation = this.SharedTransferData.TransferJob.Destination as AzureBlobLocation;
this.blockBlob = this.destLocation.Blob as CloudBlockBlob;
Debug.Assert(null != this.blockBlob, "The destination is not a block blob while initializing a BlockBlobWriter instance.");
this.state = State.FetchAttributes;
this.hasWork = true;
}
public static AsyncCopyController CreateAsyncCopyController(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken)
{
if (transferJob.Destination.Type == TransferLocationType.AzureFile)
{
return(new FileAsyncCopyController(transferScheduler, transferJob, cancellationToken));
}
if (transferJob.Destination.Type == TransferLocationType.AzureBlob)
{
return(new BlobAsyncCopyController(transferScheduler, transferJob, cancellationToken));
}
throw new InvalidOperationException(Resources.CanOnlyCopyToFileOrBlobException);
}
public BlockBlobWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.location = this.SharedTransferData.TransferJob.Destination;
this.blockBlob = this.location.Blob as CloudBlockBlob;
Debug.Assert(null != this.blockBlob, "The destination is not a block blob while initializing a BlockBlobWriter instance.");
this.state = State.FetchAttributes;
this.hasWork = true;
}
public AppendBlobWriter(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.destLocation = this.SharedTransferData.TransferJob.Destination as AzureBlobLocation;
this.appendBlob = this.destLocation.Blob as CloudAppendBlob;
Debug.Assert(null != this.appendBlob, "The destination is not an append blob while initializing a AppendBlobWriter instance.");
this.state = State.FetchAttributes;
this.workToken = 1;
}
public BlockBasedBlobReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferLocation = this.SharedTransferData.TransferJob.Source;
this.transferJob = this.SharedTransferData.TransferJob;
this.blob = this.transferLocation.Blob;
Debug.Assert(
(this.blob is CloudBlockBlob) ||(this.blob is CloudAppendBlob),
"Initializing BlockBlobReader while source location is not a block blob or an append blob.");
this.hasWork = true;
}
public BlockBasedBlobReader(
TransferScheduler scheduler,
SyncTransferController controller,
CancellationToken cancellationToken)
: base(scheduler, controller, cancellationToken)
{
this.transferJob = this.SharedTransferData.TransferJob;
this.sourceLocation = this.transferJob.Source as AzureBlobLocation;
this.sourceBlob = this.sourceLocation.Blob;
Debug.Assert(
(this.sourceBlob is CloudBlockBlob) || (this.sourceBlob is CloudAppendBlob),
"Initializing BlockBlobReader while source location is not a block blob or an append blob.");
this.hasWork = true;
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncCopyController"/> class.
/// </summary>
/// <param name="scheduler">Scheduler object which creates this object.</param>
/// <param name="transferJob">Instance of job to start async copy.</param>
/// <param name="userCancellationToken">Token user input to notify about cancellation.</param>
internal AsyncCopyController(
TransferScheduler scheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(scheduler, transferJob, userCancellationToken)
{
if (null == transferJob.Destination)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest"),
"transferJob");
}
if ((null == transferJob.Source.SourceUri && null == transferJob.Source.Blob && null == transferJob.Source.AzureFile) ||
(null != transferJob.Source.SourceUri && null != transferJob.Source.Blob) ||
(null != transferJob.Source.Blob && null != transferJob.Source.AzureFile) ||
(null != transferJob.Source.SourceUri && null != transferJob.Source.AzureFile))
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ProvideExactlyOneOfThreeParameters,
"Source.SourceUri",
"Source.Blob",
"Source.AzureFile"),
"transferJob");
}
this.SourceUri = this.TransferJob.Source.SourceUri;
this.SourceBlob = this.TransferJob.Source.Blob;
this.SourceFile = this.TransferJob.Source.AzureFile;
// initialize the status refresh timer
this.statusRefreshTimer = new Timer(
new TimerCallback(
delegate(object timerState)
{
this.hasWork = true;
}));
this.SetInitialStatus();
}
protected ServiceSideSyncCopyController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
if (null == transferJob.Destination)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"transferJob"),
"transferJob");
}
this.state = State.FetchSourceAttributes;
}
public BlobAsyncCopyController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken cancellationToken)
: base(transferScheduler, transferJob, cancellationToken)
{
this.destLocation = transferJob.Destination as AzureBlobLocation;
CloudBlob transferDestBlob = this.destLocation.Blob;
if (null == transferDestBlob)
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
Resources.ParameterCannotBeNullException,
"Dest.Blob"),
"transferJob");
}
if (transferDestBlob.IsSnapshot)
{
throw new ArgumentException(Resources.DestinationMustBeBaseBlob, "transferJob");
}
AzureBlobLocation sourceBlobLocation = transferJob.Source as AzureBlobLocation;
if (sourceBlobLocation != null)
{
if (sourceBlobLocation.Blob.BlobType != transferDestBlob.BlobType)
{
throw new ArgumentException(Resources.SourceAndDestinationBlobTypeDifferent, "transferJob");
}
if (StorageExtensions.Equals(sourceBlobLocation.Blob, transferDestBlob))
{
throw new InvalidOperationException(Resources.SourceAndDestinationLocationCannotBeEqualException);
}
}
this.destBlob = transferDestBlob;
}
public DummyTransferController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException(nameof(transferScheduler));
}
if (null == transferJob)
{
throw new ArgumentNullException(nameof(transferJob));
}
if (null == transferJob.CheckPoint)
{
transferJob.CheckPoint = new SingleObjectCheckpoint();
}
}
protected TransferControllerBase(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException("transferScheduler");
}
if (null == transferJob)
{
throw new ArgumentNullException("transferJob");
}
this.Scheduler = transferScheduler;
this.TransferJob = transferJob;
this.transferSchedulerCancellationTokenRegistration =
this.Scheduler.CancellationTokenSource.Token.Register(this.CancelWork);
this.userCancellationTokenRegistration = userCancellationToken.Register(this.CancelWork);
this.TaskCompletionSource = new TaskCompletionSource <object>();
}
public static AsyncCopyController CreateAsyncCopyController(TransferScheduler transferScheduler, TransferJob transferJob, CancellationToken cancellationToken)
{
if (transferJob.Destination.Type == TransferLocationType.AzureFile)
{
return new FileAsyncCopyController(transferScheduler, transferJob, cancellationToken);
}
if (transferJob.Destination.Type == TransferLocationType.AzureBlob)
{
return new BlobAsyncCopyController(transferScheduler, transferJob, cancellationToken);
}
throw new InvalidOperationException(Resources.CanOnlyCopyToFileOrBlobException);
}
public SyncTransferController(
TransferScheduler transferScheduler,
TransferJob transferJob,
CancellationToken userCancellationToken)
: base(transferScheduler, transferJob, userCancellationToken)
{
if (null == transferScheduler)
{
throw new ArgumentNullException(nameof(transferScheduler));
}
if (null == transferJob)
{
throw new ArgumentNullException(nameof(transferJob));
}
this.SharedTransferData = new SharedTransferData()
{
TransferJob = this.TransferJob,
AvailableData = new ConcurrentDictionary <long, TransferData>(),
};
if (null == transferJob.CheckPoint)
{
transferJob.CheckPoint = new SingleObjectCheckpoint();
}
this.reader = this.GetReader(transferJob.Source);
this.writer = this.GetWriter(transferJob.Destination);
this.CheckAndEnableSmallFileOptimization();
this.SharedTransferData.OnTotalLengthChanged += (sender, args) =>
{
// For large block blob uploading, we need to re-calculate the BlockSize according to the total size
// The formula: Ceiling(TotalSize / (50000 * DefaultBlockSize)) * DefaultBlockSize. This will make sure the
// new block size will be mutiple of DefaultBlockSize(aka MemoryManager's chunk size)
if (this.writer is BlockBlobWriter)
{
var normalMaxBlockBlobSize = (long)50000 * Constants.DefaultBlockSize;
// Calculate the min block size according to the blob total length
var memoryChunksRequiredEachTime = (int)Math.Ceiling((double)this.SharedTransferData.TotalLength / normalMaxBlockBlobSize);
var blockSize = memoryChunksRequiredEachTime * Constants.DefaultBlockSize;
// Take the block size user specified when it's greater than the calculated value
if (TransferManager.Configurations.BlockSize > blockSize)
{
blockSize = TransferManager.Configurations.BlockSize;
memoryChunksRequiredEachTime = (int)Math.Ceiling((double)blockSize / Constants.DefaultBlockSize);
}
else
{
// Try to increase the memory pool size
this.Scheduler.TransferOptions.UpdateMaximumCacheSize(blockSize);
}
// If data size is smaller than block size, fit block size according to total length, in order to minimize buffer allocation,
// and save space and time.
if (this.SharedTransferData.TotalLength < blockSize)
{
// Note total length could be 0, in this case, use default block size.
memoryChunksRequiredEachTime = Math.Max(1,
(int)Math.Ceiling((double)this.SharedTransferData.TotalLength / Constants.DefaultBlockSize));
blockSize = memoryChunksRequiredEachTime * Constants.DefaultBlockSize;
}
this.SharedTransferData.BlockSize = blockSize;
this.SharedTransferData.MemoryChunksRequiredEachTime = memoryChunksRequiredEachTime;
}
else
{
// For normal directions, we'll use default block size 4MB for transfer.
this.SharedTransferData.BlockSize = Constants.DefaultBlockSize;
this.SharedTransferData.MemoryChunksRequiredEachTime = 1;
}
};
}
