/// <summary>
/// Initializes a new instance of the <see cref="SingleObjectTransfer"/> class.
/// </summary>
/// <param name="other">Another <see cref="SingleObjectTransfer"/> object. </param>
private SingleObjectTransfer(SingleObjectTransfer other)
: base(other)
{
this.ProgressTracker = other.ProgressTracker.Copy();
this.transferJob = other.transferJob.Copy();
this.transferJob.Transfer = this;
}
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken)
{
// return all existing transfers in subTransfers
foreach (var transfer in this.subTransfers.GetEnumerator())
{
Utils.CheckCancellation(cancellationToken);
transfer.Context = this.Context;
transfer.ContentType = this.ContentType;
this.UpdateTransfer(transfer);
yield return(transfer as SingleObjectTransfer);
}
// list new transfers
if (this.enumerateContinuationToken != null)
{
this.SourceEnumerator.EnumerateContinuationToken = this.enumerateContinuationToken.ListContinuationToken;
}
var enumerator = this.SourceEnumerator.EnumerateLocation(cancellationToken).GetEnumerator();
while (true)
{
Utils.CheckCancellation(cancellationToken);
// lock enumerator
if (!enumerator.MoveNext())
{
yield break;
}
TransferEntry entry = enumerator.Current;
ErrorEntry errorEntry = entry as ErrorEntry;
if (errorEntry != null)
{
TransferException exception = errorEntry.Exception as TransferException;
if (null != exception)
{
throw exception;
}
else
{
throw new TransferException(
TransferErrorCode.FailToEnumerateDirectory,
errorEntry.Exception.GetExceptionMessage(),
errorEntry.Exception);
}
}
SingleObjectTransfer transfer = this.CreateTransfer(entry);
lock (this.lockEnumerateContinuationToken)
{
this.subTransfers.AddTransfer(transfer);
this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken);
}
yield return(transfer);
}
}
protected override SingleObjectTransfer CreateTransfer(TransferEntry entry)
{
TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry);
TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry);
SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, this.TransferMethod);
transfer.Context = this.Context;
return(transfer);
}
internal void AddSubtransfer(SingleObjectTransfer transfer)
{
lock (this.journalLock)
{
long offset = this.SearchFreeOffset();
transfer.Journal = this;
transfer.StreamJournalOffset = offset + 2 * sizeof(long);
transfer.ProgressTracker.Journal = this;
transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize;
this.stream.Position = transfer.StreamJournalOffset;
#if BINARY_SERIALIZATION
this.formatter.Serialize(this.stream, transfer);
#else
transfer.IsStreamJournal = true;
this.WriteObject(this.transferSerializer, transfer);
#endif
this.stream.Position = transfer.ProgressTracker.StreamJournalOffset;
#if BINARY_SERIALIZATION
this.formatter.Serialize(this.stream, transfer.ProgressTracker);
#else
this.WriteObject(this.progressCheckerSerializer, transfer.ProgressTracker);
#endif
if (0 == this.usedChunkHead)
{
this.usedChunkHead = offset;
this.usedChunkTail = this.usedChunkHead;
// Set the transferEntry's previous and next trunk to 0.
this.stream.Position = offset;
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
}
else
{
// Set current tail's next to the transferEntry's offset.
this.stream.Position = this.usedChunkTail + sizeof(long);
this.stream.Write(BitConverter.GetBytes(offset), 0, sizeof(long));
// Set the transferEntry's previous trunk to current tail.
this.stream.Position = offset;
this.stream.Write(BitConverter.GetBytes(this.usedChunkTail), 0, sizeof(long));
// Set the transferEntry's next trunk to 0.
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
this.usedChunkTail = offset;
}
this.WriteJournalHead();
this.stream.Flush();
}
}
private void ListNewTransfers(CancellationToken cancellationToken)
{
// list new transfers
if (this.enumerateContinuationToken != null)
{
this.SourceEnumerator.EnumerateContinuationToken = this.enumerateContinuationToken.ListContinuationToken;
}
ShouldTransferCallbackAsync shouldTransferCallback = this.DirectoryContext?.ShouldTransferCallbackAsync;
try
{
var enumerator = this.SourceEnumerator.EnumerateLocation(cancellationToken).GetEnumerator();
while (true)
{
Utils.CheckCancellation(cancellationToken);
if (!enumerator.MoveNext())
{
break;
}
TransferEntry entry = enumerator.Current;
ErrorEntry errorEntry = entry as ErrorEntry;
if (errorEntry != null)
{
TransferException exception = errorEntry.Exception as TransferException;
if (null != exception)
{
throw exception;
}
else
{
throw new TransferException(
TransferErrorCode.FailToEnumerateDirectory,
errorEntry.Exception.GetExceptionMessage(),
errorEntry.Exception);
}
}
this.shouldTransferQueue.EnqueueJob(async() =>
{
SingleObjectTransfer candidate = this.CreateTransfer(entry);
bool shouldTransfer = shouldTransferCallback == null || await shouldTransferCallback(candidate.Source.Instance, candidate.Destination.Instance);
return(new Tuple <SingleObjectTransfer, TransferEntry>(shouldTransfer ? candidate : null, entry));
});
}
}
finally
{
this.shouldTransferQueue.CompleteAdding();
}
}
protected override SingleObjectTransfer CreateTransfer(TransferEntry entry)
{
TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry);
sourceLocation.IsInstanceInfoFetched = true;
TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry);
var transferMethod = IsDummyCopy(entry) ? TransferMethod.DummyCopy : this.TransferMethod;
SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod);
transfer.Context = this.Context;
return(transfer);
}
/// <summary>
/// Gets a static snapshot of this transfer checkpoint
/// </summary>
/// <returns>A snapshot of current transfer checkpoint</returns>
internal TransferCheckpoint Copy()
{
TransferCheckpoint copyObj = new TransferCheckpoint();
foreach (var kvPair in this.transfers)
{
SingleObjectTransfer transfer = kvPair.Value as SingleObjectTransfer;
if (transfer != null)
{
copyObj.AddTransfer(transfer.Copy());
}
}
return(copyObj);
}
protected override void CreateParentDirectory(SingleObjectTransfer transfer)
{
switch (transfer.Destination.Type)
{
case TransferLocationType.FilePath:
var filePath = (transfer.Destination as FileLocation).FilePath;
ValidateDestinationPath(transfer.Source.Instance.ConvertToString(), filePath);
CreateParentDirectoryIfNotExists(filePath);
break;
case TransferLocationType.AzureFile:
try
{
CreateParentDirectoryIfNotExists((transfer.Destination as AzureFileLocation).AzureFile);
}
catch (Exception ex)
{
AggregateException aggregateException = ex as AggregateException;
StorageException storageException = null;
if (aggregateException != null)
{
storageException = aggregateException.Flatten().InnerExceptions[0] as StorageException;
}
if (storageException == null)
{
storageException = ex as StorageException;
}
if (storageException != null)
{
throw new TransferException(TransferErrorCode.FailToVadlidateDestination,
string.Format(CultureInfo.CurrentCulture,
Resources.FailedToValidateDestinationException,
storageException.ToErrorDetail()),
storageException);
}
throw;
}
break;
default:
break;
}
}
/// <summary>
/// Serializes the checkpoint.
/// </summary>
/// <param name="info">Serialization info object.</param>
/// <param name="context">Streaming context.</param>
public virtual void GetObjectData(SerializationInfo info, StreamingContext context)
{
if (info == null)
{
throw new ArgumentNullException("info");
}
List <SingleObjectTransfer> singleObjectTransfers = new List <SingleObjectTransfer>();
foreach (var kvPair in this.transfers)
{
SingleObjectTransfer transfer = kvPair.Value as SingleObjectTransfer;
if (transfer != null)
{
singleObjectTransfers.Add(transfer);
}
}
info.AddValue(SingleObjectTransfersName, singleObjectTransfers, typeof(List <SingleObjectTransfer>));
}
/// <summary>
/// Serializes the checkpoint.
/// </summary>
/// <param name="info">Serialization info object.</param>
/// <param name="context">Streaming context.</param>
public virtual void GetObjectData(SerializationInfo info, StreamingContext context)
{
if (info == null)
{
throw new ArgumentNullException("info");
}
List <SingleObjectTransfer> singleObjectTransfers = new List <SingleObjectTransfer>();
List <DirectoryTransfer> directoryTransfers = new List <DirectoryTransfer>();
foreach (var kv in this.transfers)
{
SingleObjectTransfer transfer = kv.Value as SingleObjectTransfer;
if (transfer != null)
{
singleObjectTransfers.Add(transfer);
continue;
}
DirectoryTransfer transfer2 = kv.Value as DirectoryTransfer;
if (transfer2 != null)
{
directoryTransfers.Add(transfer2);
continue;
}
}
info.AddValue(SingleObjectTransfersName, singleObjectTransfers.Count);
for (int i = 0; i < singleObjectTransfers.Count; ++i)
{
info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", SingleObjectTransfersName, i), singleObjectTransfers[i], typeof(SingleObjectTransfer));
}
info.AddValue(DirectoryTransfersName, directoryTransfers.Count);
for (int i = 0; i < directoryTransfers.Count; ++i)
{
info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", DirectoryTransfersName, i), directoryTransfers[i], typeof(DirectoryTransfer));
}
}
/// <summary>
/// Gets a static snapshot of this transfer checkpoint
/// </summary>
/// <returns>A snapshot of current transfer checkpoint</returns>
public TransferCollection Copy()
{
TransferCollection copyObj = new TransferCollection();
foreach (var kv in this.transfers)
{
SingleObjectTransfer transfer = kv.Value as SingleObjectTransfer;
if (transfer != null)
{
copyObj.AddTransfer(transfer.Copy());
continue;
}
DirectoryTransfer transfer2 = kv.Value as DirectoryTransfer;
if (transfer2 != null)
{
copyObj.AddTransfer(transfer2.Copy());
continue;
}
}
return(copyObj);
}
protected abstract void CreateParentDirectory(SingleObjectTransfer transfer);
public IEnumerable <SingleObjectTransfer> ListSubTransfers()
{
long currentOffset = this.usedChunkHead;
bool shouldBreak = false;
while (true)
{
SingleObjectTransfer transfer = null;
lock (this.journalLock)
{
if (0 == this.usedChunkHead)
{
shouldBreak = true;
}
else
{
this.stream.Position = currentOffset;
long previousUsedChunk = this.ReadLong();
long nextUsedChunk = this.ReadLong();
if (0 == previousUsedChunk)
{
if (this.usedChunkHead != currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
else
{
if (this.usedChunkHead == currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
try
{
#if BINARY_SERIALIZATION
transfer = this.formatter.Deserialize(this.stream) as SingleObjectTransfer;
#else
transfer = this.ReadObject(this.transferSerializer) as SingleObjectTransfer;
if (transfer.Source is FileLocation)
{
(transfer.Source as FileLocation).SetDirectoryPath(this.DirectoryPath);
}
else if (transfer.Destination is FileLocation)
{
(transfer.Destination as FileLocation).SetDirectoryPath(this.DirectoryPath);
}
#endif
}
catch (Exception)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
if (null == transfer)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
transfer.StreamJournalOffset = currentOffset + 2 * sizeof(long);
transfer.Journal = this;
this.stream.Position = transfer.StreamJournalOffset + TransferItemContentSize;
TransferProgressTracker progressTracker = null;
try
{
#if BINARY_SERIALIZATION
progressTracker = this.formatter.Deserialize(this.stream) as TransferProgressTracker;
#else
progressTracker = this.ReadObject(this.progressCheckerSerializer) as TransferProgressTracker;
#endif
}
catch (Exception)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
if (null == progressTracker)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
transfer.ProgressTracker.AddProgress(progressTracker);
transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize;
transfer.ProgressTracker.Journal = this;
if (0 == nextUsedChunk)
{
if (this.usedChunkTail != currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
shouldBreak = true;
}
else
{
if (this.usedChunkTail == currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
currentOffset = nextUsedChunk;
}
}
if (null != transfer)
{
yield return(transfer);
}
if (shouldBreak)
{
yield break;
}
}
}
protected override Transfer CreateTransfer(TransferEntry entry)
{
TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry);
TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry);
SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, this.TransferMethod);
transfer.Context = this.Context;
transfer.ContentType = this.ContentType;
return transfer;
}
private static Transfer CreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext)
{
Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext);
if (transfer == null)
{
transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod);
if (transferContext != null)
{
transferContext.Checkpoint.AddTransfer(transfer);
}
}
if (transferContext != null)
{
transfer.ProgressTracker.Parent = transferContext.OverallProgressTracker;
transfer.Context = transferContext;
}
return transfer;
}
/// <summary>
/// Initializes a new instance of the <see cref="SingleObjectTransfer"/> class.
/// </summary>
/// <param name="other">Another <see cref="SingleObjectTransfer"/> object. </param>
private SingleObjectTransfer(SingleObjectTransfer other)
: base(other)
{
this.transferJob = other.transferJob.Copy();
this.transferJob.Transfer = this;
}
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken)
{
if (null == this.Journal)
{
// return all existing transfers in subTransfers
foreach (var transfer in this.subTransfers.GetEnumerator())
{
Utils.CheckCancellation(cancellationToken);
transfer.Context = this.Context;
this.UpdateTransfer(transfer);
yield return(transfer as SingleObjectTransfer);
}
}
else
{
foreach (var transfer in this.Journal.ListSubTransfers())
{
Utils.CheckCancellation(cancellationToken);
transfer.Context = this.Context;
this.UpdateTransfer(transfer);
this.subTransfers.AddTransfer(transfer, false);
yield return(transfer);
}
}
while (true)
{
Utils.CheckCancellation(cancellationToken);
Tuple <SingleObjectTransfer, TransferEntry> pair;
try
{
pair = this.shouldTransferQueue.DequeueResult();
}
catch (InvalidOperationException)
{
// Task queue is empty and CompleteAdding. No more transfer to dequeue.
break;
}
catch (AggregateException aggregateException)
{
// Unwrap the AggregateException.
ExceptionDispatchInfo.Capture(aggregateException.Flatten().InnerExceptions[0]).Throw();
break;
}
SingleObjectTransfer transfer = pair.Item1;
TransferEntry entry = pair.Item2;
lock (this.lockEnumerateContinuationToken)
{
if (null != transfer)
{
this.subTransfers.AddTransfer(transfer);
}
this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken);
}
if (null != transfer)
{
this.Journal?.AddSubtransfer(transfer);
}
this.Journal?.UpdateJournalItem(this);
if (null == transfer)
{
continue;
}
try
{
this.CreateParentDirectory(transfer);
}
catch (Exception ex)
{
transfer.OnTransferFailed(ex);
// Don't keep failed transfers in memory if they can be persisted to a journal.
if (null != this.Journal)
{
this.subTransfers.RemoveTransfer(transfer);
}
continue;
}
#if DEBUG
FaultInjectionPoint fip = new FaultInjectionPoint(FaultInjectionPoint.FIP_ThrowExceptionAfterEnumerated);
string fiValue;
string filePath = entry.RelativePath;
CloudBlob sourceBlob = transfer.Source.Instance as CloudBlob;
if (sourceBlob != null && sourceBlob.IsSnapshot)
{
filePath = Utils.AppendSnapShotTimeToFileName(filePath, sourceBlob.SnapshotTime);
}
if (fip.TryGetValue(out fiValue) &&
string.Equals(fiValue, filePath, StringComparison.OrdinalIgnoreCase))
{
throw new TransferException(TransferErrorCode.Unknown, "test exception thrown because of ThrowExceptionAfterEnumerated is enabled", null);
}
#endif
yield return(transfer);
}
}
private static SingleObjectTransfer GetOrCreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext)
{
SingleObjectTransfer singleObjectTransfer = null;
Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext);
if (transfer == null)
{
singleObjectTransfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod);
if (transferContext != null)
{
transferContext.Checkpoint.AddTransfer(singleObjectTransfer);
}
}
else
{
singleObjectTransfer = transfer as SingleObjectTransfer;
Debug.Assert(singleObjectTransfer != null, "singleObjectTransfer");
}
return singleObjectTransfer;
}
