/// <summary> /// Initializes a new instance of the <see cref="SingleObjectTransfer"/> class. /// </summary> /// <param name="other">Another <see cref="SingleObjectTransfer"/> object. </param> private SingleObjectTransfer(SingleObjectTransfer other) : base(other) { this.ProgressTracker = other.ProgressTracker.Copy(); this.transferJob = other.transferJob.Copy(); this.transferJob.Transfer = this; }
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken) { // return all existing transfers in subTransfers foreach (var transfer in this.subTransfers.GetEnumerator()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; transfer.ContentType = this.ContentType; this.UpdateTransfer(transfer); yield return(transfer as SingleObjectTransfer); } // list new transfers if (this.enumerateContinuationToken != null) { this.SourceEnumerator.EnumerateContinuationToken = this.enumerateContinuationToken.ListContinuationToken; } var enumerator = this.SourceEnumerator.EnumerateLocation(cancellationToken).GetEnumerator(); while (true) { Utils.CheckCancellation(cancellationToken); // lock enumerator if (!enumerator.MoveNext()) { yield break; } TransferEntry entry = enumerator.Current; ErrorEntry errorEntry = entry as ErrorEntry; if (errorEntry != null) { TransferException exception = errorEntry.Exception as TransferException; if (null != exception) { throw exception; } else { throw new TransferException( TransferErrorCode.FailToEnumerateDirectory, errorEntry.Exception.GetExceptionMessage(), errorEntry.Exception); } } SingleObjectTransfer transfer = this.CreateTransfer(entry); lock (this.lockEnumerateContinuationToken) { this.subTransfers.AddTransfer(transfer); this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); } yield return(transfer); } }
protected override SingleObjectTransfer CreateTransfer(TransferEntry entry) { TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry); TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry); SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, this.TransferMethod); transfer.Context = this.Context; return(transfer); }
internal void AddSubtransfer(SingleObjectTransfer transfer) { lock (this.journalLock) { long offset = this.SearchFreeOffset(); transfer.Journal = this; transfer.StreamJournalOffset = offset + 2 * sizeof(long); transfer.ProgressTracker.Journal = this; transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize; this.stream.Position = transfer.StreamJournalOffset; #if BINARY_SERIALIZATION this.formatter.Serialize(this.stream, transfer); #else transfer.IsStreamJournal = true; this.WriteObject(this.transferSerializer, transfer); #endif this.stream.Position = transfer.ProgressTracker.StreamJournalOffset; #if BINARY_SERIALIZATION this.formatter.Serialize(this.stream, transfer.ProgressTracker); #else this.WriteObject(this.progressCheckerSerializer, transfer.ProgressTracker); #endif if (0 == this.usedChunkHead) { this.usedChunkHead = offset; this.usedChunkTail = this.usedChunkHead; // Set the transferEntry's previous and next trunk to 0. this.stream.Position = offset; this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); } else { // Set current tail's next to the transferEntry's offset. this.stream.Position = this.usedChunkTail + sizeof(long); this.stream.Write(BitConverter.GetBytes(offset), 0, sizeof(long)); // Set the transferEntry's previous trunk to current tail. this.stream.Position = offset; this.stream.Write(BitConverter.GetBytes(this.usedChunkTail), 0, sizeof(long)); // Set the transferEntry's next trunk to 0. this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); this.usedChunkTail = offset; } this.WriteJournalHead(); this.stream.Flush(); } }
private void ListNewTransfers(CancellationToken cancellationToken) { // list new transfers if (this.enumerateContinuationToken != null) { this.SourceEnumerator.EnumerateContinuationToken = this.enumerateContinuationToken.ListContinuationToken; } ShouldTransferCallbackAsync shouldTransferCallback = this.DirectoryContext?.ShouldTransferCallbackAsync; try { var enumerator = this.SourceEnumerator.EnumerateLocation(cancellationToken).GetEnumerator(); while (true) { Utils.CheckCancellation(cancellationToken); if (!enumerator.MoveNext()) { break; } TransferEntry entry = enumerator.Current; ErrorEntry errorEntry = entry as ErrorEntry; if (errorEntry != null) { TransferException exception = errorEntry.Exception as TransferException; if (null != exception) { throw exception; } else { throw new TransferException( TransferErrorCode.FailToEnumerateDirectory, errorEntry.Exception.GetExceptionMessage(), errorEntry.Exception); } } this.shouldTransferQueue.EnqueueJob(async() => { SingleObjectTransfer candidate = this.CreateTransfer(entry); bool shouldTransfer = shouldTransferCallback == null || await shouldTransferCallback(candidate.Source.Instance, candidate.Destination.Instance); return(new Tuple <SingleObjectTransfer, TransferEntry>(shouldTransfer ? candidate : null, entry)); }); } } finally { this.shouldTransferQueue.CompleteAdding(); } }
protected override SingleObjectTransfer CreateTransfer(TransferEntry entry) { TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry); sourceLocation.IsInstanceInfoFetched = true; TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry); var transferMethod = IsDummyCopy(entry) ? TransferMethod.DummyCopy : this.TransferMethod; SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod); transfer.Context = this.Context; return(transfer); }
/// <summary> /// Gets a static snapshot of this transfer checkpoint /// </summary> /// <returns>A snapshot of current transfer checkpoint</returns> internal TransferCheckpoint Copy() { TransferCheckpoint copyObj = new TransferCheckpoint(); foreach (var kvPair in this.transfers) { SingleObjectTransfer transfer = kvPair.Value as SingleObjectTransfer; if (transfer != null) { copyObj.AddTransfer(transfer.Copy()); } } return(copyObj); }
protected override void CreateParentDirectory(SingleObjectTransfer transfer) { switch (transfer.Destination.Type) { case TransferLocationType.FilePath: var filePath = (transfer.Destination as FileLocation).FilePath; ValidateDestinationPath(transfer.Source.Instance.ConvertToString(), filePath); CreateParentDirectoryIfNotExists(filePath); break; case TransferLocationType.AzureFile: try { CreateParentDirectoryIfNotExists((transfer.Destination as AzureFileLocation).AzureFile); } catch (Exception ex) { AggregateException aggregateException = ex as AggregateException; StorageException storageException = null; if (aggregateException != null) { storageException = aggregateException.Flatten().InnerExceptions[0] as StorageException; } if (storageException == null) { storageException = ex as StorageException; } if (storageException != null) { throw new TransferException(TransferErrorCode.FailToVadlidateDestination, string.Format(CultureInfo.CurrentCulture, Resources.FailedToValidateDestinationException, storageException.ToErrorDetail()), storageException); } throw; } break; default: break; } }
/// <summary> /// Serializes the checkpoint. /// </summary> /// <param name="info">Serialization info object.</param> /// <param name="context">Streaming context.</param> public virtual void GetObjectData(SerializationInfo info, StreamingContext context) { if (info == null) { throw new ArgumentNullException("info"); } List <SingleObjectTransfer> singleObjectTransfers = new List <SingleObjectTransfer>(); foreach (var kvPair in this.transfers) { SingleObjectTransfer transfer = kvPair.Value as SingleObjectTransfer; if (transfer != null) { singleObjectTransfers.Add(transfer); } } info.AddValue(SingleObjectTransfersName, singleObjectTransfers, typeof(List <SingleObjectTransfer>)); }
/// <summary> /// Serializes the checkpoint. /// </summary> /// <param name="info">Serialization info object.</param> /// <param name="context">Streaming context.</param> public virtual void GetObjectData(SerializationInfo info, StreamingContext context) { if (info == null) { throw new ArgumentNullException("info"); } List <SingleObjectTransfer> singleObjectTransfers = new List <SingleObjectTransfer>(); List <DirectoryTransfer> directoryTransfers = new List <DirectoryTransfer>(); foreach (var kv in this.transfers) { SingleObjectTransfer transfer = kv.Value as SingleObjectTransfer; if (transfer != null) { singleObjectTransfers.Add(transfer); continue; } DirectoryTransfer transfer2 = kv.Value as DirectoryTransfer; if (transfer2 != null) { directoryTransfers.Add(transfer2); continue; } } info.AddValue(SingleObjectTransfersName, singleObjectTransfers.Count); for (int i = 0; i < singleObjectTransfers.Count; ++i) { info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", SingleObjectTransfersName, i), singleObjectTransfers[i], typeof(SingleObjectTransfer)); } info.AddValue(DirectoryTransfersName, directoryTransfers.Count); for (int i = 0; i < directoryTransfers.Count; ++i) { info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", DirectoryTransfersName, i), directoryTransfers[i], typeof(DirectoryTransfer)); } }
/// <summary> /// Gets a static snapshot of this transfer checkpoint /// </summary> /// <returns>A snapshot of current transfer checkpoint</returns> public TransferCollection Copy() { TransferCollection copyObj = new TransferCollection(); foreach (var kv in this.transfers) { SingleObjectTransfer transfer = kv.Value as SingleObjectTransfer; if (transfer != null) { copyObj.AddTransfer(transfer.Copy()); continue; } DirectoryTransfer transfer2 = kv.Value as DirectoryTransfer; if (transfer2 != null) { copyObj.AddTransfer(transfer2.Copy()); continue; } } return(copyObj); }
protected abstract void CreateParentDirectory(SingleObjectTransfer transfer);
public IEnumerable <SingleObjectTransfer> ListSubTransfers() { long currentOffset = this.usedChunkHead; bool shouldBreak = false; while (true) { SingleObjectTransfer transfer = null; lock (this.journalLock) { if (0 == this.usedChunkHead) { shouldBreak = true; } else { this.stream.Position = currentOffset; long previousUsedChunk = this.ReadLong(); long nextUsedChunk = this.ReadLong(); if (0 == previousUsedChunk) { if (this.usedChunkHead != currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } else { if (this.usedChunkHead == currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } try { #if BINARY_SERIALIZATION transfer = this.formatter.Deserialize(this.stream) as SingleObjectTransfer; #else transfer = this.ReadObject(this.transferSerializer) as SingleObjectTransfer; if (transfer.Source is FileLocation) { (transfer.Source as FileLocation).SetDirectoryPath(this.DirectoryPath); } else if (transfer.Destination is FileLocation) { (transfer.Destination as FileLocation).SetDirectoryPath(this.DirectoryPath); } #endif } catch (Exception) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } if (null == transfer) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } transfer.StreamJournalOffset = currentOffset + 2 * sizeof(long); transfer.Journal = this; this.stream.Position = transfer.StreamJournalOffset + TransferItemContentSize; TransferProgressTracker progressTracker = null; try { #if BINARY_SERIALIZATION progressTracker = this.formatter.Deserialize(this.stream) as TransferProgressTracker; #else progressTracker = this.ReadObject(this.progressCheckerSerializer) as TransferProgressTracker; #endif } catch (Exception) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } if (null == progressTracker) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } transfer.ProgressTracker.AddProgress(progressTracker); transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize; transfer.ProgressTracker.Journal = this; if (0 == nextUsedChunk) { if (this.usedChunkTail != currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } shouldBreak = true; } else { if (this.usedChunkTail == currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } currentOffset = nextUsedChunk; } } if (null != transfer) { yield return(transfer); } if (shouldBreak) { yield break; } } }
protected override Transfer CreateTransfer(TransferEntry entry) { TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry); TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry); SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, this.TransferMethod); transfer.Context = this.Context; transfer.ContentType = this.ContentType; return transfer; }
private static Transfer CreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext) { Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext); if (transfer == null) { transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod); if (transferContext != null) { transferContext.Checkpoint.AddTransfer(transfer); } } if (transferContext != null) { transfer.ProgressTracker.Parent = transferContext.OverallProgressTracker; transfer.Context = transferContext; } return transfer; }
/// <summary> /// Initializes a new instance of the <see cref="SingleObjectTransfer"/> class. /// </summary> /// <param name="other">Another <see cref="SingleObjectTransfer"/> object. </param> private SingleObjectTransfer(SingleObjectTransfer other) : base(other) { this.transferJob = other.transferJob.Copy(); this.transferJob.Transfer = this; }
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken) { if (null == this.Journal) { // return all existing transfers in subTransfers foreach (var transfer in this.subTransfers.GetEnumerator()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); yield return(transfer as SingleObjectTransfer); } } else { foreach (var transfer in this.Journal.ListSubTransfers()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); this.subTransfers.AddTransfer(transfer, false); yield return(transfer); } } while (true) { Utils.CheckCancellation(cancellationToken); Tuple <SingleObjectTransfer, TransferEntry> pair; try { pair = this.shouldTransferQueue.DequeueResult(); } catch (InvalidOperationException) { // Task queue is empty and CompleteAdding. No more transfer to dequeue. break; } catch (AggregateException aggregateException) { // Unwrap the AggregateException. ExceptionDispatchInfo.Capture(aggregateException.Flatten().InnerExceptions[0]).Throw(); break; } SingleObjectTransfer transfer = pair.Item1; TransferEntry entry = pair.Item2; lock (this.lockEnumerateContinuationToken) { if (null != transfer) { this.subTransfers.AddTransfer(transfer); } this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); } if (null != transfer) { this.Journal?.AddSubtransfer(transfer); } this.Journal?.UpdateJournalItem(this); if (null == transfer) { continue; } try { this.CreateParentDirectory(transfer); } catch (Exception ex) { transfer.OnTransferFailed(ex); // Don't keep failed transfers in memory if they can be persisted to a journal. if (null != this.Journal) { this.subTransfers.RemoveTransfer(transfer); } continue; } #if DEBUG FaultInjectionPoint fip = new FaultInjectionPoint(FaultInjectionPoint.FIP_ThrowExceptionAfterEnumerated); string fiValue; string filePath = entry.RelativePath; CloudBlob sourceBlob = transfer.Source.Instance as CloudBlob; if (sourceBlob != null && sourceBlob.IsSnapshot) { filePath = Utils.AppendSnapShotTimeToFileName(filePath, sourceBlob.SnapshotTime); } if (fip.TryGetValue(out fiValue) && string.Equals(fiValue, filePath, StringComparison.OrdinalIgnoreCase)) { throw new TransferException(TransferErrorCode.Unknown, "test exception thrown because of ThrowExceptionAfterEnumerated is enabled", null); } #endif yield return(transfer); } }
private static SingleObjectTransfer GetOrCreateSingleObjectTransfer(TransferLocation sourceLocation, TransferLocation destLocation, TransferMethod transferMethod, TransferContext transferContext) { SingleObjectTransfer singleObjectTransfer = null; Transfer transfer = GetTransfer(sourceLocation, destLocation, transferMethod, transferContext); if (transfer == null) { singleObjectTransfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod); if (transferContext != null) { transferContext.Checkpoint.AddTransfer(singleObjectTransfer); } } else { singleObjectTransfer = transfer as SingleObjectTransfer; Debug.Assert(singleObjectTransfer != null, "singleObjectTransfer"); } return singleObjectTransfer; }