public void CreateDestinationParentDirectoryRecursively(SingleObjectTransfer transferItem) { switch (transferItem.Destination.Type) { case TransferLocationType.FilePath: var filePath = (transferItem.Destination as FileLocation).FilePath; Utils.ValidateDestinationPath(transferItem.Source.Instance.ConvertToString(), filePath); Utils.CreateParentDirectoryIfNotExists(filePath); break; case TransferLocationType.AzureFile: var parent = (transferItem.Destination as AzureFileLocation).AzureFile.Parent; CloudFileDirectory destDirectory = this.dest.Instance as CloudFileDirectory; if (!string.Equals(parent.SnapshotQualifiedUri.AbsolutePath, destDirectory.SnapshotQualifiedUri.AbsolutePath)) { if (this.baseDirectoryTransfer.IsForceOverwrite || !parent.ExistsAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null).Result) { Utils.CreateCloudFileDirectoryRecursively(parent); } } break; default: break; } }
/// <summary> /// Initializes a new instance of the <see cref="SingleObjectTransfer"/> class. /// </summary> /// <param name="other">Another <see cref="SingleObjectTransfer"/> object. </param> private SingleObjectTransfer(SingleObjectTransfer other) : base(other) { this.ProgressTracker = other.ProgressTracker.Copy(); this.transferJob = other.transferJob.Copy(); this.transferJob.Transfer = this; }
internal SingleObjectTransfer CreateTransfer(TransferEntry entry) { TransferLocation sourceLocation = GetSourceTransferLocation(this.Source, entry); sourceLocation.IsInstanceInfoFetched = true; TransferLocation destLocation = GetDestinationTransferLocation(this.Destination, entry); var transferMethod = IsDummyCopy(entry) ? TransferMethod.DummyCopy : this.TransferMethod; SingleObjectTransfer transfer = new SingleObjectTransfer(sourceLocation, destLocation, transferMethod); transfer.Context = this.Context; return(transfer); }
public void CreateParentDirectory(SingleObjectTransfer transfer) { switch (transfer.Destination.Type) { case TransferLocationType.FilePath: var filePath = (transfer.Destination as FileLocation).FilePath; Utils.ValidateDestinationPath(transfer.Source.Instance.ConvertToString(), filePath); filePath = filePath.ToLongPath(); Utils.CreateParentDirectoryIfNotExists(filePath); break; case TransferLocationType.AzureFile: try { CreateParentDirectoryIfNotExists((transfer.Destination as AzureFileLocation).AzureFile); } catch (Exception ex) { AggregateException aggregateException = ex as AggregateException; StorageException storageException = null; if (aggregateException != null) { storageException = aggregateException.Flatten().InnerExceptions[0] as StorageException; } if (storageException == null) { storageException = ex as StorageException; } if (storageException != null) { throw new TransferException(TransferErrorCode.FailToVadlidateDestination, string.Format(CultureInfo.CurrentCulture, Resources.FailedToValidateDestinationException, storageException.ToErrorDetail()), storageException); } throw; } break; default: break; } }
public static void HandleFaultInjection(string relativePath, SingleObjectTransfer transfer) { FaultInjectionPoint fip = new FaultInjectionPoint(FaultInjectionPoint.FIP_ThrowExceptionAfterEnumerated); string fiValue; string filePath = relativePath; CloudBlob sourceBlob = transfer.Source.Instance as CloudBlob; if (sourceBlob != null && sourceBlob.IsSnapshot) { filePath = Utils.AppendSnapShotTimeToFileName(filePath, sourceBlob.SnapshotTime); } if (fip.TryGetValue(out fiValue) && string.Equals(fiValue, filePath, StringComparison.OrdinalIgnoreCase)) { throw new TransferException(TransferErrorCode.Unknown, "test exception thrown because of ThrowExceptionAfterEnumerated is enabled", null); } }
/// <summary> /// Serializes the checkpoint. /// </summary> /// <param name="info">Serialization info object.</param> /// <param name="context">Streaming context.</param> public virtual void GetObjectData(SerializationInfo info, StreamingContext context) { if (info == null) { throw new ArgumentNullException("info"); } List <SingleObjectTransfer> singleObjectTransfers = new List <SingleObjectTransfer>(); List <DirectoryTransfer> directoryTransfers = new List <DirectoryTransfer>(); foreach (var kv in this.transfers) { SingleObjectTransfer transfer = kv.Value as SingleObjectTransfer; if (transfer != null) { singleObjectTransfers.Add(transfer); continue; } DirectoryTransfer transfer2 = kv.Value as DirectoryTransfer; if (transfer2 != null) { directoryTransfers.Add(transfer2); continue; } } info.AddValue(SingleObjectTransfersName, singleObjectTransfers.Count); for (int i = 0; i < singleObjectTransfers.Count; ++i) { info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", SingleObjectTransfersName, i), singleObjectTransfers[i], typeof(SingleObjectTransfer)); } info.AddValue(DirectoryTransfersName, directoryTransfers.Count); for (int i = 0; i < directoryTransfers.Count; ++i) { info.AddValue(string.Format(CultureInfo.InvariantCulture, "{0}{1}", DirectoryTransfersName, i), directoryTransfers[i], typeof(DirectoryTransfer)); } }
internal void AddSingleObjectTransfer( SingleObjectTransfer singleObjectTransfer, Func <SerializableListContinuationToken> updateContinuationToken) { SerializableListContinuationToken listContinuationToken = null; // Add to subtransfers in the TransferCollection for checkpoint (journal without stream) lock (this.continuationTokenLock) { this.subTransfers.AddTransfer(singleObjectTransfer); listContinuationToken = updateContinuationToken(); } if (null != this.Journal) { this.Journal.AddSubtransfer(singleObjectTransfer); this.Journal.UpdateJournalItem(listContinuationToken); } this.TransferFile(singleObjectTransfer, this.currentScheduler, this.currentCancellationToken); }
public async Task ExecuteAsync(CancellationToken cancellationToken) { await Task.Yield(); this.CreateDestinationDirectory(cancellationToken); var enumerator = this.transferEnumerator.EnumerateLocation(cancellationToken).GetEnumerator(); while (true) { if (!enumerator.MoveNext()) { break; } TransferEntry entry = enumerator.Current; ErrorEntry errorEntry = entry as ErrorEntry; if (errorEntry != null) { TransferException exception = errorEntry.Exception as TransferException; if (null != exception) { throw exception; } else { throw new TransferException( TransferErrorCode.FailToEnumerateDirectory, errorEntry.Exception.GetExceptionMessage(), errorEntry.Exception); } } if (entry.IsDirectory) { this.baseDirectoryTransfer.AddSubDir(entry.RelativePath, () => { var currentContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); currentContinuationToken.Journal = this.enumerateContinuationToken.Journal; currentContinuationToken.StreamJournalOffset = this.enumerateContinuationToken.StreamJournalOffset; this.enumerateContinuationToken = currentContinuationToken; return(this.enumerateContinuationToken); }); } else { SingleObjectTransfer transferItem = this.baseDirectoryTransfer.CreateTransfer(entry); #if DEBUG Utils.HandleFaultInjection(entry.RelativePath, transferItem); #endif this.CreateDestinationParentDirectoryRecursively(transferItem); this.baseDirectoryTransfer.AddSingleObjectTransfer(transferItem, () => { var currentContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); currentContinuationToken.Journal = this.enumerateContinuationToken.Journal; currentContinuationToken.StreamJournalOffset = this.enumerateContinuationToken.StreamJournalOffset; this.enumerateContinuationToken = currentContinuationToken; return(this.enumerateContinuationToken); }); } } }
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken) { if (null == this.Journal) { // return all existing transfers in subTransfers foreach (var transfer in this.subTransfers.GetEnumerator()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); yield return(transfer as SingleObjectTransfer); } } else { foreach (var transfer in this.Journal.ListSubTransfers()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); this.subTransfers.AddTransfer(transfer, false); yield return(transfer); } } while (true) { Utils.CheckCancellation(cancellationToken); Tuple <SingleObjectTransfer, TransferEntry> pair; try { pair = this.shouldTransferQueue.DequeueResult(); } catch (InvalidOperationException) { // Task queue is empty and CompleteAdding. No more transfer to dequeue. break; } catch (AggregateException aggregateException) { // Unwrap the AggregateException. ExceptionDispatchInfo.Capture(aggregateException.Flatten().InnerExceptions[0]).Throw(); break; } SingleObjectTransfer transfer = pair.Item1; TransferEntry entry = pair.Item2; lock (this.lockEnumerateContinuationToken) { if (null != transfer) { this.subTransfers.AddTransfer(transfer); } this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); } if (null != transfer) { this.Journal?.AddSubtransfer(transfer); } this.Journal?.UpdateJournalItem(this); if (null == transfer) { continue; } try { this.CreateParentDirectory(transfer); } catch (Exception ex) { transfer.OnTransferFailed(ex); // Don't keep failed transfers in memory if they can be persisted to a journal. if (null != this.Journal) { this.subTransfers.RemoveTransfer(transfer); } continue; } #if DEBUG FaultInjectionPoint fip = new FaultInjectionPoint(FaultInjectionPoint.FIP_ThrowExceptionAfterEnumerated); string fiValue; string filePath = entry.RelativePath; CloudBlob sourceBlob = transfer.Source.Instance as CloudBlob; if (sourceBlob != null && sourceBlob.IsSnapshot) { filePath = Utils.AppendSnapShotTimeToFileName(filePath, sourceBlob.SnapshotTime); } if (fip.TryGetValue(out fiValue) && string.Equals(fiValue, filePath, StringComparison.OrdinalIgnoreCase)) { throw new TransferException(TransferErrorCode.Unknown, "test exception thrown because of ThrowExceptionAfterEnumerated is enabled", null); } #endif yield return(transfer); } }
private void ListNewTransfers(CancellationToken cancellationToken) { // list new transfers if (this.enumerateContinuationToken != null) { this.SourceEnumerator.EnumerateContinuationToken = this.enumerateContinuationToken.ListContinuationToken; } ShouldTransferCallbackAsync shouldTransferCallback = this.DirectoryContext?.ShouldTransferCallbackAsync; try { var enumerator = this.SourceEnumerator.EnumerateLocation(cancellationToken).GetEnumerator(); while (true) { Utils.CheckCancellation(cancellationToken); if (!enumerator.MoveNext()) { break; } TransferEntry entry = enumerator.Current; ErrorEntry errorEntry = entry as ErrorEntry; if (errorEntry != null) { TransferException exception = errorEntry.Exception as TransferException; if (null != exception) { throw exception; } else { throw new TransferException( TransferErrorCode.FailToEnumerateDirectory, errorEntry.Exception.GetExceptionMessage(), errorEntry.Exception); } } this.shouldTransferQueue.EnqueueJob(async() => { try { SingleObjectTransfer candidate = this.CreateTransfer(entry); bool shouldTransfer = shouldTransferCallback == null || await shouldTransferCallback(candidate.Source.Instance, candidate.Destination.Instance); return(new Tuple <SingleObjectTransfer, TransferEntry>(shouldTransfer ? candidate : null, entry)); } catch (Exception ex) { throw new TransferException(TransferErrorCode.FailToEnumerateDirectory, string.Format(CultureInfo.CurrentCulture, "Error happens when handling entry {0}: {1}", entry.ToString(), ex.Message), ex); } }); } } finally { this.shouldTransferQueue.CompleteAdding(); } }
protected abstract void CreateParentDirectory(SingleObjectTransfer transfer);
/// <summary> /// Initializes a new instance of the <see cref="TransferJob"/> class. /// </summary> /// <param name="transfer">Transfer object.</param> public TransferJob(SingleObjectTransfer transfer) { this.Transfer = transfer; this.CheckPoint = new SingleObjectCheckpoint(); }
internal async void TransferFile(SingleObjectTransfer transferItem, TransferScheduler scheduler, CancellationToken cancellationToken) { try { this.maxConcurrencyControl.Wait(cancellationToken); } catch (OperationCanceledException) { // No need to report exception here, OperationCanceledException will be handled in other place. return; } catch (ObjectDisposedException) { return; } await Task.Yield(); bool hasError = false; bool shouldStopTransfer = false; transferItem.UpdateProgressLock(this.progressUpdateLock); try { using (transferItem) { await transferItem.ExecuteAsync(scheduler, cancellationToken); } } catch (TransferException ex) { if (ex.ErrorCode == TransferErrorCode.FailedCheckingShouldTransfer) { shouldStopTransfer = true; this.enumerateException = new TransferException( TransferErrorCode.FailToEnumerateDirectory, string.Format(CultureInfo.CurrentCulture, Resources.EnumerateDirectoryException, this.Destination.Instance.ConvertToString()), ex.InnerException); } hasError = true; } catch { hasError = true; } finally { if ((!hasError) || (null != this.Journal)) { this.subTransfers.RemoveTransfer(transferItem); } if (this.maxConcurrency == this.maxConcurrencyControl.Release()) { this.transfersCompleteSource.TrySetResult(null); } } if (shouldStopTransfer) { this.cancellationTokenSource?.Cancel(); } }
public IEnumerable <SingleObjectTransfer> ListSubTransfers() { long currentOffset = this.singleTransferChunkHead; bool shouldBreak = false; while (true) { SingleObjectTransfer transfer = null; lock (this.journalLock) { if (0 == this.singleTransferChunkHead) { shouldBreak = true; } else { this.stream.Position = currentOffset; long previousUsedChunk = this.ReadLong(); long nextUsedChunk = this.ReadLong(); if (0 == previousUsedChunk) { if (this.singleTransferChunkHead != currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } else { if (this.singleTransferChunkHead == currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } try { #if BINARY_SERIALIZATION transfer = this.formatter.Deserialize(this.stream) as SingleObjectTransfer; #else transfer = this.ReadObject(this.transferSerializer) as SingleObjectTransfer; if (transfer.Source is FileLocation) { (transfer.Source as FileLocation).SetDirectoryPath(this.DirectoryPath); } else if (transfer.Destination is FileLocation) { (transfer.Destination as FileLocation).SetDirectoryPath(this.DirectoryPath); } #endif } catch (Exception) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } if (null == transfer) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } transfer.StreamJournalOffset = currentOffset + 2 * sizeof(long); transfer.Journal = this; this.stream.Position = transfer.StreamJournalOffset + TransferItemContentSize; TransferProgressTracker progressTracker = null; try { #if BINARY_SERIALIZATION progressTracker = this.formatter.Deserialize(this.stream) as TransferProgressTracker; #else progressTracker = this.ReadObject(this.progressCheckerSerializer) as TransferProgressTracker; #endif } catch (Exception) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } if (null == progressTracker) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } transfer.ProgressTracker.AddProgress(progressTracker); transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize; transfer.ProgressTracker.Journal = this; if (0 == nextUsedChunk) { if (this.singleTransferChunkTail != currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } shouldBreak = true; } else { if (this.singleTransferChunkTail == currentOffset) { #if !NO_FILEFORMAT_EX throw new FileFormatException(Resources.RestartableLogCorrupted); #else throw new InvalidOperationException(Resources.RestartableLogCorrupted); #endif } } currentOffset = nextUsedChunk; } } if (null != transfer) { yield return(transfer); } if (shouldBreak) { yield break; } } }
internal void AddSubtransfer(SingleObjectTransfer transfer) { lock (this.journalLock) { long offset = this.SearchFreeOffset(); transfer.Journal = this; transfer.StreamJournalOffset = offset + 2 * sizeof(long); transfer.ProgressTracker.Journal = this; transfer.ProgressTracker.StreamJournalOffset = transfer.StreamJournalOffset + TransferItemContentSize; this.stream.Position = transfer.StreamJournalOffset; #if BINARY_SERIALIZATION this.formatter.Serialize(this.stream, transfer); #else transfer.IsStreamJournal = true; if (transfer.Source is FileLocation) { (transfer.Source as FileLocation).IsStreamJournal = true; } else if (transfer.Destination is FileLocation) { (transfer.Destination as FileLocation).IsStreamJournal = true; } this.WriteObject(this.transferSerializer, transfer); #endif this.stream.Position = transfer.ProgressTracker.StreamJournalOffset; #if BINARY_SERIALIZATION this.formatter.Serialize(this.stream, transfer.ProgressTracker); #else this.WriteObject(this.progressCheckerSerializer, transfer.ProgressTracker); #endif if (0 == this.singleTransferChunkHead) { this.singleTransferChunkHead = offset; this.singleTransferChunkTail = this.singleTransferChunkHead; // Set the transferEntry's previous and next trunk to 0. this.stream.Position = offset; this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); } else { // Set current tail's next to the transferEntry's offset. this.stream.Position = this.singleTransferChunkTail + sizeof(long); this.stream.Write(BitConverter.GetBytes(offset), 0, sizeof(long)); // Set the transferEntry's previous trunk to current tail. this.stream.Position = offset; this.stream.Write(BitConverter.GetBytes(this.singleTransferChunkTail), 0, sizeof(long)); // Set the transferEntry's next trunk to 0. this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); this.singleTransferChunkTail = offset; } this.WriteJournalHead(); this.stream.Flush(); } }
private IEnumerable <SingleObjectTransfer> AllTransfers(CancellationToken cancellationToken) { if (null == this.Journal) { // return all existing transfers in subTransfers foreach (var transfer in this.subTransfers.GetEnumerator()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); yield return(transfer as SingleObjectTransfer); } } else { foreach (var transfer in this.Journal.ListSubTransfers()) { Utils.CheckCancellation(cancellationToken); transfer.Context = this.Context; this.UpdateTransfer(transfer); this.subTransfers.AddTransfer(transfer, false); yield return(transfer); } } while (true) { Utils.CheckCancellation(cancellationToken); Tuple <SingleObjectTransfer, TransferEntry> pair; try { pair = this.shouldTransferQueue.DequeueResult(); } catch (InvalidOperationException) { // Task queue is empty and CompleteAdding. No more transfer to dequeue. break; } catch (AggregateException aggregateException) { // Unwrap the AggregateException. ExceptionDispatchInfo.Capture(aggregateException.Flatten().InnerExceptions[0]).Throw(); break; } SingleObjectTransfer transfer = pair.Item1; TransferEntry entry = pair.Item2; lock (this.lockEnumerateContinuationToken) { if (null != transfer) { this.subTransfers.AddTransfer(transfer); } this.enumerateContinuationToken = new SerializableListContinuationToken(entry.ContinuationToken); } if (null != transfer) { this.Journal?.AddSubtransfer(transfer); } this.Journal?.UpdateJournalItem(this); if (null == transfer) { continue; } try { this.CreateParentDirectory(transfer); } catch (Exception ex) { transfer.OnTransferFailed(ex); // Don't keep failed transfers in memory if they can be persisted to a journal. if (null != this.Journal) { this.subTransfers.RemoveTransfer(transfer); } continue; } #if DEBUG Utils.HandleFaultInjection(entry.RelativePath, transfer); #endif yield return(transfer); } }