private static void CopyContainer(CloudBlobDirectory sourceDirectory, CloudBlobDirectory destDirectory)
{
// This function copies an entire Storage container using the data movement library.
// See https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.storage.datamovement.copydirectoryoptions?view=azure-dotnet for the API
// and https://docs.microsoft.com/en-us/azure/storage/common/storage-use-data-movement-library for a general overview.
ProgressRecorder recorder = new ProgressRecorder();
Stopwatch stopWatch = Stopwatch.StartNew();
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
// Needs to be true to copy the contents of the directory / container. If this is false the contents will
// not be copied.
Recursive = true
};
// Copy all files under root folder
DirectoryTransferContext context = new DirectoryTransferContext
{
ProgressHandler = recorder
};
TransferStatus copyStatus = TransferManager.CopyDirectoryAsync(sourceDirectory, destDirectory, CopyMethod.ServiceSideAsyncCopy, copyDirectoryOptions, context).Result;
stopWatch.Stop();
// Get the elapsed time as a TimeSpan value.
TimeSpan ts = stopWatch.Elapsed;
// Format and display the TimeSpan value.
string elapsedTime = string.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine($"Elapsed:{elapsedTime} T File transferred: {copyStatus.NumberOfFilesTransferred} total bytes：{copyStatus.BytesTransferred}, failed: {copyStatus.NumberOfFilesFailed}, skipped:{copyStatus.NumberOfFilesSkipped}");
}
private static async Task BlobDirectoryCopySample()
{
var sourceBlobDir = await Util.GetCloudBlobDirectoryAsync("sourcecontainer", "dir1");
var destBlobDir = await Util.GetCloudBlobDirectoryAsync("targetcontainer", "dir2");
var options = new CopyDirectoryOptions()
{
Recursive = true,
};
var context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
TransferManager.Configurations.ParallelOperations = 50;
Console.WriteLine("Transfer started");
try
{
Task task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false, options, context);
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
Console.WriteLine("The transfer is completed.");
}
public static object GetDefaultTransferDirectoryOptions(DMLibDataType sourceType, DMLibDataType destType)
{
if (DMLibTestBase.IsLocal(sourceType))
{
var result = new UploadDirectoryOptions();
if (IsCloudBlob(destType))
{
result.BlobType = MapBlobDataTypeToXSCLBlobType(destType);
}
return(result);
}
else if (DMLibTestBase.IsLocal(destType))
{
return(new DownloadDirectoryOptions());
}
else
{
var result = new CopyDirectoryOptions();
if (IsCloudBlob(destType))
{
result.BlobType = MapBlobDataTypeToXSCLBlobType(destType);
}
return(result);
}
}
/// <summary>
/// Legacy convenience function that exposes transform & Retry count
/// </summary>
/// <param name="SourceDirPath"></param>
/// <param name="DestDirPath"></param>
/// <param name="Mode"></param>
/// <param name="Transform"></param>
/// <param name="RetryCount"></param>
public static void CopyDirectory(string SourceDirPath, string DestDirPath, CopyOptions Mode, Func <string, string> Transform, int RetryCount = 5)
{
CopyDirectoryOptions Options = new CopyDirectoryOptions();
Options.Retries = RetryCount;
Options.Mode = Mode;
Options.Transform = Transform;
CopyDirectory(SourceDirPath, DestDirPath, Options);
}
public async Task <List <BlobInfo> > CopyDirectory(
IBlobContainer sourceContainerName,
string sourceDirectoryPath,
IBlobContainer destinationContainerName,
string destinationDirectoryPath,
IBlobStorageService.CopyDirectoryOptions?options = null)
{
_logger.LogInformation(
"Copying directory from {0}/{1} to {2}/{3}",
sourceContainerName,
sourceDirectoryPath,
destinationContainerName,
destinationDirectoryPath
);
var sourceContainer = await GetCloudBlobContainer(sourceContainerName);
var destinationContainer = await GetCloudBlobContainer(
destinationContainerName,
connectionString : options?.DestinationConnectionString
);
var sourceDirectory = sourceContainer.GetDirectoryReference(sourceDirectoryPath);
var destinationDirectory = destinationContainer.GetDirectoryReference(destinationDirectoryPath);
var copyDirectoryOptions = new CopyDirectoryOptions
{
Recursive = true
};
var filesTransferred = new List <BlobInfo>();
var context = new DirectoryTransferContext();
context.FileTransferred += (sender, args) => FileTransferredCallback(sender, args, filesTransferred);
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallbackAsync += options?.SetAttributesCallbackAsync;
context.ShouldTransferCallbackAsync += options?.ShouldTransferCallbackAsync;
context.ShouldOverwriteCallbackAsync += options?.ShouldOverwriteCallbackAsync;
await TransferManager.CopyDirectoryAsync(
sourceDirectory,
destinationDirectory,
CopyMethod.ServiceSideAsyncCopy,
copyDirectoryOptions,
context
);
return(filesTransferred);
}
private Task <TransferStatus> CopyDirectory(dynamic sourceObject, dynamic destObject, TransferItem item)
{
CopyDirectoryOptions copyDirectoryOptions = item.Options as CopyDirectoryOptions;
DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
if (cancellationToken == null || cancellationToken == CancellationToken.None)
{
return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext));
}
else
{
return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext, cancellationToken));
}
}
/// <summary>
/// Moves a folder with its contents to the path given
/// </summary>
/// <param name="folder"></param>
/// <param name="path"></param>
/// <returns></returns>
public async Task <BlobDto> MoveFolder(BlobDto folder, string path)
{
var container = await GetContainer();
if (path[path.Length - 1] != '/')
{
path = path + "/";
}
string newPath = $"{path}{folder.Name}";
var sourceBlobDir = container.GetDirectoryReference(folder.Path);
var destBlobDir = container.GetDirectoryReference(newPath);
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upoload progress
DirectoryTransferContext context = new DirectoryTransferContext
{
ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
})
};
var copyDirOptions = new CopyDirectoryOptions
{
Recursive = true,
IncludeSnapshots = true
};
await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, true, copyDirOptions, context);
await DeleteFile(folder.Path);
folder.Path = newPath;
folder.StoragePath = destBlobDir.Uri.ToString();
folder.DateModified = DateTime.UtcNow;
return(folder);
}
public async Task RunAsync(CancellationToken cancellationToken = default(CancellationToken))
{
var copyDirectoryOptions = new CopyDirectoryOptions
{
Recursive = true
};
var directoryTransferContext = new DirectoryTransferContext
{
//ProgressHandler = new Progress<TransferStatus>(progress => Log.Trace(() => $"Progress: transferred: {progress.NumberOfFilesTransferred}, failed: {progress.NumberOfFilesFailed}, skipped: {progress.NumberOfFilesSkipped}, bytes transferred: {progress.BytesTransferred}"))
};
directoryTransferContext.FileTransferred += (sender, args) => Log.Trace(() => $"Transferred {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
directoryTransferContext.FileSkipped += (sender, args) => Log.Trace(() => $"Skipped {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
directoryTransferContext.FileFailed += (sender, args) => Log.Error(() => $"Failed {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
Log.Info(() => $"Starting server side copy from {this.sourceDirectory.Uri} to {this.targetDirectory.Uri}");
var status = await TransferManager.CopyDirectoryAsync(sourceDirectory, targetDirectory, true, copyDirectoryOptions, directoryTransferContext, cancellationToken);
Log.Info(() => $"Finished server side copy, transferred: {status.NumberOfFilesTransferred}, failed: {status.NumberOfFilesFailed}, skipped: {status.NumberOfFilesSkipped}, bytes transferred: {status.BytesTransferred}");
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlobDirectory
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint into a file after transfer being cancelled
/// 4. Reload checkpoint from the file
/// 4. Resume the transfer with the loaded checkpoint
/// </summary>
private static async Task BlobDirectoryCopySample()
{
CloudBlobDirectory sourceBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
CloudBlobDirectory destBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir2");
// When source is CloudBlobDirectory:
// 1. If recursive is set to true, data movement library matches the source blob name against SearchPattern as prefix.
// 2. Otherwise, data movement library matches the blob with the exact name specified by SearchPattern.
//
// You can also replace the source directory with a CloudFileDirectory instance to copy data from Azure File Storage. If so:
// 1. If recursive is set to true, SearchPattern is not supported. Data movement library simply transfer all azure files
// under the source CloudFileDirectory and its sub-directories.
// 2. Otherwise, data movement library matches the azure file with the exact name specified by SearchPattern.
//
// In the following case, data movement library will copy all blobs with the prefix "azure" in source blob directory.
CopyDirectoryOptions options = new CopyDirectoryOptions()
{
SearchPattern = "azure",
Recursive = true,
};
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferStatus trasferStatus = null;
try
{
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Serialize the checkpoint into a file
#if DOTNET5_4
var formatter = new DataContractSerializer(typeof(TransferCheckpoint));
#else
IFormatter formatter = new BinaryFormatter();
#endif
string tempFileName = Guid.NewGuid().ToString();
using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
formatter.Serialize(stream, checkpoint);
}
// Deserialize the checkpoint from the file
using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
checkpoint = formatter.Deserialize(stream) as TransferCheckpoint;
}
File.Delete(tempFileName);
// Create a new TransferContext with the store checkpoint
DirectoryTransferContext resumeContext = new DirectoryTransferContext(checkpoint);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
resumeContext.ProgressHandler = recorder;
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
trasferStatus = await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, resumeContext);
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
}
/// <summary>
/// Copies the content of one directory to another on the server side.
/// Avoids having to download all items and reupload them to somewhere else on the client side.
/// </summary>
/// <param name="sourceDirectoryPath">The path to the directory containing the files to be transferred</param>
/// <param name="destinationDirectoryPath">The path to the destination directory </param>
/// <param name="transferEvent">Action callback for tracking progress of each file when transferring.</param>
/// <returns>Result from the server side transfer.</returns>
public async Task <ITransferResult> CopyDirectory(string sourceDirectoryPath, string destinationDirectoryPath, Action <Core.TransferEventType, ITransferEvent> transferEvent = null)
{
var sourceContainerName = GetContainerFromPath(sourceDirectoryPath);
var destinationContainerName = GetContainerFromPath(destinationDirectoryPath);
// Ensure destination folder exists if we've configured to create automatically.
if (CreateFolderIfNotExists)
{
GetContainer(destinationContainerName, true);
}
var directoryTransferContext = new DirectoryTransferContext();
// Subscribe to the transfer events if an action method was passed.
if (transferEvent != null)
{
directoryTransferContext.FileTransferred += (fileTransferSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Transferred, i);
};
directoryTransferContext.FileFailed += (fileFailedSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Failed, i);
};
directoryTransferContext.FileSkipped += (fileSkippedSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Skipped, i);
};
}
directoryTransferContext.ShouldOverwriteCallbackAsync = (source, destination) => Task.FromResult(true);
var copyOptions = new CopyDirectoryOptions {
BlobType = BlobType.AppendBlob, Recursive = true
};
var sourceContainer = CloudBlobClient.GetContainerReference(sourceContainerName);
var sourceRelativeUrl = GetPathWithoutContainer(sourceDirectoryPath);
var sourceDirectory = sourceContainer.GetDirectoryReference(sourceRelativeUrl);
var destinationContainer = CloudBlobClient.GetContainerReference(destinationContainerName);
var destinationRelativeUrl = GetPathWithoutContainer(destinationDirectoryPath);
var destinationDirectory = destinationContainer.GetDirectoryReference(destinationRelativeUrl);
var transferTask = TransferManager.CopyDirectoryAsync(sourceDirectory, destinationDirectory, CopyMethod.ServiceSideSyncCopy, copyOptions, directoryTransferContext);
var result = await transferTask;
// Output the result from the transfer.
return(new TransferResult
{
BytesTransferred = result.BytesTransferred,
NumberOfFilesFailed = result.NumberOfFilesFailed,
NumberOfFilesSkipped = result.NumberOfFilesSkipped,
NumberOfFilesTransferred = result.NumberOfFilesTransferred
});
}
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
// Copy TextWrite into Log Helper class
Logger.log = log;
// Log Job Start
await Logger.JobStartAsync(copyItem.JobName);
// This class accumulates transfer data during the copy
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
_opContext = new OperationContext();
_opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
_blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML.
// This allows it to copy multiple items at once when copying a container or directory
// The best (and default value) is Environment.ProcessorCount * 8
int parallelTasks = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks;
// Set the number of connections.
// This should match ParallelOperations so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = parallelTasks;
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// User Agent for tracing
TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup";
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Represents a checkpoint from which a transfer may be resumed and continued.
// This is initalized as null first time then hydrated within CopyDirectoryAsync().
// However if this job is being resumed from a previous failure this function will hydrate
// from a serialized checkpoint saved to blob storage.
TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId);
// Context object for the transfer, provides additional runtime information about its execution
// If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy
TransferContext transferContext = new TransferContext(transferCheckpoint)
{
// Pipe transfer progress data to ProgressRecorder
// ProgressRecorder is used to log the results of the copy operation
ProgressHandler = progressRecorder,
// If the destination already exists this delegate is called.
// Return true to overwrite or false to skip the file during the transfer
OverwriteCallback = (source, destination) =>
{
return(OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental));
}
};
// This event is used to log files skipped during the transfer
transferContext.FileSkipped += TransferContext_FileSkipped;
// This event is used to catch exceptions for files that fail during a transfer
transferContext.FileFailed += TransferContext_FileFailed;
// Set Options for copying the container such as search patterns, recursive, etc.
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory);
// Copy the container
await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
// Check if any files failed during transfer
if (_failedFiles.Count > 0)
{
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint);
// Throw an exception to fail the job so WebJobs will rerun it
throw new Exception("One or more errors occurred during the transfer.");
}
// Log job completion
await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles);
}
catch (Exception ex)
{
// Log Job Error
await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles);
// Rethrow the error to fail the web job
throw ex;
}
}
private async static Task CopyDirectoryAsync(string jobId, CloudBlobDirectory sourceDirectory, CloudBlobDirectory destinationDirectory, CopyDirectoryOptions copyDirectoryOptions, TransferContext transferContext, TransferCheckpoint transferCheckpoint, CancellationTokenSource cancellationTokenSource)
{
// Start the transfer
try
{
await TransferManager.CopyDirectoryAsync(
sourceBlobDir : sourceDirectory,
destBlobDir : destinationDirectory,
isServiceCopy : false,
options : copyDirectoryOptions,
context : transferContext,
cancellationToken : cancellationTokenSource.Token);
// Store the transfer checkpoint to record the completed copy operation
transferCheckpoint = transferContext.LastCheckpoint;
}
catch (TransferException)
{
// Swallow all transfer exceptions here. Files skipped in the OverwriteCallback throw an exception here
// even in an Incremental copy where the source is skipped because it and destination are identical
// Instead all exceptions from transfers are handled in the FileFailed event handler.
}
catch (Exception ex)
{
// Fatal or other exceptions resulting in the transfer being cancelled will still appear here
// Save a Checkpoint so we can restart the transfer
transferCheckpoint = transferContext.LastCheckpoint;
SaveTransferCheckpoint(jobId, transferCheckpoint);
throw new Exception("Error in CopyDirectoryAsync(): " + ex.Message);
}
}
/// <summary>
/// Copies src to dest by comparing files sizes and time stamps and only copying files that are different in src. Basically a more flexible
/// robocopy
/// </summary>
/// <param name="SourcePath"></param>
/// <param name="DestPath"></param>
/// <param name="Verbose"></param>
public static void CopyDirectory(string SourceDirPath, string DestDirPath, CopyDirectoryOptions Options)
{
DateTime StartTime = DateTime.Now;
DirectoryInfo SourceDir = new DirectoryInfo(SourceDirPath);
DirectoryInfo DestDir = new DirectoryInfo(DestDirPath);
if (DestDir.Exists == false)
{
DestDir = Directory.CreateDirectory(DestDir.FullName);
}
bool IsMirroring = (Options.Mode & CopyOptions.Mirror) == CopyOptions.Mirror;
if (IsMirroring && !Options.IsDirectoryPattern)
{
Log.Warning("Can only use mirror with pattern that includes whole directories (e.g. '*')");
IsMirroring = false;
}
IEnumerable <FileInfo> SourceFiles = null;
FileInfo[] DestFiles = null;
// find all files. If a directory get them all, else use the pattern/regex
if (Options.IsDirectoryPattern)
{
SourceFiles = SourceDir.GetFiles("*", Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
}
else
{
if (Options.Regex == null)
{
SourceFiles = SourceDir.GetFiles(Options.Pattern, Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
}
else
{
SourceFiles = SourceDir.GetFiles("*", Options.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
SourceFiles = SourceFiles.Where(F => Options.Regex.IsMatch(F.Name));
}
}
// Convert dest into a map of relative paths to absolute
Dictionary <string, System.IO.FileInfo> DestStructure = new Dictionary <string, System.IO.FileInfo>();
if (IsMirroring)
{
DestFiles = DestDir.GetFiles("*", SearchOption.AllDirectories);
foreach (FileInfo Info in DestFiles)
{
string RelativePath = Info.FullName.Replace(DestDir.FullName, "");
// remove leading seperator
if (RelativePath.First() == Path.DirectorySeparatorChar)
{
RelativePath = RelativePath.Substring(1);
}
DestStructure[RelativePath] = Info;
}
}
// List of relative-path files to copy to dest
List <string> CopyList = new List <string>();
// List of relative path files in dest to delete
List <string> DeletionList = new List <string>();
foreach (FileInfo SourceInfo in SourceFiles)
{
string SourceFilePath = SourceInfo.FullName.Replace(SourceDir.FullName, "");
// remove leading seperator
if (SourceFilePath.First() == Path.DirectorySeparatorChar)
{
SourceFilePath = SourceFilePath.Substring(1);
}
string DestFilePath = Options.Transform(SourceFilePath);
FileInfo DestInfo = null;
// We may have destination info if mirroring where we prebuild it all, if not
// grab it now
if (DestStructure.ContainsKey(DestFilePath))
{
DestInfo = DestStructure[DestFilePath];
}
else
{
string FullDestPath = Path.Combine(DestDir.FullName, DestFilePath);
if (File.Exists(FullDestPath))
{
DestInfo = new FileInfo(FullDestPath);
}
}
if (DestInfo == null)
{
// No copy in dest, add it to the list
CopyList.Add(SourceFilePath);
}
else
{
// Check the file is the same version
// Difference in ticks. Even though we set the dest to the src there still appears to be minute
// differences in ticks. 1ms is 10k ticks...
Int64 TimeDelta = Math.Abs(DestInfo.LastWriteTime.Ticks - SourceInfo.LastWriteTime.Ticks);
Int64 Threshhold = 100000;
if (DestInfo.Length != SourceInfo.Length ||
TimeDelta > Threshhold)
{
CopyList.Add(SourceFilePath);
}
else
{
if (Options.Verbose)
{
Log.Info("Will skip copy to {0}. File up to date.", DestInfo.FullName);
}
else
{
Log.Verbose("Will skip copy to {0}. File up to date.", DestInfo.FullName);
}
}
// Remove it from the map
DestStructure.Remove(DestFilePath);
}
}
// If set to mirror, delete all the files that were not in source
if (IsMirroring)
{
// Now go through the remaining map items and delete them
foreach (var Pair in DestStructure)
{
DeletionList.Add(Pair.Key);
}
foreach (string RelativePath in DeletionList)
{
FileInfo DestInfo = new FileInfo(Path.Combine(DestDir.FullName, RelativePath));
if (Options.Verbose)
{
Log.Info("Deleting extra file {0}", DestInfo.FullName);
}
else
{
Log.Verbose("Deleting extra file {0}", DestInfo.FullName);
}
try
{
// avoid an UnauthorizedAccessException by making sure file isn't read only
DestInfo.IsReadOnly = false;
DestInfo.Delete();
}
catch (Exception Ex)
{
Log.Warning("Failed to delete file {0}. {1}", DestInfo.FullName, Ex);
}
}
// delete empty directories
DirectoryInfo DestDirInfo = new DirectoryInfo(DestDirPath);
DirectoryInfo[] AllSubDirs = DestDirInfo.GetDirectories("*", SearchOption.AllDirectories);
foreach (DirectoryInfo SubDir in AllSubDirs)
{
try
{
if (SubDir.GetFiles().Length == 0 && SubDir.GetDirectories().Length == 0)
{
if (Options.Verbose)
{
Log.Info("Deleting empty dir {0}", SubDir.FullName);
}
else
{
Log.Verbose("Deleting empty dir {0}", SubDir.FullName);
}
SubDir.Delete(true);
}
}
catch (Exception Ex)
{
// handle the case where a file is locked
Log.Info("Failed to delete directory {0}. {1}", SubDir.FullName, Ex);
}
}
}
CancellationTokenSource CTS = new CancellationTokenSource();
// todo - make param..
var POptions = new ParallelOptions {
MaxDegreeOfParallelism = 1, CancellationToken = CTS.Token
};
// install a cancel handler so we can stop parallel-for gracefully
Action CancelHandler = delegate()
{
CTS.Cancel();
};
Globals.AbortHandlers.Add(CancelHandler);
// now do the work
Parallel.ForEach(CopyList, POptions, RelativePath =>
{
// ensure path exists
string DestPath = Path.Combine(DestDir.FullName, RelativePath);
if (Options.Transform != null)
{
DestPath = Options.Transform(DestPath);
}
string SourcePath = Path.Combine(SourceDir.FullName, RelativePath);
FileInfo DestInfo;
FileInfo SrcInfo;
// wrap FileInfo creation with exception handler as can throw and want informative error
try
{
DestInfo = new FileInfo(DestPath);
SrcInfo = new FileInfo(SourcePath);
}
catch (Exception Ex)
{
throw new Exception(string.Format("FileInfo creation failed for Source:{0}, Dest:{1}, with: {2}", SourcePath, DestPath, Ex.Message));
}
// ensure directory exists
DestInfo.Directory.Create();
string DestFile = DestInfo.FullName;
if (Options.Transform != null)
{
DestFile = Options.Transform(DestFile);
}
int Tries = 0;
bool Copied = false;
do
{
try
{
if (Options.Verbose)
{
Log.Info("Copying to {0}", DestFile);
}
else
{
Log.Verbose("Copying to {0}", DestFile);
}
SrcInfo.CopyTo(DestFile, true);
// Clear and read-only attrib and set last write time
FileInfo DestFileInfo = new FileInfo(DestFile);
DestFileInfo.IsReadOnly = false;
DestFileInfo.LastWriteTime = SrcInfo.LastWriteTime;
Copied = true;
}
catch (Exception ex)
{
if (Tries++ < Options.Retries)
{
Log.Info("Copy to {0} failed, retrying {1} of {2} in 30 secs..", DestFile, Tries, Options.Retries);
// todo - make param..
Thread.Sleep(30000);
}
else
{
using (var PauseEC = new ScopedSuspendECErrorParsing())
{
Log.Error("File Copy failed with {0}.", ex.Message);
}
throw new Exception(string.Format("File Copy failed with {0}.", ex.Message));
}
}
} while (Copied == false);
});
TimeSpan Duration = DateTime.Now - StartTime;
if (Duration.TotalSeconds > 10)
{
if (Options.Verbose)
{
Log.Info("Copied Directory in {0}", Duration.ToString(@"mm\m\:ss\s"));
}
else
{
Log.Verbose("Copied Directory in {0}", Duration.ToString(@"mm\m\:ss\s"));
}
}
// remove cancel handler
Globals.AbortHandlers.Remove(CancelHandler);
}
async void RenameToolStripMenuItem_ClickAsync(object sender, EventArgs e)
{
TreeNode node = Containers.SelectedNode;
if (node == null)
{
MessageBox.Show("Choose an item to rename first");
return;
}
CloudBlobDirectory sourceDir;
CloudBlobDirectory targetDir;
var options = new CopyDirectoryOptions {
Recursive = true
};
TreeNode parentNode = node.Parent;
switch (node.ImageIndex)
{
case 0:
new AskForm("Rename Container").ShowDialog();
if (Answer == null)
{
return;
}
if (!Answer.Any(n => n.IsLetter()))
{
MessageBox.Show("Russian is not allowed");
return;
}
ProgressBar.Visible = true;
CloudBlobContainer sourceC = Client.GetContainerReference(node.Text);
CloudBlobContainer targetC = Client.GetContainerReference(Answer);
await targetC.CreateIfNotExistsAsync();
foreach (var blob in sourceC.ListBlobs())
{
switch (blob)
{
case CloudBlobDirectory cbd:
sourceDir = sourceC.GetDirectoryReference(blob.Uri.Directory());
targetDir = targetC.GetDirectoryReference(blob.Uri.Directory());
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
break;
case CloudBlob cb:
CloudBlob sourceBlob = sourceC.GetBlobReference(blob.Uri.File());
CloudBlob targetBlob = targetC.GetBlobReference(blob.Uri.File());
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
break;
}
}
await sourceC.DeleteIfExistsAsync();
break;
case 1:
switch (parentNode.ImageIndex)
{
case 0:
CloudBlobContainer container = Client.GetContainerReference(parentNode.Text);
sourceDir = container.GetDirectoryReference(node.Text);
new AskForm("Rename Directory").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetDir = container.GetDirectoryReference(Answer);
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
await DeleteDirectory(sourceDir);
break;
case 1:
(TreeNode containerNode, int count, string[] hier) = GetContainerNode(parentNode);
Array.Reverse(hier);
container = Client.GetContainerReference(containerNode.Text);
sourceDir = container.GetDirectoryReference(hier[0]);
for (int i = 0; i < count; i++)
{
sourceDir = sourceDir.GetDirectoryReference(hier[i + 1]);
}
sourceDir = sourceDir.GetDirectoryReference(node.Text);
new AskForm("Rename Directory").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetDir = sourceDir.Parent.GetDirectoryReference(Answer);
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
await DeleteDirectory(sourceDir);
break;
}
break;
case 2:
switch (parentNode.ImageIndex)
{
case 0:
CloudBlobContainer container = Client.GetContainerReference(parentNode.Text);
CloudBlob sourceBlob = container.GetBlobReference(node.Text);
new AskForm("Rename Cloud Blob").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
CloudBlob targetBlob = container.GetBlobReference(Answer);
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
await sourceBlob.DeleteIfExistsAsync();
break;
case 1:
(TreeNode containerNode, int count, string[] hier) = GetContainerNode(parentNode);
Array.Reverse(hier);
container = Client.GetContainerReference(containerNode.Text);
sourceDir = container.GetDirectoryReference(hier[0]);
for (int i = 0; i < count; i++)
{
sourceDir = sourceDir.GetDirectoryReference(hier[i + 1]);
}
sourceBlob = sourceDir.GetBlobReference(node.Text);
new AskForm("Rename Cloud Blob").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetBlob = sourceDir.GetBlobReference(Answer);
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
await sourceBlob.DeleteIfExistsAsync();
break;
}
break;
}
RefreshAll();
ProgressBar.Visible = false;
}
private async static Task CopyDirectoryAsync(CloudBlobDirectory sourceDirectory, CloudBlobDirectory destinationDirectory, CopyDirectoryOptions copyDirectoryOptions, TransferContext transferContext, TransferCheckpoint transferCheckpoint, CancellationTokenSource cancellationTokenSource)
{
// Start the transfer
try
{
await TransferManager.CopyDirectoryAsync(
sourceBlobDir : sourceDirectory,
destBlobDir : destinationDirectory,
isServiceCopy : true,
options : copyDirectoryOptions,
context : transferContext,
cancellationToken : cancellationTokenSource.Token);
// Store the transfer checkpoint.
transferCheckpoint = transferContext.LastCheckpoint;
}
catch (TransferException te)
{
// Swallow Exceptions from skipped files in Overwrite Callback
// Log any other Transfer Exceptions
if (te.ErrorCode != TransferErrorCode.SubTransferFails)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Transfer Error: " + te.Message);
sb.AppendLine("Transfer Error Code: " + te.ErrorCode);
await _log.WriteLineAsync(sb.ToString());
}
}
}
// This version uses CopyDirectoryAsync in DML 0.2. I'm not sure it is faster than what I did above copying them manually in DML 0.1
public async static Task ProcessQueueMessage2([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken)
{
_log = log;
log.WriteLine("Job Start: " + copyItem.JobName);
// This class accumulates transfer data during the process
ProgressRecorder progressRecorder = new ProgressRecorder();
try
{
// OpContext to track PreCopy Retries on Azure Storage
// DML has its own context object and retry
OperationContext opContext = new OperationContext();
opContext.Retrying += StorageRequest_Retrying;
// Define Blob Request Options
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
// Defined Exponential Retry Policy above
RetryPolicy = _retryPolicy
};
// Set the number of parallel tasks in DML. This allows it to copy multiple
// items at once when copying a container or directory
//int parallelTasks = Environment.ProcessorCount * 8;
int parallelTasks = Convert.ToInt32(ConfigurationManager.AppSettings["ParallelTasks"]);
// Set the number of connections so each DML copy task has its own connection to Azure Storage
ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
TransferManager.Configurations.ParallelOperations = parallelTasks; //64;
log.WriteLine("Parallel Operations = " + parallelTasks.ToString());
// Short circuit additional request round trips. We are not chunking and
// uploading large amounts of data where we'd send 100's so set to false
ServicePointManager.Expect100Continue = false;
// CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
// Represents a checkpoint from which a transfer may be resumed and continued
// This is set within the CopyContainerAsync function
TransferCheckpoint transferCheckpoint = null;
// Open connections to both storage accounts
CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken);
CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken);
// Context object for the transfer, provides additional runtime information about its execution
TransferContext transferContext = new TransferContext
{
// Pipe transfer progress data to ProgressRecorder
ProgressHandler = progressRecorder,
// Callback to overwrite destination if it exists
OverwriteCallback = (source, destination) =>
{
return(OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem, blobRequestOptions, opContext));
}
};
CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions
{
IncludeSnapshots = true,
Recursive = true
};
// Get the root source and destination directories for the two containers to be copied
CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, blobRequestOptions);
CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, blobRequestOptions);
// Copy the container
await CopyDirectoryAsync(sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource);
log.WriteLine(progressRecorder.ToString());
log.WriteLine("Job Complete: " + copyItem.JobName);
}
catch (Exception ex)
{
log.WriteLine("Backup Job error: " + copyItem.JobName + ", Error: " + ex.Message);
log.WriteLine(progressRecorder.ToString());
}
}
