public void TestTransfer() { var from = new BankAccountImpl(new BankAccountId("A"), new Money(10)); var to = new BankAccountImpl(new BankAccountId("B"), new Money(0)); var result = new TransferContext<BankAccount>(from, to).Transfer(new Money(10)); Assert.AreEqual(0, result.From.Barance.Value); Assert.AreEqual(10, result.To.Barance.Value); }
internal static TransferContext.Result Send(this TransferContext.Sender sender, Money money, TransferContext.Receiver receiver) { var sender_ = sender as BankAccount; var receiver_ = receiver as BankAccount; var newFrom = sender_.Decrease(money); var newTo = receiver_.Increase(money); return new TransferContext.Result() { From = newFrom, To = newTo }; }
private Task <TransferStatus> UploadDirectory(dynamic destObject, TransferItem item) { UploadDirectoryOptions uploadDirectoryOptions = item.Options as UploadDirectoryOptions; TransferContext transferContrext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string sourcePath = item.SourceObject as string; if (cancellationToken != null && cancellationToken != CancellationToken.None) { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext, cancellationToken)); } else if (transferContrext != null || uploadDirectoryOptions != null) { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext)); } else { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject)); } }
private Task DownloadDirectory(dynamic sourceObject, TransferItem item) { DownloadDirectoryOptions downloadDirectoryOptions = item.Options as DownloadDirectoryOptions; TransferContext transferContext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string destPath = item.DestObject as string; if (cancellationToken != null && cancellationToken != CancellationToken.None) { return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext, cancellationToken)); } else if (transferContext != null || downloadDirectoryOptions != null) { return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext)); } else { return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath)); } }
protected TransferContext GetTransferContext(ProgressRecord record, long totalTransferLength) { TransferContext transferContext = new TransferContext(); transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId; transferContext.OverwriteCallback = ConfirmOverwrite; transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) => { if (record != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. record.PercentComplete = (totalTransferLength == 0) ? 100 : (int)(transferProgress.BytesTransferred * 100 / totalTransferLength); record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, record.PercentComplete); this.OutputStream.WriteProgress(record); } }); return(transferContext); }
protected TransferContext GetTransferContext(DataMovementUserData userData) { TransferContext transferContext = new TransferContext(); transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId; transferContext.OverwriteCallback = ConfirmOverwrite; transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) => { if (userData.Record != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. userData.Record.PercentComplete = 0 == userData.TotalSize ? 100 : (int)(transferProgress.BytesTransferred * 100 / userData.TotalSize); userData.Record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, userData.Record.PercentComplete); this.OutputStream.WriteProgress(userData.Record); } }); return(transferContext); }
private static async Task Download( ITransferClient client, AutoDeleteDirectory directory, TransferContext context, IList <TransferPath> localSourcePaths, string uploadTargetPath, SampleRunner sampleRunner, CancellationToken token) { // Create a job-based download transfer request. Console2.WriteStartHeader("Transfer - Download"); string downloadTargetPath = directory.Path; TransferRequest downloadJobRequest = TransferRequest.ForDownloadJob(downloadTargetPath, context); downloadJobRequest.Application = "Github Sample"; downloadJobRequest.Name = "Download Sample"; Console2.WriteLine("Download started."); // Create a transfer job to download the sample dataset to the target local path. using (ITransferJob job = await client.CreateJobAsync(downloadJobRequest, token).ConfigureAwait(false)) { IEnumerable <TransferPath> remotePaths = localSourcePaths.Select( localPath => new TransferPath { SourcePath = uploadTargetPath + "\\" + Path.GetFileName(localPath.SourcePath), PathAttributes = TransferPathAttributes.File, TargetPath = downloadTargetPath }); await job.AddPathsAsync(remotePaths, token).ConfigureAwait(false); await sampleRunner.ChangeDataRateAsync(job, token).ConfigureAwait(false); // Await completion of the job. ITransferResult result = await job.CompleteAsync(token).ConfigureAwait(false); Console2.WriteLine("Download completed."); sampleRunner.DisplayTransferResult(result); Console2.WriteEndHeader(); } }
public void Apply(TransferContext context) { context.FileTransferred += (sender, transferEventArgs) => { Test.Info("Transfer succeeds: {0} -> {1}", DMLibTestHelper.TransferInstanceToString(transferEventArgs.Source), DMLibTestHelper.TransferInstanceToString(transferEventArgs.Destination)); this.Increase(TransferEventType.Transferred); }; context.FileSkipped += (sender, transferEventArgs) => { Test.Info("Transfer skips: {0} -> {1}", DMLibTestHelper.TransferInstanceToString(transferEventArgs.Source), DMLibTestHelper.TransferInstanceToString(transferEventArgs.Destination)); this.Increase(TransferEventType.Skippied); }; context.FileFailed += (sender, transferEventArgs) => { Test.Info("Transfer fails: {0} -> {1}", DMLibTestHelper.TransferInstanceToString(transferEventArgs.Source), DMLibTestHelper.TransferInstanceToString(transferEventArgs.Destination)); Test.Info("Exception: {0}", transferEventArgs.Exception.ToString()); this.Increase(TransferEventType.Failed); }; }
internal int freeAsync(TransferContext transferContext) { lock (oLockTransferContext) { int ret = 0; if (!transferContext.mContext.IsValid) { return(ret); } cancelAsync_NL(transferContext); ret = LibUsbAPI.usb_free_async(ref transferContext.mContext); if (ret < 0 || transferContext.mContext.IsValid) { UsbGlobals.Error(this, UsbGlobals.LastError, "freeAsync", ret); } return(ret); } }
internal int transferSync(TransferContext transferContext) { int ret; ret = setupAsync(transferContext); if (ret < 0) { return(ret); } bool bContinue = true; while (bContinue) { ret = submitAsync(transferContext); if (ret < 0) { freeAsync(transferContext); return(ret); } ret = reapAsync(transferContext); if (ret < 0) { freeAsync(transferContext); return(ret); } lock (oLockTransferContext) bContinue = ((transferContext.mCurrentRemaining > 0) && (ret == transferContext.mRequested)); } freeAsync(transferContext); return(transferContext.currentTransmitted); }
private static async Task DemoTransferAsync(IRelativityTransferHost host, CancellationToken token, SampleRunner sampleRunner) { // Search for the first logical file share. const int LogicalFileShareNumber = 1; RelativityFileShare fileShare = await sampleRunner.GetFileShareAsync(host, LogicalFileShareNumber, token).ConfigureAwait(false); // Assigning the file share bypasses auto-configuration that will normally use the default workspace repository. sampleRunner.AssignFileshare(fileShare); // Prepare transfer setup common for upload and download. IList <TransferPath> localSourcePaths = await sampleRunner.SearchLocalSourcePathsAsync(token).ConfigureAwait(false); string uploadTargetPath = sampleRunner.GetUniqueRemoteTargetPath(fileShare); TransferContext context = sampleRunner.CreateTransferContext(); using (ITransferClient client = await sampleRunner.CreateClientAsync(host, token).ConfigureAwait(false)) using (AutoDeleteDirectory directory = new AutoDeleteDirectory()) { await Upload(client, context, localSourcePaths, uploadTargetPath, sampleRunner, token); await Download(client, directory, context, localSourcePaths, uploadTargetPath, sampleRunner, token); } }
public void TestProgressHandlerTest() { long fileSize = 10 * 1024 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize); var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { TransferContext transferContext = new TransferContext(); ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public override Task DownloadAsync(CloudFile sourceFile, string destFilePath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { validateAction(sourceFile, destFilePath); return(TaskEx.FromResult(true)); }
internal static TransferContext<BankAccount>.Result Send(this TransferContext<BankAccount>.Sender sender, Money money, TransferContext<BankAccount>.Receiver to) { var newFrom = sender._.Decrease(money); var newTo = to.OnReceived(money); return new TransferContext<BankAccount>.Result() { From = newFrom, To = newTo }; }
private void ResumeInAllDirectionsHelper(bool directoryTransfer) { List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true) : AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true); int fileCount = expectedFileNodes.Keys.Count; // Execution and store checkpoints CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferContext transferContext = null; if (directoryTransfer) { transferContext = new DirectoryTransferContext(); } else { transferContext = new SingleTransferContext(); } var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); allItems.ForEach(item => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; }); var options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; // Checkpoint names const string PartialStarted = "PartialStarted", AllStarted = "AllStarted", AllStartedAndWait = "AllStartedAndWait", BeforeCancel = "BeforeCancel", AfterCancel = "AfterCancel"; Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>(); TransferItem randomItem = allItems[random.Next(0, allItems.Count)]; randomItem.AfterStarted = () => { Test.Info("Store check point after transfer item: {0}.", randomItem.ToString()); checkpoints.Add(PartialStarted, transferContext.LastCheckpoint); }; options.AfterAllItemAdded = () => { if (!progressChecker.DataTransferred.WaitOne(30000)) { Test.Error("No progress in 30s."); } checkpoints.Add(AllStarted, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint); tokenSource.Cancel(); checkpoints.Add(AfterCancel, transferContext.LastCheckpoint); }; var result = this.RunTransferItems(allItems, options); // Resume with stored checkpoints in random order var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >(); checkpointList.AddRange(checkpoints); checkpointList.Shuffle(); foreach (var pair in checkpointList) { Test.Info("===Resume with checkpoint '{0}'===", pair.Key); options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; progressChecker.Reset(); if (directoryTransfer) { transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value)) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; } else { transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value)) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; } int expectedFailureCount = 0; transferContext.FileFailed += (resource, eventArgs) => { TransferException exception = eventArgs.Exception as TransferException; if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId) { Interlocked.Increment(ref expectedFailureCount); } }; TransferEventChecker eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); List <TransferItem> itemsToResume = allItems.Select(item => { TransferItem itemToResume = item.Clone(); itemToResume.TransferContext = transferContext; return(itemToResume); }).ToList(); result = this.RunTransferItems(itemsToResume, options); foreach (DMLibDataType destDataType in DataTypes) { if (DMLibDataType.URI == destDataType) { continue; } DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { string fileName = destFileNode.Name; FileNode sourceFileNode = expectedFileNodes[fileName]; Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } } if (!directoryTransfer) { Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count); } else { Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count); Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber); } } }
public void TestDirectoryResume() { int bigFileSizeInKB = 5 * 1024; // 5 MB int smallFileSizeInKB = 1; // 1 KB int bigFileNum = 5; int smallFileNum = 50; long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024; int totalFileNum = bigFileNum + smallFileNum; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB); DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; options.IsDirectoryTransfer = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; Thread.Sleep(100); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); secondCheckpoint = transferContext.LastCheckpoint; if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(), }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); }
public async static Task ProcessMessage([QueueTrigger("backupqueue")] CopyItem copyItem, TextWriter log, CancellationToken cancelToken) { // Copy TextWrite into Log Helper class Logger.log = log; // Log Job Start await Logger.JobStartAsync(copyItem.JobName); // This class accumulates transfer data during the copy ProgressRecorder progressRecorder = new ProgressRecorder(); try { // OpContext to track PreCopy Retries on Azure Storage // DML has its own context object and retry _opContext = new OperationContext(); _opContext.Retrying += StorageRequest_Retrying; // Define Blob Request Options _blobRequestOptions = new BlobRequestOptions { // Defined Exponential Retry Policy above RetryPolicy = _retryPolicy }; // Set the number of parallel tasks in DML. // This allows it to copy multiple items at once when copying a container or directory // The best (and default value) is Environment.ProcessorCount * 8 int parallelTasks = Environment.ProcessorCount * 8; TransferManager.Configurations.ParallelOperations = parallelTasks; // Set the number of connections. // This should match ParallelOperations so each DML copy task has its own connection to Azure Storage ServicePointManager.DefaultConnectionLimit = parallelTasks; // Short circuit additional request round trips. We are not chunking and // uploading large amounts of data where we'd send 100's so set to false ServicePointManager.Expect100Continue = false; // User Agent for tracing TransferManager.Configurations.UserAgentPrefix = "AzureDmlBackup"; // CancellationTokenSource used to cancel the transfer CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); // Open connections to both storage accounts CloudStorageAccount sourceAccount = GetAccount(copyItem.SourceAccountToken); CloudStorageAccount destinationAccount = GetAccount(copyItem.DestinationAccountToken); // Represents a checkpoint from which a transfer may be resumed and continued. // This is initalized as null first time then hydrated within CopyDirectoryAsync(). // However if this job is being resumed from a previous failure this function will hydrate // from a serialized checkpoint saved to blob storage. TransferCheckpoint transferCheckpoint = await GetTransferCheckpoint(copyItem.JobId); // Context object for the transfer, provides additional runtime information about its execution // If this is a resumed copy operation then pass the checkpoint to the TransferContext so it can resume the copy TransferContext transferContext = new TransferContext(transferCheckpoint) { // Pipe transfer progress data to ProgressRecorder // ProgressRecorder is used to log the results of the copy operation ProgressHandler = progressRecorder, // If the destination already exists this delegate is called. // Return true to overwrite or false to skip the file during the transfer OverwriteCallback = (source, destination) => { return(OverwriteFile(source, destination, sourceAccount, destinationAccount, copyItem.IsIncremental)); } }; // This event is used to log files skipped during the transfer transferContext.FileSkipped += TransferContext_FileSkipped; // This event is used to catch exceptions for files that fail during a transfer transferContext.FileFailed += TransferContext_FileFailed; // Set Options for copying the container such as search patterns, recursive, etc. CopyDirectoryOptions copyDirectoryOptions = new CopyDirectoryOptions { IncludeSnapshots = true, Recursive = true }; // Get the root source and destination directories for the two containers to be copied CloudBlobDirectory sourceDirectory = await GetDirectoryAsync(sourceAccount, copyItem.SourceContainer, copyItem.SourceDirectory); CloudBlobDirectory destinationDirectory = await GetDirectoryAsync(destinationAccount, copyItem.DestinationContainer, copyItem.DestinationDirectory); // Copy the container await CopyDirectoryAsync(copyItem.JobId, sourceDirectory, destinationDirectory, copyDirectoryOptions, transferContext, transferCheckpoint, cancellationTokenSource); // Check if any files failed during transfer if (_failedFiles.Count > 0) { // Save a Checkpoint so we can restart the transfer transferCheckpoint = transferContext.LastCheckpoint; SaveTransferCheckpoint(copyItem.JobId, transferCheckpoint); // Throw an exception to fail the job so WebJobs will rerun it throw new Exception("One or more errors occurred during the transfer."); } // Log job completion await Logger.JobCompleteAsync(copyItem.JobName, progressRecorder, _skippedFiles); } catch (Exception ex) { // Log Job Error await Logger.JobErrorAsync(copyItem.JobName, ex.Message, progressRecorder, _failedFiles, _skippedFiles); // Rethrow the error to fail the web job throw ex; } }
/// <summary> /// Download an Azure blob from Azure Blob Storage. /// </summary> /// <param name="sourceBlob">The Microsoft.WindowsAzure.Storage.Blob.CloudBlob that is the source Azure blob.</param> /// <param name="destPath">Path to the destination file.</param> /// <param name="options">A Microsoft.WindowsAzure.Storage.DataMovement.DownloadOptions object that specifies /// additional options for the operation.</param> /// <param name="context">A Microsoft.WindowsAzure.Storage.DataMovement.TransferContext object that represents /// the context for the current operation.</param> /// <param name="cancellationToken">A System.Threading.CancellationToken object to observe while waiting for a task /// to complete.</param> /// <returns>A System.Threading.Tasks.Task object that represents the asynchronous operation.</returns> public Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { return(TransferManager.DownloadAsync(sourceBlob, destPath, options, context, cancellationToken)); }
public TransferRepository(TransferContext context) { _context = context; }
public void TestDirectoryCheckContentMD5() { long fileSize = 5 * 1024 * 1024; long totalSize = fileSize * 4; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); TransferContext context = new TransferContext(); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } }; ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); context.FileFailed += (eventSource, eventArgs) => { transferExceptions.Add(eventArgs.Exception); }; TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, }; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); DMLibDataInfo actualDataInfo = testResult.DataInfo; actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1) { Test.Error("Expect one exception but actually no exception is thrown."); } else { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } }
private void ReadData(object obj) { int ret = 0; TransferContext transferContext = (TransferContext)obj; UsbEndpointReader reader = (UsbEndpointReader)transferContext.mEndpointBase; reader.mDataReceivedEnabled = true; try { while (ret >= 0 && !reader.mEventCancelReadThread.WaitOne(0, false)) { lock (oLockTransferContext) transferContext.Reset(); ret = setupAsync(transferContext); if (ret >= 0) { bool bContinue = true; while (bContinue) { ret = submitAsync(transferContext); if (ret >= 0) { ReapRetry: ret = reapAsyncNoCancel(transferContext); if (ret < 0) { if (ret == (int)ErrorCodes.ETIMEDOUT) { if (!transferContext.mbAsyncCancelled && !reader.mEventCancelReadThread.WaitOne(0, false)) { goto ReapRetry; } else { ret = (int)ErrorCodes.EINTR; } } } } lock (oLockTransferContext) bContinue = ((transferContext.mCurrentRemaining > 0) && (ret == transferContext.mRequested)); } freeAsync(transferContext); } if (ret >= 0) { EventHandler <DataReceivedArgs> temp = reader.DataReceived; if (temp != null) { temp(this, new DataReceivedArgs(transferContext.Buffer, transferContext.currentTransmitted)); } } } } catch (ThreadAbortException) { Debug.Print("ThreadAbortException:" + GetType().FullName + ".ReadData"); } finally { if (transferContext.mContext.IsValid) { freeAsync(transferContext); } reader.mDataReceivedEnabled = false; reader.TransferLock.Release(); } }
public AgencyRepository(TransferContext ctx) : base(ctx) { ctx.Agencies.Include(a => a.AgencyPartners).ToList(); }
public void TestDirectoryResume() { int bigFileSizeInKB = 5 * 1024; // 5 MB int smallFileSizeInKB = 1; // 1 KB int bigFileNum = 5; int smallFileNum = 50; long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024; int totalFileNum = bigFileNum + smallFileNum; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB); DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; options.IsDirectoryTransfer = true; using (Stream journalStream = new MemoryStream()) { bool IsStreamJournal = random.Next(0, 2) == 0; var transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext(); var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Helper.VerifyCancelException(e.Exception); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { dirOptions.PreserveSMBAttributes = true; dirOptions.PreserveSMBPermissions = true; } item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!IsStreamJournal) { // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; if (!IsStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = null; if (IsStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream) { ProgressHandler = progressChecker.GetProgressHandler() }; } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; } eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (!IsStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(), }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } } }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; bool IsStreamJournal = random.Next(0, 2) == 0; using (Stream journalStream = new MemoryStream()) { TransferContext transferContext = IsStreamJournal ? new SingleTransferContext(journalStream) : new SingleTransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { dynamic transferOptions = DefaultTransferOptions; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; item.DisableStreamDispose = true; item.Options = transferOptions; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { if (IsStreamJournal && (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)) { return; } // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint if (!IsStreamJournal) { firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; if ((DMLibTestContext.SourceType == DMLibDataType.Stream) || (DMLibTestContext.DestType == DMLibDataType.Stream)) { options.DisableDestinationFetch = true; } // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (!IsStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; } else { if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify job is failed"); Exception jobException = result.Exceptions[0]; Test.Info("{0}", jobException); VerificationHelper.VerifyExceptionErrorMessage(jobException, "Cannot deserialize to TransferLocation when its TransferLocationType is Stream."); transferItem.DisableStreamDispose = false; transferItem.CloseStreamIfNecessary(); return; } } Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; Helper.VerifyCancelException(exception); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; ProgressChecker firstProgressChecker = null, secondProgressChecker = null; if (!IsStreamJournal) { // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy))) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } } // first progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { // The destination is already created, will cause a transfer skip firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob)) { firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else { firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024); } // second progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream) { // The destination is already created, will cause a transfer skip secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream) { secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024); } else { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024); } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); TransferContext resumeContext = null; if (IsStreamJournal) { Exception deserializeEX = null; try { resumeContext = new SingleTransferContext(journalStream) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; } catch (Exception ex) { if ((DMLibTestContext.SourceType != DMLibDataType.Stream) && (DMLibTestContext.DestType != DMLibDataType.Stream)) { Test.Error("Should no exception in deserialization when no target is stream."); } deserializeEX = ex; } } else { resumeContext = new SingleTransferContext(IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; } resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported"); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } if (!IsStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); resumeContext = new SingleTransferContext( IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = secondProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported"); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } } }
public virtual Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken) { throw new NotImplementedException(); }
public UserRepository(TransferContext context) : base(context) { this.Context = context ?? throw new ArgumentNullException("DbContext cannot be null."); }
public virtual Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { throw new NotImplementedException(); }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); secondCheckpoint = transferContext.LastCheckpoint; Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled."); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; ProgressChecker firstProgressChecker = null, secondProgressChecker = null; // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync)) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // first progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { // The destination is already created, will cause a transfer skip firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob)) { firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else { firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024); } // second progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream) { // The destination is already created, will cause a transfer skip secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream) { secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024); } else { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024); } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); TransferContext resumeContext = new TransferContext( IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); resumeContext = new TransferContext( IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = secondProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } }
private void TestDirectorySetAttribute_Restart( int bigFileSizeInKB, int smallFileSizeInKB, int bigFileNum, int smallFileNum, Action <DirNode> bigFileDirAddFileAction, Action <DirNode> smallFileDirAddFileAction, SetAttributesCallbackAsync setAttributesCallback = null, Action <DMLibDataInfo> sourceDataInfoDecorator = null) { int totalFileNum = bigFileNum + smallFileNum; long totalSizeInBytes = ((bigFileSizeInKB * bigFileNum) + (smallFileSizeInKB * smallFileNum)) * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); bigFileDirAddFileAction(bigFileDirNode); smallFileDirAddFileAction(smallFileDirNode); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo> { LimitSpeed = true, IsDirectoryTransfer = true }; using (Stream journalStream = new MemoryStream()) { bool isStreamJournal = random.Next(0, 2) == 0; var transferContext = isStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext(); transferContext.SetAttributesCallbackAsync = setAttributesCallback; var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Helper.VerifyCancelException(e.Exception); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!isStreamJournal) { // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; if (!isStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = null; if (isStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream) { ProgressHandler = progressChecker.GetProgressHandler() }; } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; } resumeContext.SetAttributesCallbackAsync = setAttributesCallback; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); sourceDataInfoDecorator?.Invoke(sourceDataInfo); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (!isStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(), SetAttributesCallbackAsync = setAttributesCallback }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); } } }
/// <summary> /// Upload a file to Azure Blob Storage. /// </summary> /// <param name="sourcePath">Path to the source file.</param> /// <param name="destBlob">The Microsoft.WindowsAzure.Storage.Blob.CloudBlob that is the destination Azure blob.</param> /// <param name="options">An Microsoft.WindowsAzure.Storage.DataMovement.UploadOptions object that specifies /// additional options for the operation.</param> /// <param name="context">A Microsoft.WindowsAzure.Storage.DataMovement.TransferContext object that represents /// the context for the current operation.</param> /// <param name="cancellationToken">A System.Threading.CancellationToken object to observe while waiting for a task /// to complete.</param> /// <returns>A System.Threading.Tasks.Task object that represents the asynchronous operation.</returns> public Task UploadAsync(string sourcePath, CloudBlob destBlob, UploadOptions options, TransferContext context, CancellationToken cancellationToken) { return(TransferManager.UploadAsync(sourcePath, destBlob, options, context, cancellationToken)); }
public void DirectoryOverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); TransferContext transferContext = new TransferContext(); transferContext.OverwriteCallback = (string sourcePath, string destinationPath) => { if (sourcePath.EndsWith(destExistNName)) { return(false); } else { return(true); } }; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); TransferException transferException = args.Exception as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 1, "Verify skipped transfer"); } else { Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } }
public GenericRepository(TransferContext context) { Context = context ?? throw new ApplicationException("DbContext cannot be null."); this.DbSet = Context.Set <T>(); }
public void OverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; TransferContext transferContext = new TransferContext(); if (fileName.Equals(destExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } else if (fileName.Equals(destExistNName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackN(); } else if (fileName.Equals(destNotExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions."); TransferException transferException = result.Exceptions[0] as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); } }