コード例 #1
0
        public void TestDirectoryResume()
        {
            int  bigFileSizeInKB   = 5 * 1024; // 5 MB
            int  smallFileSizeInKB = 1;        // 1 KB
            int  bigFileNum        = 5;
            int  smallFileNum      = 50;
            long totalSizeInBytes  = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
            int  totalFileNum      = bigFileNum + smallFileNum;

            DMLibDataInfo sourceDataInfo   = new DMLibDataInfo(string.Empty);
            DirNode       bigFileDirNode   = new DirNode("big");
            DirNode       smallFileDirNode = new DirNode("small");

            sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
            sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);

            DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
            DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);

            CancellationTokenSource tokenSource = new CancellationTokenSource();

            TransferItem transferItem = null;
            var          options      = new TestExecutionOptions <DMLibDataInfo>();

            options.LimitSpeed          = true;
            options.IsDirectoryTransfer = true;

            var transferContext = new TransferContext();
            var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);

            transferContext.ProgressHandler = progressChecker.GetProgressHandler();
            var eventChecker = new TransferEventChecker();

            eventChecker.Apply(transferContext);

            transferContext.FileFailed += (sender, e) =>
            {
                Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message);
            };

            options.TransferItemModifier = (fileName, item) =>
            {
                dynamic dirOptions = DefaultTransferDirectoryOptions;
                dirOptions.Recursive = true;

                item.Options           = dirOptions;
                item.CancellationToken = tokenSource.Token;
                item.TransferContext   = transferContext;
                transferItem           = item;
            };

            TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;

            options.AfterAllItemAdded = () =>
            {
                // Wait until there are data transferred
                progressChecker.DataTransferred.WaitOne();

                // Store the first checkpoint
                firstCheckpoint = transferContext.LastCheckpoint;
                Thread.Sleep(100);

                // Cancel the transfer and store the second checkpoint
                tokenSource.Cancel();
            };

            // Cancel and store checkpoint for resume
            var result = this.ExecuteTestCase(sourceDataInfo, options);

            secondCheckpoint = transferContext.LastCheckpoint;

            if (progressChecker.FailedFilesNumber <= 0)
            {
                Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
            }

            TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;

            Test.Info("Resume with the second checkpoint first.");
            firstResumeCheckpoint  = secondCheckpoint;
            secondResumeCheckpoint = firstCheckpoint;

            // resume with firstResumeCheckpoint
            TransferItem resumeItem = transferItem.Clone();

            progressChecker.Reset();
            TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
            {
                ProgressHandler = progressChecker.GetProgressHandler()
            };

            eventChecker.Reset();
            eventChecker.Apply(resumeContext);

            resumeItem.TransferContext = resumeContext;

            result = this.RunTransferItems(new List <TransferItem>()
            {
                resumeItem
            }, new TestExecutionOptions <DMLibDataInfo>());

            VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
            VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
            VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);

            // resume with secondResumeCheckpoint
            resumeItem = transferItem.Clone();

            progressChecker.Reset();
            resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
            {
                ProgressHandler = progressChecker.GetProgressHandler(),

                // Need this overwrite callback since some files is already transferred to destination
                OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
            };

            eventChecker.Reset();
            eventChecker.Apply(resumeContext);

            resumeItem.TransferContext = resumeContext;

            result = this.RunTransferItems(new List <TransferItem>()
            {
                resumeItem
            }, new TestExecutionOptions <DMLibDataInfo>());

            VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
            VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
            VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
        }
コード例 #2
0
        private void ResumeInAllDirectionsHelper(bool directoryTransfer)
        {
            List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true)
                : AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true);

            int fileCount = expectedFileNodes.Keys.Count;

            // Execution and store checkpoints
            CancellationTokenSource tokenSource = new CancellationTokenSource();

            TransferContext transferContext = null;

            if (directoryTransfer)
            {
                transferContext = new DirectoryTransferContext();
            }
            else
            {
                transferContext = new SingleTransferContext();
            }

            var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);

            transferContext.ProgressHandler = progressChecker.GetProgressHandler();
            allItems.ForEach(item =>
            {
                item.CancellationToken = tokenSource.Token;
                item.TransferContext   = transferContext;
            });

            var options = new TestExecutionOptions <DMLibDataInfo>();

            options.DisableDestinationFetch = true;

            // Checkpoint names
            const string PartialStarted    = "PartialStarted",
                         AllStarted        = "AllStarted",
                         AllStartedAndWait = "AllStartedAndWait",
                         BeforeCancel      = "BeforeCancel",
                         AfterCancel       = "AfterCancel";
            Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();

            TransferItem randomItem = allItems[random.Next(0, allItems.Count)];

            randomItem.AfterStarted = () =>
            {
                Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
                checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
            };

            options.AfterAllItemAdded = () =>
            {
                if (!progressChecker.DataTransferred.WaitOne(30000))
                {
                    Test.Error("No progress in 30s.");
                }

                checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
                Thread.Sleep(1000);
                checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
                Thread.Sleep(1000);
                checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
                tokenSource.Cancel();
                checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
            };

            var result = this.RunTransferItems(allItems, options);

            // Resume with stored checkpoints in random order
            var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();

            checkpointList.AddRange(checkpoints);
            checkpointList.Shuffle();

            foreach (var pair in checkpointList)
            {
                Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
                options = new TestExecutionOptions <DMLibDataInfo>();
                options.DisableDestinationFetch = true;

                progressChecker.Reset();

                if (directoryTransfer)
                {
                    transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
                    {
                        ProgressHandler = progressChecker.GetProgressHandler(),

                        // The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
                        // So it will case an skip file error if the desination file already exists and
                        // We don't have overwrite callback here.
                        ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
                    };
                }
                else
                {
                    transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
                    {
                        ProgressHandler = progressChecker.GetProgressHandler(),

                        // The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
                        // So it will case an skip file error if the desination file already exists and
                        // We don't have overwrite callback here.
                        ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
                    };
                }

                int expectedFailureCount = 0;

                transferContext.FileFailed += (resource, eventArgs) =>
                {
                    TransferException exception = eventArgs.Exception as TransferException;
                    if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId)
                    {
                        Interlocked.Increment(ref expectedFailureCount);
                    }
                };

                TransferEventChecker eventChecker = new TransferEventChecker();
                eventChecker.Apply(transferContext);

                List <TransferItem> itemsToResume = allItems.Select(item =>
                {
                    TransferItem itemToResume    = item.Clone();
                    itemToResume.TransferContext = transferContext;
                    return(itemToResume);
                }).ToList();

                result = this.RunTransferItems(itemsToResume, options);

                foreach (DMLibDataType destDataType in DataTypes)
                {
                    if (DMLibDataType.URI == destDataType)
                    {
                        continue;
                    }

                    DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
                    DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);

                    foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
                    {
                        string   fileName       = destFileNode.Name;
                        FileNode sourceFileNode = expectedFileNodes[fileName];

                        Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
                    }
                }

                if (!directoryTransfer)
                {
                    Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count);
                }
                else
                {
                    Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count);
                    Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber);
                }
            }
        }
コード例 #3
0
        public void TestDirectoryResume()
        {
            int  bigFileSizeInKB   = 5 * 1024; // 5 MB
            int  smallFileSizeInKB = 1;        // 1 KB
            int  bigFileNum        = 5;
            int  smallFileNum      = 50;
            long totalSizeInBytes  = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
            int  totalFileNum      = bigFileNum + smallFileNum;

            DMLibDataInfo sourceDataInfo   = new DMLibDataInfo(string.Empty);
            DirNode       bigFileDirNode   = new DirNode("big");
            DirNode       smallFileDirNode = new DirNode("small");

            sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
            sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);

            DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
            DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);

            CancellationTokenSource tokenSource = new CancellationTokenSource();

            TransferItem transferItem = null;
            var          options      = new TestExecutionOptions <DMLibDataInfo>();

            options.LimitSpeed          = true;
            options.IsDirectoryTransfer = true;

            using (Stream journalStream = new MemoryStream())
            {
                bool IsStreamJournal = random.Next(0, 2) == 0;
                var  transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
                var  progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
                transferContext.ProgressHandler = progressChecker.GetProgressHandler();
                var eventChecker = new TransferEventChecker();
                eventChecker.Apply(transferContext);

                transferContext.FileFailed += (sender, e) =>
                {
                    Helper.VerifyCancelException(e.Exception);
                };

                options.TransferItemModifier = (fileName, item) =>
                {
                    dynamic dirOptions = DefaultTransferDirectoryOptions;
                    dirOptions.Recursive = true;

                    if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
                        DMLibTestContext.DestType == DMLibDataType.CloudFile)
                    {
                        dirOptions.PreserveSMBAttributes  = true;
                        dirOptions.PreserveSMBPermissions = true;
                    }

                    item.Options           = dirOptions;
                    item.CancellationToken = tokenSource.Token;
                    item.TransferContext   = transferContext;
                    transferItem           = item;
                };

                TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
                options.AfterAllItemAdded = () =>
                {
                    // Wait until there are data transferred
                    progressChecker.DataTransferred.WaitOne();

                    if (!IsStreamJournal)
                    {
                        // Store the first checkpoint
                        firstCheckpoint = transferContext.LastCheckpoint;
                    }

                    Thread.Sleep(1000);

                    // Cancel the transfer and store the second checkpoint
                    tokenSource.Cancel();
                };

                // Cancel and store checkpoint for resume
                var result = this.ExecuteTestCase(sourceDataInfo, options);

                if (progressChecker.FailedFilesNumber <= 0)
                {
                    Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
                }

                TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;

                if (!IsStreamJournal)
                {
                    secondCheckpoint = transferContext.LastCheckpoint;

                    Test.Info("Resume with the second checkpoint first.");
                    firstResumeCheckpoint  = secondCheckpoint;
                    secondResumeCheckpoint = firstCheckpoint;
                }

                // resume with firstResumeCheckpoint
                TransferItem resumeItem = transferItem.Clone();

                progressChecker.Reset();
                TransferContext resumeContext = null;

                if (IsStreamJournal)
                {
                    resumeContext = new DirectoryTransferContext(journalStream)
                    {
                        ProgressHandler = progressChecker.GetProgressHandler()
                    };
                }
                else
                {
                    resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
                    {
                        ProgressHandler = progressChecker.GetProgressHandler()
                    };
                }

                eventChecker.Reset();
                eventChecker.Apply(resumeContext);

                resumeItem.TransferContext = resumeContext;

                result = this.RunTransferItems(new List <TransferItem>()
                {
                    resumeItem
                }, new TestExecutionOptions <DMLibDataInfo>());

                VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
                VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
                VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);

                if (!IsStreamJournal)
                {
                    // resume with secondResumeCheckpoint
                    resumeItem = transferItem.Clone();

                    progressChecker.Reset();
                    resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
                    {
                        ProgressHandler = progressChecker.GetProgressHandler(),

                        // Need this overwrite callback since some files is already transferred to destination
                        ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
                    };

                    eventChecker.Reset();
                    eventChecker.Apply(resumeContext);

                    resumeItem.TransferContext = resumeContext;

                    result = this.RunTransferItems(new List <TransferItem>()
                    {
                        resumeItem
                    }, new TestExecutionOptions <DMLibDataInfo>());

                    VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
                    VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
                    VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
                }

                if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
                    DMLibTestContext.DestType == DMLibDataType.CloudFile)
                {
                    Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
                    Helper.CompareSMBPermissions(
                        sourceDataInfo.RootNode,
                        result.DataInfo.RootNode,
                        PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
                }
            }
        }
コード例 #4
0
        public void ResumeInAllDirections()
        {
            long fileSizeInByte = 10 * 1024 * 1024;
            Dictionary <string, FileNode> sourceFileNodes = this.PrepareSourceData(fileSizeInByte);
            List <TransferItem>           allItems        = this.GetTransformItemsForAllDirections(sourceFileNodes);

            int fileCount = sourceFileNodes.Keys.Count;

            // Execution and store checkpoints
            CancellationTokenSource tokenSource = new CancellationTokenSource();

            var transferContext = new TransferContext();
            var progressChecker = new ProgressChecker(fileCount, fileSizeInByte * fileCount);

            transferContext.ProgressHandler = progressChecker.GetProgressHandler();
            allItems.ForEach(item =>
            {
                item.CancellationToken = tokenSource.Token;
                item.TransferContext   = transferContext;
            });

            var options = new TestExecutionOptions <DMLibDataInfo>();

            options.DisableDestinationFetch = true;

            // Checkpoint names
            const string PartialStarted    = "PartialStarted",
                         AllStarted        = "AllStarted",
                         AllStartedAndWait = "AllStartedAndWait",
                         BeforeCancel      = "BeforeCancel",
                         AfterCancel       = "AfterCancel";
            Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();

            TransferItem randomItem = allItems[random.Next(0, allItems.Count)];

            randomItem.AfterStarted = () =>
            {
                Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
                checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
            };

            options.AfterAllItemAdded = () =>
            {
                progressChecker.DataTransferred.WaitOne();
                checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
                Thread.Sleep(1000);
                checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
                Thread.Sleep(1000);
                checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
                tokenSource.Cancel();
                checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
            };

            var result = this.RunTransferItems(allItems, options);

            // Resume with stored checkpoints in random order
            var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();

            checkpointList.AddRange(checkpoints);
            checkpointList.Shuffle();

            foreach (var pair in checkpointList)
            {
                Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
                options = new TestExecutionOptions <DMLibDataInfo>();
                options.DisableDestinationFetch = true;

                progressChecker.Reset();
                transferContext = new TransferContext(pair.Value)
                {
                    ProgressHandler = progressChecker.GetProgressHandler(),

                    // The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
                    // So it will case an skip file error if the desination file already exists and
                    // We don't have overwrite callback here.
                    OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
                };

                List <TransferItem> itemsToResume = allItems.Select(item =>
                {
                    TransferItem itemToResume    = item.Clone();
                    itemToResume.TransferContext = transferContext;
                    return(itemToResume);
                }).ToList();

                result = this.RunTransferItems(itemsToResume, options);

                int resumeFailCount = 0;
                foreach (DMLibDataType destDataType in DataTypes)
                {
                    DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
                    DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);

                    foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
                    {
                        string fileName = destFileNode.Name;
                        if (!fileName.Contains(DMLibDataType.Stream.ToString()))
                        {
                            FileNode sourceFileNode = sourceFileNodes[fileName];
                            Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
                        }
                        else
                        {
                            resumeFailCount++;
                        }
                    }
                }

                Test.Assert(result.Exceptions.Count == resumeFailCount, "Verify resume failure count: expected {0}, actual {1}.", resumeFailCount, result.Exceptions.Count);

                foreach (var resumeException in result.Exceptions)
                {
                    Test.Assert(resumeException is NotSupportedException, "Verify resume exception is NotSupportedException.");
                }
            }
        }