Exemple #1
0
        public void DataLakeUploader_CancelUpload()
        {
            CancellationTokenSource myTokenSource = new CancellationTokenSource();
            var cancelToken    = myTokenSource.Token;
            var frontEnd       = new InMemoryFrontEnd();
            var mockedFrontend = new MockableFrontEnd(frontEnd);

            mockedFrontend.GetStreamLengthImplementation = (streamPath, isDownload) =>
            {
                // sleep for 2 second to allow for the cancellation to actual happen
                Thread.Sleep(2000);
                return(frontEnd.GetStreamLength(streamPath, isDownload));
            };

            mockedFrontend.StreamExistsImplementation = (streamPath, isDownload) =>
            {
                // sleep for 2 second to allow for the cancellation to actual happen
                Thread.Sleep(2000);
                return(frontEnd.StreamExists(streamPath, isDownload));
            };
            var                        up              = CreateParameters(isResume: false);
            UploadProgress             progress        = null;
            var                        syncRoot        = new object();
            IProgress <UploadProgress> progressTracker = new Progress <UploadProgress>(
                (p) =>
            {
                lock (syncRoot)
                {
                    //it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
                    if (progress == null || progress.UploadedByteCount < p.UploadedByteCount)
                    {
                        progress = p;
                    }
                }
            });
            var uploader = new DataLakeStoreUploader(up, mockedFrontend, cancelToken, progressTracker);

            Task uploadTask = Task.Run(() =>
            {
                uploader.Execute();
                Thread.Sleep(2000);
            }, cancelToken);

            myTokenSource.Cancel();
            Assert.True(cancelToken.IsCancellationRequested);

            while (uploadTask.Status == TaskStatus.Running || uploadTask.Status == TaskStatus.WaitingToRun)
            {
                Thread.Sleep(250);
            }

            // Verify that the file did not get uploaded completely.
            Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Uploaded stream exists when it should not yet have been completely created");
        }
Exemple #2
0
        public void TestRetryBlock(int failCount)
        {
            bool expectSuccess = failCount < SingleSegmentUploader.MaxBufferUploadAttemptCount;

            int callCount = 0;

            var workingFrontEnd = new InMemoryFrontEnd();
            var fe = new MockableFrontEnd(workingFrontEnd);

            fe.CreateStreamImplementation =
                (streamPath, overwrite, data, byteCount) =>
            {
                callCount++;
                if (callCount <= failCount)
                {
                    throw new IntentionalException();
                }
                workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount);
            };

            fe.AppendToStreamImplementation =
                (streamPath, data, offset, byteCount) =>
            {
                callCount++;
                if (callCount <= failCount)
                {
                    throw new IntentionalException();
                }
                workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount);
            };

            var metadata        = CreateMetadata(_smallFilePath, _smallFileContents.Length);
            var progressTracker = new TestProgressTracker();
            var ssu             = new SingleSegmentUploader(0, metadata, fe, progressTracker);

            ssu.UseBackOffRetryStrategy = false;

            if (expectSuccess)
            {
                ssu.Upload();
                var actualContents = workingFrontEnd.GetStreamContents(StreamPath);
                AssertExtensions.AreEqual(_smallFileContents, actualContents, "Unexpected uploaded stream contents.");
            }
            else
            {
                Assert.Throws <IntentionalException>(() => { ssu.Upload(); });
            }
            VerifyTracker(progressTracker, expectSuccess);
        }
Exemple #3
0
        public void DataLakeUploader_ResumePartialUpload()
        {
            //attempt to load the file fully, but only allow creating 1 target stream
            var backingFrontEnd = new InMemoryFrontEnd();
            var frontEnd        = new MockableFrontEnd(backingFrontEnd);

            int createStreamCount = 0;

            frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
            {
                createStreamCount++;
                if (createStreamCount > 1)
                {
                    //we only allow 1 file to be created
                    throw new IntentionalException();
                }
                backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
            };
            var up       = CreateParameters(isResume: false);
            var uploader = new DataLakeStoreUploader(up, frontEnd);

            uploader.DeleteMetadataFile();

            Assert.Throws <AggregateException>(() => uploader.Execute());
            Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
            Assert.Equal(1, backingFrontEnd.StreamCount);

            //resume the upload but point it to the real back-end, which doesn't throw exceptions
            up       = CreateParameters(isResume: true);
            uploader = new DataLakeStoreUploader(up, backingFrontEnd);

            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, backingFrontEnd);
        }
Exemple #4
0
        public void SingleSegmentUploader_VerifyUploadStreamFails()
        {
            //create a mock front end which doesn't do anything
            var fe = new MockableFrontEnd();

            fe.CreateStreamImplementation    = (streamPath, overwrite, data, byteCount) => { };
            fe.DeleteStreamImplementation    = (streamPath, recurse) => { };
            fe.StreamExistsImplementation    = (streamPath) => { return(true); };
            fe.AppendToStreamImplementation  = (streamPath, data, offset, byteCount) => { };
            fe.GetStreamLengthImplementation = (streamPath) => { return(0); };

            //upload some data
            var metadata = CreateMetadata(_smallFilePath, _smallFileContents.Length);
            var ssu      = new SingleSegmentUploader(0, metadata, fe);

            ssu.UseBackOffRetryStrategy = false;

            //the Upload method should fail if it cannot verify that the stream was uploaded after the upload (i.e., it will get a length of 0 at the end)
            Assert.Throws <UploadFailedException>(() => { ssu.Upload(); });
        }
Exemple #5
0
        public void DataLakeUploader_ResumeUploadWithAllMissingFiles()
        {
            //this scenario is achieved by refusing to execute the concat command on the front end for the initial upload (which will interrupt it)
            //and then resuming the upload against a fresh front-end (which obviously has no files there)

            var backingFrontEnd1 = new InMemoryFrontEnd();
            var frontEnd1        = new MockableFrontEnd(backingFrontEnd1);

            frontEnd1.ConcatenateImplementation = (target, inputs) => { throw new IntentionalException(); }; //fail the concatenation

            //attempt full upload
            var up       = CreateParameters(isResume: false);
            var uploader = new DataLakeStoreUploader(up, frontEnd1);

            uploader.DeleteMetadataFile();

            Assert.Throws <IntentionalException>(() => uploader.Execute());
            Assert.False(frontEnd1.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
            Assert.True(0 < backingFrontEnd1.StreamCount, "No temporary streams seem to have been created");

            //attempt to resume the upload
            var frontEnd2 = new InMemoryFrontEnd();

            up       = CreateParameters(isResume: true);
            uploader = new DataLakeStoreUploader(up, frontEnd2);

            //at this point the metadata exists locally but there are no target files in frontEnd2
            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, frontEnd2);
        }
Exemple #6
0
        public void DataLakeUploader_FreshFolderUploadDownload()
        {
            var frontEnd = new InMemoryFrontEnd();
            var up       = CreateParameters(isResume: false, isRecursive: true);
            UploadFolderProgress progress = null;
            var syncRoot = new object();
            IProgress <UploadFolderProgress> progressTracker = new Progress <UploadFolderProgress>(
                (p) =>
            {
                lock (syncRoot)
                {
                    //it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
                    if (progress == null || progress.UploadedByteCount < p.UploadedByteCount)
                    {
                        progress = p;
                    }
                }
            });
            var uploader = new DataLakeStoreUploader(up, frontEnd, null, progressTracker);

            uploader.Execute();

            VerifyFileUploadedSuccessfully(up, frontEnd);
            VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);

            // now download
            var downloadFrontEnd = new MockableFrontEnd(frontEnd);

            // replace the isDirectory implementation to return true
            downloadFrontEnd.IsDirectoryImplementation = (streamPath) => { return(true); };
            progress = null;
            up       = CreateParameters(isRecursive: true, isResume: false, isDownload: true, targetStreamPath: Path.GetDirectoryName(_downloadFilePath), isOverwrite: true, filePath: TargetStreamPath);
            uploader = new DataLakeStoreUploader(up, downloadFrontEnd, null, progressTracker);

            uploader.Execute();
            VerifyFileUploadedSuccessfully(up, downloadFrontEnd.BaseAdapter);
            VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
        }
Exemple #7
0
        public void DataLakeUploader_UploadSingleSegment()
        {
            var frontEnd     = new InMemoryFrontEnd();
            var mockFrontEnd = new MockableFrontEnd(frontEnd);

            mockFrontEnd.ConcatenateImplementation = (target, inputs) => { Assert.True(false, "Concatenate should not be called when using 1 segment"); };

            var up = new UploadParameters(
                inputFilePath: _smallFilePath,
                targetStreamPath: "1",
                threadCount: ThreadCount,
                accountName: "foo",
                isResume: false,
                maxSegmentLength: 4 * 1024 * 1024,
                localMetadataLocation: Path.GetTempPath());

            File.WriteAllBytes(_smallFilePath, _smallFileData);

            var uploader = new DataLakeStoreUploader(up, frontEnd);

            uploader.Execute();

            VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
        }
        private void TestRetry(int segmentFailCount)
        {
            //we only have access to the underlying FrontEnd, so we need to simulate many exceptions in order to force a segment to fail the upload (multiply by SingleSegmentUploader.MaxBufferUploadAttemptAccount)
            //this only works because we have a small file, which we know will fit in only one buffer (for a larger file, more complex operations are necessary)
            int  actualfailCount = segmentFailCount * SingleSegmentUploader.MaxBufferUploadAttemptCount;
            bool expectSuccess   = segmentFailCount < MultipleSegmentUploader.MaxUploadAttemptCount;

            int callCount = 0;

            //create a mock front end sitting on top of a working front end that simulates some erros for some time
            var workingFrontEnd = new InMemoryFrontEnd();
            var fe = new MockableFrontEnd(workingFrontEnd);

            fe.CreateStreamImplementation =
                (streamPath, overwrite, data, byteCount) =>
            {
                callCount++;
                if (callCount <= actualfailCount)
                {
                    throw new IntentionalException();
                }
                workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount);
            };

            fe.AppendToStreamImplementation =
                (streamPath, data, offset, byteCount) =>
            {
                callCount++;
                if (callCount <= actualfailCount)
                {
                    throw new IntentionalException();
                }
                workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount);
            };

            var metadata = CreateMetadata(1);

            try
            {
                var msu = new MultipleSegmentUploader(metadata, 1, fe);
                msu.UseSegmentBlockBackOffRetryStrategy = false;

                if (expectSuccess)
                {
                    //the Upload method should not throw any exceptions in this case
                    msu.Upload();

                    //if we are expecting success, verify that both the metadata and the target streams are complete
                    VerifyTargetStreamsAreComplete(metadata, workingFrontEnd);
                }
                else
                {
                    //the Upload method should throw an aggregate exception in this case
                    Assert.Throws <AggregateException>(() => { msu.Upload(); });

                    //if we do not expect success, verify that at least 1 segment was marked as Failed
                    Assert.True(metadata.Segments.Any(s => s.Status == SegmentUploadStatus.Failed), "Could not find any failed segments");

                    //for every other segment, verify it was completed OK
                    foreach (var segment in metadata.Segments.Where(s => s.Status != SegmentUploadStatus.Failed))
                    {
                        VerifyTargetStreamIsComplete(segment, metadata, workingFrontEnd);
                    }
                }
            }
            finally
            {
                metadata.DeleteFile();
            }
        }
        public void DataLakeUploader_UploadSingleSegment()
        {
            var frontEnd = new InMemoryFrontEnd();
            var mockFrontEnd = new MockableFrontEnd(frontEnd);
            mockFrontEnd.ConcatenateImplementation = (target, inputs) => { Assert.True(false, "Concatenate should not be called when using 1 segment"); };

            var up = new UploadParameters(
                inputFilePath: _smallFilePath,
                targetStreamPath: "1",
                threadCount: ThreadCount,
                accountName: "foo",
                isResume: false,
                maxSegmentLength: 4 * 1024 * 1024,
                localMetadataLocation: Path.GetTempPath());

            File.WriteAllBytes(_smallFilePath, _smallFileData);

            var uploader = new DataLakeStoreUploader(up, frontEnd);
            uploader.Execute();

            VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
        }
        public void DataLakeUploader_ResumePartialUpload()
        {
            //attempt to load the file fully, but only allow creating 1 target stream
            var backingFrontEnd = new InMemoryFrontEnd();
            var frontEnd = new MockableFrontEnd(backingFrontEnd);

            int createStreamCount = 0;
            frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
            {
                createStreamCount++;
                if (createStreamCount > 1)
                {
                    //we only allow 1 file to be created
                    throw new IntentionalException();
                }
                backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
            };
            var up = CreateParameters(isResume: false);
            var uploader = new DataLakeStoreUploader(up, frontEnd);
            uploader.DeleteMetadataFile();

            Assert.Throws<AggregateException>(() => uploader.Execute());
            Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
            Assert.Equal(1, backingFrontEnd.StreamCount);

            //resume the upload but point it to the real back-end, which doesn't throw exceptions
            up = CreateParameters(isResume: true);
            uploader = new DataLakeStoreUploader(up, backingFrontEnd);

            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, backingFrontEnd);
        }
        public void DataLakeUploader_ResumeUploadWithAllMissingFiles()
        {
            //this scenario is achieved by refusing to execute the concat command on the front end for the initial upload (which will interrupt it)
            //and then resuming the upload against a fresh front-end (which obviously has no files there)
            
            var backingFrontEnd1 = new InMemoryFrontEnd();
            var frontEnd1 = new MockableFrontEnd(backingFrontEnd1);
            frontEnd1.ConcatenateImplementation = (target, inputs) => { throw new IntentionalException(); }; //fail the concatenation
            
            //attempt full upload
            var up = CreateParameters(isResume: false);
            var uploader = new DataLakeStoreUploader(up, frontEnd1);
            uploader.DeleteMetadataFile();

            Assert.Throws<IntentionalException>(() => uploader.Execute());
            Assert.False(frontEnd1.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
            Assert.True(0 < backingFrontEnd1.StreamCount, "No temporary streams seem to have been created");

            //attempt to resume the upload
            var frontEnd2 = new InMemoryFrontEnd();
            up = CreateParameters(isResume: true);
            uploader = new DataLakeStoreUploader(up, frontEnd2);

            //at this point the metadata exists locally but there are no target files in frontEnd2
            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, frontEnd2);
        }
Exemple #12
0
        public void DataLakeUploader_ResumePartialUploadDownload()
        {
            //attempt to load the file fully, but only allow creating 1 target stream
            var backingFrontEnd = new InMemoryFrontEnd();
            var frontEnd        = new MockableFrontEnd(backingFrontEnd);

            int createStreamCount = 0;

            frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
            {
                createStreamCount++;
                if (createStreamCount > 1)
                {
                    //we only allow 1 file to be created
                    throw new IntentionalException();
                }
                backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
            };

            var up       = CreateParameters(isResume: false);
            var uploader = new DataLakeStoreUploader(up, frontEnd);

            uploader.DeleteMetadataFile();

            Assert.Throws <AggregateException>(() => uploader.Execute());
            Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count);
            Assert.Equal(1, backingFrontEnd.StreamCount);

            //resume the upload but point it to the real back-end, which doesn't throw exceptions
            up       = CreateParameters(isResume: true);
            uploader = new DataLakeStoreUploader(up, backingFrontEnd);

            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, backingFrontEnd);

            // now download the same way.
            var frontEnd2 = new MockableFrontEnd(backingFrontEnd); // need to have data from the successful upload available.

            createStreamCount = 0;
            frontEnd2.ReadStreamImplementation = (path, data, byteCount, isDownload) =>
            {
                createStreamCount++;
                if (createStreamCount > 1)
                {
                    //we only allow 1 file to be created
                    throw new IntentionalException();
                }
                return(backingFrontEnd.ReadStream(path, data, byteCount, isDownload));
            };

            up       = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath);
            uploader = new DataLakeStoreUploader(up, frontEnd2);

            Assert.Throws <AggregateException>(() => uploader.Execute());
            Assert.False(frontEnd2.StreamExists(up.TargetStreamPath), "Target stream should not have been created");

            // now use the good front end
            up       = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath);
            uploader = new DataLakeStoreUploader(up, backingFrontEnd);

            //resume the download but point it to the real back-end, which doesn't throw exceptions
            try
            {
                uploader.Execute();
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, backingFrontEnd);
        }
Exemple #13
0
        public void DataLakeUploader_ResumePartialFolderUploadWithProgress()
        {
            //attempt to load the file fully, but only allow creating 1 target stream
            var backingFrontEnd           = new InMemoryFrontEnd();
            var frontEnd                  = new MockableFrontEnd(backingFrontEnd);
            UploadFolderProgress progress = null;
            var syncRoot                  = new object();
            IProgress <UploadFolderProgress> progressTracker = new Progress <UploadFolderProgress>(
                (p) =>
            {
                lock (syncRoot)
                {
                    //it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
                    if (progress == null || progress.UploadedByteCount < p.UploadedByteCount)
                    {
                        progress = p;
                    }
                }
            });
            int createStreamCount = 0;

            frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
            {
                createStreamCount++;
                if (createStreamCount > 1)
                {
                    //we only allow 1 file to be created
                    throw new IntentionalException();
                }
                backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
            };
            var up       = CreateParameters(isResume: false, isRecursive: true);
            var uploader = new DataLakeStoreUploader(up, frontEnd, folderProgressTracker: progressTracker);

            uploader.DeleteMetadataFile();

            // Verifies that a bug in folder upload with progress hung on failure is fixed.
            try
            {
                var uploadTask = Task.Run(() =>
                {
                    uploader.Execute();
                });

                uploadTask.Wait(TimeSpan.FromSeconds(60));
                Assert.True(false, "Folder upload did not fail after error in less than 60 seconds");
            }
            catch (Exception ex)
            {
                Assert.True(ex is AggregateException, "The exception thrown by upload was not the expected aggregate exception.");
            }

            Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count);
            Assert.Equal(1, backingFrontEnd.StreamCount);

            //resume the upload but point it to the real back-end, which doesn't throw exceptions
            up       = CreateParameters(isResume: true, isRecursive: true);
            uploader = new DataLakeStoreUploader(up, backingFrontEnd, folderProgressTracker: progressTracker);

            try
            {
                var uploadTask = Task.Run(() =>
                {
                    uploader.Execute();
                });

                uploadTask.Wait(TimeSpan.FromSeconds(60));
                Assert.True(uploadTask.IsCompleted, "Folder upload did not complete after error in less than 60 seconds");
            }
            finally
            {
                uploader.DeleteMetadataFile();
            }

            VerifyFileUploadedSuccessfully(up, backingFrontEnd);
            VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
        }
        private void TestRetry(int segmentFailCount)
        {
            //we only have access to the underlying FrontEnd, so we need to simulate many exceptions in order to force a segment to fail the upload (multiply by SingleSegmentUploader.MaxBufferUploadAttemptAccount)
            //this only works because we have a small file, which we know will fit in only one buffer (for a larger file, more complex operations are necessary)
            int actualfailCount = segmentFailCount * SingleSegmentUploader.MaxBufferUploadAttemptCount;
            bool expectSuccess = segmentFailCount < MultipleSegmentUploader.MaxUploadAttemptCount;

            int callCount = 0;

            //create a mock front end sitting on top of a working front end that simulates some erros for some time
            var workingFrontEnd = new InMemoryFrontEnd();
            var fe = new MockableFrontEnd(workingFrontEnd);
            fe.CreateStreamImplementation =
                (streamPath, overwrite, data, byteCount) =>
                {
                    callCount++;
                    if (callCount <= actualfailCount)
                    {
                        throw new IntentionalException();
                    }
                    workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount);
                };

            fe.AppendToStreamImplementation =
                (streamPath, data, offset, byteCount) =>
                {
                    callCount++;
                    if (callCount <= actualfailCount)
                    {
                        throw new IntentionalException();
                    }
                    workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount);
                };

            var metadata = CreateMetadata(1);
            try
            {
                var msu = new MultipleSegmentUploader(metadata, 1, fe);
                msu.UseSegmentBlockBackOffRetryStrategy = false;

                if (expectSuccess)
                {
                    //the Upload method should not throw any exceptions in this case
                    Assert.DoesNotThrow(() => { msu.Upload(); });
                    
                    //if we are expecting success, verify that both the metadata and the target streams are complete
                    VerifyTargetStreamsAreComplete(metadata, workingFrontEnd);
                }
                else
                {
                    //the Upload method should throw an aggregate exception in this case
                    Assert.Throws<AggregateException>(() => { msu.Upload(); });

                    //if we do not expect success, verify that at least 1 segment was marked as Failed
                    Assert.True(metadata.Segments.Any(s => s.Status == SegmentUploadStatus.Failed), "Could not find any failed segments");

                    //for every other segment, verify it was completed OK
                    foreach (var segment in metadata.Segments.Where(s => s.Status != SegmentUploadStatus.Failed))
                    {
                        VerifyTargetStreamIsComplete(segment, metadata, workingFrontEnd);
                    }
                }
            }
            finally
            {
                metadata.DeleteFile();
            }
        }