public void DataLakeUploader_TargetExistsNoOverwrite() { var frontEnd = new InMemoryFrontEnd(); frontEnd.CreateStream(TargetStreamPath, true, null, 0); //no resume, no overwrite var up = CreateParameters(filePath: _smallFilePath, isResume: false); var uploader = new DataLakeStoreUploader(up, frontEnd); Assert.Throws <InvalidOperationException>(() => uploader.Execute()); //resume, no overwrite up = CreateParameters(filePath: _smallFilePath, isResume: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.Throws <InvalidOperationException>(() => uploader.Execute()); //resume, overwrite up = CreateParameters(filePath: _smallFilePath, isResume: true, isOverwrite: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.DoesNotThrow(() => uploader.Execute()); //no resume, overwrite up = CreateParameters(filePath: _smallFilePath, isResume: false, isOverwrite: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.DoesNotThrow(() => uploader.Execute()); }
public void TestRetryBlock(int failCount) { bool expectSuccess = failCount < SingleSegmentUploader.MaxBufferUploadAttemptCount; int callCount = 0; var workingFrontEnd = new InMemoryFrontEnd(); var fe = new MockableFrontEnd(workingFrontEnd); fe.CreateStreamImplementation = (streamPath, overwrite, data, byteCount) => { callCount++; if (callCount <= failCount) { throw new IntentionalException(); } workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount); }; fe.AppendToStreamImplementation = (streamPath, data, offset, byteCount) => { callCount++; if (callCount <= failCount) { throw new IntentionalException(); } workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount); }; var metadata = CreateMetadata(_smallFilePath, _smallFileContents.Length); var progressTracker = new TestProgressTracker(); var ssu = new SingleSegmentUploader(0, metadata, fe, progressTracker); ssu.UseBackOffRetryStrategy = false; if (expectSuccess) { ssu.Upload(); var actualContents = workingFrontEnd.GetStreamContents(StreamPath); AssertExtensions.AreEqual(_smallFileContents, actualContents, "Unexpected uploaded stream contents."); } else { Assert.Throws <IntentionalException>(() => { ssu.Upload(); }); } VerifyTracker(progressTracker, expectSuccess); }
public void DataLakeUploader_ResumePartialUpload() { //attempt to load the file fully, but only allow creating 1 target stream var backingFrontEnd = new InMemoryFrontEnd(); var frontEnd = new MockableFrontEnd(backingFrontEnd); int createStreamCount = 0; frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } backingFrontEnd.CreateStream(path, overwrite, data, byteCount); }; var up = CreateParameters(isResume: false); var uploader = new DataLakeStoreUploader(up, frontEnd); uploader.DeleteMetadataFile(); Assert.Throws <AggregateException>(() => uploader.Execute()); Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Target stream should not have been created"); Assert.Equal(1, backingFrontEnd.StreamCount); //resume the upload but point it to the real back-end, which doesn't throw exceptions up = CreateParameters(isResume: true); uploader = new DataLakeStoreUploader(up, backingFrontEnd); try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); }
public void SingleSegmentUploader_TargetStreamExists() { var fe = new InMemoryFrontEnd(); //load up an existing stream fe.CreateStream(StreamPath, true, null, 0); var data = Encoding.UTF8.GetBytes("random"); fe.AppendToStream(StreamPath, data, 0, (int)data.Length); //force a re-upload of the stream var metadata = CreateMetadata(_smallFilePath, _smallFileContents.Length); var ssu = new SingleSegmentUploader(0, metadata, fe); ssu.UseBackOffRetryStrategy = false; ssu.Upload(); var actualContents = fe.GetStreamContents(StreamPath); AssertExtensions.AreEqual(_smallFileContents, actualContents, "Unexpected uploaded stream contents."); }
private void TestRetry(int segmentFailCount) { //we only have access to the underlying FrontEnd, so we need to simulate many exceptions in order to force a segment to fail the upload (multiply by SingleSegmentUploader.MaxBufferUploadAttemptAccount) //this only works because we have a small file, which we know will fit in only one buffer (for a larger file, more complex operations are necessary) int actualfailCount = segmentFailCount * SingleSegmentUploader.MaxBufferUploadAttemptCount; bool expectSuccess = segmentFailCount < MultipleSegmentUploader.MaxUploadAttemptCount; int callCount = 0; //create a mock front end sitting on top of a working front end that simulates some erros for some time var workingFrontEnd = new InMemoryFrontEnd(); var fe = new MockableFrontEnd(workingFrontEnd); fe.CreateStreamImplementation = (streamPath, overwrite, data, byteCount) => { callCount++; if (callCount <= actualfailCount) { throw new IntentionalException(); } workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount); }; fe.AppendToStreamImplementation = (streamPath, data, offset, byteCount) => { callCount++; if (callCount <= actualfailCount) { throw new IntentionalException(); } workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount); }; var metadata = CreateMetadata(1); try { var msu = new MultipleSegmentUploader(metadata, 1, fe); msu.UseSegmentBlockBackOffRetryStrategy = false; if (expectSuccess) { //the Upload method should not throw any exceptions in this case msu.Upload(); //if we are expecting success, verify that both the metadata and the target streams are complete VerifyTargetStreamsAreComplete(metadata, workingFrontEnd); } else { //the Upload method should throw an aggregate exception in this case Assert.Throws <AggregateException>(() => { msu.Upload(); }); //if we do not expect success, verify that at least 1 segment was marked as Failed Assert.True(metadata.Segments.Any(s => s.Status == SegmentUploadStatus.Failed), "Could not find any failed segments"); //for every other segment, verify it was completed OK foreach (var segment in metadata.Segments.Where(s => s.Status != SegmentUploadStatus.Failed)) { VerifyTargetStreamIsComplete(segment, metadata, workingFrontEnd); } } } finally { metadata.DeleteFile(); } }
public void DataLakeUploader_ResumePartialUpload() { //attempt to load the file fully, but only allow creating 1 target stream var backingFrontEnd = new InMemoryFrontEnd(); var frontEnd = new MockableFrontEnd(backingFrontEnd); int createStreamCount = 0; frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } backingFrontEnd.CreateStream(path, overwrite, data, byteCount); }; var up = CreateParameters(isResume: false); var uploader = new DataLakeStoreUploader(up, frontEnd); uploader.DeleteMetadataFile(); Assert.Throws<AggregateException>(() => uploader.Execute()); Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Target stream should not have been created"); Assert.Equal(1, backingFrontEnd.StreamCount); //resume the upload but point it to the real back-end, which doesn't throw exceptions up = CreateParameters(isResume: true); uploader = new DataLakeStoreUploader(up, backingFrontEnd); try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); }
public void DataLakeUploader_TargetExistsNoOverwrite() { var frontEnd = new InMemoryFrontEnd(); frontEnd.CreateStream(TargetStreamPath, true, null, 0); //no resume, no overwrite var up = CreateParameters(filePath: _smallFilePath, isResume: false); var uploader = new DataLakeStoreUploader(up, frontEnd); Assert.Throws<InvalidOperationException>(() => uploader.Execute()); //resume, no overwrite up = CreateParameters(filePath: _smallFilePath, isResume: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.Throws<InvalidOperationException>(() => uploader.Execute()); //resume, overwrite up = CreateParameters(filePath: _smallFilePath, isResume: true, isOverwrite: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.DoesNotThrow(() => uploader.Execute()); //no resume, overwrite up = CreateParameters(filePath: _smallFilePath, isResume: false, isOverwrite: true); uploader = new DataLakeStoreUploader(up, frontEnd); Assert.DoesNotThrow(() => uploader.Execute()); }
public void DataLakeUploader_ResumePartialUploadDownload() { //attempt to load the file fully, but only allow creating 1 target stream var backingFrontEnd = new InMemoryFrontEnd(); var frontEnd = new MockableFrontEnd(backingFrontEnd); int createStreamCount = 0; frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } backingFrontEnd.CreateStream(path, overwrite, data, byteCount); }; var up = CreateParameters(isResume: false); var uploader = new DataLakeStoreUploader(up, frontEnd); uploader.DeleteMetadataFile(); Assert.Throws <AggregateException>(() => uploader.Execute()); Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count); Assert.Equal(1, backingFrontEnd.StreamCount); //resume the upload but point it to the real back-end, which doesn't throw exceptions up = CreateParameters(isResume: true); uploader = new DataLakeStoreUploader(up, backingFrontEnd); try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); // now download the same way. var frontEnd2 = new MockableFrontEnd(backingFrontEnd); // need to have data from the successful upload available. createStreamCount = 0; frontEnd2.ReadStreamImplementation = (path, data, byteCount, isDownload) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } return(backingFrontEnd.ReadStream(path, data, byteCount, isDownload)); }; up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath); uploader = new DataLakeStoreUploader(up, frontEnd2); Assert.Throws <AggregateException>(() => uploader.Execute()); Assert.False(frontEnd2.StreamExists(up.TargetStreamPath), "Target stream should not have been created"); // now use the good front end up = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath); uploader = new DataLakeStoreUploader(up, backingFrontEnd); //resume the download but point it to the real back-end, which doesn't throw exceptions try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); }
public void DataLakeUploader_ResumePartialFolderUploadWithProgress() { //attempt to load the file fully, but only allow creating 1 target stream var backingFrontEnd = new InMemoryFrontEnd(); var frontEnd = new MockableFrontEnd(backingFrontEnd); UploadFolderProgress progress = null; var syncRoot = new object(); IProgress <UploadFolderProgress> progressTracker = new Progress <UploadFolderProgress>( (p) => { lock (syncRoot) { //it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress if (progress == null || progress.UploadedByteCount < p.UploadedByteCount) { progress = p; } } }); int createStreamCount = 0; frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } backingFrontEnd.CreateStream(path, overwrite, data, byteCount); }; var up = CreateParameters(isResume: false, isRecursive: true); var uploader = new DataLakeStoreUploader(up, frontEnd, folderProgressTracker: progressTracker); uploader.DeleteMetadataFile(); // Verifies that a bug in folder upload with progress hung on failure is fixed. try { var uploadTask = Task.Run(() => { uploader.Execute(); }); uploadTask.Wait(TimeSpan.FromSeconds(60)); Assert.True(false, "Folder upload did not fail after error in less than 60 seconds"); } catch (Exception ex) { Assert.True(ex is AggregateException, "The exception thrown by upload was not the expected aggregate exception."); } Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count); Assert.Equal(1, backingFrontEnd.StreamCount); //resume the upload but point it to the real back-end, which doesn't throw exceptions up = CreateParameters(isResume: true, isRecursive: true); uploader = new DataLakeStoreUploader(up, backingFrontEnd, folderProgressTracker: progressTracker); try { var uploadTask = Task.Run(() => { uploader.Execute(); }); uploadTask.Wait(TimeSpan.FromSeconds(60)); Assert.True(uploadTask.IsCompleted, "Folder upload did not complete after error in less than 60 seconds"); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3); }
private void TestRetry(int segmentFailCount) { //we only have access to the underlying FrontEnd, so we need to simulate many exceptions in order to force a segment to fail the upload (multiply by SingleSegmentUploader.MaxBufferUploadAttemptAccount) //this only works because we have a small file, which we know will fit in only one buffer (for a larger file, more complex operations are necessary) int actualfailCount = segmentFailCount * SingleSegmentUploader.MaxBufferUploadAttemptCount; bool expectSuccess = segmentFailCount < MultipleSegmentUploader.MaxUploadAttemptCount; int callCount = 0; //create a mock front end sitting on top of a working front end that simulates some erros for some time var workingFrontEnd = new InMemoryFrontEnd(); var fe = new MockableFrontEnd(workingFrontEnd); fe.CreateStreamImplementation = (streamPath, overwrite, data, byteCount) => { callCount++; if (callCount <= actualfailCount) { throw new IntentionalException(); } workingFrontEnd.CreateStream(streamPath, overwrite, data, byteCount); }; fe.AppendToStreamImplementation = (streamPath, data, offset, byteCount) => { callCount++; if (callCount <= actualfailCount) { throw new IntentionalException(); } workingFrontEnd.AppendToStream(streamPath, data, offset, byteCount); }; var metadata = CreateMetadata(1); try { var msu = new MultipleSegmentUploader(metadata, 1, fe); msu.UseSegmentBlockBackOffRetryStrategy = false; if (expectSuccess) { //the Upload method should not throw any exceptions in this case Assert.DoesNotThrow(() => { msu.Upload(); }); //if we are expecting success, verify that both the metadata and the target streams are complete VerifyTargetStreamsAreComplete(metadata, workingFrontEnd); } else { //the Upload method should throw an aggregate exception in this case Assert.Throws<AggregateException>(() => { msu.Upload(); }); //if we do not expect success, verify that at least 1 segment was marked as Failed Assert.True(metadata.Segments.Any(s => s.Status == SegmentUploadStatus.Failed), "Could not find any failed segments"); //for every other segment, verify it was completed OK foreach (var segment in metadata.Segments.Where(s => s.Status != SegmentUploadStatus.Failed)) { VerifyTargetStreamIsComplete(segment, metadata, workingFrontEnd); } } } finally { metadata.DeleteFile(); } }