public void DataLakeUploader_ResumePartialUploadDownload() { //attempt to load the file fully, but only allow creating 1 target stream var backingFrontEnd = new InMemoryFrontEnd(); var frontEnd = new MockableFrontEnd(backingFrontEnd); int createStreamCount = 0; frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } backingFrontEnd.CreateStream(path, overwrite, data, byteCount); }; var up = CreateParameters(isResume: false); var uploader = new DataLakeStoreTransferClient(up, frontEnd); uploader.DeleteMetadataFile(); Assert.Throws <AggregateException>(() => uploader.Execute()); Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count); Assert.Equal(1, backingFrontEnd.StreamCount); //resume the upload but point it to the real back-end, which doesn't throw exceptions up = CreateParameters(isResume: true); uploader = new DataLakeStoreTransferClient(up, backingFrontEnd); try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); // now download the same way. var frontEnd2 = new MockableFrontEnd(backingFrontEnd); // need to have data from the successful upload available. createStreamCount = 0; frontEnd2.ReadStreamImplementation = (path, data, byteCount, isDownload) => { createStreamCount++; if (createStreamCount > 1) { //we only allow 1 file to be created throw new IntentionalException(); } return(backingFrontEnd.ReadStream(path, data, byteCount, isDownload)); }; up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath); uploader = new DataLakeStoreTransferClient(up, frontEnd2); Assert.Throws <AggregateException>(() => uploader.Execute()); Assert.False(frontEnd2.StreamExists(up.TargetStreamPath), "Target stream should not have been created"); // now use the good front end up = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath); uploader = new DataLakeStoreTransferClient(up, backingFrontEnd); //resume the download but point it to the real back-end, which doesn't throw exceptions try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, backingFrontEnd); }
public void DataLakeUploader_ResumeUploadDownloadWithAllMissingFiles() { //this scenario is achieved by refusing to execute the concat command on the front end for the initial upload (which will interrupt it) //and then resuming the upload against a fresh front-end (which obviously has no files there) var backingFrontEnd1 = new InMemoryFrontEnd(); var frontEnd1 = new MockableFrontEnd(backingFrontEnd1); frontEnd1.ConcatenateImplementation = (target, inputs, isDownload) => { throw new IntentionalException(); }; //fail the concatenation //attempt full upload var up = CreateParameters(isResume: false); var uploader = new DataLakeStoreTransferClient(up, frontEnd1); uploader.DeleteMetadataFile(); Assert.Throws <IntentionalException>(() => uploader.Execute()); Assert.False(frontEnd1.StreamExists(up.TargetStreamPath), "Target stream should not have been created"); Assert.True(0 < backingFrontEnd1.StreamCount, "No temporary streams seem to have been created"); //attempt to resume the upload var frontEnd2 = new InMemoryFrontEnd(); up = CreateParameters(isResume: true); uploader = new DataLakeStoreTransferClient(up, frontEnd2); //at this point the metadata exists locally but there are no target files in frontEnd2 try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, frontEnd2); // now download the same way. var frontEnd3 = new MockableFrontEnd(frontEnd2); // need to have data from the successful upload available. frontEnd3.ConcatenateImplementation = (target, inputs, isDownload) => { throw new IntentionalException(); }; //fail the concatenation up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath); uploader = new DataLakeStoreTransferClient(up, frontEnd3); Assert.Throws <IntentionalException>(() => uploader.Execute()); Assert.False(frontEnd1.StreamExists(up.TargetStreamPath, true), "Target stream should not have been created"); // now use the good front end up = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath); uploader = new DataLakeStoreTransferClient(up, frontEnd2); //at this point the metadata exists locally but there are no target files in frontEnd2 try { uploader.Execute(); } finally { uploader.DeleteMetadataFile(); } VerifyFileUploadedSuccessfully(up, frontEnd2); }