public async Task <ISaveResult> SaveFileAsync(string key, Stream content, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress, AmazonS3CabinetConfig config)
{
Contract.NotNullOrEmpty(key, nameof(key));
Contract.NotNull(content, nameof(content));
Contract.NotNull(config, nameof(config));
if (handleExisting != HandleExistingMethod.Overwrite)
{
throw new NotImplementedException();
}
using (var s3Client = GetS3Client(config)) {
try {
// Use the transfer utility as it handle large files in a better way
var uploadRequest = new TransferUtilityUploadRequest {
InputStream = content,
};
await UploadInternal(key, config, s3Client, progress, uploadRequest);
return(new SaveResult(key));
} catch (Exception e) {
return(new SaveResult(key, e));
}
}
}
public async Task <ISaveResult> SaveFileAsync(string key, string filePath, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress, FileSystemCabinetConfig config)
{
if (String.IsNullOrWhiteSpace(key))
{
throw new ArgumentNullException(nameof(key));
}
if (String.IsNullOrWhiteSpace(filePath))
{
throw new ArgumentNullException(nameof(filePath));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
try {
var fs = GetFileSystem(config);
using (var readStream = fs.File.OpenRead(filePath)) {
return(await this.SaveFileAsync(key, readStream, handleExisting, progress, config));
}
} catch (FileNotFoundException e) {
return(new SaveResult(key, success: false, errorMsg: $"File does not exist at path {filePath}"));
} catch (Exception e) {
return(new SaveResult(key, e));
}
}
public async Task<UploadSaveResult[]> SaveInCabinet(HandleExistingMethod handleExisting = HandleExistingMethod.Throw, IFileScanner fileScanner = null) {
var saveTasks = this.FileData.Select(async (fd) => {
string uploadFileName = fd.Headers.ContentDisposition.FileName?.Trim('"')?.Trim('\\');
string uploadExtension = Path.GetExtension(uploadFileName)?.TrimStart('.');
string uploadMediaType = fd.Headers.ContentType?.MediaType;
if(!this.fileValidator.IsFileTypeWhitelisted(uploadExtension, uploadMediaType)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "The file type is not allowed");
}
string fileName = fd.LocalFileName;
var uploadedFileInfo = new FileInfo(fileName);
if(this.fileValidator.IsFileTooLarge(uploadedFileInfo.Length)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "The file is too large");
}
if(this.fileValidator.IsFileTooSmall(uploadedFileInfo.Length)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "The file is too small");
}
string delimiter = this.fileCabinet.GetKeyDelimiter();
string key = this.keyProvider.GetKey(uploadFileName, uploadMediaType, delimiter);
if(String.IsNullOrWhiteSpace(key)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "No key was provided");
}
if(!File.Exists(fileName)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "Could not find uploaded file.");
}
// File scanner to optionally check the file an remove if it's unsafe
// note the c# 6 fileScanner?.ScanFileAsync syntax doesn't seem to work
if(fileScanner != null) {
await fileScanner.ScanFileAsync(fileName);
}
if(!File.Exists(fileName)) {
return new UploadSaveResult(uploadFileName, uploadMediaType, "File has been removed as it is unsafe.");
}
var cabinetResult = await this.fileCabinet.SaveFileAsync(key, fileName, handleExisting, this.CabinetFileSaveProgress);
return new UploadSaveResult(uploadFileName, uploadMediaType, cabinetResult);
});
var saveResults = await Task.WhenAll(saveTasks);
// cleanup temp files
foreach(var file in this.FileData) {
File.Delete(file.LocalFileName);
}
return saveResults;
}
public async Task <ISaveResult> SaveFileAsync(string key, Stream content, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress, ReplicatedProviderConfig config)
{
Contract.NotNullOrEmpty(key, nameof(key));
Contract.NotNull(content, nameof(content));
Contract.NotNull(config, nameof(config));
return(await SaveReplica(async (cabinet) => {
return await cabinet.SaveFileAsync(key, content, handleExisting, progress);
}, config));
}
public async Task Save_File_Path(string key, string filePath, HandleExistingMethod handleExisting)
{
var mockSaveResult = new Mock <ISaveResult>();
this.mockToCabinet.Setup(c => c.SaveFileAsync(key, filePath, handleExisting, null)).ReturnsAsync(mockSaveResult.Object);
await provider.SaveFileAsync(key, filePath, handleExisting, null, config);
this.mockToCabinet.Verify(c => c.SaveFileAsync(key, filePath, handleExisting, null), Times.Once);
}
public SaveCommand() {
IsCommand("save", "Saves a file into the cabinet");
HasRequiredOption("cabinet=|c=", "Cabinet to save file to", c => configName = c);
HasRequiredOption("file-path=|f=", "Path of the File to save", f => filePath = f);
HasRequiredOption("key=|k=", "Key to save file to", k => key = k);
HasOption("overwrite|o", "Overwrite an existing file in the cabinet", o => handleExisting = HandleExistingMethod.Overwrite);
HasOption("skip|s", "Skip if there is an existing file in the cabinet", o => handleExisting = HandleExistingMethod.Skip);
}
public async Task <ISaveResult> SaveFileAsync(string key, string filePath, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress, MigrationProviderConfig config)
{
Contract.NotNullOrEmpty(key, nameof(key));
Contract.NotNullOrEmpty(filePath, nameof(filePath));
Contract.NotNull(config, nameof(config));
var to = GetToCabinet(config);
return(await to.SaveFileAsync(key, filePath, handleExisting, progress));
}
public async Task Move_File(string sourceKey, string destKey, HandleExistingMethod handleExisting)
{
var mockResult = new Mock <IMoveResult>();
this.mockStorageProvider.Setup(s => s.MoveFileAsync(sourceKey, destKey, handleExisting, config)).ReturnsAsync(mockResult.Object);
var actualResult = await this.fileCabinet.MoveFileAsync(sourceKey, destKey, handleExisting);
this.mockStorageProvider.Verify(s => s.MoveFileAsync(sourceKey, destKey, handleExisting, config), Times.Once);
Assert.Equal(mockResult.Object, actualResult);
}
public async Task Move_File_Not_Overwrite_Throws(HandleExistingMethod handleExisting)
{
var provider = GetProvider();
var config = GetConfig(ValidBucketName);
string sourceKey = @"source.txt";
string destKey = @"dest.txt";
var mockProgress = new Mock <IProgress <IWriteProgress> >();
await Assert.ThrowsAsync <NotImplementedException>(async() =>
await provider.MoveFileAsync(sourceKey, destKey, handleExisting, config)
);
}
public async Task Save_File_Path(string key, HandleExistingMethod handleExisting)
{
string filePath = "C:\test";
var mockResult = new Mock <ISaveResult>();
var mockProgress = new Mock <IProgress <IWriteProgress> >();
this.mockStorageProvider.Setup(s => s.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object, config)).ReturnsAsync(mockResult.Object);
var actualResult = await this.fileCabinet.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object);
this.mockStorageProvider.Verify(s => s.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object, config), Times.Once);
Assert.Equal(mockResult.Object, actualResult);
}
public async Task Save_File_Path(string key, string filePath, HandleExistingMethod handleExisting, bool masterSuccess)
{
var mockMasterSaveResult = new Mock <ISaveResult>();
var mockReplicaSaveResult = new Mock <ISaveResult>();
mockMasterSaveResult.SetupGet(r => r.Success).Returns(masterSuccess);
this.mockMasterCabinet.Setup(c => c.SaveFileAsync(key, filePath, handleExisting, null)).ReturnsAsync(mockMasterSaveResult.Object);
this.mockReplicaCabinet.Setup(c => c.SaveFileAsync(key, filePath, handleExisting, null)).ReturnsAsync(mockReplicaSaveResult.Object);
var saveResult = await provider.SaveFileAsync(key, filePath, handleExisting, null, config);
this.mockMasterCabinet.Verify(c => c.SaveFileAsync(key, filePath, handleExisting, null), Times.Once);
this.mockReplicaCabinet.Verify(c => c.SaveFileAsync(key, filePath, handleExisting, null), masterSuccess ? Times.Once() : Times.Never());
Assert.Equal(mockMasterSaveResult.Object, saveResult);
}
private Task <IMoveResult> MoveInternal(string sourceKey, string destKey, HandleExistingMethod handleExisting, FileSystemCabinetConfig config)
{
var destFileInfo = this.GetFileInfo(destKey, config);
try {
var fs = GetFileSystem(config);
var fileInfo = this.GetFileInfo(sourceKey, config);
if (!destFileInfo.Directory.Exists)
{
destFileInfo.Directory.Create();
}
// Do file system move
fs.File.Move(fileInfo.FullName, destFileInfo.FullName);
return(Task.FromResult <IMoveResult>(new MoveResult(sourceKey, destKey, success: true)));
} catch (DirectoryNotFoundException e) {
return(Task.FromResult <IMoveResult>(new MoveResult(sourceKey, destKey, e)));
} catch (IOException e) {
if (handleExisting == HandleExistingMethod.Throw)
{
throw new ApplicationException($"File exists at {destKey} and handleExisting is set to throw");
}
if (handleExisting == HandleExistingMethod.Skip)
{
return(Task.FromResult <IMoveResult>(new MoveResult(sourceKey, destKey, success: true)
{
AlreadyExists = true
}));
}
return(Task.FromResult <IMoveResult>(new MoveResult(sourceKey, destKey, e)));
} catch (Exception e) {
return(Task.FromResult <IMoveResult>(new MoveResult(sourceKey, destKey, e)));
}
}
public async Task Save_File_Stream(string key, HandleExistingMethod handleExisting) {
var mockStream = new Mock<Stream>();
var mockSaveResult = new Mock<ISaveResult>();
this.mockToCabinet.Setup(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null)).ReturnsAsync(mockSaveResult.Object);
await provider.SaveFileAsync(key, mockStream.Object, handleExisting, null, config);
this.mockToCabinet.Verify(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null), Times.Once);
}
public async Task <IMoveResult> MoveFileAsync(string sourceKey, string destKey, HandleExistingMethod handleExisting)
{
return(await provider.MoveFileAsync(sourceKey, destKey, handleExisting, config));
}
public async Task <IMoveResult> MoveFileAsync(string sourceKey, string destKey, HandleExistingMethod handleExisting, FileSystemCabinetConfig config)
{
if (String.IsNullOrWhiteSpace(sourceKey))
{
throw new ArgumentNullException(nameof(sourceKey));
}
if (String.IsNullOrWhiteSpace(destKey))
{
throw new ArgumentNullException(nameof(destKey));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (handleExisting == HandleExistingMethod.Overwrite)
{
return(await MoveAndOverwrite(sourceKey, destKey, config));
}
return(await MoveInternal(sourceKey, destKey, handleExisting, config));
}
public async Task <ISaveResult> SaveFileAsync(string key, string filePath, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress = null)
{
return(await provider.SaveFileAsync(key, filePath, handleExisting, progress, config));
}
public async Task <ISaveResult> SaveFileAsync(string key, Stream content, HandleExistingMethod handleExisting, IProgress <IWriteProgress> progress, FileSystemCabinetConfig config)
{
if (String.IsNullOrWhiteSpace(key))
{
throw new ArgumentNullException(nameof(key));
}
if (content == null)
{
throw new ArgumentNullException(nameof(content));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
// If for some reason, a file with the same key is created,
// only allow it to overwrite if overwriting is allowed
var openMode = handleExisting == HandleExistingMethod.Overwrite ? FileMode.Create : FileMode.CreateNew;
try {
var fileInfo = this.GetFileInfo(key, config);
var fs = GetFileSystem(config);
if (!fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
using (var writeStream = fs.File.Open(fileInfo.FullName, openMode, FileAccess.Write)) {
// no using block as this is simply a wrapper
var progressWriteStream = new ProgressStream(key, writeStream, content.Length, progress);
await content.CopyToAsync(progressWriteStream);
return(new SaveResult(key, success: true));
}
} catch (DirectoryNotFoundException e) {
return(new SaveResult(key, e));
} catch (IOException e) {
// We tried to create a new file but it already exists
if (openMode == FileMode.CreateNew)
{
if (handleExisting == HandleExistingMethod.Throw)
{
throw new ApplicationException($"File exists at {key} and handleExisting is set to throw");
}
if (handleExisting == HandleExistingMethod.Skip)
{
return(new SaveResult(key, success: true)
{
AlreadyExists = true
});
}
}
// Failed for some other reason so return an error
return(new SaveResult(key, e));
} catch (Exception e) {
return(new SaveResult(key, e));
}
}
public async Task Move_File_Not_Overwrite_Throws(HandleExistingMethod handleExisting) {
var provider = GetProvider();
var config = GetConfig(ValidBucketName);
string sourceKey = @"source.txt";
string destKey = @"dest.txt";
var mockProgress = new Mock<IProgress<IWriteProgress>>();
await Assert.ThrowsAsync<NotImplementedException>(async () =>
await provider.MoveFileAsync(sourceKey, destKey, handleExisting, config)
);
}
public async Task <IMoveResult> MoveFileAsync(string sourceKey, string destKey, HandleExistingMethod handleExisting, AmazonS3CabinetConfig config)
{
Contract.NotNullOrEmpty(sourceKey, nameof(sourceKey));
Contract.NotNullOrEmpty(destKey, nameof(destKey));
Contract.NotNull(config, nameof(config));
if (handleExisting != HandleExistingMethod.Overwrite)
{
throw new NotImplementedException();
}
using (var s3Client = GetS3Client(config)) {
try {
var copyRequest = new CopyObjectRequest {
SourceBucket = config.BucketName,
DestinationBucket = config.BucketName,
SourceKey = GetKey(sourceKey, config),
DestinationKey = GetKey(destKey, config)
};
var response = await s3Client.CopyObjectAsync(copyRequest);
if (response.HttpStatusCode == HttpStatusCode.OK)
{
var deleteResult = await DeleteInternal(sourceKey, config, s3Client);
if (!deleteResult.Success)
{
return(new MoveResult(sourceKey, destKey, deleteResult.Exception, deleteResult.GetErrorMessage()));
}
}
return(new MoveResult(sourceKey, destKey, response.HttpStatusCode));
} catch (Exception e) {
return(new MoveResult(sourceKey, destKey, e));
}
}
}
public async Task Move_File(string sourceKey, string destKey, HandleExistingMethod handleExisting) {
var mockResult = new Mock<IMoveResult>();
this.mockStorageProvider.Setup(s => s.MoveFileAsync(sourceKey, destKey, handleExisting, config)).ReturnsAsync(mockResult.Object);
var actualResult = await this.fileCabinet.MoveFileAsync(sourceKey, destKey, handleExisting);
this.mockStorageProvider.Verify(s => s.MoveFileAsync(sourceKey, destKey, handleExisting, config), Times.Once);
Assert.Equal(mockResult.Object, actualResult);
}
public SaveCommand()
{
IsCommand("save", "Saves a file into the cabinet");
HasRequiredOption("cabinet=|c=", "Cabinet to save file to", c => configName = c);
HasRequiredOption("file-path=|f=", "Path of the File to save", f => filePath = f);
HasRequiredOption("key=|k=", "Key to save file to", k => key = k);
HasOption("overwrite|o", "Overwrite an existing file in the cabinet", o => handleExisting = HandleExistingMethod.Overwrite);
HasOption("skip|s", "Skip if there is an existing file in the cabinet", o => handleExisting = HandleExistingMethod.Skip);
}
public async Task <UploadSaveResult[]> SaveInCabinet(HandleExistingMethod handleExisting = HandleExistingMethod.Throw, IFileScanner fileScanner = null)
{
var saveTasks = this.FileData.Select(async(fd) => {
string uploadFileName = fd.Headers.ContentDisposition.FileName?.Trim('"')?.Trim('\\');
string uploadExtension = Path.GetExtension(uploadFileName)?.TrimStart('.');
string uploadMediaType = fd.Headers.ContentType?.MediaType;
if (!this.fileValidator.IsFileTypeWhitelisted(uploadExtension))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "The file type is not allowed"));
}
string fileName = fd.LocalFileName;
var uploadedFileInfo = new FileInfo(fileName);
if (this.fileValidator.IsFileTooLarge(uploadedFileInfo.Length))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "The file is too large"));
}
if (this.fileValidator.IsFileTooSmall(uploadedFileInfo.Length))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "The file is too small"));
}
string delimiter = this.fileCabinet.GetKeyDelimiter();
string key = this.keyProvider.GetKey(uploadFileName, uploadMediaType, delimiter);
if (String.IsNullOrWhiteSpace(key))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "No key was provided"));
}
if (!File.Exists(fileName))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "Could not find uploaded file."));
}
// File scanner to optionally check the file an remove if it's unsafe
// note the c# 6 fileScanner?.ScanFileAsync syntax doesn't seem to work
if (fileScanner != null)
{
await fileScanner.ScanFileAsync(fileName);
}
if (!File.Exists(fileName))
{
return(new UploadSaveResult(uploadFileName, uploadMediaType, "File has been removed as it is unsafe."));
}
var cabinetResult = await this.fileCabinet.SaveFileAsync(key, fileName, handleExisting, this.CabinetFileSaveProgress);
return(new UploadSaveResult(uploadFileName, uploadMediaType, cabinetResult));
});
var saveResults = await Task.WhenAll(saveTasks);
// cleanup temp files
foreach (var file in this.FileData)
{
File.Delete(file.LocalFileName);
}
return(saveResults);
}
public async Task Save_File_Path(string key, HandleExistingMethod handleExisting, bool masterSuccess) {
var mockStream = new Mock<Stream>();
var mockMasterSaveResult = new Mock<ISaveResult>();
var mockReplicaSaveResult = new Mock<ISaveResult>();
mockMasterSaveResult.SetupGet(r => r.Success).Returns(masterSuccess);
this.mockMasterCabinet.Setup(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null)).ReturnsAsync(mockMasterSaveResult.Object);
this.mockReplicaCabinet.Setup(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null)).ReturnsAsync(mockReplicaSaveResult.Object);
var saveResult = await provider.SaveFileAsync(key, mockStream.Object, handleExisting, null, config);
this.mockMasterCabinet.Verify(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null), Times.Once);
this.mockReplicaCabinet.Verify(c => c.SaveFileAsync(key, mockStream.Object, handleExisting, null), masterSuccess ? Times.Once() : Times.Never());
Assert.Equal(mockMasterSaveResult.Object, saveResult);
}
public async Task <IMoveResult> MoveFileAsync(string sourceKey, string destKey, HandleExistingMethod handleExisting, MigrationProviderConfig config)
{
Contract.NotNullOrEmpty(sourceKey, nameof(sourceKey));
Contract.NotNullOrEmpty(destKey, nameof(destKey));
Contract.NotNull(config, nameof(config));
var from = GetFromCabinet(config);
var to = GetToCabinet(config);
bool sourceExistsInTo = await to.ExistsAsync(sourceKey);
if (sourceExistsInTo)
{
return(await to.MoveFileAsync(sourceKey, destKey, handleExisting));
}
var fromFile = await from.GetItemAsync(sourceKey);
if (!fromFile.Exists)
{
return(new MoveResult(sourceKey, destKey, false, errorMsg: "Source file does not exist"));
}
using (var stream = await from.OpenReadStreamAsync(fromFile.Key)) {
var saveResult = await to.SaveFileAsync(destKey, stream, handleExisting);
return(new MoveResult(
sourceKey, destKey,
success: saveResult.Success,
errorMsg: saveResult.GetErrorMessage()
));
}
}
public async Task Save_File_Path(string key, HandleExistingMethod handleExisting) {
string filePath = "C:\test";
var mockResult = new Mock<ISaveResult>();
var mockProgress = new Mock<IProgress<IWriteProgress>>();
this.mockStorageProvider.Setup(s => s.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object, config)).ReturnsAsync(mockResult.Object);
var actualResult = await this.fileCabinet.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object);
this.mockStorageProvider.Verify(s => s.SaveFileAsync(key, filePath, handleExisting, mockProgress.Object, config), Times.Once);
Assert.Equal(mockResult.Object, actualResult);
}
public async Task <IMoveResult> MoveFileAsync(string sourceKey, string destKey, HandleExistingMethod handleExisting, ReplicatedProviderConfig config)
{
Contract.NotNullOrEmpty(sourceKey, nameof(sourceKey));
Contract.NotNullOrEmpty(destKey, nameof(destKey));
Contract.NotNull(config, nameof(config));
var master = GetMasterConfig(config);
var masterResult = await master.MoveFileAsync(sourceKey, destKey, handleExisting);
if (!masterResult.Success)
{
return(masterResult);
}
var replica = GetReplicaConfig(config);
// Ignore result, this should be used with the CabinetReplicator which will handle failures
await replica.MoveFileAsync(sourceKey, destKey, handleExisting);
return(masterResult);
}