private async Task <BackblazeApiResponse <FileDataOutputDTO> > ProcessSmallUploadAsync(
AuthorizationOutputDTO auth,
UploadFile file,
IDictionary <string, FileProgress> fileProgress,
IProgress <TransferProgress> progressData = null,
int retryTimeoutCount = 5,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var result = await B2Client.UploadSmallFileAsync(
auth : auth,
bucketId : file.bucketId,
sourcePath : file.sourceFilePath,
destinationFilename : file.destinationFilename,
retryTimeoutCount : retryTimeoutCount,
cancellationToken : cancellationToken,
progressData : new Progress <FileProgress>((FileProgress progress) => {
fileProgress[progress.filename] = progress;
progressData?.Report(new TransferProgress(
fileProgress: new Dictionary <string, FileProgress>(fileProgress)
));
})
);
return(result);
}
public async Task <BackblazeApiResponse <ListFileVersionsOutputDTO> > ListFileVersionsAsync(
AuthorizationOutputDTO auth,
string bucketId,
string startFileName = null,
string startFileId = null,
int?maxFileCount = null,
string prefix = null,
string delimiter = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new ListFileVersionsInputDTO()
{
BucketId = bucketId,
StartFileName = startFileName,
StartFileId = startFileId,
MaxFileCount = maxFileCount,
Prefix = prefix,
Delimiter = delimiter,
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_list_file_versions"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <ListFileVersionsOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <ListFileVersionsOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <ListFileVersionsOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <FileDataOutputDTO> > UploadDynamicAsync(
AuthorizationOutputDTO auth,
UploadFile file,
IProgress <TransferProgress> progressData = null,
int retryTimeoutCount = 5,
int concurrentUploads = 4,
string tempDir = "/tmp",
CancellationToken cancellationToken = default(CancellationToken)
)
{
ConcurrentDictionary <string, FileProgress> fileProgress = new ConcurrentDictionary <string, FileProgress>();
long fileSize = new FileInfo(file.sourceFilePath).Length;
fileProgress.TryAdd(file.destinationFilename, new FileProgress(filename: file.destinationFilename, totalBytes: fileSize));
Task <BackblazeApiResponse <FileDataOutputDTO> > uploadTask;
if (fileSize > auth.RecommendedPartSize)
{
uploadTask = ProcessLargeUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
fileSize: fileSize,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
else
{
uploadTask = ProcessSmallUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
return(await uploadTask);
}
public async Task <BackblazeApiResponse <FileVersionDeletedOutputDTO> > DeleteFileVersionAsync(
AuthorizationOutputDTO auth,
string fileName,
string fileId,
bool?bypassGovernance = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new DeleteFileVersionInputDTO()
{
FileName = fileName,
FileId = fileId,
BypassGovernance = bypassGovernance,
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_delete_file_version"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <FileVersionDeletedOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <FileVersionDeletedOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <FileVersionDeletedOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <FileDataOutputDTO> > StartLargeFileAsync(
AuthorizationOutputDTO auth,
string bucketId,
string destinationFilename,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new StartLargeFileUploadInputDTO()
{
BucketId = bucketId,
FileName = destinationFilename,
ContentType = "b2/x-auto",
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_start_large_file"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <FileDataOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <FileDataOutputDTO> > FinishLargeFileAsync(
AuthorizationOutputDTO auth,
string fileId,
IEnumerable <string> partHashes,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new FinishLargeFileUploadInputDTO()
{
FileId = fileId,
PartSha1Array = (string[])partHashes
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_finish_large_file"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <FileDataOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <UploadUrlOutputDTO> > GetUploadUrlAsync(
AuthorizationOutputDTO auth,
string bucketId,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var body = SerializeToJsonContent(new UploadUrlInputDTO()
{
BucketId = bucketId
});
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"{auth.ApiUrl}/b2api/v2/b2_get_upload_url"),
Content = body
};
request.Headers.TryAddWithoutValidation("Authorization", auth.AuthorizationToken);
var response = await _client.SendAsync(request, cancellationToken);
string responseJson = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
return(new BackblazeApiResponse <UploadUrlOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Error = JsonSerializer.Deserialize <B2ErrorResponseOutputDTO>(responseJson)
});
}
return(new BackblazeApiResponse <UploadUrlOutputDTO>()
{
Succeeded = response.IsSuccessStatusCode,
Data = JsonSerializer.Deserialize <UploadUrlOutputDTO>(responseJson)
});
}
public async Task <BackblazeApiResponse <FileDataOutputDTO>[]> UploadDynamicBatchAsync(
AuthorizationOutputDTO auth,
IEnumerable <UploadFile> files,
IProgress <TransferProgress> progressData = null,
int retryTimeoutCount = 5,
int concurrentUploads = 4,
string tempDir = "/tmp",
CancellationToken cancellationToken = default(CancellationToken)
)
{
FileDataOutputDTO[] result = new FileDataOutputDTO[files.Count()];
Task <BackblazeApiResponse <FileDataOutputDTO> >[] uploadTasks = new Task <BackblazeApiResponse <FileDataOutputDTO> > [files.Count()];
Dictionary <string, FileProgress> fileProgress = new Dictionary <string, FileProgress>();
foreach (var file in files)
{
long fileSize = new FileInfo(file.sourceFilePath).Length;
fileProgress.Add(file.destinationFilename, new FileProgress(filename: file.destinationFilename, totalBytes: fileSize));
}
int fileI = 0;
foreach (var file in files)
{
long fileSize = new FileInfo(file.sourceFilePath).Length;
if (fileSize > auth.RecommendedPartSize)
{
uploadTasks[fileI] = ProcessLargeUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
fileSize: fileSize,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
else
{
uploadTasks[fileI] = ProcessSmallUploadAsync(
auth: auth,
file: file,
fileProgress: fileProgress,
progressData: progressData,
retryTimeoutCount: retryTimeoutCount,
cancellationToken: cancellationToken
);
}
fileI++;
}
SemaphoreSlim throttler = new SemaphoreSlim(concurrentUploads, concurrentUploads);
IEnumerable <Task <BackblazeApiResponse <FileDataOutputDTO> > > tasks = uploadTasks.Select(async input =>
{
await throttler.WaitAsync();
try
{
return(await input);
}
finally
{
throttler.Release();
}
});
return(await Task.WhenAll(tasks));
}
private async Task <BackblazeApiResponse <FileDataOutputDTO> > ProcessLargeUploadAsync(
AuthorizationOutputDTO auth,
UploadFile file,
IDictionary <string, FileProgress> fileProgress,
long fileSize,
IProgress <TransferProgress> progressData = null,
int retryTimeoutCount = 5,
string tempDir = "/tmp",
CancellationToken cancellationToken = default(CancellationToken)
)
{
int partCount = (int)Math.Ceiling((double)fileSize / (double)auth.RecommendedPartSize);
var initDataResponse = await B2Client.StartLargeFileAsync(auth, file.bucketId, file.destinationFilename, cancellationToken);
if (!initDataResponse.Succeeded)
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = false,
Error = initDataResponse.Error
});
}
string[] partHashes = new string[partCount];
List <UploadedPartOutputDTO> completeParts = new List <UploadedPartOutputDTO>();
for (int i = 0; i < partCount; i++)
{
int partNumber = i + 1;
long partSize = auth.RecommendedPartSize;
if (partNumber == partCount)
{
partSize = fileSize - ((partCount - 1) * auth.RecommendedPartSize);
}
long offset = i * auth.RecommendedPartSize;
string tmpFilePath = Path.Combine(tempDir, Guid.NewGuid().ToString());
FileSplitter.Extract(file.sourceFilePath, tmpFilePath, offset, partSize);
using (var stream = System.IO.File.OpenRead(tmpFilePath))
{
var uploadPartResponse = await B2Client.UploadLargeFilePartAsync(
auth : auth,
fileData : initDataResponse.Data,
partStream : stream,
partNumber : partNumber,
retryTimeoutCount : retryTimeoutCount,
cancellationToken : cancellationToken,
progressData : new Progress <FilePartProgress>((FilePartProgress progress) => {
var oldProgress = fileProgress[progress.filename];
long completeBytes = 0;
foreach (var completePart in completeParts)
{
completeBytes += completePart.ContentLength;
}
var newProgress = new FileProgress(
filename: progress.filename,
totalBytes: oldProgress.totalBytes,
bytesTransferred: completeBytes + progress.bytesTransferred
);
fileProgress[progress.filename] = newProgress;
progressData?.Report(new TransferProgress(
fileProgress: new Dictionary <string, FileProgress>(fileProgress)
));
})
);
stream.Close();
if (!uploadPartResponse.Succeeded)
{
throw new B2Exception(uploadPartResponse.Error);
}
else
{
completeParts.Add(uploadPartResponse.Data);
partHashes[i] = uploadPartResponse.Data.ContentSha1;
}
}
File.Delete(tmpFilePath);
}
return(await B2Client.FinishLargeFileAsync(auth, initDataResponse.Data.FileId, partHashes, cancellationToken));
}
public async Task <BackblazeApiResponse <UploadedPartOutputDTO> > UploadLargeFilePartAsync(
AuthorizationOutputDTO auth,
FileDataOutputDTO fileData,
Stream partStream,
int partNumber,
int retryTimeoutCount = 5,
IProgress <FilePartProgress> progressData = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
if (partStream == null)
{
throw new NullReferenceException("partStream");
}
if (!partStream.CanRead)
{
throw new InvalidDataException();
}
int retryCount = 0;
Retry:
var uploadUrlResponse = await GetUploadPartUrlAsync(auth, fileData, cancellationToken);
if (!uploadUrlResponse.Succeeded)
{
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Error = uploadUrlResponse.Error
});
}
UploadedPartOutputDTO uploadPart = null;
var uploadPartResponse = await UploadLargeFilePartAsync(uploadUrlResponse.Data, partStream, fileData.FileName, partNumber, progressData, cancellationToken);
if (uploadPartResponse.Succeeded)
{
uploadPart = uploadPartResponse.Data;
}
else
{
if (ErrorHelper.IsRecoverableError(uploadPartResponse.Error))
{
if (retryCount > retryTimeoutCount)
{
uploadPartResponse.Error.Message = $"Hit retry max on error: {uploadPartResponse.Error.Message}";
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = false,
Error = uploadPartResponse.Error
});
}
retryCount++;
goto Retry;
}
else
{
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = false,
Error = uploadPartResponse.Error
});
}
}
return(new BackblazeApiResponse <UploadedPartOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Data = uploadPart
});
}
/// <summary>
/// Upload a file with attempted recovery from errors
/// </summary>
public async Task <BackblazeApiResponse <FileDataOutputDTO> > UploadSmallFileAsync(
AuthorizationOutputDTO auth,
string bucketId,
string sourcePath,
string destinationFilename,
int retryTimeoutCount = 5,
IProgress <FileProgress> progressData = null,
CancellationToken cancellationToken = default(CancellationToken)
)
{
int retryCount = 0;
Retry:
var uploadUrlResponse = await GetUploadUrlAsync(auth, bucketId, cancellationToken);
if (!uploadUrlResponse.Succeeded)
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = uploadUrlResponse.Succeeded,
Error = uploadUrlResponse.Error
});
}
FileDataOutputDTO uploadData = null;
var uploadDataResponse = await UploadSmallFileAsync(uploadUrlResponse.Data, sourcePath, destinationFilename, progressData, cancellationToken);
if (uploadDataResponse.Succeeded)
{
uploadData = uploadDataResponse.Data;
}
else
{
if (ErrorHelper.IsRecoverableError(uploadDataResponse.Error))
{
if (retryCount > retryTimeoutCount)
{
uploadDataResponse.Error.Message = $"Hit retry max on error: {uploadDataResponse.Error.Message}";
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = false,
Error = uploadDataResponse.Error
});
}
retryCount++;
goto Retry;
}
else
{
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = false,
Error = uploadDataResponse.Error
});
}
}
return(new BackblazeApiResponse <FileDataOutputDTO>()
{
Succeeded = uploadDataResponse.Succeeded,
Data = uploadData
});
}
