/// <summary>
/// A more efficient function for deleting all files off the server. This will delete all
/// files on the B2 server, including those that aren't on the file database manifest and
/// the file database manifest itself!
/// </summary>
/// <param name="authorizationSessionGenerator">A generator function for the authorization session</param>
public void DeleteAllFiles(Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator)
{
this.Debug("Begin deleting all files");
RemoveAllFiles();
IEnumerable <FileResult> rawB2FileList = GetRawB2FileNames(authorizationSessionGenerator());
object lockObject = new object();
Parallel.ForEach(rawB2FileList, rawB2File =>
{
CancellationEventRouter.GlobalCancellationToken.ThrowIfCancellationRequested();
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), rawB2File.FileID, rawB2File.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
lock (lockObject)
{
this.Critical($"Failed to delete file {rawB2File.FileName}. Reason: {deletionResult}");
}
}
else
{
lock (lockObject)
{
this.Info($"Deleted file: {rawB2File.FileName}");
}
}
});
}
private BackblazeB2ActionResult <BackblazeB2UploadFileResult> UploadFile(
BackblazeB2ActionResult <GetUploadFileURLResponse> getUploadFileUrlResult
)
{
if (getUploadFileUrlResult.HasResult)
{
GetUploadFileURLResponse unwrappedResult = getUploadFileUrlResult.MaybeResult.Value;
string sha1Hash = ComputeSHA1Hash(_bytesToUpload);
// Loop because we want to retry to upload the file part should it fail for
// recoverable reasons. We will break out of this loop should the upload succeed
// or throw an exception should we determine we can't upload to the server
int attemptNumber = 0;
while (true)
{
// Throw an exception is we are being interrupted
CancellationToken.ThrowIfCancellationRequested();
// Sleep the current thread so that we can give the server some time to recover
if (attemptNumber > 0)
{
TimeSpan backoffSleepTime = CalculateExponentialBackoffSleepTime(attemptNumber);
_exponentialBackoffCallback(backoffSleepTime);
Thread.Sleep(backoffSleepTime);
}
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(unwrappedResult.UploadURL);
webRequest.Method = "POST";
webRequest.Headers.Add("Authorization", unwrappedResult.AuthorizationToken);
webRequest.Headers.Add("X-Bz-File-Name", Uri.EscapeDataString(_fileDestination));
webRequest.Headers.Add("X-Bz-Content-Sha1", sha1Hash);
webRequest.ContentType = "b2/x-auto";
webRequest.ContentLength = _bytesToUpload.Length;
attemptNumber++;
BackblazeB2ActionResult <BackblazeB2UploadFileResult> response = SendWebRequestAndDeserialize(webRequest, _bytesToUpload);
if (response.HasResult)
{
return(response);
}
else if (response.Errors.Any(e => e.Status >= 500 && e.Status < 600) && attemptNumber < _maxUploadAttempts)
{
continue;
}
else
{
return(response);
}
}
}
else
{
return(new BackblazeB2ActionResult <BackblazeB2UploadFileResult>(Maybe <BackblazeB2UploadFileResult> .Nothing, getUploadFileUrlResult.Errors));
}
}
/// <summary>
/// Checks the file manifest against the file shards on the server
/// </summary>
/// <param name="authorizationSessionGenerator">The generator for an authorization session</param>
public void CheckFileManifest(Func<BackblazeB2AuthorizationSession> authorizationSessionGenerator)
{
this.Info("Checking file manifest");
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult<BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
ISet<string> allShardIDsPresent = listFilesActionResult.Result.Files.Select(t => t.FileName).ToHashSet();
IDictionary<string, ISet<Database.File>> shardIDToDatbaseFile = new Dictionary<string, ISet<Database.File>>();
foreach (Database.File file in FileDatabaseManifestFiles)
{
foreach (string shardID in file.FileShardIDs)
{
if (shardIDToDatbaseFile.TryGetValue(shardID, out ISet<Database.File> setOfFiles) == false)
{
setOfFiles = new HashSet<Database.File>();
shardIDToDatbaseFile[shardID] = setOfFiles;
}
setOfFiles.Add(file);
}
}
IEnumerable<string> allShardIDsNotAccountedFor = from file in FileDatabaseManifestFiles
from shardID in file.FileShardIDs
where allShardIDsPresent.Contains(shardID) == false
select shardID;
bool allShardIDsAccountedFor = true;
foreach (string shardIDNotAccountedFor in allShardIDsNotAccountedFor)
{
allShardIDsAccountedFor = false;
StringBuilder stringBuilder = new StringBuilder();
foreach (Database.File file in shardIDToDatbaseFile[shardIDNotAccountedFor])
{
stringBuilder.Append($"{file} | ");
}
this.Critical($"Shard ID {shardIDNotAccountedFor} for file(s): {stringBuilder.ToString()}");
}
if (allShardIDsAccountedFor)
{
this.Info("All Shard IDs accounted for");
}
else
{
this.Info("Some Shard IDs Are Not Accounted For");
}
}
private BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> FinishUploadingLargeFile(
string fileId,
IEnumerable <BackblazeB2ActionResult <UploadFilePartResponse> > uploadResponses
)
{
if (uploadResponses.Any(t => t.HasErrors) || Interlocked.Read(ref _totalNumberOfChunks) != uploadResponses.Count())
{
IEnumerable <BackblazeB2ActionErrorDetails> errors = from uploadResponse in uploadResponses
where uploadResponse.HasErrors
from error in uploadResponse.Errors
select error;
if (errors.Any() == false)
{
errors = new[]
{
new BackblazeB2ActionErrorDetails
{
Code = "INTERNAL_UPLOAD_ERROR",
Message = "Unable to upload all file chunks. Most likely due to all upload consumer threads failing",
Status = -1,
},
};
}
return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>(Maybe <BackblazeB2UploadMultipartFileResult> .Nothing, errors));
}
IList <string> sha1Hashes = (from uploadResponse in uploadResponses
let uploadResponseValue = uploadResponse.MaybeResult.Value
orderby uploadResponseValue.PartNumber ascending
select uploadResponseValue.ContentSHA1).ToList();
FinishLargeFileRequest finishLargeFileRequest = new FinishLargeFileRequest
{
FileID = fileId,
FilePartHashes = sha1Hashes,
};
string serializedFileRequest = JsonConvert.SerializeObject(finishLargeFileRequest);
byte[] requestBytes = Encoding.UTF8.GetBytes(serializedFileRequest);
HttpWebRequest webRequest = GetHttpWebRequest(_authorizationSession.APIURL + FinishLargeFileURL);
webRequest.ContentLength = requestBytes.Length;
webRequest.Method = "POST";
webRequest.Headers.Add("Authorization", _authorizationSession.AuthorizationToken);
BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> response =
SendWebRequestAndDeserialize(webRequest, requestBytes);
response.MaybeResult.Do(r => r.FileHashes = sha1Hashes);
return(response);
}
/// <summary>
/// Initializes this authorization session
/// </summary>
/// <param name="applicationID">The application key ID</param>
/// <param name="applicationKey">The application key</param>
private BackblazeB2AuthorizationSession CreateNewAuthorizationSession()
{
this.Debug("Creating new authorization session");
AuthorizeAccountAction authorizeAccountAction =
new AuthorizeAccountAction(_config.ApplicationKeyID, _config.ApplicationKey);
BackblazeB2ActionResult <BackblazeB2AuthorizationSession> authorizationSessionResult = authorizeAccountAction.Execute();
if (authorizationSessionResult.HasErrors)
{
throw new AuthorizationException
{
BackblazeErrorDetails = authorizationSessionResult.Errors,
};
}
return(authorizationSessionResult.Result);
}
/// <summary>
/// Prunes any shards on the server that are not accounted for in the database manifest
/// </summary>
/// <param name="authorizationSessionGenerator">The generator for an authorization session</param>
public void PruneShards(Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator)
{
this.Debug("Pruning shards");
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
object lockObject = new object();
IDictionary <string, FileResult> fileNameToFileResultMap = listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v);
ISet <string> allDatabaseFileShardIds = FileDatabaseManifestFiles.SelectMany(t => t.FileShardIDs).ToHashSet();
ISet <string> allRawFileNamesOnServer = fileNameToFileResultMap.Keys.ToHashSet();
ISet <string> allFilesNotAccountedFor = allRawFileNamesOnServer.Except(allDatabaseFileShardIds).Where(t => t.Equals(RemoteFileDatabaseManifestName, StringComparison.OrdinalIgnoreCase) == false).ToHashSet();
Parallel.ForEach(allFilesNotAccountedFor, fileNameNotAccountedFor =>
{
CancellationEventRouter.GlobalCancellationToken.ThrowIfCancellationRequested();
FileResult fileNotAccountedFor = fileNameToFileResultMap[fileNameNotAccountedFor];
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), fileNotAccountedFor.FileID, fileNotAccountedFor.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
lock (lockObject)
{
this.Critical($"Failed to prune file. Reason: {deletionResult}");
}
}
else
{
lock (lockObject)
{
this.Info($"Pruned shard: {fileNameNotAccountedFor}");
}
}
});
}
public override BackblazeB2ActionResult <BackblazeB2AuthorizationSession> Execute()
{
HttpWebRequest webRequest = GetHttpWebRequest(APIURL);
string credentialsHeader = Convert.ToBase64String(
Encoding.UTF8.GetBytes(_keyID + ":" + _applicationKey)
);
webRequest.Headers.Add("Authorization", "Basic " + credentialsHeader);
BackblazeB2ActionResult <BackblazeB2AuthorizationSession> response = SendWebRequestAndDeserialize(webRequest, null);
response.MaybeResult.Do(r =>
{
r.ApplicationKey = _applicationKey;
r.SessionExpirationDate = DateTime.Now + TimeSpan.FromHours(24);
});
return(response);
}
protected IEnumerable <FileResult> GetRawB2FileNames(BackblazeB2AuthorizationSession authorizationSession)
{
ListFilesAction listFilesAction =
ListFilesAction.CreateListFileActionForFileVersions(authorizationSession, Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
return(rawB2FileList);
}
private bool TryDownloadFileShard(
BackblazeB2AuthorizationSession authorizationSession,
string fileShardID,
string filePathDestination,
FileResult remoteFile
)
{
this.Verbose($"Starting download of: {fileShardID}");
using (DownloadFileAction fileShardDownload =
new DownloadFileAction(authorizationSession, filePathDestination, remoteFile.FileID))
{
BackblazeB2ActionResult <BackblazeB2DownloadFileResult> downloadResult = fileShardDownload.Execute();
if (downloadResult.HasErrors)
{
this.Critical($"Exception occurred during downloading a file shard: {downloadResult}.");
return(false);
}
}
return(true);
}
private IDictionary <string, FileResult> GetShardIDToFileResultMapping(
BackblazeB2AuthorizationSession authorizationSession
)
{
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(
authorizationSession,
_config.BucketID,
true
);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
return(listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v));
}
private BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> CancelFileUpload(string fileID)
{
CancelLargeFileUploadAction cancelAction = new CancelLargeFileUploadAction(_authorizationSession, fileID);
BackblazeB2ActionResult <BackblazeB2CancelLargeFileUploadResult> cancelResult = cancelAction.Execute();
if (cancelResult.HasErrors)
{
return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>(
Maybe <BackblazeB2UploadMultipartFileResult> .Nothing,
cancelResult.Errors
));
}
return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>(
Maybe <BackblazeB2UploadMultipartFileResult> .Nothing,
new BackblazeB2ActionErrorDetails
{
Status = -1,
Code = "MULTIPART_UPLOAD_CANCELLED",
Message = "The user cancalled the upload",
}
));
}
protected void UploadFileDatabaseManifest(
BackblazeB2AuthorizationSession authorizationSession
)
{
UploadWithSingleConnectionAction uploadAction = new UploadWithSingleConnectionAction(
authorizationSession,
Config.BucketID,
SerializeManifest(FileDatabaseManifest, Config),
RemoteFileDatabaseManifestName,
MaxAttemptsToUploadFileManifest,
CancellationToken.None,
_ => { } // NoOp for exponential backoff callback
);
BackblazeB2ActionResult <BackblazeB2UploadFileResult> result = uploadAction.Execute();
if (result.HasErrors)
{
throw new FailedToUploadFileDatabaseManifestException
{
BackblazeErrorDetails = result.Errors,
};
}
}
public override BackblazeB2ActionResult <UploadFilePartResponse> Execute()
{
// Loop because we want to retry to upload the file part should it fail for
// recoverable reasons. We will break out of this loop should the upload succeed
// or throw an exception should we determine we can't upload to the server
int attemptNumber = 0;
while (true)
{
// Throw an exception is we are being interrupted
CancellationToken.ThrowIfCancellationRequested();
// Sleep the current thread so that we can give the server some time to recover
if (attemptNumber > 0)
{
TimeSpan backoffSleepTime = CalculateExponentialBackoffSleepTime(attemptNumber);
_exponentialBackoffCallback(backoffSleepTime);
Thread.Sleep(backoffSleepTime);
}
HttpWebRequest webRequest = GetHttpWebRequest(_getUploadPartUrl.UploadURL);
webRequest.Method = "POST";
webRequest.Headers.Add("Authorization", _getUploadPartUrl.AuthorizationToken);
webRequest.Headers.Add("X-Bz-Part-Number", _filePartNumber.ToString());
webRequest.Headers.Add("X-Bz-Content-Sha1", _sha1);
webRequest.ContentLength = _rawBytes.Length;
attemptNumber++;
BackblazeB2ActionResult <UploadFilePartResponse> uploadResponse = SendWebRequestAndDeserialize(webRequest, _rawBytes);
if (uploadResponse.HasResult)
{
// Verify result
UploadFilePartResponse unwrappedResponse = uploadResponse.MaybeResult.Value;
if (
unwrappedResponse.ContentLength != _rawBytes.Length ||
unwrappedResponse.ContentSHA1.Equals(_sha1, StringComparison.Ordinal) == false ||
unwrappedResponse.PartNumber != _filePartNumber
)
{
return(new BackblazeB2ActionResult <UploadFilePartResponse>(
Maybe <UploadFilePartResponse> .Nothing,
new BackblazeB2ActionErrorDetails
{
Code = "CUSTOM_ERROR",
Message = string.Format("File part number {0} uploaded successfully but could not be verified", _filePartNumber),
Status = -1,
}
));
}
else
{
return(uploadResponse);
}
}
else if (uploadResponse.Errors.Any(e => e.Status >= 500 && e.Status < 600) && attemptNumber < _maxUploadAttempts)
{
continue;
}
else
{
return(uploadResponse);
}
}
}
/// <summary>
/// Initializes this file database manifest
/// </summary>
private static FileDatabaseManifest GetOrCreateFileDatabaseManifest(
BackblazeB2AuthorizationSession authorizationSession,
Config config
)
{
if (SharedFileDatabaseManifest == null)
{
lock (SharedFileDatabaseManifestLock)
{
if (SharedFileDatabaseManifest == null)
{
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(
authorizationSession,
config.BucketID,
true
);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
FileResult fileDatabaseManifestFileResult = listFilesActionResult.Result.Files
.Where(f => f.FileName.Equals(RemoteFileDatabaseManifestName, StringComparison.Ordinal))
.SingleOrDefault();
if (fileDatabaseManifestFileResult == null)
{
SharedFileDatabaseManifest = new FileDatabaseManifest
{
DataFormatVersionNumber = CurrentDatabaseDataFormatVersion,
Files = new Database.File[0]
};
}
else
{
using (MemoryStream outputStream = new MemoryStream())
using (DownloadFileAction manifestFileDownloadAction = new DownloadFileAction(
authorizationSession,
outputStream,
fileDatabaseManifestFileResult.FileID
))
{
BackblazeB2ActionResult <BackblazeB2DownloadFileResult> manifestResultOption =
manifestFileDownloadAction.Execute();
if (manifestResultOption.HasResult)
{
// Now, read string from manifest
outputStream.Flush();
SharedFileDatabaseManifest = DeserializeManifest(
outputStream.ToArray(),
config
);
}
else
{
SharedFileDatabaseManifest = new FileDatabaseManifest
{
DataFormatVersionNumber = CurrentDatabaseDataFormatVersion,
Files = new Database.File[0]
};
}
}
}
}
}
}
return(SharedFileDatabaseManifest);
}
/// <summary>
/// Deletes a file off the remote file system
/// </summary>
/// <param name="authorizationSessionGenerator">The generator function that returns an authorization session</param>
/// <param name="file">The file to delete</param>
public void DeleteFile(
Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator,
Database.File file
)
{
this.Debug($"Begin deleting file: {file.FileName}");
// Remove entry in the File Manifest
RemoveFile(file);
// Determine if another file shares the same file shards
IEnumerable <Database.File> filesThatShareTheSameShards = from currentFile in FileDatabaseManifestFiles
where file.FileLength == currentFile.FileLength &&
file.FileShardIDs.Length == currentFile.FileShardIDs.Length &&
file.SHA1.Equals(currentFile.SHA1, StringComparison.OrdinalIgnoreCase) &&
// Even if a single shard ID is shared by another file, that's enough to count it
// as being equal (or at least not eligible for hard-deletion)
file.FileShardIDs.Any(t => currentFile.FileShardIDs.Contains(t))
select currentFile;
// If there are no files that share the same Shard IDs
if (filesThatShareTheSameShards.Any() == false)
{
// Get the raw B2 File List so we can get the B2 file IDs of the file shards
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
IDictionary <string, FileResult> fileNameToFileResult = rawB2FileList.ToDictionary(k => k.FileName, v => v);
foreach (string shardID in file.FileShardIDs)
{
if (fileNameToFileResult.TryGetValue(shardID, out FileResult fileShardToDelete))
{
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), fileShardToDelete.FileID, fileShardToDelete.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
this.Critical($"Failed to delete file. Reason: {deletionResult}");
}
}
}
}
else
{
this.Info($"File {file.FileName} shares file shares with another file. Will not perform a hard-delete. Removing from file manifest instead");
}
while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false)
{
Thread.Sleep(5);
}
this.Info($"Deleted file: {file.FileName}");
}
