/// <summary> /// A more efficient function for deleting all files off the server. This will delete all /// files on the B2 server, including those that aren't on the file database manifest and /// the file database manifest itself! /// </summary> /// <param name="authorizationSessionGenerator">A generator function for the authorization session</param> public void DeleteAllFiles(Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator) { this.Debug("Begin deleting all files"); RemoveAllFiles(); IEnumerable <FileResult> rawB2FileList = GetRawB2FileNames(authorizationSessionGenerator()); object lockObject = new object(); Parallel.ForEach(rawB2FileList, rawB2File => { CancellationEventRouter.GlobalCancellationToken.ThrowIfCancellationRequested(); DeleteFileAction deleteFileAction = new DeleteFileAction(authorizationSessionGenerator(), rawB2File.FileID, rawB2File.FileName); BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute(); if (deletionResult.HasErrors) { lock (lockObject) { this.Critical($"Failed to delete file {rawB2File.FileName}. Reason: {deletionResult}"); } } else { lock (lockObject) { this.Info($"Deleted file: {rawB2File.FileName}"); } } }); }
private BackblazeB2ActionResult <BackblazeB2UploadFileResult> UploadFile( BackblazeB2ActionResult <GetUploadFileURLResponse> getUploadFileUrlResult ) { if (getUploadFileUrlResult.HasResult) { GetUploadFileURLResponse unwrappedResult = getUploadFileUrlResult.MaybeResult.Value; string sha1Hash = ComputeSHA1Hash(_bytesToUpload); // Loop because we want to retry to upload the file part should it fail for // recoverable reasons. We will break out of this loop should the upload succeed // or throw an exception should we determine we can't upload to the server int attemptNumber = 0; while (true) { // Throw an exception is we are being interrupted CancellationToken.ThrowIfCancellationRequested(); // Sleep the current thread so that we can give the server some time to recover if (attemptNumber > 0) { TimeSpan backoffSleepTime = CalculateExponentialBackoffSleepTime(attemptNumber); _exponentialBackoffCallback(backoffSleepTime); Thread.Sleep(backoffSleepTime); } HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(unwrappedResult.UploadURL); webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", unwrappedResult.AuthorizationToken); webRequest.Headers.Add("X-Bz-File-Name", Uri.EscapeDataString(_fileDestination)); webRequest.Headers.Add("X-Bz-Content-Sha1", sha1Hash); webRequest.ContentType = "b2/x-auto"; webRequest.ContentLength = _bytesToUpload.Length; attemptNumber++; BackblazeB2ActionResult <BackblazeB2UploadFileResult> response = SendWebRequestAndDeserialize(webRequest, _bytesToUpload); if (response.HasResult) { return(response); } else if (response.Errors.Any(e => e.Status >= 500 && e.Status < 600) && attemptNumber < _maxUploadAttempts) { continue; } else { return(response); } } } else { return(new BackblazeB2ActionResult <BackblazeB2UploadFileResult>(Maybe <BackblazeB2UploadFileResult> .Nothing, getUploadFileUrlResult.Errors)); } }
/// <summary> /// Checks the file manifest against the file shards on the server /// </summary> /// <param name="authorizationSessionGenerator">The generator for an authorization session</param> public void CheckFileManifest(Func<BackblazeB2AuthorizationSession> authorizationSessionGenerator) { this.Info("Checking file manifest"); // Get just the file names on the server ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true); BackblazeB2ActionResult<BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } ISet<string> allShardIDsPresent = listFilesActionResult.Result.Files.Select(t => t.FileName).ToHashSet(); IDictionary<string, ISet<Database.File>> shardIDToDatbaseFile = new Dictionary<string, ISet<Database.File>>(); foreach (Database.File file in FileDatabaseManifestFiles) { foreach (string shardID in file.FileShardIDs) { if (shardIDToDatbaseFile.TryGetValue(shardID, out ISet<Database.File> setOfFiles) == false) { setOfFiles = new HashSet<Database.File>(); shardIDToDatbaseFile[shardID] = setOfFiles; } setOfFiles.Add(file); } } IEnumerable<string> allShardIDsNotAccountedFor = from file in FileDatabaseManifestFiles from shardID in file.FileShardIDs where allShardIDsPresent.Contains(shardID) == false select shardID; bool allShardIDsAccountedFor = true; foreach (string shardIDNotAccountedFor in allShardIDsNotAccountedFor) { allShardIDsAccountedFor = false; StringBuilder stringBuilder = new StringBuilder(); foreach (Database.File file in shardIDToDatbaseFile[shardIDNotAccountedFor]) { stringBuilder.Append($"{file} | "); } this.Critical($"Shard ID {shardIDNotAccountedFor} for file(s): {stringBuilder.ToString()}"); } if (allShardIDsAccountedFor) { this.Info("All Shard IDs accounted for"); } else { this.Info("Some Shard IDs Are Not Accounted For"); } }
private BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> FinishUploadingLargeFile( string fileId, IEnumerable <BackblazeB2ActionResult <UploadFilePartResponse> > uploadResponses ) { if (uploadResponses.Any(t => t.HasErrors) || Interlocked.Read(ref _totalNumberOfChunks) != uploadResponses.Count()) { IEnumerable <BackblazeB2ActionErrorDetails> errors = from uploadResponse in uploadResponses where uploadResponse.HasErrors from error in uploadResponse.Errors select error; if (errors.Any() == false) { errors = new[] { new BackblazeB2ActionErrorDetails { Code = "INTERNAL_UPLOAD_ERROR", Message = "Unable to upload all file chunks. Most likely due to all upload consumer threads failing", Status = -1, }, }; } return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>(Maybe <BackblazeB2UploadMultipartFileResult> .Nothing, errors)); } IList <string> sha1Hashes = (from uploadResponse in uploadResponses let uploadResponseValue = uploadResponse.MaybeResult.Value orderby uploadResponseValue.PartNumber ascending select uploadResponseValue.ContentSHA1).ToList(); FinishLargeFileRequest finishLargeFileRequest = new FinishLargeFileRequest { FileID = fileId, FilePartHashes = sha1Hashes, }; string serializedFileRequest = JsonConvert.SerializeObject(finishLargeFileRequest); byte[] requestBytes = Encoding.UTF8.GetBytes(serializedFileRequest); HttpWebRequest webRequest = GetHttpWebRequest(_authorizationSession.APIURL + FinishLargeFileURL); webRequest.ContentLength = requestBytes.Length; webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", _authorizationSession.AuthorizationToken); BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> response = SendWebRequestAndDeserialize(webRequest, requestBytes); response.MaybeResult.Do(r => r.FileHashes = sha1Hashes); return(response); }
/// <summary> /// Initializes this authorization session /// </summary> /// <param name="applicationID">The application key ID</param> /// <param name="applicationKey">The application key</param> private BackblazeB2AuthorizationSession CreateNewAuthorizationSession() { this.Debug("Creating new authorization session"); AuthorizeAccountAction authorizeAccountAction = new AuthorizeAccountAction(_config.ApplicationKeyID, _config.ApplicationKey); BackblazeB2ActionResult <BackblazeB2AuthorizationSession> authorizationSessionResult = authorizeAccountAction.Execute(); if (authorizationSessionResult.HasErrors) { throw new AuthorizationException { BackblazeErrorDetails = authorizationSessionResult.Errors, }; } return(authorizationSessionResult.Result); }
/// <summary> /// Prunes any shards on the server that are not accounted for in the database manifest /// </summary> /// <param name="authorizationSessionGenerator">The generator for an authorization session</param> public void PruneShards(Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator) { this.Debug("Pruning shards"); // Get just the file names on the server ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } object lockObject = new object(); IDictionary <string, FileResult> fileNameToFileResultMap = listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v); ISet <string> allDatabaseFileShardIds = FileDatabaseManifestFiles.SelectMany(t => t.FileShardIDs).ToHashSet(); ISet <string> allRawFileNamesOnServer = fileNameToFileResultMap.Keys.ToHashSet(); ISet <string> allFilesNotAccountedFor = allRawFileNamesOnServer.Except(allDatabaseFileShardIds).Where(t => t.Equals(RemoteFileDatabaseManifestName, StringComparison.OrdinalIgnoreCase) == false).ToHashSet(); Parallel.ForEach(allFilesNotAccountedFor, fileNameNotAccountedFor => { CancellationEventRouter.GlobalCancellationToken.ThrowIfCancellationRequested(); FileResult fileNotAccountedFor = fileNameToFileResultMap[fileNameNotAccountedFor]; DeleteFileAction deleteFileAction = new DeleteFileAction(authorizationSessionGenerator(), fileNotAccountedFor.FileID, fileNotAccountedFor.FileName); BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute(); if (deletionResult.HasErrors) { lock (lockObject) { this.Critical($"Failed to prune file. Reason: {deletionResult}"); } } else { lock (lockObject) { this.Info($"Pruned shard: {fileNameNotAccountedFor}"); } } }); }
public override BackblazeB2ActionResult <BackblazeB2AuthorizationSession> Execute() { HttpWebRequest webRequest = GetHttpWebRequest(APIURL); string credentialsHeader = Convert.ToBase64String( Encoding.UTF8.GetBytes(_keyID + ":" + _applicationKey) ); webRequest.Headers.Add("Authorization", "Basic " + credentialsHeader); BackblazeB2ActionResult <BackblazeB2AuthorizationSession> response = SendWebRequestAndDeserialize(webRequest, null); response.MaybeResult.Do(r => { r.ApplicationKey = _applicationKey; r.SessionExpirationDate = DateTime.Now + TimeSpan.FromHours(24); }); return(response); }
protected IEnumerable <FileResult> GetRawB2FileNames(BackblazeB2AuthorizationSession authorizationSession) { ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileVersions(authorizationSession, Config.BucketID, true); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files; return(rawB2FileList); }
private bool TryDownloadFileShard( BackblazeB2AuthorizationSession authorizationSession, string fileShardID, string filePathDestination, FileResult remoteFile ) { this.Verbose($"Starting download of: {fileShardID}"); using (DownloadFileAction fileShardDownload = new DownloadFileAction(authorizationSession, filePathDestination, remoteFile.FileID)) { BackblazeB2ActionResult <BackblazeB2DownloadFileResult> downloadResult = fileShardDownload.Execute(); if (downloadResult.HasErrors) { this.Critical($"Exception occurred during downloading a file shard: {downloadResult}."); return(false); } } return(true); }
private IDictionary <string, FileResult> GetShardIDToFileResultMapping( BackblazeB2AuthorizationSession authorizationSession ) { ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames( authorizationSession, _config.BucketID, true ); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } return(listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v)); }
private BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult> CancelFileUpload(string fileID) { CancelLargeFileUploadAction cancelAction = new CancelLargeFileUploadAction(_authorizationSession, fileID); BackblazeB2ActionResult <BackblazeB2CancelLargeFileUploadResult> cancelResult = cancelAction.Execute(); if (cancelResult.HasErrors) { return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>( Maybe <BackblazeB2UploadMultipartFileResult> .Nothing, cancelResult.Errors )); } return(new BackblazeB2ActionResult <BackblazeB2UploadMultipartFileResult>( Maybe <BackblazeB2UploadMultipartFileResult> .Nothing, new BackblazeB2ActionErrorDetails { Status = -1, Code = "MULTIPART_UPLOAD_CANCELLED", Message = "The user cancalled the upload", } )); }
protected void UploadFileDatabaseManifest( BackblazeB2AuthorizationSession authorizationSession ) { UploadWithSingleConnectionAction uploadAction = new UploadWithSingleConnectionAction( authorizationSession, Config.BucketID, SerializeManifest(FileDatabaseManifest, Config), RemoteFileDatabaseManifestName, MaxAttemptsToUploadFileManifest, CancellationToken.None, _ => { } // NoOp for exponential backoff callback ); BackblazeB2ActionResult <BackblazeB2UploadFileResult> result = uploadAction.Execute(); if (result.HasErrors) { throw new FailedToUploadFileDatabaseManifestException { BackblazeErrorDetails = result.Errors, }; } }
public override BackblazeB2ActionResult <UploadFilePartResponse> Execute() { // Loop because we want to retry to upload the file part should it fail for // recoverable reasons. We will break out of this loop should the upload succeed // or throw an exception should we determine we can't upload to the server int attemptNumber = 0; while (true) { // Throw an exception is we are being interrupted CancellationToken.ThrowIfCancellationRequested(); // Sleep the current thread so that we can give the server some time to recover if (attemptNumber > 0) { TimeSpan backoffSleepTime = CalculateExponentialBackoffSleepTime(attemptNumber); _exponentialBackoffCallback(backoffSleepTime); Thread.Sleep(backoffSleepTime); } HttpWebRequest webRequest = GetHttpWebRequest(_getUploadPartUrl.UploadURL); webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", _getUploadPartUrl.AuthorizationToken); webRequest.Headers.Add("X-Bz-Part-Number", _filePartNumber.ToString()); webRequest.Headers.Add("X-Bz-Content-Sha1", _sha1); webRequest.ContentLength = _rawBytes.Length; attemptNumber++; BackblazeB2ActionResult <UploadFilePartResponse> uploadResponse = SendWebRequestAndDeserialize(webRequest, _rawBytes); if (uploadResponse.HasResult) { // Verify result UploadFilePartResponse unwrappedResponse = uploadResponse.MaybeResult.Value; if ( unwrappedResponse.ContentLength != _rawBytes.Length || unwrappedResponse.ContentSHA1.Equals(_sha1, StringComparison.Ordinal) == false || unwrappedResponse.PartNumber != _filePartNumber ) { return(new BackblazeB2ActionResult <UploadFilePartResponse>( Maybe <UploadFilePartResponse> .Nothing, new BackblazeB2ActionErrorDetails { Code = "CUSTOM_ERROR", Message = string.Format("File part number {0} uploaded successfully but could not be verified", _filePartNumber), Status = -1, } )); } else { return(uploadResponse); } } else if (uploadResponse.Errors.Any(e => e.Status >= 500 && e.Status < 600) && attemptNumber < _maxUploadAttempts) { continue; } else { return(uploadResponse); } } }
/// <summary> /// Initializes this file database manifest /// </summary> private static FileDatabaseManifest GetOrCreateFileDatabaseManifest( BackblazeB2AuthorizationSession authorizationSession, Config config ) { if (SharedFileDatabaseManifest == null) { lock (SharedFileDatabaseManifestLock) { if (SharedFileDatabaseManifest == null) { // Get just the file names on the server ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames( authorizationSession, config.BucketID, true ); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } FileResult fileDatabaseManifestFileResult = listFilesActionResult.Result.Files .Where(f => f.FileName.Equals(RemoteFileDatabaseManifestName, StringComparison.Ordinal)) .SingleOrDefault(); if (fileDatabaseManifestFileResult == null) { SharedFileDatabaseManifest = new FileDatabaseManifest { DataFormatVersionNumber = CurrentDatabaseDataFormatVersion, Files = new Database.File[0] }; } else { using (MemoryStream outputStream = new MemoryStream()) using (DownloadFileAction manifestFileDownloadAction = new DownloadFileAction( authorizationSession, outputStream, fileDatabaseManifestFileResult.FileID )) { BackblazeB2ActionResult <BackblazeB2DownloadFileResult> manifestResultOption = manifestFileDownloadAction.Execute(); if (manifestResultOption.HasResult) { // Now, read string from manifest outputStream.Flush(); SharedFileDatabaseManifest = DeserializeManifest( outputStream.ToArray(), config ); } else { SharedFileDatabaseManifest = new FileDatabaseManifest { DataFormatVersionNumber = CurrentDatabaseDataFormatVersion, Files = new Database.File[0] }; } } } } } } return(SharedFileDatabaseManifest); }
/// <summary> /// Deletes a file off the remote file system /// </summary> /// <param name="authorizationSessionGenerator">The generator function that returns an authorization session</param> /// <param name="file">The file to delete</param> public void DeleteFile( Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator, Database.File file ) { this.Debug($"Begin deleting file: {file.FileName}"); // Remove entry in the File Manifest RemoveFile(file); // Determine if another file shares the same file shards IEnumerable <Database.File> filesThatShareTheSameShards = from currentFile in FileDatabaseManifestFiles where file.FileLength == currentFile.FileLength && file.FileShardIDs.Length == currentFile.FileShardIDs.Length && file.SHA1.Equals(currentFile.SHA1, StringComparison.OrdinalIgnoreCase) && // Even if a single shard ID is shared by another file, that's enough to count it // as being equal (or at least not eligible for hard-deletion) file.FileShardIDs.Any(t => currentFile.FileShardIDs.Contains(t)) select currentFile; // If there are no files that share the same Shard IDs if (filesThatShareTheSameShards.Any() == false) { // Get the raw B2 File List so we can get the B2 file IDs of the file shards ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files; IDictionary <string, FileResult> fileNameToFileResult = rawB2FileList.ToDictionary(k => k.FileName, v => v); foreach (string shardID in file.FileShardIDs) { if (fileNameToFileResult.TryGetValue(shardID, out FileResult fileShardToDelete)) { DeleteFileAction deleteFileAction = new DeleteFileAction(authorizationSessionGenerator(), fileShardToDelete.FileID, fileShardToDelete.FileName); BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute(); if (deletionResult.HasErrors) { this.Critical($"Failed to delete file. Reason: {deletionResult}"); } } } } else { this.Info($"File {file.FileName} shares file shares with another file. Will not perform a hard-delete. Removing from file manifest instead"); } while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false) { Thread.Sleep(5); } this.Info($"Deleted file: {file.FileName}"); }