private string GenerateLocalFilePath(Database.File remoteFile) { int counter = 0; while (true) { string localDownloadPath = Path.Combine( Directory.GetCurrentDirectory(), Path.GetFileName(remoteFile.FileName) ); // Resolve conflicts automatically if (counter > 0) { localDownloadPath = localDownloadPath + $"_{counter}"; } if (File.Exists(localDownloadPath) == false) { return(localDownloadPath); } counter++; } }
public void CompactShards( BackblazeB2AuthorizationSession authorizationSession, bool dryRun ) { this.Debug("Compacting file shards"); ISet <ISet <Database.File> > fileGroupsByContents = new HashSet <ISet <Database.File> >(); foreach (Database.File file in FileDatabaseManifestFiles) { bool foundAGroup = false; foreach (ISet <Database.File> auxGroup in fileGroupsByContents) { Database.File auxFile = auxGroup.First(); if (file.FileLength == auxFile.FileLength && file.FileShardHashes.Length == auxFile.FileShardHashes.Length && file.SHA1.Equals(auxFile.SHA1, StringComparison.OrdinalIgnoreCase) && file.FileShardHashes.SequenceEqual(auxFile.FileShardHashes)) { foundAGroup = true; auxGroup.Add(file); break; } } if (foundAGroup == false) { HashSet <Database.File> newAuxGroup = new HashSet <Database.File> { file, }; fileGroupsByContents.Add(newAuxGroup); } } // Go through groups and rewrite the file manifest foreach (ISet <Database.File> fileGroup in fileGroupsByContents.Where(g => g.Count > 1)) { Database.File prototypeFile = fileGroup.First(); foreach (Database.File otherFile in fileGroup.Where(f => ReferenceEquals(f, prototypeFile) == false)) { this.Verbose($"{otherFile.FileName} is now using shards from {prototypeFile.FileName}"); RemoveFile(otherFile); otherFile.FileShardIDs = prototypeFile.FileShardIDs; AddFile(otherFile); } } if (dryRun == false) { while (TryUploadFileDatabaseManifest(authorizationSession) == false) { Thread.Sleep(TimeSpan.FromSeconds(5)); } } this.Info("Finished compacting file shards"); }
/// <summary> /// Adds a file to the FileDatabaseManifest in a thread-safe manner. It does not check to see /// if the file already exists /// </summary> /// <param name="file">The file to add</param> protected void AddFile(Database.File file) { lock (SharedFileDatabaseManifestLock) { this.Debug($"Adding file: {file}"); FileDatabaseManifest.Files = FileDatabaseManifest.Files.Append(file).ToArray(); } }
/// <summary> /// Removes a file from the FileDatabaseManifest in a thread-safe manner /// </summary> /// <param name="file">The file to remove</param> protected void RemoveFile(Database.File file) { lock (SharedFileDatabaseManifestLock) { this.Debug($"Removing file: {file}"); FileDatabaseManifest.Files = FileDatabaseManifest.Files.Where(t => t.Equals(file) == false).ToArray(); } }
/// <summary> /// Downloads a file from the B2 Backblaze server, throwing an exception if /// this fails /// </summary> /// <param name="file">The file to download</param> /// <param name="destination">The destination of the downloaded file</param> public void DownloadFile( BackblazeB2AuthorizationSession authorizationSession, Database.File file, string destination ) { this.Verbose($"Downloading file: {file.FileName}"); if (System.IO.File.Exists(destination)) { throw new InvalidOperationException($"Cannot override file {destination}."); } if (_shardIDToFilePath == null) { _shardIDToFilePath = GetShardIDToFileResultMapping(authorizationSession); } ConcurrentBag <Tuple <string, long> > localFileShardIDPathsAndIndices = new ConcurrentBag <Tuple <string, long> >(); long currentShardsDownloaded = 0; Parallel.ForEach( file.FileShardIDs, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (fileShardID, loopState, currentShardIndex) => { if (loopState.ShouldExitCurrentIteration || loopState.IsExceptional || loopState.IsStopped) { return; } string shardFilePath = GetShardIDFilePath(fileShardID); localFileShardIDPathsAndIndices.Add(Tuple.Create(shardFilePath, currentShardIndex)); if (_shardIDToFilePath.TryGetValue(fileShardID, out FileResult b2FileShard)) { if (TryDownloadFileShard(authorizationSession, shardFilePath, fileShardID, b2FileShard)) { long totalDownloaded = Interlocked.Increment(ref currentShardsDownloaded); this.Info($"{file.FileName} download progress: {totalDownloaded} / {file.FileShardIDs.Length} downloaded"); } else { loopState.Stop(); throw new FailedToDownloadFileException($"Could not download shard due to a B2 exception"); } } else { loopState.Stop(); throw new FailedToDownloadFileException($"Could not find the file shard: {fileShardID}"); } }); ReconstructFile(destination, localFileShardIDPathsAndIndices); VerifyFile(file, destination); }
private void VerifyFile(Database.File remoteFile, string localFilePath) { string sha1File = SHA1FileHashStore.Instance.ComputeSHA1(localFilePath); if (remoteFile.SHA1.Equals(sha1File, StringComparison.OrdinalIgnoreCase) == false) { this.Critical($"File SHA-1 does not match!"); } }
public override void Execute(INotification notification) { this.Debug(CommandNotification); Database.File file = GetFile(); string newFileName = GetNewFileName(); AuthorizationSessionProxy authorizationProxy = (AuthorizationSessionProxy)Facade.RetrieveProxy(AuthorizationSessionProxy.Name); RenameFileProxy renameFileProxy = (RenameFileProxy)Facade.RetrieveProxy(RenameFileProxy.Name); renameFileProxy.RenameFile(authorizationProxy.AuthorizationSession, file, newFileName); }
/// <summary> /// Tries to get a file by name /// </summary> /// <param name="fileName">The file name to fetch</param> /// <param name="file">The reference to write the result to</param> /// <returns>True if a result was found. False otherwise</returns> public bool TryGetFileByName( string fileName, out Database.File file ) { // Do linear search since this doesn't happen often file = FileDatabaseManifestFiles .Where(f => f.FileName.Equals(fileName, StringComparison.Ordinal)) .SingleOrDefault(); return(file != null); }
/// <summary> /// Tries to get a file by its ID /// </summary> /// <param name="fileID">The file ID to retrieve</param> /// <param name="file">The reference to write to</param> /// <returns>True if the result was found. False otherwise</returns> public bool TryGetFileByID( string fileID, out Database.File file ) { // Linear search since this doesn't happen often file = FileDatabaseManifestFiles .Where(f => f.FileID.Equals(fileID, StringComparison.OrdinalIgnoreCase)) .SingleOrDefault(); return(file != null); }
public override void Execute(INotification notification) { this.Debug(CommandNotification); AuthorizationSessionProxy authorizationProxy = (AuthorizationSessionProxy)Facade.RetrieveProxy(AuthorizationSessionProxy.Name); Database.File fileToDownload = GetFile(); string localFileDestination = GetDestinationOfFile(fileToDownload); DownloadFileProxy downloadFileProxy = (DownloadFileProxy)Facade.RetrieveProxy(DownloadFileProxy.Name); downloadFileProxy.DownloadFile(authorizationProxy.AuthorizationSession, fileToDownload, localFileDestination); this.Info($"Finished downloading file: {fileToDownload}"); }
private string GetDestinationOfFile(Database.File remoteFileToDownload) { this.Debug($"Getting destination of file {remoteFileToDownload}"); ProgramArgumentsProxy programArgsProxy = (ProgramArgumentsProxy)Facade.RetrieveProxy(ProgramArgumentsProxy.Name); if (programArgsProxy.TryGetArgument(DestinationOption, out string localFileDestination) == false) { localFileDestination = Path.Combine( Directory.GetCurrentDirectory(), Path.GetFileName(remoteFileToDownload.FileName) ); } this.Debug($"Destination file is: {localFileDestination}"); return(localFileDestination); }
/// <summary> /// Renames a file in the File Database Manifest /// </summary> /// <param name="file"></param> /// <param name="newFilePath"></param> public void RenameFile( BackblazeB2AuthorizationSession authorizationSession, Database.File file, string newFilePath ) { this.Info($"Renaming file: {file.FileName} -> {newFilePath}"); // Ensure that the new file name doesn't conflict with something else if (TryGetFileByName(newFilePath, out Database.File _)) { throw new FailedToRenameFileException($"{newFilePath} already exists"); } RemoveFile(file); file.FileName = newFilePath; AddFile(file); while (TryUploadFileDatabaseManifest(authorizationSession) == false) { Thread.Sleep(5); } ; this.Info("Finished renaming file"); }
private void UploadFiles( Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator, IDictionary <string, string> absoluteLocalFilePathsToDestinationFilePaths, bool dryRun ) { if (dryRun) { StringBuilder builder = new StringBuilder(); builder.AppendLine("Would have uploaded the following files"); foreach (KeyValuePair <string, string> absoluteToDestinationPath in absoluteLocalFilePathsToDestinationFilePaths) { builder.AppendLine($"{absoluteToDestinationPath.Key} -> {absoluteToDestinationPath.Value}"); } this.Info($"DRY RUN. {builder}"); return; } using (TieredUploadManager uploadManager = new TieredUploadManager(authorizationSessionGenerator, Config, CancellationEventRouter.GlobalCancellationToken)) { IDictionary <string, ISet <string> > localFilePathToUploadIDs = new Dictionary <string, ISet <string> >(); IDictionary <string, ISet <string> > localFilePathToFinishedUploadIDs = new Dictionary <string, ISet <string> >(); IDictionary <string, string> uploadIDsToLocalFilePath = new Dictionary <string, string>(); foreach (KeyValuePair <string, string> localPathToRemotePath in absoluteLocalFilePathsToDestinationFilePaths) { string localFilePath = localPathToRemotePath.Key; localFilePathToUploadIDs[localFilePath] = new HashSet <string>(); localFilePathToFinishedUploadIDs[localFilePath] = new HashSet <string>(); foreach (Lazy <FileShard> lazyFileShard in FileShardFactory.CreateLazyFileShards(localFilePath)) { string uploadID = uploadManager.AddLazyFileShard(lazyFileShard); localFilePathToUploadIDs[localFilePath].Add(uploadID); uploadIDsToLocalFilePath[uploadID] = localFilePath; } } // This part is tricky because all of these event handlers will be executed on background threads. Because of this, // we will need to lock the SendNotification call in order to prevent any race conditions or data corruption. We // know the main thread is blocked on these background threads because we have called the "Wait()" method. object localLockObject = new object(); IDictionary <string, ISet <UploadManagerEventArgs> > localFileToFileShardIDs = new Dictionary <string, ISet <UploadManagerEventArgs> >(); ISet <string> localFilesThatHaveAlreadyStarted = new HashSet <string>(); DateTime lastFileManifestUpload = DateTime.Now; bool uploadedManifest = true; void HandleOnUploadBegin(object sender, UploadManagerEventArgs eventArgs) { lock (localLockObject) { string localFile = uploadIDsToLocalFilePath[eventArgs.UploadID]; if (localFilesThatHaveAlreadyStarted.Contains(localFile) == false) { localFilesThatHaveAlreadyStarted.Add(localFile); this.Info($"Begin uploading file: {localFile}"); } } } void HandleOnUploadFailed(object sender, UploadManagerEventArgs eventArgs) { lock (localLockObject) { this.Critical($"Failed to upload file. {uploadIDsToLocalFilePath[eventArgs.UploadID]} | {eventArgs.UploadResult.ToString()}"); } } void HandleOnUploadFinished(object sender, UploadManagerEventArgs eventArgs) { lock (localLockObject) { string localFilePath = uploadIDsToLocalFilePath[eventArgs.UploadID]; if (localFileToFileShardIDs.TryGetValue(localFilePath, out ISet <UploadManagerEventArgs> uploadEvents) == false) { uploadEvents = new HashSet <UploadManagerEventArgs>(); localFileToFileShardIDs[localFilePath] = uploadEvents; } uploadEvents.Add(eventArgs); localFilePathToFinishedUploadIDs[localFilePath].Add(eventArgs.UploadID); int currentNumberOfUploadedShards = localFilePathToFinishedUploadIDs[localFilePath].Count; int totalNumberOfShards = localFilePathToUploadIDs[localFilePath].Count; if (totalNumberOfShards == currentNumberOfUploadedShards) { this.Info($"Finished uploading file: {localFilePath}"); string[] orderedShardIDs = new string[uploadEvents.Count]; string[] orderedShardHashes = new string[uploadEvents.Count]; foreach (UploadManagerEventArgs uploadEvent in uploadEvents) { orderedShardIDs[(int)uploadEvent.FileShardPieceNumber] = uploadEvent.FileShardID; orderedShardHashes[(int)uploadEvent.FileShardPieceNumber] = uploadEvent.FileShardSHA1; } string destinationPath = absoluteLocalFilePathsToDestinationFilePaths[localFilePath]; // Create file FileInfo info = new FileInfo(localFilePath); Database.File file = new Database.File { FileID = Guid.NewGuid().ToString(), FileLength = info.Length, FileName = destinationPath, FileShardIDs = orderedShardIDs, FileShardHashes = orderedShardHashes, LastModified = info.LastWriteTimeUtc.ToBinary(), SHA1 = SHA1FileHashStore.Instance.ComputeSHA1(localFilePath), }; // Update manifest if (TryGetFileByName(destinationPath, out Database.File fileThatExists)) { // Remove old file first RemoveFile(fileThatExists); } uploadedManifest = false; AddFile(file); if (DateTime.Now - lastFileManifestUpload >= MaxTimeBetweenFileManifestUploads) { if (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false) { this.Critical("Failed to upload file manifest. Going to do it on the next try"); uploadedManifest = true; } } } else { double percentFinished = (double)currentNumberOfUploadedShards / totalNumberOfShards * 100.0; this.Info($"{localFilePath} - {percentFinished:N2}% uploaded"); } } } void HandleOnUploadTierChanged(object sender, UploadManagerEventArgs eventArgs) { lock (localLockObject) { this.Warning($"File tier changed. {uploadIDsToLocalFilePath[eventArgs.UploadID]} -> {eventArgs.NewUploadTier}"); } } // Hook up events uploadManager.OnUploadBegin += HandleOnUploadBegin; uploadManager.OnUploadFailed += HandleOnUploadFailed; uploadManager.OnUploadFinished += HandleOnUploadFinished; uploadManager.OnUploadTierChanged += HandleOnUploadTierChanged; uploadManager.SealUploadManager(); uploadManager.Execute(); uploadManager.Wait(); // Specifically lock this part so that there is not possibility of a rogue thread // firing an event while this main thread is in the middle of cleaning it up lock (localLockObject) { // Unsubscribe from events uploadManager.OnUploadBegin -= HandleOnUploadBegin; uploadManager.OnUploadFailed -= HandleOnUploadFailed; uploadManager.OnUploadFinished -= HandleOnUploadFinished; uploadManager.OnUploadTierChanged -= HandleOnUploadTierChanged; } // Ensure that we've uploaded the file manifest and if not upload it again if (uploadedManifest == false) { this.Verbose("Uploading file mainfest"); while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false) { Thread.Sleep(TimeSpan.FromSeconds(10)); } this.Verbose("Successfully uploaded file manifest"); } } }
/// <summary> /// Deletes a file off the remote file system /// </summary> /// <param name="authorizationSessionGenerator">The generator function that returns an authorization session</param> /// <param name="file">The file to delete</param> public void DeleteFile( Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator, Database.File file ) { this.Debug($"Begin deleting file: {file.FileName}"); // Remove entry in the File Manifest RemoveFile(file); // Determine if another file shares the same file shards IEnumerable <Database.File> filesThatShareTheSameShards = from currentFile in FileDatabaseManifestFiles where file.FileLength == currentFile.FileLength && file.FileShardIDs.Length == currentFile.FileShardIDs.Length && file.SHA1.Equals(currentFile.SHA1, StringComparison.OrdinalIgnoreCase) && // Even if a single shard ID is shared by another file, that's enough to count it // as being equal (or at least not eligible for hard-deletion) file.FileShardIDs.Any(t => currentFile.FileShardIDs.Contains(t)) select currentFile; // If there are no files that share the same Shard IDs if (filesThatShareTheSameShards.Any() == false) { // Get the raw B2 File List so we can get the B2 file IDs of the file shards ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true); BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute(); if (listFilesActionResult.HasErrors) { throw new FailedToGetListOfFilesOnB2Exception { BackblazeErrorDetails = listFilesActionResult.Errors, }; } IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files; IDictionary <string, FileResult> fileNameToFileResult = rawB2FileList.ToDictionary(k => k.FileName, v => v); foreach (string shardID in file.FileShardIDs) { if (fileNameToFileResult.TryGetValue(shardID, out FileResult fileShardToDelete)) { DeleteFileAction deleteFileAction = new DeleteFileAction(authorizationSessionGenerator(), fileShardToDelete.FileID, fileShardToDelete.FileName); BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute(); if (deletionResult.HasErrors) { this.Critical($"Failed to delete file. Reason: {deletionResult}"); } } } } else { this.Info($"File {file.FileName} shares file shares with another file. Will not perform a hard-delete. Removing from file manifest instead"); } while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false) { Thread.Sleep(5); } this.Info($"Deleted file: {file.FileName}"); }