private string GenerateLocalFilePath(Database.File remoteFile)
{
int counter = 0;
while (true)
{
string localDownloadPath = Path.Combine(
Directory.GetCurrentDirectory(),
Path.GetFileName(remoteFile.FileName)
);
// Resolve conflicts automatically
if (counter > 0)
{
localDownloadPath = localDownloadPath + $"_{counter}";
}
if (File.Exists(localDownloadPath) == false)
{
return(localDownloadPath);
}
counter++;
}
}
public void CompactShards(
BackblazeB2AuthorizationSession authorizationSession,
bool dryRun
)
{
this.Debug("Compacting file shards");
ISet <ISet <Database.File> > fileGroupsByContents = new HashSet <ISet <Database.File> >();
foreach (Database.File file in FileDatabaseManifestFiles)
{
bool foundAGroup = false;
foreach (ISet <Database.File> auxGroup in fileGroupsByContents)
{
Database.File auxFile = auxGroup.First();
if (file.FileLength == auxFile.FileLength &&
file.FileShardHashes.Length == auxFile.FileShardHashes.Length &&
file.SHA1.Equals(auxFile.SHA1, StringComparison.OrdinalIgnoreCase) &&
file.FileShardHashes.SequenceEqual(auxFile.FileShardHashes))
{
foundAGroup = true;
auxGroup.Add(file);
break;
}
}
if (foundAGroup == false)
{
HashSet <Database.File> newAuxGroup = new HashSet <Database.File>
{
file,
};
fileGroupsByContents.Add(newAuxGroup);
}
}
// Go through groups and rewrite the file manifest
foreach (ISet <Database.File> fileGroup in fileGroupsByContents.Where(g => g.Count > 1))
{
Database.File prototypeFile = fileGroup.First();
foreach (Database.File otherFile in fileGroup.Where(f => ReferenceEquals(f, prototypeFile) == false))
{
this.Verbose($"{otherFile.FileName} is now using shards from {prototypeFile.FileName}");
RemoveFile(otherFile);
otherFile.FileShardIDs = prototypeFile.FileShardIDs;
AddFile(otherFile);
}
}
if (dryRun == false)
{
while (TryUploadFileDatabaseManifest(authorizationSession) == false)
{
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
this.Info("Finished compacting file shards");
}
/// <summary>
/// Adds a file to the FileDatabaseManifest in a thread-safe manner. It does not check to see
/// if the file already exists
/// </summary>
/// <param name="file">The file to add</param>
protected void AddFile(Database.File file)
{
lock (SharedFileDatabaseManifestLock)
{
this.Debug($"Adding file: {file}");
FileDatabaseManifest.Files = FileDatabaseManifest.Files.Append(file).ToArray();
}
}
/// <summary>
/// Removes a file from the FileDatabaseManifest in a thread-safe manner
/// </summary>
/// <param name="file">The file to remove</param>
protected void RemoveFile(Database.File file)
{
lock (SharedFileDatabaseManifestLock)
{
this.Debug($"Removing file: {file}");
FileDatabaseManifest.Files = FileDatabaseManifest.Files.Where(t => t.Equals(file) == false).ToArray();
}
}
/// <summary>
/// Downloads a file from the B2 Backblaze server, throwing an exception if
/// this fails
/// </summary>
/// <param name="file">The file to download</param>
/// <param name="destination">The destination of the downloaded file</param>
public void DownloadFile(
BackblazeB2AuthorizationSession authorizationSession,
Database.File file,
string destination
)
{
this.Verbose($"Downloading file: {file.FileName}");
if (System.IO.File.Exists(destination))
{
throw new InvalidOperationException($"Cannot override file {destination}.");
}
if (_shardIDToFilePath == null)
{
_shardIDToFilePath = GetShardIDToFileResultMapping(authorizationSession);
}
ConcurrentBag <Tuple <string, long> > localFileShardIDPathsAndIndices = new ConcurrentBag <Tuple <string, long> >();
long currentShardsDownloaded = 0;
Parallel.ForEach(
file.FileShardIDs,
new ParallelOptions {
MaxDegreeOfParallelism = 3
},
(fileShardID, loopState, currentShardIndex) =>
{
if (loopState.ShouldExitCurrentIteration || loopState.IsExceptional || loopState.IsStopped)
{
return;
}
string shardFilePath = GetShardIDFilePath(fileShardID);
localFileShardIDPathsAndIndices.Add(Tuple.Create(shardFilePath, currentShardIndex));
if (_shardIDToFilePath.TryGetValue(fileShardID, out FileResult b2FileShard))
{
if (TryDownloadFileShard(authorizationSession, shardFilePath, fileShardID, b2FileShard))
{
long totalDownloaded = Interlocked.Increment(ref currentShardsDownloaded);
this.Info($"{file.FileName} download progress: {totalDownloaded} / {file.FileShardIDs.Length} downloaded");
}
else
{
loopState.Stop();
throw new FailedToDownloadFileException($"Could not download shard due to a B2 exception");
}
}
else
{
loopState.Stop();
throw new FailedToDownloadFileException($"Could not find the file shard: {fileShardID}");
}
});
ReconstructFile(destination, localFileShardIDPathsAndIndices);
VerifyFile(file, destination);
}
private void VerifyFile(Database.File remoteFile, string localFilePath)
{
string sha1File = SHA1FileHashStore.Instance.ComputeSHA1(localFilePath);
if (remoteFile.SHA1.Equals(sha1File, StringComparison.OrdinalIgnoreCase) == false)
{
this.Critical($"File SHA-1 does not match!");
}
}
public override void Execute(INotification notification)
{
this.Debug(CommandNotification);
Database.File file = GetFile();
string newFileName = GetNewFileName();
AuthorizationSessionProxy authorizationProxy = (AuthorizationSessionProxy)Facade.RetrieveProxy(AuthorizationSessionProxy.Name);
RenameFileProxy renameFileProxy = (RenameFileProxy)Facade.RetrieveProxy(RenameFileProxy.Name);
renameFileProxy.RenameFile(authorizationProxy.AuthorizationSession, file, newFileName);
}
/// <summary>
/// Tries to get a file by name
/// </summary>
/// <param name="fileName">The file name to fetch</param>
/// <param name="file">The reference to write the result to</param>
/// <returns>True if a result was found. False otherwise</returns>
public bool TryGetFileByName(
string fileName,
out Database.File file
)
{
// Do linear search since this doesn't happen often
file = FileDatabaseManifestFiles
.Where(f => f.FileName.Equals(fileName, StringComparison.Ordinal))
.SingleOrDefault();
return(file != null);
}
/// <summary>
/// Tries to get a file by its ID
/// </summary>
/// <param name="fileID">The file ID to retrieve</param>
/// <param name="file">The reference to write to</param>
/// <returns>True if the result was found. False otherwise</returns>
public bool TryGetFileByID(
string fileID,
out Database.File file
)
{
// Linear search since this doesn't happen often
file = FileDatabaseManifestFiles
.Where(f => f.FileID.Equals(fileID, StringComparison.OrdinalIgnoreCase))
.SingleOrDefault();
return(file != null);
}
public override void Execute(INotification notification)
{
this.Debug(CommandNotification);
AuthorizationSessionProxy authorizationProxy = (AuthorizationSessionProxy)Facade.RetrieveProxy(AuthorizationSessionProxy.Name);
Database.File fileToDownload = GetFile();
string localFileDestination = GetDestinationOfFile(fileToDownload);
DownloadFileProxy downloadFileProxy = (DownloadFileProxy)Facade.RetrieveProxy(DownloadFileProxy.Name);
downloadFileProxy.DownloadFile(authorizationProxy.AuthorizationSession, fileToDownload, localFileDestination);
this.Info($"Finished downloading file: {fileToDownload}");
}
private string GetDestinationOfFile(Database.File remoteFileToDownload)
{
this.Debug($"Getting destination of file {remoteFileToDownload}");
ProgramArgumentsProxy programArgsProxy = (ProgramArgumentsProxy)Facade.RetrieveProxy(ProgramArgumentsProxy.Name);
if (programArgsProxy.TryGetArgument(DestinationOption, out string localFileDestination) == false)
{
localFileDestination = Path.Combine(
Directory.GetCurrentDirectory(),
Path.GetFileName(remoteFileToDownload.FileName)
);
}
this.Debug($"Destination file is: {localFileDestination}");
return(localFileDestination);
}
/// <summary>
/// Renames a file in the File Database Manifest
/// </summary>
/// <param name="file"></param>
/// <param name="newFilePath"></param>
public void RenameFile(
BackblazeB2AuthorizationSession authorizationSession,
Database.File file,
string newFilePath
)
{
this.Info($"Renaming file: {file.FileName} -> {newFilePath}");
// Ensure that the new file name doesn't conflict with something else
if (TryGetFileByName(newFilePath, out Database.File _))
{
throw new FailedToRenameFileException($"{newFilePath} already exists");
}
RemoveFile(file);
file.FileName = newFilePath;
AddFile(file);
while (TryUploadFileDatabaseManifest(authorizationSession) == false)
{
Thread.Sleep(5);
}
;
this.Info("Finished renaming file");
}
private void UploadFiles(
Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator,
IDictionary <string, string> absoluteLocalFilePathsToDestinationFilePaths,
bool dryRun
)
{
if (dryRun)
{
StringBuilder builder = new StringBuilder();
builder.AppendLine("Would have uploaded the following files");
foreach (KeyValuePair <string, string> absoluteToDestinationPath in absoluteLocalFilePathsToDestinationFilePaths)
{
builder.AppendLine($"{absoluteToDestinationPath.Key} -> {absoluteToDestinationPath.Value}");
}
this.Info($"DRY RUN. {builder}");
return;
}
using (TieredUploadManager uploadManager = new TieredUploadManager(authorizationSessionGenerator, Config, CancellationEventRouter.GlobalCancellationToken))
{
IDictionary <string, ISet <string> > localFilePathToUploadIDs = new Dictionary <string, ISet <string> >();
IDictionary <string, ISet <string> > localFilePathToFinishedUploadIDs = new Dictionary <string, ISet <string> >();
IDictionary <string, string> uploadIDsToLocalFilePath = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> localPathToRemotePath in absoluteLocalFilePathsToDestinationFilePaths)
{
string localFilePath = localPathToRemotePath.Key;
localFilePathToUploadIDs[localFilePath] = new HashSet <string>();
localFilePathToFinishedUploadIDs[localFilePath] = new HashSet <string>();
foreach (Lazy <FileShard> lazyFileShard in FileShardFactory.CreateLazyFileShards(localFilePath))
{
string uploadID = uploadManager.AddLazyFileShard(lazyFileShard);
localFilePathToUploadIDs[localFilePath].Add(uploadID);
uploadIDsToLocalFilePath[uploadID] = localFilePath;
}
}
// This part is tricky because all of these event handlers will be executed on background threads. Because of this,
// we will need to lock the SendNotification call in order to prevent any race conditions or data corruption. We
// know the main thread is blocked on these background threads because we have called the "Wait()" method.
object localLockObject = new object();
IDictionary <string, ISet <UploadManagerEventArgs> > localFileToFileShardIDs = new Dictionary <string, ISet <UploadManagerEventArgs> >();
ISet <string> localFilesThatHaveAlreadyStarted = new HashSet <string>();
DateTime lastFileManifestUpload = DateTime.Now;
bool uploadedManifest = true;
void HandleOnUploadBegin(object sender, UploadManagerEventArgs eventArgs)
{
lock (localLockObject)
{
string localFile = uploadIDsToLocalFilePath[eventArgs.UploadID];
if (localFilesThatHaveAlreadyStarted.Contains(localFile) == false)
{
localFilesThatHaveAlreadyStarted.Add(localFile);
this.Info($"Begin uploading file: {localFile}");
}
}
}
void HandleOnUploadFailed(object sender, UploadManagerEventArgs eventArgs)
{
lock (localLockObject)
{
this.Critical($"Failed to upload file. {uploadIDsToLocalFilePath[eventArgs.UploadID]} | {eventArgs.UploadResult.ToString()}");
}
}
void HandleOnUploadFinished(object sender, UploadManagerEventArgs eventArgs)
{
lock (localLockObject)
{
string localFilePath = uploadIDsToLocalFilePath[eventArgs.UploadID];
if (localFileToFileShardIDs.TryGetValue(localFilePath, out ISet <UploadManagerEventArgs> uploadEvents) == false)
{
uploadEvents = new HashSet <UploadManagerEventArgs>();
localFileToFileShardIDs[localFilePath] = uploadEvents;
}
uploadEvents.Add(eventArgs);
localFilePathToFinishedUploadIDs[localFilePath].Add(eventArgs.UploadID);
int currentNumberOfUploadedShards = localFilePathToFinishedUploadIDs[localFilePath].Count;
int totalNumberOfShards = localFilePathToUploadIDs[localFilePath].Count;
if (totalNumberOfShards == currentNumberOfUploadedShards)
{
this.Info($"Finished uploading file: {localFilePath}");
string[] orderedShardIDs = new string[uploadEvents.Count];
string[] orderedShardHashes = new string[uploadEvents.Count];
foreach (UploadManagerEventArgs uploadEvent in uploadEvents)
{
orderedShardIDs[(int)uploadEvent.FileShardPieceNumber] = uploadEvent.FileShardID;
orderedShardHashes[(int)uploadEvent.FileShardPieceNumber] = uploadEvent.FileShardSHA1;
}
string destinationPath = absoluteLocalFilePathsToDestinationFilePaths[localFilePath];
// Create file
FileInfo info = new FileInfo(localFilePath);
Database.File file = new Database.File
{
FileID = Guid.NewGuid().ToString(),
FileLength = info.Length,
FileName = destinationPath,
FileShardIDs = orderedShardIDs,
FileShardHashes = orderedShardHashes,
LastModified = info.LastWriteTimeUtc.ToBinary(),
SHA1 = SHA1FileHashStore.Instance.ComputeSHA1(localFilePath),
};
// Update manifest
if (TryGetFileByName(destinationPath, out Database.File fileThatExists))
{
// Remove old file first
RemoveFile(fileThatExists);
}
uploadedManifest = false;
AddFile(file);
if (DateTime.Now - lastFileManifestUpload >= MaxTimeBetweenFileManifestUploads)
{
if (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false)
{
this.Critical("Failed to upload file manifest. Going to do it on the next try");
uploadedManifest = true;
}
}
}
else
{
double percentFinished = (double)currentNumberOfUploadedShards / totalNumberOfShards * 100.0;
this.Info($"{localFilePath} - {percentFinished:N2}% uploaded");
}
}
}
void HandleOnUploadTierChanged(object sender, UploadManagerEventArgs eventArgs)
{
lock (localLockObject)
{
this.Warning($"File tier changed. {uploadIDsToLocalFilePath[eventArgs.UploadID]} -> {eventArgs.NewUploadTier}");
}
}
// Hook up events
uploadManager.OnUploadBegin += HandleOnUploadBegin;
uploadManager.OnUploadFailed += HandleOnUploadFailed;
uploadManager.OnUploadFinished += HandleOnUploadFinished;
uploadManager.OnUploadTierChanged += HandleOnUploadTierChanged;
uploadManager.SealUploadManager();
uploadManager.Execute();
uploadManager.Wait();
// Specifically lock this part so that there is not possibility of a rogue thread
// firing an event while this main thread is in the middle of cleaning it up
lock (localLockObject)
{
// Unsubscribe from events
uploadManager.OnUploadBegin -= HandleOnUploadBegin;
uploadManager.OnUploadFailed -= HandleOnUploadFailed;
uploadManager.OnUploadFinished -= HandleOnUploadFinished;
uploadManager.OnUploadTierChanged -= HandleOnUploadTierChanged;
}
// Ensure that we've uploaded the file manifest and if not upload it again
if (uploadedManifest == false)
{
this.Verbose("Uploading file mainfest");
while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
}
this.Verbose("Successfully uploaded file manifest");
}
}
}
/// <summary>
/// Deletes a file off the remote file system
/// </summary>
/// <param name="authorizationSessionGenerator">The generator function that returns an authorization session</param>
/// <param name="file">The file to delete</param>
public void DeleteFile(
Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator,
Database.File file
)
{
this.Debug($"Begin deleting file: {file.FileName}");
// Remove entry in the File Manifest
RemoveFile(file);
// Determine if another file shares the same file shards
IEnumerable <Database.File> filesThatShareTheSameShards = from currentFile in FileDatabaseManifestFiles
where file.FileLength == currentFile.FileLength &&
file.FileShardIDs.Length == currentFile.FileShardIDs.Length &&
file.SHA1.Equals(currentFile.SHA1, StringComparison.OrdinalIgnoreCase) &&
// Even if a single shard ID is shared by another file, that's enough to count it
// as being equal (or at least not eligible for hard-deletion)
file.FileShardIDs.Any(t => currentFile.FileShardIDs.Contains(t))
select currentFile;
// If there are no files that share the same Shard IDs
if (filesThatShareTheSameShards.Any() == false)
{
// Get the raw B2 File List so we can get the B2 file IDs of the file shards
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
IDictionary <string, FileResult> fileNameToFileResult = rawB2FileList.ToDictionary(k => k.FileName, v => v);
foreach (string shardID in file.FileShardIDs)
{
if (fileNameToFileResult.TryGetValue(shardID, out FileResult fileShardToDelete))
{
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), fileShardToDelete.FileID, fileShardToDelete.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
this.Critical($"Failed to delete file. Reason: {deletionResult}");
}
}
}
}
else
{
this.Info($"File {file.FileName} shares file shares with another file. Will not perform a hard-delete. Removing from file manifest instead");
}
while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false)
{
Thread.Sleep(5);
}
this.Info($"Deleted file: {file.FileName}");
}
