/// <summary>
/// Checks the file manifest against the file shards on the server
/// </summary>
/// <param name="authorizationSessionGenerator">The generator for an authorization session</param>
public void CheckFileManifest(Func<BackblazeB2AuthorizationSession> authorizationSessionGenerator)
{
this.Info("Checking file manifest");
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult<BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
ISet<string> allShardIDsPresent = listFilesActionResult.Result.Files.Select(t => t.FileName).ToHashSet();
IDictionary<string, ISet<Database.File>> shardIDToDatbaseFile = new Dictionary<string, ISet<Database.File>>();
foreach (Database.File file in FileDatabaseManifestFiles)
{
foreach (string shardID in file.FileShardIDs)
{
if (shardIDToDatbaseFile.TryGetValue(shardID, out ISet<Database.File> setOfFiles) == false)
{
setOfFiles = new HashSet<Database.File>();
shardIDToDatbaseFile[shardID] = setOfFiles;
}
setOfFiles.Add(file);
}
}
IEnumerable<string> allShardIDsNotAccountedFor = from file in FileDatabaseManifestFiles
from shardID in file.FileShardIDs
where allShardIDsPresent.Contains(shardID) == false
select shardID;
bool allShardIDsAccountedFor = true;
foreach (string shardIDNotAccountedFor in allShardIDsNotAccountedFor)
{
allShardIDsAccountedFor = false;
StringBuilder stringBuilder = new StringBuilder();
foreach (Database.File file in shardIDToDatbaseFile[shardIDNotAccountedFor])
{
stringBuilder.Append($"{file} | ");
}
this.Critical($"Shard ID {shardIDNotAccountedFor} for file(s): {stringBuilder.ToString()}");
}
if (allShardIDsAccountedFor)
{
this.Info("All Shard IDs accounted for");
}
else
{
this.Info("Some Shard IDs Are Not Accounted For");
}
}
/// <summary>
/// Prunes any shards on the server that are not accounted for in the database manifest
/// </summary>
/// <param name="authorizationSessionGenerator">The generator for an authorization session</param>
public void PruneShards(Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator)
{
this.Debug("Pruning shards");
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
object lockObject = new object();
IDictionary <string, FileResult> fileNameToFileResultMap = listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v);
ISet <string> allDatabaseFileShardIds = FileDatabaseManifestFiles.SelectMany(t => t.FileShardIDs).ToHashSet();
ISet <string> allRawFileNamesOnServer = fileNameToFileResultMap.Keys.ToHashSet();
ISet <string> allFilesNotAccountedFor = allRawFileNamesOnServer.Except(allDatabaseFileShardIds).Where(t => t.Equals(RemoteFileDatabaseManifestName, StringComparison.OrdinalIgnoreCase) == false).ToHashSet();
Parallel.ForEach(allFilesNotAccountedFor, fileNameNotAccountedFor =>
{
CancellationEventRouter.GlobalCancellationToken.ThrowIfCancellationRequested();
FileResult fileNotAccountedFor = fileNameToFileResultMap[fileNameNotAccountedFor];
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), fileNotAccountedFor.FileID, fileNotAccountedFor.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
lock (lockObject)
{
this.Critical($"Failed to prune file. Reason: {deletionResult}");
}
}
else
{
lock (lockObject)
{
this.Info($"Pruned shard: {fileNameNotAccountedFor}");
}
}
});
}
protected IEnumerable <FileResult> GetRawB2FileNames(BackblazeB2AuthorizationSession authorizationSession)
{
ListFilesAction listFilesAction =
ListFilesAction.CreateListFileActionForFileVersions(authorizationSession, Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
return(rawB2FileList);
}
private IDictionary <string, FileResult> GetShardIDToFileResultMapping(
BackblazeB2AuthorizationSession authorizationSession
)
{
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(
authorizationSession,
_config.BucketID,
true
);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
return(listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v));
}
/// <summary>
/// Initializes this file database manifest
/// </summary>
private static FileDatabaseManifest GetOrCreateFileDatabaseManifest(
BackblazeB2AuthorizationSession authorizationSession,
Config config
)
{
if (SharedFileDatabaseManifest == null)
{
lock (SharedFileDatabaseManifestLock)
{
if (SharedFileDatabaseManifest == null)
{
// Get just the file names on the server
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(
authorizationSession,
config.BucketID,
true
);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
FileResult fileDatabaseManifestFileResult = listFilesActionResult.Result.Files
.Where(f => f.FileName.Equals(RemoteFileDatabaseManifestName, StringComparison.Ordinal))
.SingleOrDefault();
if (fileDatabaseManifestFileResult == null)
{
SharedFileDatabaseManifest = new FileDatabaseManifest
{
DataFormatVersionNumber = CurrentDatabaseDataFormatVersion,
Files = new Database.File[0]
};
}
else
{
using (MemoryStream outputStream = new MemoryStream())
using (DownloadFileAction manifestFileDownloadAction = new DownloadFileAction(
authorizationSession,
outputStream,
fileDatabaseManifestFileResult.FileID
))
{
BackblazeB2ActionResult <BackblazeB2DownloadFileResult> manifestResultOption =
manifestFileDownloadAction.Execute();
if (manifestResultOption.HasResult)
{
// Now, read string from manifest
outputStream.Flush();
SharedFileDatabaseManifest = DeserializeManifest(
outputStream.ToArray(),
config
);
}
else
{
SharedFileDatabaseManifest = new FileDatabaseManifest
{
DataFormatVersionNumber = CurrentDatabaseDataFormatVersion,
Files = new Database.File[0]
};
}
}
}
}
}
}
return(SharedFileDatabaseManifest);
}
/// <summary>
/// Deletes a file off the remote file system
/// </summary>
/// <param name="authorizationSessionGenerator">The generator function that returns an authorization session</param>
/// <param name="file">The file to delete</param>
public void DeleteFile(
Func <BackblazeB2AuthorizationSession> authorizationSessionGenerator,
Database.File file
)
{
this.Debug($"Begin deleting file: {file.FileName}");
// Remove entry in the File Manifest
RemoveFile(file);
// Determine if another file shares the same file shards
IEnumerable <Database.File> filesThatShareTheSameShards = from currentFile in FileDatabaseManifestFiles
where file.FileLength == currentFile.FileLength &&
file.FileShardIDs.Length == currentFile.FileShardIDs.Length &&
file.SHA1.Equals(currentFile.SHA1, StringComparison.OrdinalIgnoreCase) &&
// Even if a single shard ID is shared by another file, that's enough to count it
// as being equal (or at least not eligible for hard-deletion)
file.FileShardIDs.Any(t => currentFile.FileShardIDs.Contains(t))
select currentFile;
// If there are no files that share the same Shard IDs
if (filesThatShareTheSameShards.Any() == false)
{
// Get the raw B2 File List so we can get the B2 file IDs of the file shards
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(authorizationSessionGenerator(), Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
IDictionary <string, FileResult> fileNameToFileResult = rawB2FileList.ToDictionary(k => k.FileName, v => v);
foreach (string shardID in file.FileShardIDs)
{
if (fileNameToFileResult.TryGetValue(shardID, out FileResult fileShardToDelete))
{
DeleteFileAction deleteFileAction =
new DeleteFileAction(authorizationSessionGenerator(), fileShardToDelete.FileID, fileShardToDelete.FileName);
BackblazeB2ActionResult <BackblazeB2DeleteFileResult> deletionResult = deleteFileAction.Execute();
if (deletionResult.HasErrors)
{
this.Critical($"Failed to delete file. Reason: {deletionResult}");
}
}
}
}
else
{
this.Info($"File {file.FileName} shares file shares with another file. Will not perform a hard-delete. Removing from file manifest instead");
}
while (TryUploadFileDatabaseManifest(authorizationSessionGenerator()) == false)
{
Thread.Sleep(5);
}
this.Info($"Deleted file: {file.FileName}");
}
