public void CompactShards(
BackblazeB2AuthorizationSession authorizationSession,
bool dryRun
)
{
this.Debug("Compacting file shards");
ISet <ISet <Database.File> > fileGroupsByContents = new HashSet <ISet <Database.File> >();
foreach (Database.File file in FileDatabaseManifestFiles)
{
bool foundAGroup = false;
foreach (ISet <Database.File> auxGroup in fileGroupsByContents)
{
Database.File auxFile = auxGroup.First();
if (file.FileLength == auxFile.FileLength &&
file.FileShardHashes.Length == auxFile.FileShardHashes.Length &&
file.SHA1.Equals(auxFile.SHA1, StringComparison.OrdinalIgnoreCase) &&
file.FileShardHashes.SequenceEqual(auxFile.FileShardHashes))
{
foundAGroup = true;
auxGroup.Add(file);
break;
}
}
if (foundAGroup == false)
{
HashSet <Database.File> newAuxGroup = new HashSet <Database.File>
{
file,
};
fileGroupsByContents.Add(newAuxGroup);
}
}
// Go through groups and rewrite the file manifest
foreach (ISet <Database.File> fileGroup in fileGroupsByContents.Where(g => g.Count > 1))
{
Database.File prototypeFile = fileGroup.First();
foreach (Database.File otherFile in fileGroup.Where(f => ReferenceEquals(f, prototypeFile) == false))
{
this.Verbose($"{otherFile.FileName} is now using shards from {prototypeFile.FileName}");
RemoveFile(otherFile);
otherFile.FileShardIDs = prototypeFile.FileShardIDs;
AddFile(otherFile);
}
}
if (dryRun == false)
{
while (TryUploadFileDatabaseManifest(authorizationSession) == false)
{
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
this.Info("Finished compacting file shards");
}
/// <summary>
/// Construct a new DownloadFileAction
/// </summary>
/// <param name="authorizationSession">The authorization session to use</param>
/// <param name="fileDestination">The file destination for the downloaded file</param>
/// <param name="identifier">The identifier</param>
/// <param name="downloadIdentifierType">The type of identifier</param>
private DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
string fileDestination,
string identifier,
IdentifierType downloadIdentifierType
) : this(authorizationSession, new FileStream(fileDestination, FileMode.CreateNew), identifier, downloadIdentifierType)
{
}
/// <summary>
/// Construct a DownloadFileAction with the given file ID
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="fileDestination">The file path to download to</param>
/// <param name="fileID">The file ID to download</param>
public DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
Stream outputStream,
string fileID
)
: this(authorizationSession, outputStream, fileID, IdentifierType.ID)
{
}
/// <summary>
/// Construct a new cancellation request for a large file upload
/// </summary>
/// <param name="authorizationSession">The authorization session to use</param>
/// <param name="fileId">The file to cancel the upload of</param>
public CancelLargeFileUploadAction(
BackblazeB2AuthorizationSession authorizationSession,
string fileId
) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
_fileId = fileId;
}
public GetUploadFileURLAction(
BackblazeB2AuthorizationSession authorizationSession,
string bucketID
) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
_bucketID = bucketID;
}
/// <summary>
/// Construct a DownloadFileAction with the given file ID
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="fileDestination">The file path to download to</param>
/// <param name="fileID">The file ID to download</param>
public DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
string fileDestination,
string fileID
)
: this(authorizationSession, fileDestination, fileID, IdentifierType.ID)
{
}
/// <summary>
/// Standard ctor that must be overridden by base classes
/// </summary>
public BaseRemoteFileSystemProxy(
string proxyName,
BackblazeB2AuthorizationSession authorizationSession,
Config config
) : base(proxyName, null)
{
Config = config;
Data = GetOrCreateFileDatabaseManifest(authorizationSession, config);
}
/// <summary>
/// Downloads a file from the B2 Backblaze server, throwing an exception if
/// this fails
/// </summary>
/// <param name="file">The file to download</param>
/// <param name="destination">The destination of the downloaded file</param>
public void DownloadFile(
BackblazeB2AuthorizationSession authorizationSession,
Database.File file,
string destination
)
{
this.Verbose($"Downloading file: {file.FileName}");
if (System.IO.File.Exists(destination))
{
throw new InvalidOperationException($"Cannot override file {destination}.");
}
if (_shardIDToFilePath == null)
{
_shardIDToFilePath = GetShardIDToFileResultMapping(authorizationSession);
}
ConcurrentBag <Tuple <string, long> > localFileShardIDPathsAndIndices = new ConcurrentBag <Tuple <string, long> >();
long currentShardsDownloaded = 0;
Parallel.ForEach(
file.FileShardIDs,
new ParallelOptions {
MaxDegreeOfParallelism = 3
},
(fileShardID, loopState, currentShardIndex) =>
{
if (loopState.ShouldExitCurrentIteration || loopState.IsExceptional || loopState.IsStopped)
{
return;
}
string shardFilePath = GetShardIDFilePath(fileShardID);
localFileShardIDPathsAndIndices.Add(Tuple.Create(shardFilePath, currentShardIndex));
if (_shardIDToFilePath.TryGetValue(fileShardID, out FileResult b2FileShard))
{
if (TryDownloadFileShard(authorizationSession, shardFilePath, fileShardID, b2FileShard))
{
long totalDownloaded = Interlocked.Increment(ref currentShardsDownloaded);
this.Info($"{file.FileName} download progress: {totalDownloaded} / {file.FileShardIDs.Length} downloaded");
}
else
{
loopState.Stop();
throw new FailedToDownloadFileException($"Could not download shard due to a B2 exception");
}
}
else
{
loopState.Stop();
throw new FailedToDownloadFileException($"Could not find the file shard: {fileShardID}");
}
});
ReconstructFile(destination, localFileShardIDPathsAndIndices);
VerifyFile(file, destination);
}
/// <summary>
/// Construct a DownloadFileAction with the given Bucket Name and File Name and output to the
/// specified stream
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="fileDestination">The file path to download to</param>
/// <param name="bucketName">The name of the bucket the file is in</param>
/// <param name="fileName">The name of the file</param>
public DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
Stream outputStream,
string bucketName,
string fileName
)
: this(authorizationSession, outputStream, bucketName + "/" + fileName, IdentifierType.Name)
{
}
/// <summary>
/// Construct a DownloadFileAction with the given Bucket Name and File Name
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="fileDestination">The file path to download to</param>
/// <param name="bucketName">The name of the bucket the file is in</param>
/// <param name="fileName">The name of the file</param>
public DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
string fileDestination,
string bucketName,
string fileName
)
: this(authorizationSession, fileDestination, bucketName + "/" + fileName, IdentifierType.Name)
{
}
/// <summary>
/// Constructs a new DeleteFileAction
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="fileId">The file ID to delete</param>
/// <param name="remoteFilePath">The file name to delete</param>
public DeleteFileAction(
BackblazeB2AuthorizationSession authorizationSession,
string fileId,
string remoteFilePath
) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
_fileId = fileId;
_remoteFilePath = remoteFilePath;
}
public StartLargeFileAction(
BackblazeB2AuthorizationSession authorizationSession,
string bucketID,
string remoteFilePath
) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
_bucketID = bucketID;
_remoteFilePath = remoteFilePath;
}
private ListFilesAction(
BackblazeB2AuthorizationSession authorizationSession,
string bucketID,
ListFileMethod method,
bool shouldFetchAllFiles
) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
_bucketID = bucketID;
_method = method;
_shouldFetchAllFiles = shouldFetchAllFiles;
}
/// <summary>
/// Construct a DownloadFileAction with the given identifier and identifier type
/// </summary>
/// <param name="authorizationSession">The authorization session to use</param>
/// <param name="outputStream">The output stream of the download file</param>
/// <param name="identifier">The identifier</param>
/// <param name="downloadIdentifierType">The type of identifier</param>
private DownloadFileAction(
BackblazeB2AuthorizationSession authorizationSession,
Stream outputStream,
string identifier,
IdentifierType downloadIdentifierType
) : base(CancellationToken.None)
{
MaxRetries = 3;
_disposedValue = false;
_authorizationSession = authorizationSession;
_outputStream = outputStream;
_downloadIdentifierType = downloadIdentifierType;
_identifier = identifier;
}
protected bool TryUploadFileDatabaseManifest(
BackblazeB2AuthorizationSession authorizationSession
)
{
this.Debug("Attempting to upload file manifest");
try
{
UploadFileDatabaseManifest(authorizationSession);
}
catch (FailedToUploadFileDatabaseManifestException e)
{
this.Debug($"Failed to upload file manifest: {e.ToString()}");
return(false);
}
this.Debug("Successfully uploaded file manifest");
return(true);
}
public override void Execute(INotification notification)
{
this.Debug(CommandNotification);
AuthorizationSessionProxy authorizationSessionProxy = (AuthorizationSessionProxy)Facade.RetrieveProxy(AuthorizationSessionProxy.Name);
ConfigProxy configProxy = (ConfigProxy)Facade.RetrieveProxy(ConfigProxy.Name);
BackblazeB2AuthorizationSession authorizationSession = authorizationSessionProxy.AuthorizationSession;
Config config = configProxy.Config;
Facade.RegisterProxy(new CheckFileManifestProxy(authorizationSession, config));
Facade.RegisterProxy(new CompactShardsProxy(authorizationSession, config));
Facade.RegisterProxy(new DeleteFileProxy(authorizationSession, config));
Facade.RegisterProxy(new DownloadFileManifestProxy(authorizationSession, config));
Facade.RegisterProxy(new RemoteFileSystemProxy(authorizationSession, config));
Facade.RegisterProxy(new RenameFileProxy(authorizationSession, config));
Facade.RegisterProxy(new PruneFileShardProxy(authorizationSession, config));
Facade.RegisterProxy(new UploadFileProxy(authorizationSession, config));
}
protected IEnumerable <FileResult> GetRawB2FileNames(BackblazeB2AuthorizationSession authorizationSession)
{
ListFilesAction listFilesAction =
ListFilesAction.CreateListFileActionForFileVersions(authorizationSession, Config.BucketID, true);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
IEnumerable <FileResult> rawB2FileList = listFilesActionResult.Result.Files;
return(rawB2FileList);
}
/// <summary>
/// Construct an UploadWithSingleConnectionAction using the provided bytes
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="bytesToUpload">The bytes to upload</param>
/// <param name="fileDestination">The remote file path to upload to</param>
/// <param name="bucketID">The Bucket ID to upload to</param>
/// <param name="maxUploadAttempts">The maximum number of times to try to upload this file</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <param name="exponentialBackoffCallback">A callback to invoke when this upload uses exponential backoff</param>
public UploadWithSingleConnectionAction(
BackblazeB2AuthorizationSession authorizationSession,
string bucketID,
byte[] bytesToUpload,
string fileDestination,
int maxUploadAttempts,
CancellationToken cancellationToken,
Action <TimeSpan> exponentialBackoffCallback
) : base(cancellationToken)
{
ValidateRawPath(fileDestination);
_authorizationSession = authorizationSession ?? throw new ArgumentNullException("The authorization session object must not be null");
_bucketID = bucketID;
_bytesToUpload = bytesToUpload;
_fileDestination = fileDestination;
_maxUploadAttempts = maxUploadAttempts;
_exponentialBackoffCallback = exponentialBackoffCallback;
}
private bool TryDownloadFileShard(
BackblazeB2AuthorizationSession authorizationSession,
string fileShardID,
string filePathDestination,
FileResult remoteFile
)
{
this.Verbose($"Starting download of: {fileShardID}");
using (DownloadFileAction fileShardDownload =
new DownloadFileAction(authorizationSession, filePathDestination, remoteFile.FileID))
{
BackblazeB2ActionResult <BackblazeB2DownloadFileResult> downloadResult = fileShardDownload.Execute();
if (downloadResult.HasErrors)
{
this.Critical($"Exception occurred during downloading a file shard: {downloadResult}.");
return(false);
}
}
return(true);
}
private IDictionary <string, FileResult> GetShardIDToFileResultMapping(
BackblazeB2AuthorizationSession authorizationSession
)
{
ListFilesAction listFilesAction = ListFilesAction.CreateListFileActionForFileNames(
authorizationSession,
_config.BucketID,
true
);
BackblazeB2ActionResult <BackblazeB2ListFilesResult> listFilesActionResult = listFilesAction.Execute();
if (listFilesActionResult.HasErrors)
{
throw new FailedToGetListOfFilesOnB2Exception
{
BackblazeErrorDetails = listFilesActionResult.Errors,
};
}
return(listFilesActionResult.Result.Files.ToDictionary(k => k.FileName, v => v));
}
public UploadFilePartAction(
BackblazeB2AuthorizationSession authorizationSession,
CancellationToken cancellationToken,
string bucketID,
long filePart,
int maxUploadAttempts,
GetUploadPartURLResponse getUploadPartUrl,
byte[] rawBytes,
string sha1,
Action <TimeSpan> exponentialBackoffCallback
) : base(cancellationToken)
{
_authorizationSession = authorizationSession;
_bucketID = bucketID;
_filePartNumber = filePart;
_maxUploadAttempts = maxUploadAttempts;
_getUploadPartUrl = getUploadPartUrl;
_rawBytes = rawBytes;
_sha1 = sha1;
_exponentialBackoffCallback = exponentialBackoffCallback;
}
/// <summary>
/// Adds a local file to the Remote File System
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="localFilePath">The local path to the </param>
/// <param name="remoteDestinationPath">The destination to upload to</param>
/// <param name="shouldOverride">Whether to overide old files</param>
public void AddLocalFile(
BackblazeB2AuthorizationSession authorizationSession,
string localFilePath,
string remoteDestinationPath,
bool shouldOverride
)
{
this.Debug($"Adding local file: {localFilePath} to {remoteDestinationPath}");
if (shouldOverride == false && TryGetFileByName(remoteDestinationPath, out Database.File existingFile))
{
throw new FailedToUploadFileException("Cannot override existing remote file");
}
UploadFiles(
() => authorizationSession,
new Dictionary <string, string> {
{ Path.GetFullPath(localFilePath), remoteDestinationPath }
},
false
);
}
/// <summary>
/// Renames a file in the File Database Manifest
/// </summary>
/// <param name="file"></param>
/// <param name="newFilePath"></param>
public void RenameFile(
BackblazeB2AuthorizationSession authorizationSession,
Database.File file,
string newFilePath
)
{
this.Info($"Renaming file: {file.FileName} -> {newFilePath}");
// Ensure that the new file name doesn't conflict with something else
if (TryGetFileByName(newFilePath, out Database.File _))
{
throw new FailedToRenameFileException($"{newFilePath} already exists");
}
RemoveFile(file);
file.FileName = newFilePath;
AddFile(file);
while (TryUploadFileDatabaseManifest(authorizationSession) == false)
{
Thread.Sleep(5);
}
;
this.Info("Finished renaming file");
}
/// <summary>
/// Constructs a new UploadFileUsingMultipleConnectionsActions
/// </summary>
/// <param name="authorizationSession">The authorization session</param>
/// <param name="dataStream">The stream to read from when uploading</param>
/// <param name="remoteFilePath">The remote path you want to upload to</param>
/// <param name="bucketID">The B2 bucket you want to upload to</param>
/// <param name="fileChunkSizesInBytes">The size (in bytes) of the file chunks you want to use when uploading</param>
/// <param name="numberOfConnections">The number of connections to use when uploading</param>
/// <param name="maxUploadAttempts">The maximum number of times to attempt to upload a file chunk</param>
/// <param name="cancellationToken">The cancellation token to pass in when this upload needs to be cancelled</param>
/// <param name="exponentialBackoffCallback">A callback to invoke when this upload uses exponential backoff</param>
public UploadWithMultipleConnectionsAction(
BackblazeB2AuthorizationSession authorizationSession,
Stream dataStream,
string remoteFilePath,
string bucketID,
int fileChunkSizesInBytes,
int numberOfConnections,
int maxUploadAttempts,
CancellationToken cancellationToken,
Action <TimeSpan> exponentialBackoffCallback
) : base(cancellationToken)
{
if (fileChunkSizesInBytes < MinimumFileChunkSize)
{
throw new ArgumentException("The file chunk sizes must be larger than 1 mebibyte");
}
if (numberOfConnections < 1)
{
throw new ArgumentException("You must specify a positive, non-zero number of connections", "numberOfConnections");
}
ValidateRawPath(remoteFilePath);
_authorizationSession = authorizationSession ?? throw new ArgumentNullException("The authorization session object must not be mull");
_bucketID = bucketID;
_dataStream = dataStream;
_remoteFilePath = remoteFilePath;
_fileChunkSizesInBytes = fileChunkSizesInBytes;
_numberOfConnections = numberOfConnections;
_maxUploadAttempts = maxUploadAttempts;
_jobStream = new BlockingCollection <ProducerUploadJob>(MaxMemoryAllowed / _fileChunkSizesInBytes);
_exponentialBackoffCallback = exponentialBackoffCallback;
_internalThreadCancellationTokenSource = new CancellationTokenSource();
_totalNumberOfChunks = 0;
}
protected void UploadFileDatabaseManifest(
BackblazeB2AuthorizationSession authorizationSession
)
{
UploadWithSingleConnectionAction uploadAction = new UploadWithSingleConnectionAction(
authorizationSession,
Config.BucketID,
SerializeManifest(FileDatabaseManifest, Config),
RemoteFileDatabaseManifestName,
MaxAttemptsToUploadFileManifest,
CancellationToken.None,
_ => { } // NoOp for exponential backoff callback
);
BackblazeB2ActionResult <BackblazeB2UploadFileResult> result = uploadAction.Execute();
if (result.HasErrors)
{
throw new FailedToUploadFileDatabaseManifestException
{
BackblazeErrorDetails = result.Errors,
};
}
}
public CompactShardsProxy(
BackblazeB2AuthorizationSession authorizationSession,
Config config
) : base(Name, authorizationSession, config)
{
}
public DownloadFileManifestProxy(
BackblazeB2AuthorizationSession authorizationSession,
Config config
) : base(Name, authorizationSession, config)
{
}
public ListBucketsAction(BackblazeB2AuthorizationSession authorizationSession) : base(CancellationToken.None)
{
_authorizationSession = authorizationSession;
}
public RenameFileProxy(
BackblazeB2AuthorizationSession authorizationSession,
Config config
) : base(Name, authorizationSession, config)
{
}
public static ListFilesAction CreateListFileActionForFileNames(BackblazeB2AuthorizationSession authorizationSession, string bucketID, bool shouldFetchAllFiles)
{
return(new ListFilesAction(authorizationSession, bucketID, ListFileMethod.FILE_NAMES, shouldFetchAllFiles));
}
