public (string, BackupRecord) GetBackupHashAndRecord(string bsname, string prefix, int offset = 0)
{
var bset = LoadBackupSet(bsname);
var match = HashByPrefix(bsname, prefix);
if (match == null || match.Value.multiplematches == true)
{
// TODO: throw this exception out of HashByPrefix?
throw new KeyNotFoundException();
}
int pidx = 0;
for (int i = 0; i < bset.Backups.Count; i++)
{
if (bset.Backups[i].hash.SequenceEqual(match.Value.singlematchhash))
{
pidx = i;
break;
}
}
int bidx = pidx + offset;
if (bidx >= 0 && bidx < bset.Backups.Count)
{
return(HashTools.ByteArrayToHexViaLookup32(bset.Backups[bidx].hash).ToLower(), GetBackupRecord(bsname, bset.Backups[bidx].hash));
}
else
{
throw new IndexOutOfRangeException();
}
}
/// <summary>
/// Chunks and saves data to blobstore.
/// Operates on stream input, so Filestreams can be used and
/// entire files need not be loaded into memory.
/// If an error occurs (typically when reading from a stream
/// representing a file), it is thrown to the caller.
/// </summary>
/// <param name="inputstream"></param>
/// <param name="type"></param>
/// <param name="filehash"></param>
/// <param name="hashblobqueue"></param>
public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue)
{
// https://rsync.samba.org/tech_report/node3.html
List <byte> newblob = new();
byte[] alphachksum = new byte[2];
byte[] betachksum = new byte[2];
SHA1 sha1filehasher = HashTools.GetSHA1Hasher();
SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();;
if (inputstream.Length != 0)
{
int readsize = 8_388_608;
int rollwindowsize = 32;
byte[] readin;
byte[] shifted = new byte[2];
for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency
{
if (i + readsize <= inputstream.Length) // readsize or more bytes left to read
{
readin = new byte[readsize];
inputstream.Read(readin, 0, readsize);
}
else // < readsize bytes left to read
{
readin = new byte[inputstream.Length % readsize];
inputstream.Read(readin, 0, (int)(inputstream.Length % readsize));
}
for (int j = 0; j < readin.Length; j++) // Byte by byte
{
newblob.Add(readin[j]);
HashTools.ByteSum(alphachksum, newblob[^ 1]);
public IEnumerable <string> GetBackupsAndMetadataReferencesAsStrings(BackupSetReference bsname)
{
var bset = LoadBackupSet(bsname);
foreach ((byte[] backupref, bool _) in bset.Backups)
{
yield return(HashTools.ByteArrayToHexViaLookup32(backupref));
foreach (byte[] reference in Dependencies.Blobs.GetAllBlobReferences(backupref,
BlobLocation.BlobType.BackupRecord, false, false))
{
yield return(HashTools.ByteArrayToHexViaLookup32(reference));
}
}
}
/// <summary>
/// Loads the data from a blob, no special handling of multiblob references etc.
/// </summary>
/// <param name="blocation"></param>
/// <param name="hash">Null for no verification</param>
/// <returns></returns>
private byte[] LoadBlob(BlobLocation blocation, byte[] hash, int retries = 0)
{
byte[] data = Dependencies.LoadBlob(blocation.EncryptedHash);
byte[] datahash = HashTools.GetSHA1Hasher().ComputeHash(data);
if (datahash.SequenceEqual(hash))
{
return(data);
}
else if (retries > 0)
{
return(LoadBlob(blocation, hash, retries - 1));
}
// NOTE: This hash check sometimes fails and throws the error, Issue #17
throw new Exception("Blob data did not match hash.");
}
/// <summary>
///
/// </summary>
/// <param name="prefix"></param>
/// <returns>Null if no matches, (true, null) for multiple matches, (false, hashstring) for exact match.</returns>
public (bool multiplematches, byte[] singlematchhash)? HashByPrefix(string bsname, string prefix)
{
var bset = LoadBackupSet(bsname);
// TODO: This implementation is pretty slow, could be improved with a better data structure like a trie or DAFSA
// also if this becomes an issue, keep a s
prefix = prefix.ToLower();
List <string> hashes = new List <string>(from backup in bset.Backups select HashTools.ByteArrayToHexViaLookup32(backup.hash));
List <string> matches = new List <string>(from h in hashes where h.ToLower().StartsWith(prefix.ToLower()) select h);
if (matches.Count == 0)
{
return(null);
}
else if (matches.Count > 1)
{
return(true, null);
}
else
{
return(false, HashTools.HexStringToByteArray(matches[0]));
}
}
/// <summary>
/// Given a path to a file, adds it to the file database and backs it up to the target path.
/// </summary>
/// <param name="db">The database containing catalogues files</param>
/// <param name="targetPath">The path to copy the file to</param>
/// <param name="file">The file to back up</param>
private void CatalogueAndCopyNewFile(FileContext db, string targetPath, string file)
{
if (!Directory.Exists(targetPath.Replace(Path.GetFileName(file), "")))
{
Directory.CreateDirectory(targetPath.Replace(Path.GetFileName(file), "")); //ensure the directory to backup to exists
}
File.Copy(file, targetPath, false);
db.Files.Add(new ProcessedFile {
FileName = Path.GetFileName(file), FilePath = file, BackupPath = targetPath, DateModified = File.GetLastWriteTime(file), FileHash = HashTools.HashFile(file)
});
Console.WriteLine("Added new file " + Path.GetFileName(file));
}
public Task DeleteBlobAsync(byte[] hash, string fileId)
{
return(DeleteFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash)), fileId));
}
public Task <(byte[] encryptedHash, string fileId)> StoreBlobAsync(byte[] hash, byte[] data)
{
return(StoreFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash)), hash, data));
}
public Task <byte[]> LoadBlobAsync(byte[] hash)
{
return(LoadFileAsync(Path.Combine(BlobSaveDirectory, HashTools.ByteArrayToHexViaLookup32(hash))));
}
/// <summary>
///
/// </summary>
/// <param name="file"></param>
/// <param name="hash"></param>
/// <param name="data"></param>
/// <returns>fileId</returns>
private async Task <(byte[] encryptedHash, string fileId)> StoreFileAsync(string file, byte[] hash, byte[] data, bool preventEncryption = false)
{
async Task <GetUploadUrlResponse> GetUploadUrl(int attempts = 0)
{
if (AuthResp == null)
{
AuthResp = await AuthorizeAccount();
}
Delay();
try
{
var urlresp = await AuthResp.apiUrl
.AppendPathSegment("/b2api/v1/b2_get_upload_url")
.WithHeaders(new { Authorization = AuthResp.authorizationToken })
.PostJsonAsync(new { bucketId = BucketId })
.ReceiveJson <GetUploadUrlResponse>().ConfigureAwait(false);
SuccessfulTransmission();
return(urlresp);
}
catch (FlurlHttpException ex)
{
if (ex.Call.HttpStatus != null && ex.Call.HttpStatus == System.Net.HttpStatusCode.Unauthorized)
{
AuthResp = null;
}
else
{
// Other classes of errors may be congestion related so we increase the delay
FailedTransmission();
}
if (attempts < Retries)
{
return(await GetUploadUrl(attempts + 1).ConfigureAwait(false));
}
throw;
}
}
var hashFileId = await UploadData();
async Task <(byte[] encryptedHash, string fileId)> UploadData(int attempts = 0)
{
if (UploadUrlResp == null)
{
UploadUrlResp = await GetUploadUrl();
}
Delay();
try
{
if (Encryptor != null && !preventEncryption)
{
data = Encryptor.EncryptBytes(data);
hash = HashTools.GetSHA1Hasher().ComputeHash(data);
}
var filecontent = new ByteArrayContent(data);
filecontent.Headers.Add("Content-Type", "application/octet-stream");
var uploadresp = await UploadUrlResp.uploadUrl
.WithHeaders(new
{
Authorization = UploadUrlResp.authorizationToken,
X_Bz_File_Name = file,
Content_Length = data.Length,
X_Bz_Content_Sha1 = HashTools.ByteArrayToHexViaLookup32(hash)
})
.PostAsync(filecontent)
.ReceiveJson <UploadResponse>().ConfigureAwait(false);
SuccessfulTransmission();
return(hash, uploadresp.fileId);
}
catch (FlurlHttpException ex)
{
if (ex.Call.HttpStatus != null && ex.Call.HttpStatus == System.Net.HttpStatusCode.Unauthorized)
{
UploadUrlResp = null;
}
else
{
// Other classes of errors may be congestion related so we increase the delay
FailedTransmission();
}
if (attempts < Retries)
{
return(await UploadData(attempts + 1).ConfigureAwait(false));
}
throw;
}
}
return(hashFileId);
}
private Task StoreFileAsync(string file, byte[] data, bool preventEncryption = false) => StoreFileAsync(file, HashTools.GetSHA1Hasher().ComputeHash(data), data, preventEncryption);
/// <summary>
/// Chunks and saves data to blobstore.
/// Operates on stream input, so Filestreams can be used and
/// entire files need not be loaded into memory.
/// If an error occurs (typically when reading from a stream
/// representing a file), it is thrown to the caller.
/// </summary>
/// <param name="inputstream"></param>
/// <param name="type"></param>
/// <param name="filehash"></param>
/// <param name="hashblobqueue"></param>
public static void SplitData(Stream inputstream, byte[] filehash, BlockingCollection <HashBlobPair> hashblobqueue)
{
// https://rsync.samba.org/tech_report/node3.html
List <byte> newblob = new List <byte>();
byte[] alphachksum = new byte[2];
byte[] betachksum = new byte[2];
SHA1 sha1filehasher = HashTools.GetSHA1Hasher();
SHA1 sha1blobhasher = HashTools.GetSHA1Hasher();;
if (inputstream.Length != 0)
{
int readsize = 8_388_608;
int rollwindowsize = 32;
byte[] readin;
byte[] shifted = new byte[2];
for (int i = 0; i < inputstream.Length; i += readsize) // read the file in larger chunks for efficiency
{
if (i + readsize <= inputstream.Length) // readsize or more bytes left to read
{
readin = new byte[readsize];
inputstream.Read(readin, 0, readsize);
}
else // < readsize bytes left to read
{
readin = new byte[inputstream.Length % readsize];
inputstream.Read(readin, 0, (int)(inputstream.Length % readsize));
}
for (int j = 0; j < readin.Length; j++) // Byte by byte
{
newblob.Add(readin[j]);
HashTools.ByteSum(alphachksum, newblob[newblob.Count - 1]);
if (newblob.Count > rollwindowsize)
{
HashTools.ByteDifference(alphachksum, newblob[newblob.Count - rollwindowsize - 1]);
shifted[0] = (byte)((newblob[newblob.Count - 1] << 5) & 0xFF); // rollwindowsize = 32 = 2^5 => 5
shifted[1] = (byte)((newblob[newblob.Count - 1] >> 3) & 0xFF); // 8-5 = 3
HashTools.BytesDifference(betachksum, shifted);
}
HashTools.BytesSum(betachksum, alphachksum);
if (alphachksum[0] == 0xFF && betachksum[0] == 0xFF && betachksum[1] < 0x02) // (256*256*128)^-1 => expected value (/2) = ~4MB
{
byte[] blob = newblob.ToArray();
if (i + readsize >= inputstream.Length && j + 1 >= readin.Length) // Need to use TransformFinalBlock if at end of input
{
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
}
else
{
sha1filehasher.TransformBlock(blob, 0, blob.Length, blob, 0);
}
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
newblob = new List <byte>();
Array.Clear(alphachksum, 0, 2);
Array.Clear(betachksum, 0, 2);
}
}
}
if (newblob.Count != 0) // Create blob from remaining bytes
{
byte[] blob = newblob.ToArray();
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
}
}
else
{
byte[] blob = new byte[0];
sha1filehasher.TransformFinalBlock(blob, 0, blob.Length);
hashblobqueue.Add(new HashBlobPair(sha1blobhasher.ComputeHash(blob), blob));
}
Array.Copy(sha1filehasher.Hash, filehash, sha1filehasher.Hash.Length);
hashblobqueue.CompleteAdding();
}
public (string, BackupRecord) GetBackupHashAndRecord(string bsname, int offset = 0)
{
var bset = LoadBackupSet(bsname);
return(GetBackupHashAndRecord(bsname, HashTools.ByteArrayToHexViaLookup32(bset.Backups[bset.Backups.Count - 1].hash).ToLower(), offset));
}
