|
static private CreateSignatureFromNewAndReusedBlocks ( List |
||
| allBlocks | List |
|
| return | ||
// updates blob if possible.
// if blob doesn't already exist OR does not have a signature file
// then we just upload as usual.
public long UploadFile(string containerName, string blobName, string localFilePath)
{
var fileLength = CommonOps.GetFileSize(localFilePath);
// not used here but is cached for later.
// WORK IN PROGRESS DONT ERASE THIS LINE.
//ConfigHelper.GetSignatureSize(fileLength, true);
// 1) Does remote blob exist?
// 2) if so, download existing signature for blob.
if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName))
{
// 3) If blob exists and have signature, then let the magic begin.
// 3.1) Download existing blob signature from Azure.
// 3.2) Search through local file for matches in existing blob signature.
// 3.3) Upload differences to Azure
// 3.4) Upload new signature.s
var blobSig = DownloadSignatureForBlob(containerName, blobName);
var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);
var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName);
var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);
UploadSignatureForBlob(blobName, containerName, sig);
long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum();
return(bytesUploaded);
}
else
{
// 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here.
// 4.1) Generate signature and upload it.
var remainingBytes = new RemainingBytes()
{
BeginOffset = 0,
EndOffset = fileLength - 1
};
var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName);
// var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName);
var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId);
PutBlockList(res.ToArray(), containerName, blobName);
var sig = CommonOps.CreateSignatureForLocalFile(localFilePath);
UploadSignatureForBlob(blobName, containerName, sig);
return(fileLength);
}
}
// updates blob if possible.
// if blob doesn't already exist OR does not have a signature file
// then we just upload as usual.
public SizeBasedCompleteSignature GenerateDeltaSigFromLocalResources(string localSigPath, string localFilePath)
{
using (var fs = new FileStream(localSigPath, FileMode.Open))
{
var sig = SerializationHelper.ReadSizeBasedBinarySignature(fs);
var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, sig);
var allBlocks = UploadDelta(localFilePath, searchResults, null, null, true);
var newSig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);
return(newSig);
}
}
// updates blob if possible.
// if blob doesn't already exist OR does not have a signature file
// then we just upload as usual.
public long UploadFile(string containerName, string blobName, string localFilePath, int parallelFactor = 2)
{
var fileLength = CommonOps.GetFileSize(localFilePath);
var sw = new Stopwatch();
sw.Start();
var md5ForFile = GetFileMD5(localFilePath);
// 1) Does remote blob exist?
// 2) if so, download existing signature for blob.
if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName))
{
var md5ForBlob = GetBlobMD5(containerName, blobName);
// only continue if files are actually different.
if (md5ForBlob != md5ForFile)
{
// 3) If blob exists and have signature, then let the magic begin.
// 3.1) Download existing blob signature from Azure.
// 3.2) Search through local file for matches in existing blob signature.
// 3.3) Upload differences to Azure
// 3.4) Upload new signature.s
var blobSig = DownloadSignatureForBlob(containerName, blobName);
Console.WriteLine(string.Format("Dowloaded sig {0}ms", sw.ElapsedMilliseconds));
var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);
Console.WriteLine(string.Format("Searched for common {0}ms", sw.ElapsedMilliseconds));
var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName, parallelFactor: parallelFactor);
var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);
UploadSignatureForBlob(blobName, containerName, sig);
// set md5 for entire blob
AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile);
long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum();
return(bytesUploaded);
}
return(0); // no bytes changed, no bytes uploaded
}
else
{
// 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here.
// 4.1) Generate signature and upload it.
var remainingBytes = new RemainingBytes()
{
BeginOffset = 0,
EndOffset = fileLength - 1
};
var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName, parallelFactor: parallelFactor);
var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId);
PutBlockList(res.ToArray(), containerName, blobName);
var sig = CommonOps.CreateSignatureForLocalFile(localFilePath);
UploadSignatureForBlob(blobName, containerName, sig);
// set md5 for entire blob
AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile);
return(fileLength);
}
}