public static SearchLocalFileForSignatures ( string localFilePath, |
||
localFilePath | string | |
sig | ||
Résultat |
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long UploadFile(string containerName, string blobName, string localFilePath) { var fileLength = CommonOps.GetFileSize(localFilePath); // not used here but is cached for later. // WORK IN PROGRESS DONT ERASE THIS LINE. //ConfigHelper.GetSignatureSize(fileLength, true); // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName); var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks); UploadSignatureForBlob(blobName, containerName, sig); long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum(); return(bytesUploaded); } else { // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here. // 4.1) Generate signature and upload it. var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName); // var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName); var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId); PutBlockList(res.ToArray(), containerName, blobName); var sig = CommonOps.CreateSignatureForLocalFile(localFilePath); UploadSignatureForBlob(blobName, containerName, sig); return(fileLength); } }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public SizeBasedCompleteSignature GenerateDeltaSigFromLocalResources(string localSigPath, string localFilePath) { using (var fs = new FileStream(localSigPath, FileMode.Open)) { var sig = SerializationHelper.ReadSizeBasedBinarySignature(fs); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, sig); var allBlocks = UploadDelta(localFilePath, searchResults, null, null, true); var newSig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks); return(newSig); } }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long CalculateDeltaSizeFromLocalSig(string localSigPath, string localFilePath) { using (var fs = new FileStream(localSigPath, FileMode.Open)) { var sig = SerializationHelper.ReadSizeBasedBinarySignature(fs); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, sig); long total = 0; foreach (var remainingBytes in searchResults.ByteRangesToUpload) { total += (remainingBytes.EndOffset - remainingBytes.BeginOffset + 1); } return(total); } }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long CalculateDeltaSize(string containerName, string blobName, string localFilePath) { // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (!string.IsNullOrEmpty(blobName) && !string.IsNullOrEmpty(containerName) && AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); long total = 0; foreach (var remainingBytes in searchResults.ByteRangesToUpload) { total += (remainingBytes.EndOffset - remainingBytes.BeginOffset); } return(total); } else { var fileLength = CommonOps.GetFileSize(localFilePath); var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; // upload all bytes of new file. UploadBytes method will break into appropriate sized blocks. var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName, true); var sizeUploaded = allUploadedBlocks.Where(b => !b.IsDuplicate).Sum(b => b.Size); return(sizeUploaded); } }
public long DownloadBlob(string containerName, string blobName, string localFilePath, int parallelFactor = 2) { long bytesDownloaded = 0; if (CommonOps.DoesFileExist(localFilePath)) { // local file exists. // 1) generate sig for local file. // 2) download sig for blob. var blobSig = DownloadSignatureForBlob(containerName, blobName); var localSig = CommonOps.CreateSignatureForLocalFile(localFilePath); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); // we now have a list of which blocks are already in the local file (searchResults.SignaturesToReuse) // We need to then determine the byteranges which are NOT covered by these blocks // and download those. // Then we need to get the blocks that already exist in the local file, read those then write them to the new file. var byteRangesToDownload = GenerateByteRangesOfBlobToDownload(searchResults.SignaturesToReuse, blobSig, containerName, blobName); RegenerateBlob(containerName, blobName, byteRangesToDownload, localFilePath, searchResults.SignaturesToReuse, blobSig, parallelFactor); foreach (var byteRange in byteRangesToDownload) { bytesDownloaded += byteRange.EndOffset - byteRange.BeginOffset; } } else { // download fresh copy. // get stream to store. using (var stream = CommonHelper.GetStream(localFilePath)) { bytesDownloaded = DownloadBlob(containerName, blobName, stream, parallelFactor); } } return(bytesDownloaded); }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long UploadFile(string containerName, string blobName, string localFilePath, int parallelFactor = 2) { var fileLength = CommonOps.GetFileSize(localFilePath); var sw = new Stopwatch(); sw.Start(); var md5ForFile = GetFileMD5(localFilePath); // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { var md5ForBlob = GetBlobMD5(containerName, blobName); // only continue if files are actually different. if (md5ForBlob != md5ForFile) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); Console.WriteLine(string.Format("Dowloaded sig {0}ms", sw.ElapsedMilliseconds)); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); Console.WriteLine(string.Format("Searched for common {0}ms", sw.ElapsedMilliseconds)); var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName, parallelFactor: parallelFactor); var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks); UploadSignatureForBlob(blobName, containerName, sig); // set md5 for entire blob AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile); long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum(); return(bytesUploaded); } return(0); // no bytes changed, no bytes uploaded } else { // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here. // 4.1) Generate signature and upload it. var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName, parallelFactor: parallelFactor); var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId); PutBlockList(res.ToArray(), containerName, blobName); var sig = CommonOps.CreateSignatureForLocalFile(localFilePath); UploadSignatureForBlob(blobName, containerName, sig); // set md5 for entire blob AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile); return(fileLength); } }