SearchLocalFileForSignatures() public static method

public static SearchLocalFileForSignatures ( string localFilePath, SizeBasedCompleteSignature sig ) : SignatureSearchResult
localFilePath string
sig BlobSync.Datatypes.SizeBasedCompleteSignature
return BlobSync.Datatypes.SignatureSearchResult
Ejemplo n.º 1
0
        // updates blob if possible.
        // if blob doesn't already exist OR does not have a signature file
        // then we just upload as usual.
        public long UploadFile(string containerName, string blobName, string localFilePath)
        {
            var fileLength = CommonOps.GetFileSize(localFilePath);

            // not used here but is cached for later.
            // WORK IN PROGRESS DONT ERASE THIS LINE.
            //ConfigHelper.GetSignatureSize(fileLength, true);

            // 1) Does remote blob exist?
            // 2) if so, download existing signature for blob.
            if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName))
            {
                // 3) If blob exists and have signature, then let the magic begin.
                // 3.1) Download existing blob signature from Azure.
                // 3.2) Search through local file for matches in existing blob signature.
                // 3.3) Upload differences to Azure
                // 3.4) Upload new signature.s

                var blobSig       = DownloadSignatureForBlob(containerName, blobName);
                var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);
                var allBlocks     = UploadDelta(localFilePath, searchResults, containerName, blobName);
                var sig           = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);

                UploadSignatureForBlob(blobName, containerName, sig);

                long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum();

                return(bytesUploaded);
            }
            else
            {
                // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here.
                // 4.1) Generate signature and upload it.

                var remainingBytes = new RemainingBytes()
                {
                    BeginOffset = 0,
                    EndOffset   = fileLength - 1
                };

                var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName);
                // var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName);

                var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId);
                PutBlockList(res.ToArray(), containerName, blobName);

                var sig = CommonOps.CreateSignatureForLocalFile(localFilePath);
                UploadSignatureForBlob(blobName, containerName, sig);

                return(fileLength);
            }
        }
Ejemplo n.º 2
0
        // updates blob if possible.
        // if blob doesn't already exist OR does not have a signature file
        // then we just upload as usual.
        public SizeBasedCompleteSignature GenerateDeltaSigFromLocalResources(string localSigPath, string localFilePath)
        {
            using (var fs = new FileStream(localSigPath, FileMode.Open))
            {
                var sig           = SerializationHelper.ReadSizeBasedBinarySignature(fs);
                var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, sig);
                var allBlocks     = UploadDelta(localFilePath, searchResults, null, null, true);

                var newSig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);


                return(newSig);
            }
        }
Ejemplo n.º 3
0
        // updates blob if possible.
        // if blob doesn't already exist OR does not have a signature file
        // then we just upload as usual.
        public long CalculateDeltaSizeFromLocalSig(string localSigPath, string localFilePath)
        {
            using (var fs = new FileStream(localSigPath, FileMode.Open))
            {
                var sig           = SerializationHelper.ReadSizeBasedBinarySignature(fs);
                var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, sig);

                long total = 0;
                foreach (var remainingBytes in searchResults.ByteRangesToUpload)
                {
                    total += (remainingBytes.EndOffset - remainingBytes.BeginOffset + 1);
                }

                return(total);
            }
        }
Ejemplo n.º 4
0
        // updates blob if possible.
        // if blob doesn't already exist OR does not have a signature file
        // then we just upload as usual.
        public long CalculateDeltaSize(string containerName, string blobName, string localFilePath)
        {
            // 1) Does remote blob exist?
            // 2) if so, download existing signature for blob.
            if (!string.IsNullOrEmpty(blobName) && !string.IsNullOrEmpty(containerName) && AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName))
            {
                // 3) If blob exists and have signature, then let the magic begin.
                // 3.1) Download existing blob signature from Azure.
                // 3.2) Search through local file for matches in existing blob signature.
                // 3.3) Upload differences to Azure
                // 3.4) Upload new signature.s

                var blobSig       = DownloadSignatureForBlob(containerName, blobName);
                var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);

                long total = 0;
                foreach (var remainingBytes in searchResults.ByteRangesToUpload)
                {
                    total += (remainingBytes.EndOffset - remainingBytes.BeginOffset);
                }

                return(total);
            }
            else
            {
                var fileLength = CommonOps.GetFileSize(localFilePath);

                var remainingBytes = new RemainingBytes()
                {
                    BeginOffset = 0,
                    EndOffset   = fileLength - 1
                };

                // upload all bytes of new file. UploadBytes method will break into appropriate sized blocks.
                var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName, true);

                var sizeUploaded = allUploadedBlocks.Where(b => !b.IsDuplicate).Sum(b => b.Size);

                return(sizeUploaded);
            }
        }
Ejemplo n.º 5
0
        public long DownloadBlob(string containerName, string blobName, string localFilePath, int parallelFactor = 2)
        {
            long bytesDownloaded = 0;

            if (CommonOps.DoesFileExist(localFilePath))
            {
                // local file exists.
                // 1) generate sig for local file.
                // 2) download sig for blob.

                var blobSig       = DownloadSignatureForBlob(containerName, blobName);
                var localSig      = CommonOps.CreateSignatureForLocalFile(localFilePath);
                var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);

                // we now have a list of which blocks are already in the local file (searchResults.SignaturesToReuse)
                // We need to then determine the byteranges which are NOT covered by these blocks
                // and download those.
                // Then we need to get the blocks that already exist in the local file, read those then write them to the new file.
                var byteRangesToDownload = GenerateByteRangesOfBlobToDownload(searchResults.SignaturesToReuse, blobSig,
                                                                              containerName, blobName);

                RegenerateBlob(containerName, blobName, byteRangesToDownload, localFilePath, searchResults.SignaturesToReuse, blobSig, parallelFactor);

                foreach (var byteRange in byteRangesToDownload)
                {
                    bytesDownloaded += byteRange.EndOffset - byteRange.BeginOffset;
                }
            }
            else
            {
                // download fresh copy.
                // get stream to store.
                using (var stream = CommonHelper.GetStream(localFilePath))
                {
                    bytesDownloaded = DownloadBlob(containerName, blobName, stream, parallelFactor);
                }
            }

            return(bytesDownloaded);
        }
Ejemplo n.º 6
0
        // updates blob if possible.
        // if blob doesn't already exist OR does not have a signature file
        // then we just upload as usual.
        public long UploadFile(string containerName, string blobName, string localFilePath, int parallelFactor = 2)
        {
            var fileLength = CommonOps.GetFileSize(localFilePath);
            var sw         = new Stopwatch();

            sw.Start();
            var md5ForFile = GetFileMD5(localFilePath);

            // 1) Does remote blob exist?
            // 2) if so, download existing signature for blob.
            if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName))
            {
                var md5ForBlob = GetBlobMD5(containerName, blobName);

                // only continue if files are actually different.
                if (md5ForBlob != md5ForFile)
                {
                    // 3) If blob exists and have signature, then let the magic begin.
                    // 3.1) Download existing blob signature from Azure.
                    // 3.2) Search through local file for matches in existing blob signature.
                    // 3.3) Upload differences to Azure
                    // 3.4) Upload new signature.s

                    var blobSig = DownloadSignatureForBlob(containerName, blobName);
                    Console.WriteLine(string.Format("Dowloaded sig {0}ms", sw.ElapsedMilliseconds));

                    var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig);

                    Console.WriteLine(string.Format("Searched for common {0}ms", sw.ElapsedMilliseconds));

                    var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName, parallelFactor: parallelFactor);
                    var sig       = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks);

                    UploadSignatureForBlob(blobName, containerName, sig);

                    // set md5 for entire blob
                    AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile);

                    long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum();

                    return(bytesUploaded);
                }

                return(0);   // no bytes changed, no bytes uploaded
            }
            else
            {
                // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here.
                // 4.1) Generate signature and upload it.

                var remainingBytes = new RemainingBytes()
                {
                    BeginOffset = 0,
                    EndOffset   = fileLength - 1
                };

                var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName, parallelFactor: parallelFactor);
                var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId);
                PutBlockList(res.ToArray(), containerName, blobName);

                var sig = CommonOps.CreateSignatureForLocalFile(localFilePath);
                UploadSignatureForBlob(blobName, containerName, sig);

                // set md5 for entire blob
                AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile);

                return(fileLength);
            }
        }