public override int GetHashCode() { // Auto-generated by Visual Studio var hashCode = -871570897; hashCode = hashCode * -1521134295 + EqualityComparer <byte[]> .Default.GetHashCode(_buffer); hashCode = hashCode * -1521134295 + _isOpen.GetHashCode(); hashCode = hashCode * -1521134295 + _readOnly.GetHashCode(); hashCode = hashCode * -1521134295 + _length.GetHashCode(); hashCode = hashCode * -1521134295 + _capacity.GetHashCode(); hashCode = hashCode * -1521134295 + _position.GetHashCode(); hashCode = hashCode * -1521134295 + _startIndex.GetHashCode(); hashCode = hashCode * -1521134295 + _isExpandable.GetHashCode(); hashCode = hashCode * -1521134295 + IsReadOnly.GetHashCode(); hashCode = hashCode * -1521134295 + IsOpen.GetHashCode(); hashCode = hashCode * -1521134295 + Length.GetHashCode(); hashCode = hashCode * -1521134295 + Capacity.GetHashCode(); hashCode = hashCode * -1521134295 + Position.GetHashCode(); hashCode = hashCode * -1521134295 + RemainingBytes.GetHashCode(); hashCode = hashCode * -1521134295 + StartIndex.GetHashCode(); hashCode = hashCode * -1521134295 + EqualityComparer <byte[]> .Default.GetHashCode(Raw); return(hashCode); }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long UploadFile(string containerName, string blobName, string localFilePath) { var fileLength = CommonOps.GetFileSize(localFilePath); // not used here but is cached for later. // WORK IN PROGRESS DONT ERASE THIS LINE. //ConfigHelper.GetSignatureSize(fileLength, true); // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName); var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks); UploadSignatureForBlob(blobName, containerName, sig); long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum(); return(bytesUploaded); } else { // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here. // 4.1) Generate signature and upload it. var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName); // var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName); var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId); PutBlockList(res.ToArray(), containerName, blobName); var sig = CommonOps.CreateSignatureForLocalFile(localFilePath); UploadSignatureForBlob(blobName, containerName, sig); return(fileLength); } }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long CalculateDeltaSize(string containerName, string blobName, string localFilePath) { // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (!string.IsNullOrEmpty(blobName) && !string.IsNullOrEmpty(containerName) && AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); long total = 0; foreach (var remainingBytes in searchResults.ByteRangesToUpload) { total += (remainingBytes.EndOffset - remainingBytes.BeginOffset); } return(total); } else { var fileLength = CommonOps.GetFileSize(localFilePath); var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; // upload all bytes of new file. UploadBytes method will break into appropriate sized blocks. var allUploadedBlocks = UploadBytes(remainingBytes, localFilePath, containerName, blobName, true); var sizeUploaded = allUploadedBlocks.Where(b => !b.IsDuplicate).Sum(b => b.Size); return(sizeUploaded); } }
// updates blob if possible. // if blob doesn't already exist OR does not have a signature file // then we just upload as usual. public long UploadFile(string containerName, string blobName, string localFilePath, int parallelFactor = 2) { var fileLength = CommonOps.GetFileSize(localFilePath); var sw = new Stopwatch(); sw.Start(); var md5ForFile = GetFileMD5(localFilePath); // 1) Does remote blob exist? // 2) if so, download existing signature for blob. if (AzureHelper.DoesBlobExist(containerName, blobName) && AzureHelper.DoesBlobSignatureExist(containerName, blobName)) { var md5ForBlob = GetBlobMD5(containerName, blobName); // only continue if files are actually different. if (md5ForBlob != md5ForFile) { // 3) If blob exists and have signature, then let the magic begin. // 3.1) Download existing blob signature from Azure. // 3.2) Search through local file for matches in existing blob signature. // 3.3) Upload differences to Azure // 3.4) Upload new signature.s var blobSig = DownloadSignatureForBlob(containerName, blobName); Console.WriteLine(string.Format("Dowloaded sig {0}ms", sw.ElapsedMilliseconds)); var searchResults = CommonOps.SearchLocalFileForSignatures(localFilePath, blobSig); Console.WriteLine(string.Format("Searched for common {0}ms", sw.ElapsedMilliseconds)); var allBlocks = UploadDelta(localFilePath, searchResults, containerName, blobName, parallelFactor: parallelFactor); var sig = CommonOps.CreateSignatureFromNewAndReusedBlocks(allBlocks); UploadSignatureForBlob(blobName, containerName, sig); // set md5 for entire blob AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile); long bytesUploaded = allBlocks.Where(b => b.IsNew).Select(b => b.Size).Sum(); return(bytesUploaded); } return(0); // no bytes changed, no bytes uploaded } else { // 4) If blob or signature does NOT exist, just upload as normal. No tricky stuff to do here. // 4.1) Generate signature and upload it. var remainingBytes = new RemainingBytes() { BeginOffset = 0, EndOffset = fileLength - 1 }; var allUploadedBlocks = UploadBytesParallel(remainingBytes, localFilePath, containerName, blobName, parallelFactor: parallelFactor); var res = (from b in allUploadedBlocks orderby b.Offset ascending select b.BlockId); PutBlockList(res.ToArray(), containerName, blobName); var sig = CommonOps.CreateSignatureForLocalFile(localFilePath); UploadSignatureForBlob(blobName, containerName, sig); // set md5 for entire blob AzureHelper.SetBlobMD5(containerName, blobName, md5ForFile); return(fileLength); } }
private List <UploadedBlock> UploadBytesParallel(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false, int parallelFactor = 2) { var uploadedBlockList = new ConcurrentBag <UploadedBlock>(); try { CloudBlockBlob blob = null; if (!testMode) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); blob = container.GetBlockBlobReference(blobName); } var blockCount = Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) / (double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero); var taskList = new List <Task>(); long offset = remainingBytes.BeginOffset; using (var stream = new FileStream(localFilePath, FileMode.Open)) { while (offset <= remainingBytes.EndOffset) { while (offset <= remainingBytes.EndOffset && taskList.Count < parallelFactor) { var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset ? ConfigHelper.SignatureSize : remainingBytes.EndOffset - offset + 1; if (sizeToRead > 0) { // seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size // we want to deal with. stream.Seek(offset, SeekOrigin.Begin); var bytesToRead = new byte[sizeToRead]; var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead); var t = WriteBytes(offset, bytesRead, bytesToRead, blob, uploadedBlockList, testMode); taskList.Add(t); offset += sizeToRead; } } // wait until we've all uploaded. var waitedIndex = Task.WaitAny(taskList.ToArray()); taskList.RemoveAt(waitedIndex); } // wait on remaining tasks. Task.WaitAll(taskList.ToArray()); } } catch (ArgumentException ex) { // probably bad container. Console.WriteLine("Argument Exception " + ex.ToString()); } finally { } return(uploadedBlockList.ToList()); //return uploadedBlockList; }
private List <UploadedBlock> UploadBytes(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false) { var uploadedBlockList = new List <UploadedBlock>(); try { CloudBlockBlob blob = null; if (!testMode) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); blob = container.GetBlockBlobReference(blobName); } var blockCount = Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) / (double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero); using (var stream = new FileStream(localFilePath, FileMode.Open)) { for (var offset = remainingBytes.BeginOffset; offset <= remainingBytes.EndOffset;) { var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset ? ConfigHelper.SignatureSize : remainingBytes.EndOffset - offset + 1; if (sizeToRead == 0) { var error = ""; } // seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size // we want to deal with. stream.Seek(offset, SeekOrigin.Begin); var bytesToRead = new byte[sizeToRead]; var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead); var sig = CommonOps.GenerateBlockSig(bytesToRead, offset, (int)sizeToRead, 0); var blockId = Convert.ToBase64String(sig.MD5Signature); var isDupe = uploadedBlockList.Any(ub => ub.BlockId == blockId); if (!testMode) { // only upload bytes IF another block hasn't already covered it. // unlikely situation I think, but possibly going to happen for // VM images etc where there is lots of "blank space". if (!isDupe) { // yes, putting into memory stream is probably a waste here. using (var ms = new MemoryStream(bytesToRead)) { var options = new BlobRequestOptions() { ServerTimeout = new TimeSpan(0, 90, 0) }; blob.PutBlock(blockId, ms, null, null, options); } } } // store the block id that is associated with this byte range. uploadedBlockList.Add(new UploadedBlock() { BlockId = blockId, Offset = offset, Sig = sig, Size = bytesRead, IsNew = true, IsDuplicate = isDupe }); offset += sizeToRead; } } } catch (ArgumentException ex) { // probably bad container. Console.WriteLine("Argument Exception " + ex.ToString()); } finally { } return(uploadedBlockList); }