static private GenerateBlockSig ( byte buffer, long offset, int blockSize, uint id ) : |
||
buffer | byte | |
offset | long | |
blockSize | int | |
id | uint | |
return |
/// <summary> /// Yes, copying the byte array to here. But given we'll not have many of these tasks going to parallel /// and each byte array is AT MOST 4M, I think I can live with the memory overhead. /// </summary> /// <param name="offset"></param> /// <param name="bytesRead"></param> /// <param name="bytesToRead"></param> /// <param name="blob"></param> /// <param name="uploadedBlockList"></param> /// <param name="testMode"></param> /// <returns></returns> private Task WriteBytes(long offset, int bytesRead, byte[] bytesToRead, CloudBlockBlob blob, ConcurrentBag <UploadedBlock> uploadedBlockList, bool testMode) { var t = Task.Factory.StartNew(() => { var sig = CommonOps.GenerateBlockSig(bytesToRead, offset, (int)bytesRead, 0); var blockId = Convert.ToBase64String(sig.MD5Signature); bool isDupe = false; lock (parallelLock) { isDupe = uploadedBlockList.Any(ub => ub.BlockId == blockId); // store the block id that is associated with this byte range. uploadedBlockList.Add(new UploadedBlock() { BlockId = blockId, Offset = offset, Sig = sig, Size = bytesRead, IsNew = true, IsDuplicate = isDupe }); } if (!testMode) { if (!isDupe) { // yes, putting into memory stream is probably a waste here. using (var ms = new MemoryStream(bytesToRead)) { var options = new BlobRequestOptions() { ServerTimeout = new TimeSpan(0, 90, 0) }; blob.PutBlock(blockId, ms, null, null, options); } } } }); return(t); }
private List <UploadedBlock> UploadBytes(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false) { var uploadedBlockList = new List <UploadedBlock>(); try { CloudBlockBlob blob = null; if (!testMode) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); blob = container.GetBlockBlobReference(blobName); } var blockCount = Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) / (double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero); using (var stream = new FileStream(localFilePath, FileMode.Open)) { for (var offset = remainingBytes.BeginOffset; offset <= remainingBytes.EndOffset;) { var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset ? ConfigHelper.SignatureSize : remainingBytes.EndOffset - offset + 1; if (sizeToRead == 0) { var error = ""; } // seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size // we want to deal with. stream.Seek(offset, SeekOrigin.Begin); var bytesToRead = new byte[sizeToRead]; var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead); var sig = CommonOps.GenerateBlockSig(bytesToRead, offset, (int)sizeToRead, 0); var blockId = Convert.ToBase64String(sig.MD5Signature); var isDupe = uploadedBlockList.Any(ub => ub.BlockId == blockId); if (!testMode) { // only upload bytes IF another block hasn't already covered it. // unlikely situation I think, but possibly going to happen for // VM images etc where there is lots of "blank space". if (!isDupe) { // yes, putting into memory stream is probably a waste here. using (var ms = new MemoryStream(bytesToRead)) { var options = new BlobRequestOptions() { ServerTimeout = new TimeSpan(0, 90, 0) }; blob.PutBlock(blockId, ms, null, null, options); } } } // store the block id that is associated with this byte range. uploadedBlockList.Add(new UploadedBlock() { BlockId = blockId, Offset = offset, Sig = sig, Size = bytesRead, IsNew = true, IsDuplicate = isDupe }); offset += sizeToRead; } } } catch (ArgumentException ex) { // probably bad container. Console.WriteLine("Argument Exception " + ex.ToString()); } finally { } return(uploadedBlockList); }