private void PutBlockList(string[] blockIdArray, string containerName, string blobName) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var blob = container.GetBlockBlobReference(blobName); blob.PutBlockList(blockIdArray); }
private string GetBlobMD5(string containerName, string blobName) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var url = AzureHelper.GenerateUrl(containerName, blobName); var blobRef = client.GetBlobReferenceFromServer(new Uri(url)); blobRef.FetchAttributes(); return(blobRef.Properties.ContentMD5); }
private void PutBlockList(string[] blockIdArray, string containerName, string blobName) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var blob = container.GetBlockBlobReference(blobName); var blobIdList = blob.DownloadBlockList(BlockListingFilter.Committed); var overlap = (from b in blobIdList where blockIdArray.Contains(b.Name) select b).ToList(); blob.PutBlockList(blockIdArray); }
// download blob to stream public long DownloadBlob(string containerName, string blobName, Stream stream, int parallelFactor = 2) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var url = AzureHelper.GenerateUrl(containerName, blobName); var blobRef = client.GetBlobReferenceFromServer(new Uri(url)); ReadBlockBlob(blobRef, stream, parallelFactor); return(stream.Length); }
private byte[] DownloadBytes(string containerName, string blobName, long beginOffset, long endOffset) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var blobRef = container.GetBlockBlobReference(blobName); var buffer = new byte[endOffset - beginOffset + 1]; blobRef.DownloadRangeToByteArray(buffer, 0, beginOffset, endOffset - beginOffset + 1); return(buffer); }
public void GetBlockListInfo(string containerName, string blobName) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var blob = container.GetBlockBlobReference(blobName); var blobIdList = blob.DownloadBlockList(BlockListingFilter.Committed); var all = blob.DownloadBlockList(BlockListingFilter.All).ToList(); foreach (var i in all) { Console.WriteLine(string.Format("{0}:{1}:{2}", i.Name, i.Length, i.Committed)); } }
private byte[] DownloadBytes(string containerName, string blobName, long beginOffset, long endOffset, int parallelFactor = 2) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); var blobRef = container.GetBlockBlobReference(blobName); var buffer = new byte[endOffset - beginOffset + 1]; blobRef.DownloadRangeToByteArray(buffer, 0, beginOffset, endOffset - beginOffset + 1, options: new BlobRequestOptions { ParallelOperationThreadCount = parallelFactor }); return(buffer); }
public void UploadSignatureForBlob(string blobName, string containerName, SizeBasedCompleteSignature sig) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); // upload sig. var sigBlobName = AzureHelper.SetSignatureName(containerName, blobName); var sigBlob = container.GetBlockBlobReference(sigBlobName); using (Stream s = new MemoryStream()) { SerializationHelper.WriteBinarySizedBasedSignature(sig, s); s.Seek(0, SeekOrigin.Begin); sigBlob.UploadFromStream(s); } }
/// <summary> /// Used to overwrite the default azure client. /// </summary> /// <param name="accountName"></param> /// <param name="accountKey"></param> public void CreateCustomAzureClient(string accountName, string accountKey) { client = AzureHelper.GetCloudBlobClient(accountName, accountKey); }
private List <UploadedBlock> UploadBytesParallel(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false, int parallelFactor = 2) { var uploadedBlockList = new ConcurrentBag <UploadedBlock>(); try { CloudBlockBlob blob = null; if (!testMode) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); blob = container.GetBlockBlobReference(blobName); } var blockCount = Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) / (double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero); var taskList = new List <Task>(); long offset = remainingBytes.BeginOffset; using (var stream = new FileStream(localFilePath, FileMode.Open)) { while (offset <= remainingBytes.EndOffset) { while (offset <= remainingBytes.EndOffset && taskList.Count < parallelFactor) { var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset ? ConfigHelper.SignatureSize : remainingBytes.EndOffset - offset + 1; if (sizeToRead > 0) { // seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size // we want to deal with. stream.Seek(offset, SeekOrigin.Begin); var bytesToRead = new byte[sizeToRead]; var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead); var t = WriteBytes(offset, bytesRead, bytesToRead, blob, uploadedBlockList, testMode); taskList.Add(t); offset += sizeToRead; } } // wait until we've all uploaded. var waitedIndex = Task.WaitAny(taskList.ToArray()); taskList.RemoveAt(waitedIndex); } // wait on remaining tasks. Task.WaitAll(taskList.ToArray()); } } catch (ArgumentException ex) { // probably bad container. Console.WriteLine("Argument Exception " + ex.ToString()); } finally { } return(uploadedBlockList.ToList()); //return uploadedBlockList; }
private List <UploadedBlock> UploadBytes(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false) { var uploadedBlockList = new List <UploadedBlock>(); try { CloudBlockBlob blob = null; if (!testMode) { var client = AzureHelper.GetCloudBlobClient(); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); blob = container.GetBlockBlobReference(blobName); } var blockCount = Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) / (double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero); using (var stream = new FileStream(localFilePath, FileMode.Open)) { for (var offset = remainingBytes.BeginOffset; offset <= remainingBytes.EndOffset;) { var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset ? ConfigHelper.SignatureSize : remainingBytes.EndOffset - offset + 1; if (sizeToRead == 0) { var error = ""; } // seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size // we want to deal with. stream.Seek(offset, SeekOrigin.Begin); var bytesToRead = new byte[sizeToRead]; var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead); var sig = CommonOps.GenerateBlockSig(bytesToRead, offset, (int)sizeToRead, 0); var blockId = Convert.ToBase64String(sig.MD5Signature); var isDupe = uploadedBlockList.Any(ub => ub.BlockId == blockId); if (!testMode) { // only upload bytes IF another block hasn't already covered it. // unlikely situation I think, but possibly going to happen for // VM images etc where there is lots of "blank space". if (!isDupe) { // yes, putting into memory stream is probably a waste here. using (var ms = new MemoryStream(bytesToRead)) { var options = new BlobRequestOptions() { ServerTimeout = new TimeSpan(0, 90, 0) }; blob.PutBlock(blockId, ms, null, null, options); } } } // store the block id that is associated with this byte range. uploadedBlockList.Add(new UploadedBlock() { BlockId = blockId, Offset = offset, Sig = sig, Size = bytesRead, IsNew = true, IsDuplicate = isDupe }); offset += sizeToRead; } } } catch (ArgumentException ex) { // probably bad container. Console.WriteLine("Argument Exception " + ex.ToString()); } finally { } return(uploadedBlockList); }