private void PutBlockList(string[] blockIdArray, string containerName, string blobName)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blob = container.GetBlockBlobReference(blobName);
blob.PutBlockList(blockIdArray);
}
private string GetBlobMD5(string containerName, string blobName)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var url = AzureHelper.GenerateUrl(containerName, blobName);
var blobRef = client.GetBlobReferenceFromServer(new Uri(url));
blobRef.FetchAttributes();
return(blobRef.Properties.ContentMD5);
}
private void PutBlockList(string[] blockIdArray, string containerName, string blobName)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blob = container.GetBlockBlobReference(blobName);
var blobIdList = blob.DownloadBlockList(BlockListingFilter.Committed);
var overlap = (from b in blobIdList where blockIdArray.Contains(b.Name) select b).ToList();
blob.PutBlockList(blockIdArray);
}
// download blob to stream
public long DownloadBlob(string containerName, string blobName, Stream stream, int parallelFactor = 2)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var url = AzureHelper.GenerateUrl(containerName, blobName);
var blobRef = client.GetBlobReferenceFromServer(new Uri(url));
ReadBlockBlob(blobRef, stream, parallelFactor);
return(stream.Length);
}
private byte[] DownloadBytes(string containerName, string blobName, long beginOffset, long endOffset)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blobRef = container.GetBlockBlobReference(blobName);
var buffer = new byte[endOffset - beginOffset + 1];
blobRef.DownloadRangeToByteArray(buffer, 0, beginOffset, endOffset - beginOffset + 1);
return(buffer);
}
public void GetBlockListInfo(string containerName, string blobName)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blob = container.GetBlockBlobReference(blobName);
var blobIdList = blob.DownloadBlockList(BlockListingFilter.Committed);
var all = blob.DownloadBlockList(BlockListingFilter.All).ToList();
foreach (var i in all)
{
Console.WriteLine(string.Format("{0}:{1}:{2}", i.Name, i.Length, i.Committed));
}
}
private byte[] DownloadBytes(string containerName, string blobName, long beginOffset, long endOffset, int parallelFactor = 2)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
var blobRef = container.GetBlockBlobReference(blobName);
var buffer = new byte[endOffset - beginOffset + 1];
blobRef.DownloadRangeToByteArray(buffer, 0, beginOffset, endOffset - beginOffset + 1, options: new BlobRequestOptions {
ParallelOperationThreadCount = parallelFactor
});
return(buffer);
}
public void UploadSignatureForBlob(string blobName, string containerName, SizeBasedCompleteSignature sig)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
// upload sig.
var sigBlobName = AzureHelper.SetSignatureName(containerName, blobName);
var sigBlob = container.GetBlockBlobReference(sigBlobName);
using (Stream s = new MemoryStream())
{
SerializationHelper.WriteBinarySizedBasedSignature(sig, s);
s.Seek(0, SeekOrigin.Begin);
sigBlob.UploadFromStream(s);
}
}
/// <summary>
/// Used to overwrite the default azure client.
/// </summary>
/// <param name="accountName"></param>
/// <param name="accountKey"></param>
public void CreateCustomAzureClient(string accountName, string accountKey)
{
client = AzureHelper.GetCloudBlobClient(accountName, accountKey);
}
private List <UploadedBlock> UploadBytesParallel(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false, int parallelFactor = 2)
{
var uploadedBlockList = new ConcurrentBag <UploadedBlock>();
try
{
CloudBlockBlob blob = null;
if (!testMode)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
blob = container.GetBlockBlobReference(blobName);
}
var blockCount =
Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) /
(double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero);
var taskList = new List <Task>();
long offset = remainingBytes.BeginOffset;
using (var stream = new FileStream(localFilePath, FileMode.Open))
{
while (offset <= remainingBytes.EndOffset)
{
while (offset <= remainingBytes.EndOffset && taskList.Count < parallelFactor)
{
var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset
? ConfigHelper.SignatureSize
: remainingBytes.EndOffset - offset + 1;
if (sizeToRead > 0)
{
// seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size
// we want to deal with.
stream.Seek(offset, SeekOrigin.Begin);
var bytesToRead = new byte[sizeToRead];
var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead);
var t = WriteBytes(offset, bytesRead, bytesToRead, blob, uploadedBlockList, testMode);
taskList.Add(t);
offset += sizeToRead;
}
}
// wait until we've all uploaded.
var waitedIndex = Task.WaitAny(taskList.ToArray());
taskList.RemoveAt(waitedIndex);
}
// wait on remaining tasks.
Task.WaitAll(taskList.ToArray());
}
}
catch (ArgumentException ex)
{
// probably bad container.
Console.WriteLine("Argument Exception " + ex.ToString());
}
finally
{
}
return(uploadedBlockList.ToList());
//return uploadedBlockList;
}
private List <UploadedBlock> UploadBytes(RemainingBytes remainingBytes, string localFilePath, string containerName, string blobName, bool testMode = false)
{
var uploadedBlockList = new List <UploadedBlock>();
try
{
CloudBlockBlob blob = null;
if (!testMode)
{
var client = AzureHelper.GetCloudBlobClient();
var container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
blob = container.GetBlockBlobReference(blobName);
}
var blockCount =
Math.Round((double)(remainingBytes.EndOffset - remainingBytes.BeginOffset + 1) /
(double)ConfigHelper.SignatureSize, MidpointRounding.AwayFromZero);
using (var stream = new FileStream(localFilePath, FileMode.Open))
{
for (var offset = remainingBytes.BeginOffset; offset <= remainingBytes.EndOffset;)
{
var sizeToRead = offset + ConfigHelper.SignatureSize <= remainingBytes.EndOffset
? ConfigHelper.SignatureSize
: remainingBytes.EndOffset - offset + 1;
if (sizeToRead == 0)
{
var error = "";
}
// seek to the offset we need. Dont forget remaining bytes may be bigger than the signature size
// we want to deal with.
stream.Seek(offset, SeekOrigin.Begin);
var bytesToRead = new byte[sizeToRead];
var bytesRead = stream.Read(bytesToRead, 0, (int)sizeToRead);
var sig = CommonOps.GenerateBlockSig(bytesToRead, offset, (int)sizeToRead, 0);
var blockId = Convert.ToBase64String(sig.MD5Signature);
var isDupe = uploadedBlockList.Any(ub => ub.BlockId == blockId);
if (!testMode)
{
// only upload bytes IF another block hasn't already covered it.
// unlikely situation I think, but possibly going to happen for
// VM images etc where there is lots of "blank space".
if (!isDupe)
{
// yes, putting into memory stream is probably a waste here.
using (var ms = new MemoryStream(bytesToRead))
{
var options = new BlobRequestOptions()
{
ServerTimeout = new TimeSpan(0, 90, 0)
};
blob.PutBlock(blockId, ms, null, null, options);
}
}
}
// store the block id that is associated with this byte range.
uploadedBlockList.Add(new UploadedBlock()
{
BlockId = blockId,
Offset = offset,
Sig = sig,
Size = bytesRead,
IsNew = true,
IsDuplicate = isDupe
});
offset += sizeToRead;
}
}
}
catch (ArgumentException ex)
{
// probably bad container.
Console.WriteLine("Argument Exception " + ex.ToString());
}
finally
{
}
return(uploadedBlockList);
}
