/// <summary>
/// Uploads a data list as a block to the block blob
/// </summary>
/// <param name="containerName">Name of the blob container</param>
/// <param name="blobName">Block blob name</param>
/// <param name="dataList">List of data to be uploaded as a block to the blob</param>
public void UploadBlockToBlob(string containerName, string blobName, IEnumerable <string> dataList)
{
try
{
CloudBlobContainer blobContainer = this.blobClient.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(blobName);
List <string> blockIds = new List <string>();
if (blockBlob.Exists())
{
blockIds.AddRange(blockBlob.DownloadBlockList().Select(id => id.Name));
}
var newId = Convert.ToBase64String(Encoding.Default.GetBytes(blockIds.Count.ToString("d6")));
using (MemoryStream ms = new MemoryStream())
{
using (StreamWriter sw = new StreamWriter(ms))
{
foreach (var dataItem in dataList)
{
sw.WriteLine(dataItem);
}
sw.Flush();
ms.Position = 0;
blockBlob.PutBlock(newId, ms, null);
}
}
blockIds.Add(newId);
blockBlob.PutBlockList(blockIds);
}
catch (Exception e)
{
Log.Error("Could not upload blob " + blobName + " " + containerName + " " + e.Message);
}
}
// Retrieve the list of uploaded blocks for a blob.
// Return true on success, false if already exists, throw exception on error.
public bool GetBlockList(string containerName, string blobName, out string[] blockIds)
{
blockIds = null;
try
{
CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(blobName);
IEnumerable <ListBlockItem> blobs = blob.DownloadBlockList();
blockIds = new string[blobs.Count()];
int i = 0;
foreach (ListBlockItem block in blobs)
{
blockIds[i++] = block.Name;
}
return(true);
}
catch (StorageClientException ex)
{
if ((int)ex.StatusCode == 404)
{
return(false);
}
throw;
}
}
public static void WaitUntilFileCreated(FileNode fileNode, DataAdaptor <DMLibDataInfo> dataAdaptor, DMLibDataType dataType, int timeoutInSec = 300)
{
Func <bool> checkFileCreated = null;
if (dataType == DMLibDataType.Local)
{
string filePath = dataAdaptor.GetAddress() + fileNode.GetLocalRelativePath();
checkFileCreated = () =>
{
return(File.Exists(filePath));
};
}
else if (dataType == DMLibDataType.PageBlob ||
dataType == DMLibDataType.AppendBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlob cloudBlob = blobAdaptor.GetCloudBlobReference(fileNode);
return(cloudBlob.Exists(options: HelperConst.DefaultBlobOptions));
};
}
else if (dataType == DMLibDataType.BlockBlob)
{
CloudBlobDataAdaptor blobAdaptor = dataAdaptor as CloudBlobDataAdaptor;
checkFileCreated = () =>
{
CloudBlockBlob blockBlob = blobAdaptor.GetCloudBlobReference(fileNode) as CloudBlockBlob;
try
{
return(blockBlob.DownloadBlockList(BlockListingFilter.All, options: HelperConst.DefaultBlobOptions).Any());
}
catch (StorageException)
{
return(false);
}
};
}
else if (dataType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor fileAdaptor = dataAdaptor as CloudFileDataAdaptor;
checkFileCreated = () =>
{
CloudFile cloudFile = fileAdaptor.GetCloudFileReference(fileNode);
return(cloudFile.Exists(options: HelperConst.DefaultFileOptions));
};
}
else
{
Test.Error("Unexpected data type: {0}", DMLibTestContext.SourceType);
}
MultiDirectionTestHelper.WaitUntil(checkFileCreated, timeoutInSec);
}
// Get all Commited Blocks and Commit them again excluding the last block (so it will be garbage collected and deleted by the storage service)
private void RemoveLastBlock(CloudBlockBlob blob)
{
try
{
List <string> blockIds = new List <string>();
blockIds.AddRange(blob.DownloadBlockList(BlockListingFilter.Committed).Select(b => b.Name));
blockIds.RemoveAt(blockIds.Count - 1);
blob.PutBlockList(blockIds);
}
catch (Exception ex)
{
// Logfile ist zu verbuggt im Moment. Hier kann ab und an eine Exception auftreten. Reicht aber fürs Erste.
}
}
// Appends a new block to a blob
private void AppendBlock()
{
try
{
List <string> blockIds = new List <string>();
blockIds.AddRange(blob.DownloadBlockList(BlockListingFilter.Committed).Select(b => b.Name));
// Achtung: BlockId's eines BlockBlobs müssen die gleiche Länge haben! --> PadLeft()
var newId = Convert.ToBase64String(Encoding.Default.GetBytes(blockIds.Count.ToString().PadLeft(32, '0')));
// TODO: der MemoryStream müsste eigentlich passend hier reinkommen, damit hier kein neuer erzeugt werden muss aus dem vorhandenen
blob.PutBlock(newId, new MemoryStream(memoryLogfile.GetBuffer(), true), null);
blockIds.Add(newId);
blob.PutBlockList(blockIds);
}
catch (Exception ex)
{
// Logfile ist zu verbuggt im Moment. Hier kann ab und an eine Exception auftreten. Reicht aber fürs Erste.
}
}
public static void AppendBlock(this CloudBlockBlob blob, Stream stream, int maxRetries = 0, int sleepMilliseconds = 0)
{
int retryCount = 0;
string leaseID = null;
var random = new Random();
var blockID = Guid.NewGuid().ToString("N").ToUpperInvariant();
while (true)
{
try
{
leaseID = blob.AcquireLease(TimeSpan.FromMinutes(1), null);
var accessCondition = new AccessCondition {
LeaseId = leaseID
};
blob.PutBlock(blockID, stream, null, accessCondition);
var blockList = blob.
DownloadBlockList(BlockListingFilter.Committed, accessCondition).
Select(x => x.Name).
Concat(new [] { blockID });
blob.PutBlockList(blockList, accessCondition);
blob.ReleaseLease(accessCondition);
break;
}
catch (Exception)
{
if (leaseID != null)
{
blob.ReleaseLease(new AccessCondition {
LeaseId = leaseID
});
}
if (++retryCount < maxRetries)
{
Thread.Sleep(random.Next(sleepMilliseconds / 2, sleepMilliseconds + sleepMilliseconds / 2));
}
else
{
throw;
}
}
}
}
public override void GetDirectoryContentSummary(Uri dfsPath, bool expandBlocks, ref long totalSize, ref int numberOfParts)
{
totalSize = -1;
numberOfParts = -1;
try
{
CloudBlockBlob blockBlob = _container.GetBlockBlobReference(UriBlob(dfsPath));
blockBlob.FetchAttributes();
totalSize = blockBlob.Properties.Length;
// TODO: can we get the length of the list without downloading it?
ICollection <ListBlockItem> blockList = (ICollection <ListBlockItem>)blockBlob.DownloadBlockList(BlockListingFilter.All);
numberOfParts = Math.Max(blockList.Count, 1);
}
catch (Exception e)
{
throw new ApplicationException("BlockAzureClient::GetContentSummary failed", e);
}
return;
}
/// <summary>
/// Uploads/ appends data to block blob.
/// </summary>
/// <param name="blobUri">Blob Uri</param>
/// <param name="data">Data to upload</param>
/// <returns></returns>
public static async Task UploadFileToBlobAsync(string blobUri, string data)
{
var blob = new CloudBlockBlob(new Uri(blobUri));
// Generate a blob id
string id = String.Empty;
int blockCount = 0;
byte[] byteArray = Encoding.UTF8.GetBytes(data);
MemoryStream stream = new MemoryStream(byteArray);
List <string> blockIdList = new List <string>();
// If blob exists, get the list of ids of uploaded blocks, else create the blob with the header row.
if (blob.Exists())
{
var blockList = blob.DownloadBlockList();
blockCount = blockList.Count();
foreach (ListBlockItem item in blockList)
{
blockIdList.Add(item.Name);
}
}
else
{
var headerId = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", (++blockCount).ToString("0000000"))));
blob.PutBlock(headerId, new MemoryStream(Encoding.UTF8.GetBytes(headerFormat + "\r\n")), null);
blockIdList.Add(headerId);
}
id = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", (++blockCount).ToString("0000000"))));
blob.PutBlock(id, stream, null);
blockIdList.Add(id);
blob.PutBlockList(blockIdList);
}
internal static async Task UploadBlockAsync(string blobUri, string headerFormat, string data)
{
var blob = new CloudBlockBlob(new Uri(blobUri));
string id = String.Empty;
int blockCount = 0;
byte[] byteArray = Encoding.UTF8.GetBytes(data);
MemoryStream stream = new MemoryStream(byteArray);
List <string> blockIdList = new List <string>();
if (blob.Exists())
{
var blockList = blob.DownloadBlockList();
blockCount = blockList.Count();
foreach (ListBlockItem item in blockList)
{
blockIdList.Add(item.Name);
}
}
else
{
var headerId = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", (++blockCount).ToString("0000000"))));
await blob.PutBlockAsync(headerId, new MemoryStream(Encoding.UTF8.GetBytes(headerFormat + "\r\n")), null);
blockIdList.Add(headerId);
}
id = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", (++blockCount).ToString("0000000"))));
await blob.PutBlockAsync(id, stream, null);
blockIdList.Add(id);
await blob.PutBlockListAsync(blockIdList);
}
/// <summary>
/// read bytes from the stream and append the content to an existed file
/// </summary>
/// <param name="filePath"></param>
/// <param name="stream"></param>
/// <returns></returns>
public bool AppendFileFromStream(string filePath, Stream stream)
{
filePath = filePath.ToAzurePath();
ICloudBlob rawBlob = _container.GetBlockBlobReference(filePath);
if (rawBlob.Properties.BlobType == BlobType.PageBlob)
{
return(false); //for page blob, append is not supported
}
CloudBlockBlob blob = _container.GetBlockBlobReference(filePath);
// store the block id of this blob
var blockList = new List <string>();
try
{
foreach (var block in blob.DownloadBlockList())
{
blockList.Add(block.Name);
}
}
catch (StorageException)
{
// do nothing, this may happen when blob doesn't exist
}
const int blockSize = 4 * 1024 * 1024; // 4M - block size
byte[] buffer = new byte[blockSize];
// append file
try
{
int nRead = 0;
while (nRead < blockSize)
{
int actualRead = stream.Read(buffer, nRead, blockSize - nRead);
if (actualRead <= 0) // stream end
{
//put last block & break
string strBlockId = GetBlockID(blockList);
blob.PutBlock(strBlockId, new MemoryStream(buffer, 0, nRead), null);
blockList.Add(strBlockId);
break;
}
else if (actualRead == (blockSize - nRead)) // buffer full
{
//put this block
string strBlockId = GetBlockID(blockList);
blob.PutBlock(strBlockId, new MemoryStream(buffer), null);
blockList.Add(strBlockId);
nRead = 0;
continue;
}
nRead += actualRead;
}
}
catch (StorageException)
{
// blob.PutBlock error
return(false);
}
// put block list
blob.PutBlockList(blockList);
return(true);
}
public static void Run(
[BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "%nsgSourceDataAccount%")] CloudBlockBlob myBlob,
[Queue("stage1", Connection = "AzureWebJobsStorage")] ICollector <Chunk> outputChunks,
[Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable,
string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac,
TraceWriter log)
{
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.Error("Value for nsgSourceDataAccount is required.");
throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting.");
}
string blobContainerName = Util.GetEnvironmentVariable("blobContainerName");
if (blobContainerName.Length == 0)
{
log.Error("Value for blobContainerName is required.");
throw new System.ArgumentNullException("blobContainerName", "Please provide setting.");
}
var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac);
// get checkpoint
Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable);
// break up the block list into 10k chunks
List <Chunk> chunks = new List <Chunk>();
long currentChunkSize = 0;
string currentChunkLastBlockName = "";
long currentStartingByteOffset = 0;
bool firstBlockItem = true;
bool foundStartingOffset = false;
bool tieOffChunk = false;
int numberOfBlocks = 0;
long sizeOfBlocks = 0;
foreach (var blockListItem in myBlob.DownloadBlockList(BlockListingFilter.Committed))
{
if (!foundStartingOffset)
{
if (firstBlockItem)
{
currentStartingByteOffset += blockListItem.Length;
firstBlockItem = false;
if (checkpoint.LastBlockName == "")
{
foundStartingOffset = true;
}
}
else
{
if (blockListItem.Name == checkpoint.LastBlockName)
{
foundStartingOffset = true;
}
currentStartingByteOffset += blockListItem.Length;
}
}
else
{
// tieOffChunk = add current chunk to the list, initialize next chunk counters
// conditions to account for:
// 1) current chunk is empty & not the last block (size > 10 I think)
// a) add blockListItem to current chunk
// b) loop
// 2) current chunk is empty & last block (size < 10 I think)
// a) do not add blockListItem to current chunk
// b) loop terminates
// c) chunk last added to the list is the last chunk
// 3) current chunk is not empty & not the last block
// a) if size of block + size of chunk >10k
// i) add chunk to list <-- tieOffChunk
// ii) reset chunk counters
// b) add blockListItem to chunk
// c) loop
// 4) current chunk is not empty & last block
// a) add chunk to list <-- tieOffChunk
// b) do not add blockListItem to chunk
// c) loop terminates
tieOffChunk = (currentChunkSize != 0) && ((blockListItem.Length < 10) || (currentChunkSize + blockListItem.Length > MAXDOWNLOADBYTES));
if (tieOffChunk)
{
// chunk complete, add it to the list & reset counters
chunks.Add(new Chunk
{
BlobName = blobContainerName + "/" + myBlob.Name,
Length = currentChunkSize,
LastBlockName = currentChunkLastBlockName,
Start = currentStartingByteOffset,
BlobAccountConnectionName = nsgSourceDataAccount
});
currentStartingByteOffset += currentChunkSize; // the next chunk starts at this offset
currentChunkSize = 0;
tieOffChunk = false;
}
if (blockListItem.Length > 10)
{
numberOfBlocks++;
sizeOfBlocks += blockListItem.Length;
currentChunkSize += blockListItem.Length;
currentChunkLastBlockName = blockListItem.Name;
}
}
}
if (currentChunkSize != 0)
{
// residual chunk
chunks.Add(new Chunk
{
BlobName = blobContainerName + "/" + myBlob.Name,
Length = currentChunkSize,
LastBlockName = currentChunkLastBlockName,
Start = currentStartingByteOffset,
BlobAccountConnectionName = nsgSourceDataAccount
});
}
if (chunks.Count > 0)
{
var lastChunk = chunks[chunks.Count - 1];
checkpoint.PutCheckpoint(checkpointTable, lastChunk.LastBlockName, lastChunk.Start + lastChunk.Length);
}
// add the chunks to output queue
// they are sent automatically by Functions configuration
foreach (var chunk in chunks)
{
outputChunks.Add(chunk);
if (chunk.Length == 0)
{
log.Error("chunk length is 0");
}
}
}
public void UploadFile(string srcFilenameFullPath, CloudBlobContainer container, string destFilename, byte[] md5Hash, bool setArchiveFlag = false)
{
if (md5Hash == null)
{
throw new ArgumentNullException("md5Hash");
}
System.IO.FileInfo fileInfo = new System.IO.FileInfo(srcFilenameFullPath);
Uri uri = new Uri(container.Uri.AbsoluteUri + '/' + destFilename);
CloudBlockBlob blob = container.GetBlockBlobReference(destFilename);
if (fileInfo.Length > 4 * 1024 * 1024)
{
const long BlockSize = 1L * 1024L * 1024L;
// Perform a block upload.
// Check to see if there are any blocks uploaded.
IEnumerable <ListBlockItem> blockList = new List <ListBlockItem>();
try
{
blockList = blob.DownloadBlockList(BlockListingFilter.Uncommitted);
}
catch (StorageException e)
{
// no blocks uploaded?
}
int lastCommittedBlock = -1;
if (blockList.Count() > 0)
{
lastCommittedBlock = blockList.Select(a => BitConverter.ToInt32(Convert.FromBase64String(a.Name), 0))
.Max();
}
int totalBlocks = (int)(fileInfo.Length / BlockSize);
if (fileInfo.Length % BlockSize > 0)
{
totalBlocks++;
}
const int BitsPerBlock = (int)BlockSize * 8;
Stopwatch sw = new Stopwatch();
using (FileStream fs = new FileStream(srcFilenameFullPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
byte[] buffer = new byte[BlockSize];
MD5 md5 = MD5.Create();
for (int i = lastCommittedBlock + 1; i < totalBlocks; i++)
{
if (stopFlag.WaitOne(0) || !YAMLConfig.Performance.IsActive)
{
return;
}
logger.Info("\tSending block {0} of {1}...", i + 1, totalBlocks);
fs.Seek((long)i * BlockSize, SeekOrigin.Begin);
int bytesRead = fs.Read(buffer, 0, (int)BlockSize);
byte[] md5HashSegment = md5.ComputeHash(buffer, 0, bytesRead);
using (MemoryStream ms = new MemoryStream(buffer, 0, bytesRead))
{
sw.Restart();
blob.PutBlock(Convert.ToBase64String(BitConverter.GetBytes(i)), ms, Convert.ToBase64String(md5HashSegment));
sw.Stop();
if (YAMLConfig.Performance?.UploadRate > 0)
{
float bitsPerSecond = (float)BitsPerBlock / ((float)sw.ElapsedMilliseconds / 1000.0f);
float targetBitsPerSecond = (float)YAMLConfig.Performance.UploadRate * 1000.0f;
float actualSeconds = (float)sw.ElapsedMilliseconds / 1000.0f;
float targetSeconds = (float)BitsPerBlock / targetBitsPerSecond;
float diff = targetSeconds - actualSeconds;
if (diff > 0)
{
Thread.Sleep((int)(diff * 1000.0f));
}
}
}
}
}
List <string> blockNamesList = new List <string>();
int[] blockNums = blockList.Select(a => BitConverter.ToInt32(Convert.FromBase64String(a.Name), 0)).ToArray();
for (int i = 0; i < totalBlocks; i++)
{
blockNamesList.Add(Convert.ToBase64String(BitConverter.GetBytes(i)));
}
blob.PutBlockList(blockNamesList);
}
else
{
blob.UploadFromFile(srcFilenameFullPath);
}
string dateTimeString = fileInfo.LastWriteTimeUtc.ToString("yyyy-MM-dd-HH-mm-ss.fff");
blob.Metadata[LastWriteTimeUTCKey] = dateTimeString;
blob.Metadata[LastWriteTimeUTCTicksKey] = fileInfo.LastWriteTimeUtc.Ticks.ToString();
blob.Metadata[ContentMD5Key] = Convert.ToBase64String(md5Hash);
blob.SetMetadata();
if (setArchiveFlag)
{
blob.SetStandardBlobTier(StandardBlobTier.Archive);
}
}
public IEnumerable <AzureCollectionPart> GetPartition()
{
string account, key, container, blobPrefix;
Utils.FromAzureUri(_uri, out account, out key, out container, out blobPrefix);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(_uri.Query);
long lowReadSize = -1;
if (query["lowReadSize"] != null)
{
lowReadSize = Int64.Parse(query["lowReadSize"]);
query.Remove("lowReadSize");
}
long highReadSize = Int64.MaxValue;
if (query["highReadSize"] != null)
{
highReadSize = Int64.Parse(query["highReadSize"]);
query.Remove("highReadSize");
}
_seekBoundaries = null;
if (query["seekBoundaries"] != null)
{
_seekBoundaries = query["seekBoundaries"];
query.Remove("seekBoundaries");
}
UriBuilder strippedQuery = new UriBuilder(_uri);
strippedQuery.Query = query.ToString();
_uri = strippedQuery.Uri;
IEnumerator <CloudBlockBlob> blobs = GetBlobEnumerable().GetEnumerator();
long totalLength = 0;
bool moreBlobs = blobs.MoveNext();
while (moreBlobs)
{
// first accumulate any small blobs that need to be combined into a single URI
List <string> smallBlobs = new List <string>();
long combinedLength = 0;
while (moreBlobs && combinedLength + blobs.Current.Properties.Length <= lowReadSize)
{
if (blobs.Current.Name.StartsWith(blobPrefix))
{
// ignore empty inputs
if (blobs.Current.Properties.Length > 0)
{
smallBlobs.Add(blobs.Current.Name.Substring(blobPrefix.Length));
combinedLength += blobs.Current.Properties.Length;
}
moreBlobs = blobs.MoveNext();
}
else
{
throw new ApplicationException("Unexpected blob " + blobs.Current.Name + " in directory " + blobPrefix);
}
}
if (smallBlobs.Count > 0)
{
totalLength += combinedLength;
yield return(new AzureCollectionPartSmallBlob(smallBlobs));
// go back around the loop to see if there are more small blobs to combine
continue;
}
if (moreBlobs)
{
// now take the next blob and split it into one or more pieces
CloudBlockBlob blob = blobs.Current;
totalLength += blob.Properties.Length;
string blobName;
if (blob.Name.StartsWith(blobPrefix))
{
blobName = blob.Name.Substring(blobPrefix.Length);
}
else
{
throw new ApplicationException("Unexpected blob " + blob.Name + " in directory " + blobPrefix);
}
bool canSplitAtBlocks = (blob.Metadata.ContainsKey("recordsRespectBlockBoundaries") &&
blob.Metadata["recordsRespectBlockBoundaries"] == "true");
bool canSplit = canSplitAtBlocks || (_seekBoundaries != null);
if (blob.Properties.Length < highReadSize || !canSplit)
{
// we either don't want to split this blob or can't, so emit the entire blob and
// move on to the next one
yield return(new AzureCollectionPartSingleBlob(blobName));
moreBlobs = blobs.MoveNext();
continue;
}
// make a blob-local copy of seekBoundaries that may be overridden below
bool thisSeekBoundaries = (_seekBoundaries != null);
IEnumerator <long> blockLengths;
if (canSplitAtBlocks)
{
if (_seekBoundaries == null)
{
// use blocks to chunk the reads
blockLengths = blob.DownloadBlockList().Select(b => b.Length).GetEnumerator();
}
else
{
// we have a choice between using blocks or record boundaries to split. Decide based on
// whether the written blocks are compatible with highReadSize or not. If the blocks are
// longer than highReadSize we would rather seek to record boundaries
long[] blockLengthArray = blob.DownloadBlockList().Select(b => b.Length).ToArray();
if (blockLengthArray.Count(l => l < highReadSize) >= blockLengthArray.Length / 2)
{
// more than half of the blocks are shorter than highReadSize so use them for
// subdividing the blob
blockLengths = blockLengthArray.AsEnumerable().GetEnumerator();
// don't tell the reader to seek for record boundaries
thisSeekBoundaries = false;
}
else
{
// use fixed chunks instead of blocks and seek for record boundaries at the reader
blockLengths = FixedBlockLengths(highReadSize, blob.Properties.Length).GetEnumerator();
}
}
}
else
{
// use fixed chunks and seek for record boundaries at the reader
blockLengths = FixedBlockLengths(highReadSize, blob.Properties.Length).GetEnumerator();
}
long offset = 0;
long chunkLength = 0;
blockLengths.MoveNext();
while (offset < blob.Properties.Length)
{
// accumulate a chunk that is at least 1 block long, but no longer than highReadSize
// unless the first block is longer than highReadSize
while (offset < blob.Properties.Length &&
(chunkLength == 0 || chunkLength + blockLengths.Current <= highReadSize))
{
chunkLength += blockLengths.Current;
offset += blockLengths.Current;
blockLengths.MoveNext();
}
// make a URI describing this portion of the blob
yield return(new AzureCollectionPartSubBlob(blobName, offset - chunkLength, chunkLength, thisSeekBoundaries));
chunkLength = 0;
}
moreBlobs = blobs.MoveNext();
} // if (moreBlobs)
} // while (moreBlobs)
_totalLength = totalLength;
}
public void AzureUploadingTest()
{
string strAccount = "testaccount";
string containerName = "imgsvr";
string strKey = "VSMnyOXembmbHGOqfbNaKj600it+xIarQXGVdaun6JCs+cs/bc6jDKooMzOKcEOJqxdU81j8UoLPg==";
StorageCredentials credential1 = new StorageCredentials(strAccount, Convert.FromBase64String(strKey));
CloudStorageAccount csa_storageAccount1 = new CloudStorageAccount(credential1, "core.windows.net", true);
CloudBlobClient client = csa_storageAccount1.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Off;
DateTime expiryTime = DateTime.UtcNow.AddYears(1);
// Add stored access policies
containerPermissions.SharedAccessPolicies.Add("rwl", new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = expiryTime,
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List
});
containerPermissions.SharedAccessPolicies.Add("rl", new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = expiryTime,
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
});
containerPermissions.SharedAccessPolicies.Add("r", new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = expiryTime,
Permissions = SharedAccessBlobPermissions.Read
});
container.SetPermissions(containerPermissions);
string rwlSAS = container.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "rwl");
Trace.WriteLine(rwlSAS);
string rlSAS = container.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "rl");
Trace.WriteLine(rlSAS);
string rSAS = container.GetSharedAccessSignature(new SharedAccessBlobPolicy(), "r");
Trace.WriteLine(rSAS);
//Test uploading
StorageCredentials credential = new StorageCredentials(rwlSAS);
string blobUriTemplate = "https://{0}.blob.core.windows.net/{1}/{2}";
foreach (var file in Directory.EnumerateFiles(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "TestFiles")))
{
CloudBlockBlob blob = new CloudBlockBlob(new Uri(string.Format(blobUriTemplate, strAccount, containerName, Path.GetFileName(file))), credential);
using (var stream = File.OpenRead(file))
{
string blockId = Convert.ToBase64String(Encoding.Unicode.GetBytes(Path.GetFileName(file)));
blob.Properties.ContentType = "image/jpeg";
blob.Properties.CacheControl = "public, max-age=2592000"; // one month
blob.PutBlock(blockId, stream, null);
blob.PutBlockList(new string[] { blockId });
Console.WriteLine("Done with {0}", Path.GetFileName(file));
}
Assert.AreEqual(blob.DownloadBlockList(BlockListingFilter.Committed).Count(), 1);
}
}
private void WriteLogLine(WriteWay writeWay, string writeLogLine, params string[] logFilePath)
{
if (logFilePath.Length < 2)
{
Console.WriteLine(invalidExistLogFilePath);
return;
}
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={Constant.STORAGE_ACCOUNT_NAME};AccountKey={Constant.Instance.StorageAccountKey};EndpointSuffix=core.windows.net";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference(logFilePath[0]);
if (!share.Exists())
{
share.Create();
}
CloudFileDirectory sampleDir = share.GetRootDirectoryReference();
for (int i = 1; i < logFilePath.Length - 1; i++)
{
CloudFileDirectory nextLevelDir = sampleDir.GetDirectoryReference("TestLogs");
if (!sampleDir.Exists())
{
sampleDir.Create();
}
sampleDir = nextLevelDir;
}
CloudFile file = sampleDir.GetFileReference(logFilePath[logFilePath.Length - 1]);
lock ("")
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference("logs");
blobContainer.CreateIfNotExistsAsync();
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("testBlob");
List <string> blockIds = new List <string>();
DateTime before = DateTime.Now;
blockIds.AddRange(blockBlob.DownloadBlockList(BlockListingFilter.Committed).Select(b => b.Name));
DateTime after = DateTime.Now;
TimeSpan ts = after.Subtract(before);
Console.WriteLine(ts.Seconds + "_" + ts.Milliseconds);
var newId = Convert.ToBase64String(Encoding.Default.GetBytes(blockIds.Count.ToString()));
blockBlob.PutBlock(newId, new MemoryStream(Encoding.Default.GetBytes(writeLogLine + "\n")), null);
blockIds.Add(newId);
blockBlob.PutBlockList(blockIds);
string writenLineContent = "";
if (file.Exists())
{
if (writeWay == WriteWay.Cover)
{
}
else if (writeWay == WriteWay.Append)
{
writenLineContent = file.DownloadTextAsync().Result;
}
}
file.UploadText(writenLineContent + writeLogLine + "\n");
}
}
