public static void CreateOrReplace(this CloudAppendBlob blob)
{
blob.CreateOrReplaceAsync();
}
private static void PrintBlobPropertiesAndMetadata(CloudBlob blob)
{
// Write out properties that are common to all blob types.
Console.WriteLine();
Console.WriteLine("-----Blob Properties-----");
Console.WriteLine("\t Name: {0}", blob.Name);
Console.WriteLine("\t Container: {0}", blob.Container.Name);
Console.WriteLine("\t BlobType: {0}", blob.Properties.BlobType);
Console.WriteLine("\t IsSnapshot: {0}", blob.IsSnapshot);
// If the blob is a snapshot, write out snapshot properties.
if (blob.IsSnapshot)
{
Console.WriteLine("\t SnapshotTime: {0}", blob.SnapshotTime);
Console.WriteLine("\t SnapshotQualifiedUri: {0}", blob.SnapshotQualifiedUri);
}
Console.WriteLine("\t LeaseState: {0}", blob.Properties.LeaseState);
// If the blob has been leased, write out lease properties.
if (blob.Properties.LeaseState != LeaseState.Available)
{
Console.WriteLine("\t LeaseDuration: {0}", blob.Properties.LeaseDuration);
Console.WriteLine("\t LeaseStatus: {0}", blob.Properties.LeaseStatus);
}
Console.WriteLine("\t CacheControl: {0}", blob.Properties.CacheControl);
Console.WriteLine("\t ContentDisposition: {0}", blob.Properties.ContentDisposition);
Console.WriteLine("\t ContentEncoding: {0}", blob.Properties.ContentEncoding);
Console.WriteLine("\t ContentLanguage: {0}", blob.Properties.ContentLanguage);
Console.WriteLine("\t ContentMD5: {0}", blob.Properties.ContentMD5);
Console.WriteLine("\t ContentType: {0}", blob.Properties.ContentType);
Console.WriteLine("\t ETag: {0}", blob.Properties.ETag);
Console.WriteLine("\t LastModified: {0}", blob.Properties.LastModified);
Console.WriteLine("\t Length: {0}", blob.Properties.Length);
// Write out properties specific to blob type.
switch (blob.BlobType)
{
case BlobType.AppendBlob:
CloudAppendBlob appendBlob = blob as CloudAppendBlob;
Console.WriteLine("\t AppendBlobCommittedBlockCount: {0}", appendBlob.Properties.AppendBlobCommittedBlockCount);
Console.WriteLine("\t StreamWriteSizeInBytes: {0}", appendBlob.StreamWriteSizeInBytes);
break;
case BlobType.BlockBlob:
CloudBlockBlob blockBlob = blob as CloudBlockBlob;
Console.WriteLine("\t StreamWriteSizeInBytes: {0}", blockBlob.StreamWriteSizeInBytes);
break;
case BlobType.PageBlob:
CloudPageBlob pageBlob = blob as CloudPageBlob;
Console.WriteLine("\t PageBlobSequenceNumber: {0}", pageBlob.Properties.PageBlobSequenceNumber);
Console.WriteLine("\t StreamWriteSizeInBytes: {0}", pageBlob.StreamWriteSizeInBytes);
break;
default:
break;
}
Console.WriteLine("\t StreamMinimumReadSizeInBytes: {0}", blob.StreamMinimumReadSizeInBytes);
Console.WriteLine();
// Enumerate the blob's metadata.
Console.WriteLine("Blob metadata:");
foreach (var metadataItem in blob.Metadata)
{
Console.WriteLine("\tKey: {0}", metadataItem.Key);
Console.WriteLine("\tValue: {0}", metadataItem.Value);
}
}
private static async Task AppendBlobProcessAsync()
{
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
string sourceFile = null;
string destinationFile = null;
// Retrieve the connection string for use with the application. The storage connection string is stored
// in an environment variable on the machine running the application called storageconnectionstring.
// If the environment variable is created after the application is launched in a console or with Visual
// Studio, the shell needs to be closed and reloaded to take the environment variable into account.
string storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"]; //Environment.GetEnvironmentVariable("StorageConnectionString");
//storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
// Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.
////CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
//127.0.0.1:10000/devstoreaccount1/testcont
// Create a container called 'quickstartblobs' and append a GUID value to it to make the name unique.
////cloudBlobContainer = cloudBlobClient.GetContainerReference("testcontappendblob");// "quickstartblobs" + Guid.NewGuid().ToString());
////await cloudBlobContainer.CreateIfNotExistsAsync();
////Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name);
////Console.WriteLine();
////// Set the permissions so the blobs are public.
////BlobContainerPermissions permissions = new BlobContainerPermissions
////{
//// PublicAccess = BlobContainerPublicAccessType.Blob
////};
////await cloudBlobContainer.SetPermissionsAsync(permissions);
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName for the blob name.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
cloudBlobContainer = cloudBlobClient.GetContainerReference("testcont");
cloudBlobContainer.CreateIfNotExists();
CloudAppendBlob cloudAppendBlob = cloudBlobContainer.GetAppendBlobReference("1.txt");
cloudAppendBlob.CreateOrReplace();
cloudAppendBlob.AppendText("Content added");
cloudAppendBlob.AppendText("More content added");
cloudAppendBlob.AppendText("Even more content added");
string appendBlobContent = cloudAppendBlob.DownloadText();
//cloudAppendBlob.UploadFromFile(sourceFile);
// List the blobs in the container.
Console.WriteLine("Listing blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
Console.WriteLine("Downloading blob to {0}", destinationFile);
Console.WriteLine();
await cloudAppendBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
File.WriteAllText(sourceFile, "Hello, World, I am good, how is your health!");
await cloudAppendBlob.UploadFromFileAsync(sourceFile);
await cloudAppendBlob.DeleteAsync();
}
catch (StorageException ex)
{
Console.WriteLine("Error returned from the service: {0}", ex.Message);
}
finally
{
// Console.WriteLine("Press any key to delete the sample files and example container.");
Console.ReadLine();
// Clean up resources. This includes the container and the two temp files.
//Console.WriteLine("Deleting the container and any blobs it contains");
if (cloudBlobContainer != null)
{
// await cloudBlobContainer.DeleteIfExistsAsync();
}
File.Delete(sourceFile);
File.Delete(destinationFile);
}
}
else
{
Console.WriteLine(
"A connection string has not been defined in the system environment variables. " +
"Add a environment variable named 'storageconnectionstring' with your storage " +
"connection string as a value.");
}
}
/// <summary>
/// This method is used mainly by unit test
/// The source code uses it is for easier to be tested through unittest
/// It is used to get the content of a cloud append blob
/// </summary>
/// <param name="blob"></param>
/// <returns></returns>
public string GetContent(CloudAppendBlob blob)
{
return(blob.DownloadText());
}
public void RequestResultErrorCode()
{
Uri baseAddressUri = new Uri(TestBase.TargetTenantConfig.BlobServiceEndpoint);
CloudBlobClient client = new CloudBlobClient(baseAddressUri, TestBase.StorageCredentials);
CloudBlobContainer container = client.GetContainerReference(Guid.NewGuid().ToString("N"));
byte[] buffer = TestBase.GetRandomBuffer(4 * 1024 * 1024);
MD5 md5 = MD5.Create();
string contentMD5 = Convert.ToBase64String(md5.ComputeHash(buffer));
try
{
RequestResult requestResult;
XmlWriterSettings settings;
StringBuilder sb;
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
List <string> blocks = new List <string>();
for (int i = 0; i < 2; i++)
{
blocks.Add(Convert.ToBase64String(Guid.NewGuid().ToByteArray()));
}
// Verify the ErrorCode property is set and that it is serialized correctly
using (MemoryStream memoryStream = new MemoryStream(buffer))
{
memoryStream.Seek(0, SeekOrigin.Begin);
blob.PutBlock(blocks[0], memoryStream, contentMD5);
int offset = buffer.Length - 1024;
memoryStream.Seek(offset, SeekOrigin.Begin);
StorageException e = TestHelper.ExpectedException <StorageException>(
() => blob.PutBlock(blocks[1], memoryStream, contentMD5),
"Invalid MD5 should fail with mismatch");
Assert.AreEqual(e.RequestInformation.ErrorCode, StorageErrorCodeStrings.Md5Mismatch);
requestResult = new RequestResult();
settings = new XmlWriterSettings();
settings.Indent = true;
sb = new StringBuilder();
using (XmlWriter writer = XmlWriter.Create(sb, settings))
{
e.RequestInformation.WriteXml(writer);
}
using (XmlReader reader = XmlReader.Create(new StringReader(sb.ToString())))
{
requestResult.ReadXml(reader);
}
// ExtendedErrorInformation.ErrorCode will be depricated, but it should still match on a non HEAD request
Assert.AreEqual(e.RequestInformation.ErrorCode, requestResult.ErrorCode);
Assert.AreEqual(e.RequestInformation.ExtendedErrorInformation.ErrorCode, requestResult.ErrorCode);
}
// Verify the ErrorCode property is set on a HEAD request
CloudAppendBlob blob2 = container.GetAppendBlobReference("blob2");
blob2.CreateOrReplace();
StorageException e2 = TestHelper.ExpectedException <StorageException>(
() => blob2.FetchAttributes(AccessCondition.GenerateIfMatchCondition("garbage")),
"Mismatched etag should fail");
Assert.AreEqual(e2.RequestInformation.ErrorCode, StorageErrorCodeStrings.ConditionNotMet);
// Verify the ErrorCode property is not set on a successful request and that it is serialized correctly
OperationContext ctx = new OperationContext();
blob2.FetchAttributes(operationContext: ctx);
Assert.AreEqual(ctx.RequestResults[0].ErrorCode, null);
requestResult = new RequestResult();
settings = new XmlWriterSettings();
settings.Indent = true;
sb = new StringBuilder();
using (XmlWriter writer = XmlWriter.Create(sb, settings))
{
ctx.RequestResults[0].WriteXml(writer);
}
using (XmlReader reader = XmlReader.Create(new StringReader(sb.ToString())))
{
requestResult.ReadXml(reader);
}
Assert.AreEqual(ctx.RequestResults[0].ErrorCode, requestResult.ErrorCode);
}
finally
{
container.DeleteIfExists();
}
}
private void SetBlobLength(CloudAppendBlob cloudAppendBlob, int newLength)
{
cloudAppendBlob.Properties.GetType().GetProperty(nameof(BlobProperties.Length)).SetValue(cloudAppendBlob.Properties, newLength, null);
}
public static void UploadFromStream(this CloudAppendBlob cloudBlob, Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
cloudBlob.UploadFromStreamAsync(source, accessCondition, options, operationContext).GetAwaiter().GetResult();
}
public static void AppendFromFile(this CloudAppendBlob blob, string path)
{
blob.AppendFromFileAsync(path).Wait();
}
/// <inheritdoc />
public IStorageAppendBlob GetAppendBlobReference(string blobName)
{
CloudAppendBlob sdkBlob = _sdk.GetAppendBlobReference(blobName);
return(new StorageAppendBlob(this, sdkBlob));
}
public static void UploadFromStream(this CloudAppendBlob blob, System.IO.Stream stream)
{
blob.UploadFromStreamAsync(stream).Wait();
}
public static void AppendFromByteArray(this CloudAppendBlob blob, byte[] buffer, int offset, int count)
{
blob.AppendFromByteArrayAsync(buffer, offset, count).Wait();
}
public static void DownloadToStream(this CloudAppendBlob blob, System.IO.Stream stream)
{
blob.DownloadToStreamAsync(stream).Wait();
}
public static int DownloadToByteArray(this CloudAppendBlob blob, byte[] buffer, int index)
{
return(blob.DownloadToByteArrayAsync(buffer, index).Result);
}
public static void Delete(this CloudAppendBlob blob)
{
blob.DeleteAsync().Wait();
}
public static void AppendText(this CloudAppendBlob blob, string content)
{
blob.AppendTextAsync(content).Wait();
}
/// <summary>Initializes a new instance of the <see cref="StorageAppendBlob"/> class.</summary>
/// <param name="parent">The parent blob container.</param>
/// <param name="sdk">The SDK blob to wrap.</param>
public StorageAppendBlob(IStorageBlobContainer parent, CloudAppendBlob sdk)
{
_parent = parent;
_sdk = sdk;
_properties = new StorageBlobProperties(sdk);
}
public static void AppendFromStream(this CloudAppendBlob blob, Stream stream)
{
blob.AppendFromStreamAsync(stream).Wait();
}
private async Task Initialize()
{
RandomNameResolver nameResolver = new RandomNameResolver();
JobHostConfiguration hostConfiguration = new JobHostConfiguration()
{
NameResolver = nameResolver,
TypeLocator = new FakeTypeLocator(typeof(BlobBindingEndToEndTests)),
};
hostConfiguration.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());
Config = hostConfiguration;
StorageAccount = CloudStorageAccount.Parse(hostConfiguration.StorageConnectionString);
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
BlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(OutputContainerName));
CloudBlobContainer pageBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
CloudBlobContainer hierarchicalBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
CloudBlobContainer appendBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
Host = new JobHost(hostConfiguration);
Host.Start();
// upload some test blobs
CloudBlockBlob blob = BlobContainer.GetBlockBlobReference("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
CloudPageBlob pageBlob = pageBlobContainer.GetPageBlobReference("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
pageBlob = pageBlobContainer.GetPageBlobReference("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
CloudAppendBlob appendBlob = appendBlobContainer.GetAppendBlobReference("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob3");
await appendBlob.UploadTextAsync(TestData);
}
public static void UploadFromFile(this CloudAppendBlob cloudBlob, string path, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
cloudBlob.UploadFromFileAsync(path, accessCondition, options, operationContext).GetAwaiter().GetResult();
}
private static async Task <int> BackUpCollection(CloudStorageAccount storageAccount, DateTime starttime)
{
Uri collectionURI = UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName);
//Our changes file daily roll example
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
//Setup our container we are going to use and create it.
CloudBlobContainer container = blobClient.GetContainerReference("dbchanges");
await container.CreateIfNotExistsAsync();
DateTime changefiledate = DateTime.Today;
CloudAppendBlob appBlob = container.GetAppendBlobReference(
string.Format("{0}_{1}_{2}{3}", DatabaseName, CollectionName, changefiledate.ToString("yyyyMMdd"), ".json"));
var exists = await appBlob.ExistsAsync();
if (!exists)
{
await appBlob.CreateOrReplaceAsync();
await appBlob.AppendTextAsync("[");
//Store file as JSON Array for easy import
}
int x = 0;
string pkRangesResponseContinuation = null;
using (var client = new DocumentClient(new Uri(endpointUrl), authorizationKey,
new ConnectionPolicy {
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp
}))
{
var pkRangesResponse = await client.ReadPartitionKeyRangeFeedAsync(collectionURI, new FeedOptions { RequestContinuation = pkRangesResponseContinuation });
List <PartitionKeyRange> partitionKeyRanges = new List <PartitionKeyRange>();
partitionKeyRanges.AddRange(pkRangesResponse);
pkRangesResponseContinuation = pkRangesResponse.ResponseContinuation;
Dictionary <string, string> checkpoints = new Dictionary <string, string>();
bool comma = exists;
foreach (PartitionKeyRange pkRange in partitionKeyRanges)
{
string continuation = null;
checkpoints.TryGetValue(pkRange.Id, out continuation);
IDocumentQuery <Document> query = client.CreateDocumentChangeFeedQuery(
collectionURI,
new ChangeFeedOptions
{
PartitionKeyRangeId = pkRange.Id,
StartFromBeginning = true,
RequestContinuation = continuation,
MaxItemCount = -1,
// Set reading time: only show change feed results modified since StartTime
StartTime = starttime
});
while (query.HasMoreResults)
{
FeedResponse <dynamic> readChangesResponse = query.ExecuteNextAsync <dynamic>().Result;
foreach (dynamic changedDocument in readChangesResponse)
{
Console.WriteLine("document: {0}", changedDocument);
await appBlob.AppendTextAsync((comma ? "," : "") + changedDocument);
x++;
comma = true;
}
checkpoints[pkRange.Id] = readChangesResponse.ResponseContinuation;
}
}
}
return(x);
}
public static void UploadFromByteArray(this CloudAppendBlob cloudBlob, byte[] buffer, int index, int count, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
cloudBlob.UploadFromByteArrayAsync(buffer, index, count, accessCondition, options, operationContext).GetAwaiter().GetResult();
}
public async Task <DriverReadResult> ReadAsync(long position, long maxBytes, CancellationToken cancel = new CancellationToken())
{
// STEP 1: PRELIMINARY CHECKS
// ==========================
// The local cache tells us there is no data available at the provided position,
// so start by refreshing the cache.
if (_blobs.Count == 0 || position >= _lastKnownPosition)
{
await RefreshCache(cancel);
}
// Even with a fresh cache, our position is beyond any available data:
// return that there is no more data available.
if (_blobs.Count == 0 || position >= _lastKnownPosition)
{
return(new DriverReadResult(_lastKnownPosition, new RawEvent[0]));
}
// STEP 2: IDENTIFY BLOB
// =====================
CloudAppendBlob blob = null;
long firstPosition = 0;
long blobSize = 0;
for (var i = _blobs.Count - 1; i >= 0; --i)
{
if (_firstPosition[i] <= position)
{
blob = _blobs[i];
firstPosition = _firstPosition[i];
blobSize = (i == _blobs.Count - 1 ? _lastKnownPosition : _firstPosition[i + 1]) - firstPosition;
break;
}
}
if (blob == null)
{
// Since _firstPosition[0] == 0, this means the position is negative
throw new ArgumentOutOfRangeException("Invalid position:" + position, "position");
}
// STEP 3: READ RAW DATA
// =====================
var startPos = position - firstPosition;
maxBytes = Math.Min(maxBytes, Math.Min(_buffer.Length, blobSize - startPos));
if (maxBytes == 0)
{
return(new DriverReadResult(_lastKnownPosition, new RawEvent[0]));
}
var length = await ReadRangeAsync(blob, startPos, maxBytes, cancel);
// STEP 4: PARSE DATA
// ==================
var events = new List <RawEvent>();
var readBytes = 0L;
using (var ms = new MemoryStream(_buffer, 0, length))
using (var reader = new BinaryReader(ms))
{
while (true)
{
try
{
events.Add(EventFormat.Read(reader));
// Only update after finishing a full read !
readBytes = ms.Position;
}
catch (EndOfStreamException)
{
break;
}
catch (InvalidDataException e)
{
throw new InvalidDataException($"{e.Message} at {position + readBytes}");
}
}
}
return(new DriverReadResult(position + readBytes, events));
}
public async Task SendSingleTemplateEmailMultipleRcptsAttachment(string from, List <EmailAddress> tos, string templateID, object templateData, CloudAppendBlob fileReference, string fileName)
{
Require.That(templateID != null, new ErrorCode("SendgridError", 501, "Required Sengrid template ID not configured in app settings"));
{
var fromEmail = new EmailAddress(from);
var msg = MailHelper.CreateSingleTemplateEmailToMultipleRecipients(fromEmail, tos, templateID, templateData);
using (var stream = await fileReference.OpenReadAsync())
{
await msg.AddAttachmentAsync(fileName, stream);
}
var response = await _client.SendEmailAsync(msg);
if (!response.IsSuccessStatusCode)
{
throw new Exception("Error sending sendgrid email");
}
}
}
private async Task <int> ProcessBufferAsync(
byte[] buffer,
int filledCount,
bool isBlobCompressed,
CloudAppendBlob blob)
{
var delimiterEndIndexes = new List <int> {
-1
};
while (true)
{
int newDelimiterEndIndex = await GetNextDelimiterEndIndexAsync(
buffer,
filledCount,
delimiterEndIndexes.Last(),
blob);
if (newDelimiterEndIndex == -1)
{
break;
}
bool foundCorrectChunk = false;
for (int j = 0; j < delimiterEndIndexes.Count; ++j)
{
int delimiterEndIndex = delimiterEndIndexes[j];
int chunkSize = newDelimiterEndIndex - delimiterEndIndex - _delimiterBytes.Length;
if (chunkSize == 0)
{
continue;
}
int chunkStart = delimiterEndIndex + 1;
if (_isNewFormat.HasValue && _isNewFormat.Value)
{
chunkSize -= 8;
chunkStart += 4;
}
var chunk = new byte[chunkSize];
Array.Copy(buffer, chunkStart, chunk, 0, chunkSize);
if (isBlobCompressed)
{
chunk = UnpackMessage(chunk);
if (chunk == null)
{
continue;
}
}
foundCorrectChunk = TryDeserialize(chunk, out var obj);
if (foundCorrectChunk)
{
int skippedBytesCount = delimiterEndIndex - delimiterEndIndexes[0];
if (j > 1 || j == 1 && skippedBytesCount > _maxAllowedSkippedBytesCount)
{
if (_skipCorrupted)
{
_log.WriteWarning(nameof(ProcessBufferAsync), null, $"Skipped {skippedBytesCount} bytes with {j} delimiters.");
}
else
{
throw new InvalidOperationException(
$"Couldn't process message(s). Found corrupted chunk - {skippedBytesCount} bytes with {j} delimiters.");
}
}
await _messageProcessor.ProcessMessageAsync(obj);
break;
}
}
if (foundCorrectChunk)
{
delimiterEndIndexes.Clear();
}
delimiterEndIndexes.Add(newDelimiterEndIndex);
if (delimiterEndIndexes.Count >= _maxUnprocessedPatternsCount)
{
throw new InvalidOperationException($"Couldn't properly process blob - {delimiterEndIndexes.Count} unprocessed patterns.");
}
}
return(delimiterEndIndexes[0]);
}
private void SetCloudBlobBlockCount(CloudAppendBlob cloudAppendBlob, int newBlockCount)
{
cloudAppendBlob.Properties.GetType().GetProperty(nameof(BlobProperties.AppendBlobCommittedBlockCount)).SetValue(cloudAppendBlob.Properties, newBlockCount, null);
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events)
{
try
{
if (events == null)
{
return;
}
IList <EventData> eventDataList = events as IList <EventData> ?? events.ToList();
// Trace individual events
foreach (EventData eventData in eventDataList)
{
try
{
if (!eventData.Properties.ContainsKey("userId"))
{
continue;
}
if (!eventData.Properties.ContainsKey("eventType"))
{
continue;
}
string userId = eventData.Properties["userId"] as string;
if (string.IsNullOrWhiteSpace(userId))
{
continue;
}
EventType eventType;
object type = eventData.Properties["eventType"];
if (type is EventType)
{
// Casting type
eventType = (EventType)type;
}
else if (type is int)
{
// Unboxing type
eventType = (EventType)(int)type;
}
else
{
string value = type as string;
if (value != null)
{
if (!Enum.TryParse(value, out eventType))
{
continue;
}
}
else
{
continue;
}
}
CloudAppendBlob cloudAppendBlob = eventType == EventType.StartSession
? this.NewCloudAppendBlob(userId)
: this.GetCloudAppendBlob(userId);
if (cloudAppendBlob == null)
{
continue;
}
using (MemoryStream stream = new MemoryStream(eventData.GetBytes()))
{
await cloudAppendBlob.AppendBlockAsync(stream);
ServiceEventSource.Current.Message($"Event appended to [{cloudAppendBlob.Name}] ");
}
// Sends a message to a Service Bus queue containing the address of the append blob
// containing the user session events, any time the user session is complete
if (eventType == EventType.StopSession)
{
ServiceEventSource.Current.Message($"User session closed: UserId=[{userId}]");
await this.SendMessageAsync(userId, cloudAppendBlob.Uri);
}
// Increase messageCount
this.messageCount++;
// Invoke CheckpointAsync when messageCount => checkpointCount
if (this.messageCount < this.checkpointCount)
{
continue;
}
await context.CheckpointAsync();
this.messageCount = 0;
}
catch (LeaseLostException ex)
{
// Trace Exception as message
ServiceEventSource.Current.Message(ex.Message);
}
catch (AggregateException ex)
{
// Trace Exception
foreach (Exception exception in ex.InnerExceptions)
{
ServiceEventSource.Current.Message(exception.Message);
}
}
catch (Exception ex)
{
// Trace Exception
ServiceEventSource.Current.Message(ex.Message);
}
}
}
catch (LeaseLostException ex)
{
// Trace Exception as message
ServiceEventSource.Current.Message(ex.Message);
}
catch (AggregateException ex)
{
// Trace Exception
foreach (Exception exception in ex.InnerExceptions)
{
ServiceEventSource.Current.Message(exception.Message);
}
}
catch (Exception ex)
{
// Trace Exception
ServiceEventSource.Current.Message(ex.Message);
}
}
/// <summary>
/// Initializes this instance.
/// </summary>
private void Initialize()
{
//set initial configuration
_blob = GetAppendBlob();
}
public static void CreateOrReplace(this CloudAppendBlob blob, AccessCondition condition, BlobRequestOptions options, OperationContext operationContext)
{
blob.CreateOrReplaceAsync(condition, options, operationContext).Wait();
}
public virtual Task <string> StartCopyAsync(CloudAppendBlob source, AccessCondition sourceAccessCondition, AccessCondition destAccessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(this.failoverContainer.ExecuteAsync(x => x.StartCopyAsync(source, sourceAccessCondition, destAccessCondition, options, operationContext, cancellationToken)));
}
public static bool Exists(this CloudAppendBlob blob)
{
return(blob.ExistsAsync().Result);
}
