public async Task WriteNothingIfNoBlocksSent() { Skip.If(targetsNetCore); await defaultAppendBlobBlockWriter.WriteBlocksToAppendBlobAsync(cloudBlobFake, noBlocksToWrite); A.CallTo(() => cloudBlobFake.AppendBlockAsync(A <Stream> .Ignored, A <string> .Ignored)).MustNotHaveHappened(); }
/// <summary> /// Appends to BLOB. /// </summary> /// <param name="record">The record.</param> /// <returns>Task.</returns> private async Task AppendToBlob(string record) { using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(record))) { await _blob.AppendBlockAsync(stream); } }
public async Task WriteBlocksToAppendBlobAsync(CloudAppendBlob cloudAppendBlob, IEnumerable <string> blocks) { if (cloudAppendBlob == null) { throw new ArgumentNullException(nameof(cloudAppendBlob)); } if (blocks == null) { throw new ArgumentNullException(nameof(blocks)); } foreach (string blockContent in blocks) { using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(blockContent))) { try { await cloudAppendBlob.AppendBlockAsync(stream); } catch (StorageException ex) { Debugging.SelfLog.WriteLine($"Exception {ex} thrown while trying to append a block. Http response code {ex.RequestInformation?.HttpStatusCode} and error code {ex.RequestInformation?.ErrorCode}. If this is the second or later block in this batch there might be duplicate log entries written due to the retry mechanism."); } catch (Exception ex) { Debugging.SelfLog.WriteLine($"Exception {ex} thrown while trying to append a block. If this is the second or later block in this batch there might be duplicate log entries written due to the retry mechanism."); } } } }
private async Task <long> AppendInternalAsync(CloudAppendBlob appendBlob, Stream block, CancellationToken cancellationToken) { if (!(block.Position < block.Length)) { throw new ArgumentException("block is empty", nameof(block)); } // this exist to be able to impose arbitrary limits for testing purposes var sequenceNumber = await appendBlob.AppendBlockAsync(block, null, cancellationToken); if (!(appendBlob.Properties.AppendBlobCommittedBlockCount.Value < Configuration.MaxBlockCount)) { // ignore this block // in this case the block has been written but it is not valid // this is expected because the append block API doesn't guarantee // that the block isn't committed twice for various reasons // the table operation is necessary for both transactional integrity // and as an indexing service throw new StorageException(new RequestResult { HttpStatusCode = 409 }, null, null); } return(sequenceNumber); }
public async Task WriteLog(ILog log, CloudBlobContainer container = null) { try { if (container == null) { container = await GetCloudBlobLogsContainer(LOGS_KEY); } var path = this.GetCloudContainerLogPath(log.LogType); var dir = container.GetDirectoryReference(path); var file = DateTime.Now.Hour.ToString("00") + "00.log"; string json = JsonConvert.SerializeObject(log, JSON.SerializationSettings); // Note: AppendBlockBlob is not available in the Storage Emulator/Explorer if (GetCloudConnectionString(LOGS_KEY) == "UseDevelopmentStorage=true") { CloudBlockBlob blob = dir.GetBlockBlobReference(file); try { await blob.UploadTextAsync(json); } catch (StorageException ex) { } catch (Exception ex) { } } else { CloudAppendBlob blob = dir.GetAppendBlobReference(DateTime.Now.Hour.ToString("00") + "00.log"); if (!(await blob.ExistsAsync())) { try { await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null); } catch (StorageException ex) { } catch (Exception ex) { } } using (var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) { await blob.AppendBlockAsync(stream); } } } catch (StorageException ex) { } catch (Exception ex) { } }
private async Task ProcessEvent(LoggingEvent loggingEvent) { CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName)); var xml = _lineFeed + loggingEvent.GetXmlString(Layout); using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xml))) { await appendBlob.AppendBlockAsync(ms); } }
private async Task SaveToBlobAsync(CloudAppendBlob blob, int count, bool withDates) { using (var stream = new MemoryStream()) { using (var writer = new StreamWriter(stream)) { for (int j = 0; j < count; j++) { if (withDates) { writer.WriteLine(_queue[j].Item1.ToString(_timeFormat)); } writer.WriteLine(_queue[j].Item2); } writer.Flush(); stream.Position = 0; await blob.AppendBlockAsync(stream, null, null, _blobRequestOptions, null); } } bool isLocked = await _lock.WaitAsync(TimeSpan.FromSeconds(1)); if (isLocked) { try { _queue.RemoveRange(0, count); } finally { _lock.Release(); } } else { await _log.WriteWarningAsync( "BlobSaver.SaveToBlobAsync", _containerName, "Using unsafe queue clearing"); _queue.RemoveRange(0, count); } if (_queue.Count > _warningQueueCount) { await _log.WriteInfoAsync( "BlobSaver.SaveToBlobAsync." + _containerName, blob?.Uri != null?blob.Uri.ToString() : "", $"{count} items were saved and removed from queue"); } }
private async Task AppendAsync(CloudAppendBlob blob, Stream stream) { try { await blob.AppendBlockAsync(stream); await stream.FlushAsync(); stream.Close(); } catch (Exception ex) { Trace.TraceWarning("Blob append failed."); Trace.TraceError(ex.Message); throw ex; } }
} // End of the DeleteByLastModifiedDate method #endregion #region Helper methods /// <summary> /// Append text to an append blob /// </summary> private async Task WriteToAppendBlob(string blob_name, string log) { // Get a blob reference CloudAppendBlob blob = this.container.GetAppendBlobReference(blob_name); // Create a blob if it doesn't exist if (await blob.ExistsAsync() == false) { await blob.CreateOrReplaceAsync(); } // Append the log to a blob using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(log))) { // Append text to the blob await blob.AppendBlockAsync(stream); } } // End of the WriteToAppendBlob method
public async Task WriteAsync(IEnumerable <LogEvent> events) { CloudAppendBlob blob = null; var sb = new StringBuilder(); foreach (LogEvent e in events) { if (blob == null) { blob = await GetBlobAsync(e.EventTime); } string line = TextFormatter.Format(e); sb.AppendLine(line); } if (blob != null) { using (var ms = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) { await blob.AppendBlockAsync(ms); } } }
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events) { try { if (events == null) { return; } IList <EventData> eventDataList = events as IList <EventData> ?? events.ToList(); // Trace individual events foreach (EventData eventData in eventDataList) { try { if (!eventData.Properties.ContainsKey("userId")) { continue; } if (!eventData.Properties.ContainsKey("eventType")) { continue; } string userId = eventData.Properties["userId"] as string; if (string.IsNullOrWhiteSpace(userId)) { continue; } EventType eventType; object type = eventData.Properties["eventType"]; if (type is EventType) { // Casting type eventType = (EventType)type; } else if (type is int) { // Unboxing type eventType = (EventType)(int)type; } else { string value = type as string; if (value != null) { if (!Enum.TryParse(value, out eventType)) { continue; } } else { continue; } } CloudAppendBlob cloudAppendBlob = eventType == EventType.StartSession ? this.NewCloudAppendBlob(userId) : this.GetCloudAppendBlob(userId); if (cloudAppendBlob == null) { continue; } using (MemoryStream stream = new MemoryStream(eventData.GetBytes())) { await cloudAppendBlob.AppendBlockAsync(stream); ServiceEventSource.Current.Message($"Event appended to [{cloudAppendBlob.Name}] "); } // Sends a message to a Service Bus queue containing the address of the append blob // containing the user session events, any time the user session is complete if (eventType == EventType.StopSession) { ServiceEventSource.Current.Message($"User session closed: UserId=[{userId}]"); await this.SendMessageAsync(userId, cloudAppendBlob.Uri); } // Increase messageCount this.messageCount++; // Invoke CheckpointAsync when messageCount => checkpointCount if (this.messageCount < this.checkpointCount) { continue; } await context.CheckpointAsync(); this.messageCount = 0; } catch (LeaseLostException ex) { // Trace Exception as message ServiceEventSource.Current.Message(ex.Message); } catch (AggregateException ex) { // Trace Exception foreach (Exception exception in ex.InnerExceptions) { ServiceEventSource.Current.Message(exception.Message); } } catch (Exception ex) { // Trace Exception ServiceEventSource.Current.Message(ex.Message); } } } catch (LeaseLostException ex) { // Trace Exception as message ServiceEventSource.Current.Message(ex.Message); } catch (AggregateException ex) { // Trace Exception foreach (Exception exception in ex.InnerExceptions) { ServiceEventSource.Current.Message(exception.Message); } } catch (Exception ex) { // Trace Exception ServiceEventSource.Current.Message(ex.Message); } }
public async Task RunPermissionsTestBlobs(SharedAccessAccountPolicy policy) { CloudBlobClient blobClient = GenerateCloudBlobClient(); string containerName = "c" + Guid.NewGuid().ToString("N"); try { CloudStorageAccount account = new CloudStorageAccount(blobClient.Credentials, false); string accountSASToken = account.GetSharedAccessSignature(policy); StorageCredentials accountSAS = new StorageCredentials(accountSASToken); CloudStorageAccount accountWithSAS = new CloudStorageAccount(accountSAS, blobClient.StorageUri, null, null, null); CloudBlobClient blobClientWithSAS = accountWithSAS.CreateCloudBlobClient(); CloudBlobContainer containerWithSAS = blobClientWithSAS.GetContainerReference(containerName); CloudBlobContainer container = blobClient.GetContainerReference(containerName); // General pattern - If current perms support doing a thing with SAS, do the thing with SAS and validate with shared // Otherwise, make sure SAS fails and then do the thing with shared key. // Things to do: // Create the container (Create / Write perms, Container RT) // List containers with prefix (List perms, Service RT) // Create an append blob (Create / Write perms, Object RT) // Append a block to append blob (Add / Write perms, Object RT) // Read the data from the append blob (Read perms, Object RT) // Delete the blob (Delete perms, Object RT) if ((((policy.Permissions & SharedAccessAccountPermissions.Create) == SharedAccessAccountPermissions.Create) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Container) == SharedAccessAccountResourceTypes.Container)) { await containerWithSAS.CreateAsync(); } else { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await containerWithSAS.CreateAsync(), "Create a container should fail with SAS without Create or Write and Container-level permissions."); await container.CreateAsync(); } Assert.IsTrue(await container.ExistsAsync()); if (((policy.Permissions & SharedAccessAccountPermissions.List) == SharedAccessAccountPermissions.List) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Service) == SharedAccessAccountResourceTypes.Service)) { ContainerResultSegment segment = null; BlobContinuationToken ct = null; IEnumerable <CloudBlobContainer> results = null; do { segment = await blobClientWithSAS.ListContainersSegmentedAsync(container.Name, ct); ct = segment.ContinuationToken; results = segment.Results; }while(ct != null && !results.Any()); Assert.AreEqual(container.Name, segment.Results.First().Name); } else { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => { ContainerResultSegment segment = await blobClientWithSAS.ListContainersSegmentedAsync(container.Name, null); segment.Results.First(); }, "List containers should fail with SAS without List and Service-level permissions."); } string blobName = "blob"; CloudAppendBlob appendBlob = container.GetAppendBlobReference(blobName); CloudAppendBlob appendBlobWithSAS = containerWithSAS.GetAppendBlobReference(blobName); //Try creating credentials using SAS Uri directly CloudAppendBlob appendBlobWithSASUri = new CloudAppendBlob(new Uri(container.Uri + accountSASToken)); if ((((policy.Permissions & SharedAccessAccountPermissions.Create) == SharedAccessAccountPermissions.Create) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object)) { await appendBlobWithSAS.CreateOrReplaceAsync(); } else { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.CreateOrReplaceAsync(), "Creating an append blob should fail with SAS without Create or Write and Object-level perms."); await appendBlob.CreateOrReplaceAsync(); } Assert.IsTrue(await appendBlob.ExistsAsync()); string blobText = "blobText"; if ((((policy.Permissions & SharedAccessAccountPermissions.Add) == SharedAccessAccountPermissions.Add) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object)) { using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(blobText))) { await appendBlobWithSAS.AppendBlockAsync(stream); } } else { using (MemoryStream memStream = new MemoryStream(Encoding.UTF8.GetBytes(blobText))) { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.AppendBlockAsync(memStream), "Append a block to an append blob should fail with SAS without Add or Write and Object-level perms."); memStream.Seek(0, SeekOrigin.Begin); await appendBlob.AppendBlockAsync(memStream); } } Assert.AreEqual(blobText, await appendBlob.DownloadTextAsync()); if (((policy.Permissions & SharedAccessAccountPermissions.Read) == SharedAccessAccountPermissions.Read) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object)) { Assert.AreEqual(blobText, await appendBlobWithSAS.DownloadTextAsync()); } else { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.DownloadTextAsync(), "Reading a blob's contents with SAS without Read and Object-level permissions should fail."); } if (((policy.Permissions & SharedAccessAccountPermissions.Delete) == SharedAccessAccountPermissions.Delete) && ((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object)) { await appendBlobWithSAS.DeleteAsync(); } else { await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.DeleteAsync(), "Deleting a blob with SAS without Delete and Object-level perms should fail."); await appendBlob.DeleteAsync(); } Assert.IsFalse(await appendBlob.ExistsAsync()); } finally { blobClient.GetContainerReference(containerName).DeleteIfExistsAsync().Wait(); } }
private async Task SaveToBlobAsync(int count) { using (var stream = new MemoryStream()) { for (int j = 0; j < count; j++) { var data = _queue[j].Item2; var lengthArray = BitConverter.GetBytes(data.Length); if (_compressData) { stream.Write(lengthArray, 0, 4); long messageStartPosition = stream.Position; using (var zipIn = new GZipStream(stream, CompressionLevel.Fastest, true)) { zipIn.Write(data, 0, data.Length); } int messageLength = (int)(stream.Position - messageStartPosition); lengthArray = BitConverter.GetBytes(messageLength); stream.Write(lengthArray, 0, 4); long returnPosition = stream.Position; stream.Position = messageStartPosition - 4; stream.Write(lengthArray, 0, 4); stream.Position = returnPosition; } else { stream.Write(lengthArray, 0, 4); stream.Write(data, 0, data.Length); stream.Write(lengthArray, 0, 4); } stream.Write(_eolBytes, 0, _eolBytes.Length); } stream.Flush(); stream.Position = 0; await _blob.AppendBlockAsync(stream, null, null, _blobRequestOptions, null); } bool isLocked = await _lock.WaitAsync(_queueClearTimeout); if (isLocked) { try { _queue.RemoveRange(0, count); } finally { _lock.Release(); } } else { _log.WriteWarning("BlobSaver.SaveToBlobAsync", _container, "Using unsafe queue clearing"); _queue.RemoveRange(0, count); } if (_queue.Count > _warningQueueCount) { _log.WriteInfo( "BlobSaver.SaveToBlobAsync", _container, $"{count} items were saved to " + (_blob?.Uri != null ? _blob.Uri.ToString() : "")); } }