public async Task WriteNothingIfNoBlocksSent()
{
Skip.If(targetsNetCore);
await defaultAppendBlobBlockWriter.WriteBlocksToAppendBlobAsync(cloudBlobFake, noBlocksToWrite);
A.CallTo(() => cloudBlobFake.AppendBlockAsync(A <Stream> .Ignored, A <string> .Ignored)).MustNotHaveHappened();
}
/// <summary>
/// Appends to BLOB.
/// </summary>
/// <param name="record">The record.</param>
/// <returns>Task.</returns>
private async Task AppendToBlob(string record)
{
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(record)))
{
await _blob.AppendBlockAsync(stream);
}
}
public async Task WriteBlocksToAppendBlobAsync(CloudAppendBlob cloudAppendBlob, IEnumerable <string> blocks)
{
if (cloudAppendBlob == null)
{
throw new ArgumentNullException(nameof(cloudAppendBlob));
}
if (blocks == null)
{
throw new ArgumentNullException(nameof(blocks));
}
foreach (string blockContent in blocks)
{
using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(blockContent)))
{
try
{
await cloudAppendBlob.AppendBlockAsync(stream);
}
catch (StorageException ex)
{
Debugging.SelfLog.WriteLine($"Exception {ex} thrown while trying to append a block. Http response code {ex.RequestInformation?.HttpStatusCode} and error code {ex.RequestInformation?.ErrorCode}. If this is the second or later block in this batch there might be duplicate log entries written due to the retry mechanism.");
}
catch (Exception ex)
{
Debugging.SelfLog.WriteLine($"Exception {ex} thrown while trying to append a block. If this is the second or later block in this batch there might be duplicate log entries written due to the retry mechanism.");
}
}
}
}
private async Task <long> AppendInternalAsync(CloudAppendBlob appendBlob, Stream block, CancellationToken cancellationToken)
{
if (!(block.Position < block.Length))
{
throw new ArgumentException("block is empty", nameof(block));
}
// this exist to be able to impose arbitrary limits for testing purposes
var sequenceNumber = await appendBlob.AppendBlockAsync(block, null, cancellationToken);
if (!(appendBlob.Properties.AppendBlobCommittedBlockCount.Value < Configuration.MaxBlockCount))
{
// ignore this block
// in this case the block has been written but it is not valid
// this is expected because the append block API doesn't guarantee
// that the block isn't committed twice for various reasons
// the table operation is necessary for both transactional integrity
// and as an indexing service
throw new StorageException(new RequestResult {
HttpStatusCode = 409
}, null, null);
}
return(sequenceNumber);
}
public async Task WriteLog(ILog log, CloudBlobContainer container = null)
{
try
{
if (container == null)
{
container = await GetCloudBlobLogsContainer(LOGS_KEY);
}
var path = this.GetCloudContainerLogPath(log.LogType);
var dir = container.GetDirectoryReference(path);
var file = DateTime.Now.Hour.ToString("00") + "00.log";
string json = JsonConvert.SerializeObject(log, JSON.SerializationSettings);
// Note: AppendBlockBlob is not available in the Storage Emulator/Explorer
if (GetCloudConnectionString(LOGS_KEY) == "UseDevelopmentStorage=true")
{
CloudBlockBlob blob = dir.GetBlockBlobReference(file);
try
{
await blob.UploadTextAsync(json);
}
catch (StorageException ex)
{
}
catch (Exception ex)
{
}
}
else
{
CloudAppendBlob blob = dir.GetAppendBlobReference(DateTime.Now.Hour.ToString("00") + "00.log");
if (!(await blob.ExistsAsync()))
{
try
{
await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null);
}
catch (StorageException ex)
{
}
catch (Exception ex)
{
}
}
using (var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)))
{
await blob.AppendBlockAsync(stream);
}
}
}
catch (StorageException ex)
{
}
catch (Exception ex)
{
}
}
private async Task ProcessEvent(LoggingEvent loggingEvent)
{
CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName));
var xml = _lineFeed + loggingEvent.GetXmlString(Layout);
using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(xml))) {
await appendBlob.AppendBlockAsync(ms);
}
}
private async Task SaveToBlobAsync(CloudAppendBlob blob, int count, bool withDates)
{
using (var stream = new MemoryStream())
{
using (var writer = new StreamWriter(stream))
{
for (int j = 0; j < count; j++)
{
if (withDates)
{
writer.WriteLine(_queue[j].Item1.ToString(_timeFormat));
}
writer.WriteLine(_queue[j].Item2);
}
writer.Flush();
stream.Position = 0;
await blob.AppendBlockAsync(stream, null, null, _blobRequestOptions, null);
}
}
bool isLocked = await _lock.WaitAsync(TimeSpan.FromSeconds(1));
if (isLocked)
{
try
{
_queue.RemoveRange(0, count);
}
finally
{
_lock.Release();
}
}
else
{
await _log.WriteWarningAsync(
"BlobSaver.SaveToBlobAsync",
_containerName,
"Using unsafe queue clearing");
_queue.RemoveRange(0, count);
}
if (_queue.Count > _warningQueueCount)
{
await _log.WriteInfoAsync(
"BlobSaver.SaveToBlobAsync." + _containerName,
blob?.Uri != null?blob.Uri.ToString() : "",
$"{count} items were saved and removed from queue");
}
}
private async Task AppendAsync(CloudAppendBlob blob, Stream stream)
{
try
{
await blob.AppendBlockAsync(stream);
await stream.FlushAsync();
stream.Close();
}
catch (Exception ex)
{
Trace.TraceWarning("Blob append failed.");
Trace.TraceError(ex.Message);
throw ex;
}
}
} // End of the DeleteByLastModifiedDate method
#endregion
#region Helper methods
/// <summary>
/// Append text to an append blob
/// </summary>
private async Task WriteToAppendBlob(string blob_name, string log)
{
// Get a blob reference
CloudAppendBlob blob = this.container.GetAppendBlobReference(blob_name);
// Create a blob if it doesn't exist
if (await blob.ExistsAsync() == false)
{
await blob.CreateOrReplaceAsync();
}
// Append the log to a blob
using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(log)))
{
// Append text to the blob
await blob.AppendBlockAsync(stream);
}
} // End of the WriteToAppendBlob method
public async Task WriteAsync(IEnumerable <LogEvent> events)
{
CloudAppendBlob blob = null;
var sb = new StringBuilder();
foreach (LogEvent e in events)
{
if (blob == null)
{
blob = await GetBlobAsync(e.EventTime);
}
string line = TextFormatter.Format(e);
sb.AppendLine(line);
}
if (blob != null)
{
using (var ms = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
await blob.AppendBlockAsync(ms);
}
}
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> events)
{
try
{
if (events == null)
{
return;
}
IList <EventData> eventDataList = events as IList <EventData> ?? events.ToList();
// Trace individual events
foreach (EventData eventData in eventDataList)
{
try
{
if (!eventData.Properties.ContainsKey("userId"))
{
continue;
}
if (!eventData.Properties.ContainsKey("eventType"))
{
continue;
}
string userId = eventData.Properties["userId"] as string;
if (string.IsNullOrWhiteSpace(userId))
{
continue;
}
EventType eventType;
object type = eventData.Properties["eventType"];
if (type is EventType)
{
// Casting type
eventType = (EventType)type;
}
else if (type is int)
{
// Unboxing type
eventType = (EventType)(int)type;
}
else
{
string value = type as string;
if (value != null)
{
if (!Enum.TryParse(value, out eventType))
{
continue;
}
}
else
{
continue;
}
}
CloudAppendBlob cloudAppendBlob = eventType == EventType.StartSession
? this.NewCloudAppendBlob(userId)
: this.GetCloudAppendBlob(userId);
if (cloudAppendBlob == null)
{
continue;
}
using (MemoryStream stream = new MemoryStream(eventData.GetBytes()))
{
await cloudAppendBlob.AppendBlockAsync(stream);
ServiceEventSource.Current.Message($"Event appended to [{cloudAppendBlob.Name}] ");
}
// Sends a message to a Service Bus queue containing the address of the append blob
// containing the user session events, any time the user session is complete
if (eventType == EventType.StopSession)
{
ServiceEventSource.Current.Message($"User session closed: UserId=[{userId}]");
await this.SendMessageAsync(userId, cloudAppendBlob.Uri);
}
// Increase messageCount
this.messageCount++;
// Invoke CheckpointAsync when messageCount => checkpointCount
if (this.messageCount < this.checkpointCount)
{
continue;
}
await context.CheckpointAsync();
this.messageCount = 0;
}
catch (LeaseLostException ex)
{
// Trace Exception as message
ServiceEventSource.Current.Message(ex.Message);
}
catch (AggregateException ex)
{
// Trace Exception
foreach (Exception exception in ex.InnerExceptions)
{
ServiceEventSource.Current.Message(exception.Message);
}
}
catch (Exception ex)
{
// Trace Exception
ServiceEventSource.Current.Message(ex.Message);
}
}
}
catch (LeaseLostException ex)
{
// Trace Exception as message
ServiceEventSource.Current.Message(ex.Message);
}
catch (AggregateException ex)
{
// Trace Exception
foreach (Exception exception in ex.InnerExceptions)
{
ServiceEventSource.Current.Message(exception.Message);
}
}
catch (Exception ex)
{
// Trace Exception
ServiceEventSource.Current.Message(ex.Message);
}
}
public async Task RunPermissionsTestBlobs(SharedAccessAccountPolicy policy)
{
CloudBlobClient blobClient = GenerateCloudBlobClient();
string containerName = "c" + Guid.NewGuid().ToString("N");
try
{
CloudStorageAccount account = new CloudStorageAccount(blobClient.Credentials, false);
string accountSASToken = account.GetSharedAccessSignature(policy);
StorageCredentials accountSAS = new StorageCredentials(accountSASToken);
CloudStorageAccount accountWithSAS = new CloudStorageAccount(accountSAS, blobClient.StorageUri, null, null, null);
CloudBlobClient blobClientWithSAS = accountWithSAS.CreateCloudBlobClient();
CloudBlobContainer containerWithSAS = blobClientWithSAS.GetContainerReference(containerName);
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
// General pattern - If current perms support doing a thing with SAS, do the thing with SAS and validate with shared
// Otherwise, make sure SAS fails and then do the thing with shared key.
// Things to do:
// Create the container (Create / Write perms, Container RT)
// List containers with prefix (List perms, Service RT)
// Create an append blob (Create / Write perms, Object RT)
// Append a block to append blob (Add / Write perms, Object RT)
// Read the data from the append blob (Read perms, Object RT)
// Delete the blob (Delete perms, Object RT)
if ((((policy.Permissions & SharedAccessAccountPermissions.Create) == SharedAccessAccountPermissions.Create) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Container) == SharedAccessAccountResourceTypes.Container))
{
await containerWithSAS.CreateAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await containerWithSAS.CreateAsync(), "Create a container should fail with SAS without Create or Write and Container-level permissions.");
await container.CreateAsync();
}
Assert.IsTrue(await container.ExistsAsync());
if (((policy.Permissions & SharedAccessAccountPermissions.List) == SharedAccessAccountPermissions.List) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Service) == SharedAccessAccountResourceTypes.Service))
{
ContainerResultSegment segment = null;
BlobContinuationToken ct = null;
IEnumerable <CloudBlobContainer> results = null;
do
{
segment = await blobClientWithSAS.ListContainersSegmentedAsync(container.Name, ct);
ct = segment.ContinuationToken;
results = segment.Results;
}while(ct != null && !results.Any());
Assert.AreEqual(container.Name, segment.Results.First().Name);
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => { ContainerResultSegment segment = await blobClientWithSAS.ListContainersSegmentedAsync(container.Name, null); segment.Results.First(); }, "List containers should fail with SAS without List and Service-level permissions.");
}
string blobName = "blob";
CloudAppendBlob appendBlob = container.GetAppendBlobReference(blobName);
CloudAppendBlob appendBlobWithSAS = containerWithSAS.GetAppendBlobReference(blobName);
//Try creating credentials using SAS Uri directly
CloudAppendBlob appendBlobWithSASUri = new CloudAppendBlob(new Uri(container.Uri + accountSASToken));
if ((((policy.Permissions & SharedAccessAccountPermissions.Create) == SharedAccessAccountPermissions.Create) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
await appendBlobWithSAS.CreateOrReplaceAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.CreateOrReplaceAsync(), "Creating an append blob should fail with SAS without Create or Write and Object-level perms.");
await appendBlob.CreateOrReplaceAsync();
}
Assert.IsTrue(await appendBlob.ExistsAsync());
string blobText = "blobText";
if ((((policy.Permissions & SharedAccessAccountPermissions.Add) == SharedAccessAccountPermissions.Add) || ((policy.Permissions & SharedAccessAccountPermissions.Write) == SharedAccessAccountPermissions.Write)) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(blobText)))
{
await appendBlobWithSAS.AppendBlockAsync(stream);
}
}
else
{
using (MemoryStream memStream = new MemoryStream(Encoding.UTF8.GetBytes(blobText)))
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.AppendBlockAsync(memStream), "Append a block to an append blob should fail with SAS without Add or Write and Object-level perms.");
memStream.Seek(0, SeekOrigin.Begin);
await appendBlob.AppendBlockAsync(memStream);
}
}
Assert.AreEqual(blobText, await appendBlob.DownloadTextAsync());
if (((policy.Permissions & SharedAccessAccountPermissions.Read) == SharedAccessAccountPermissions.Read) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
Assert.AreEqual(blobText, await appendBlobWithSAS.DownloadTextAsync());
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.DownloadTextAsync(), "Reading a blob's contents with SAS without Read and Object-level permissions should fail.");
}
if (((policy.Permissions & SharedAccessAccountPermissions.Delete) == SharedAccessAccountPermissions.Delete) &&
((policy.ResourceTypes & SharedAccessAccountResourceTypes.Object) == SharedAccessAccountResourceTypes.Object))
{
await appendBlobWithSAS.DeleteAsync();
}
else
{
await TestHelper.ExpectedExceptionAsync <StorageException>(async() => await appendBlobWithSAS.DeleteAsync(), "Deleting a blob with SAS without Delete and Object-level perms should fail.");
await appendBlob.DeleteAsync();
}
Assert.IsFalse(await appendBlob.ExistsAsync());
}
finally
{
blobClient.GetContainerReference(containerName).DeleteIfExistsAsync().Wait();
}
}
private async Task SaveToBlobAsync(int count)
{
using (var stream = new MemoryStream())
{
for (int j = 0; j < count; j++)
{
var data = _queue[j].Item2;
var lengthArray = BitConverter.GetBytes(data.Length);
if (_compressData)
{
stream.Write(lengthArray, 0, 4);
long messageStartPosition = stream.Position;
using (var zipIn = new GZipStream(stream, CompressionLevel.Fastest, true))
{
zipIn.Write(data, 0, data.Length);
}
int messageLength = (int)(stream.Position - messageStartPosition);
lengthArray = BitConverter.GetBytes(messageLength);
stream.Write(lengthArray, 0, 4);
long returnPosition = stream.Position;
stream.Position = messageStartPosition - 4;
stream.Write(lengthArray, 0, 4);
stream.Position = returnPosition;
}
else
{
stream.Write(lengthArray, 0, 4);
stream.Write(data, 0, data.Length);
stream.Write(lengthArray, 0, 4);
}
stream.Write(_eolBytes, 0, _eolBytes.Length);
}
stream.Flush();
stream.Position = 0;
await _blob.AppendBlockAsync(stream, null, null, _blobRequestOptions, null);
}
bool isLocked = await _lock.WaitAsync(_queueClearTimeout);
if (isLocked)
{
try
{
_queue.RemoveRange(0, count);
}
finally
{
_lock.Release();
}
}
else
{
_log.WriteWarning("BlobSaver.SaveToBlobAsync", _container, "Using unsafe queue clearing");
_queue.RemoveRange(0, count);
}
if (_queue.Count > _warningQueueCount)
{
_log.WriteInfo(
"BlobSaver.SaveToBlobAsync",
_container,
$"{count} items were saved to " + (_blob?.Uri != null ? _blob.Uri.ToString() : ""));
}
}
