public async Task <uint> GetLastKeyAsync(CancellationToken cancel = new CancellationToken())
{
// We need to look into the last blob, so we need to list all blobs first
await RefreshCache(cancel);
// Look into the last non-empty blob
if (_firstKey.Count == 0)
{
return(0);
}
var blob = _blobs[_firstKey.Count - 1];
// Download one event's worth of data to a local buffer.
// This will likely truncate an event somewhere at the beginning of the
// buffer, but we don't care, because looking for the *last* sequence only
// requires that the last event in the buffer isn't truncated.
var length = Math.Min(EventFormat.MaxEventFootprint, blob.Properties.Length);
var bytes = new byte[length];
await ReadSubRangeAsync(blob, bytes, blob.Properties.Length - length, 0, length, cancel);
// Look for the sequence number of the last event.
using (var stream = new MemoryStream(bytes))
return(await EventFormat.GetLastSequenceAsync(stream, cancel));
}
public Task<DriverReadResult> ReadAsync(long position, long maxBytes, CancellationToken cancel = new CancellationToken())
{
var events = new List<RawEvent>();
if (position >= _file.Length)
return Task.FromResult(new DriverReadResult(_file.Length, events));
_file.Seek(position, SeekOrigin.Begin);
var readBytes = 0L;
using (var r = new BinaryReader(_file, Encoding.UTF8, true))
{
var availableBytes = _file.Length - position;
while (readBytes < availableBytes)
{
var evt = EventFormat.Read(r);
var newReadBytes = _file.Position - position;
if (events.Count == 0 || newReadBytes < maxBytes)
{
events.Add(evt);
readBytes = newReadBytes;
}
if (newReadBytes >= maxBytes)
break;
}
}
return Task.FromResult(new DriverReadResult(position + readBytes, events));
}
public Task<DriverWriteResult> WriteAsync(long position, IEnumerable<RawEvent> events, CancellationToken cancel = new CancellationToken())
{
var written = false;
if (_file.Length == position)
{
written = true;
_file.Seek(0, SeekOrigin.End);
using (var w = new BinaryWriter(_file, Encoding.UTF8, true))
foreach (var e in events)
EventFormat.Write(w, e);
}
return Task.FromResult(new DriverWriteResult(_file.Length, written));
}
internal DriverWriteResult Write(long position, IEnumerable <RawEvent> events)
{
var written = false;
if (_file.Length == position)
{
written = true;
_file.Seek(0, SeekOrigin.End);
using (var w = new BinaryWriter(_file, Encoding.UTF8, true))
foreach (var e in events)
{
EventFormat.Write(w, e);
}
}
return(new DriverWriteResult(_file.Length, written));
}
internal DriverReadResult Read(long position, long maxBytes)
{
var events = new List <RawEvent>();
if (position >= _file.Length)
{
return(new DriverReadResult(_file.Length, events));
}
_file.Seek(position, SeekOrigin.Begin);
var readBytes = 0L;
using (var r = new BinaryReader(_file, Encoding.UTF8, true))
{
var availableBytes = _file.Length - position;
while (readBytes < availableBytes)
{
var evt = EventFormat.Read(r);
var newReadBytes = _file.Position - position;
if (events.Count == 0 || newReadBytes < maxBytes)
{
events.Add(evt);
readBytes = newReadBytes;
}
if (newReadBytes >= maxBytes)
{
break;
}
}
}
return(new DriverReadResult(position + readBytes, events));
}
public async Task <uint> GetLastKeyAsync(CancellationToken cancel = new CancellationToken()) => await EventFormat.GetLastSequenceAsync(_file, cancel);
public async Task <DriverReadResult> ReadAsync(long position, long maxBytes, CancellationToken cancel = new CancellationToken())
{
// STEP 1: PRELIMINARY CHECKS
// ==========================
// The local cache tells us there is no data available at the provided position,
// so start by refreshing the cache.
if (_blobs.Count == 0 || position >= _lastKnownPosition)
{
await RefreshCache(cancel);
}
// Even with a fresh cache, our position is beyond any available data:
// return that there is no more data available.
if (_blobs.Count == 0 || position >= _lastKnownPosition)
{
return(new DriverReadResult(_lastKnownPosition, new RawEvent[0]));
}
// STEP 2: IDENTIFY BLOB
// =====================
CloudAppendBlob blob = null;
long firstPosition = 0;
long blobSize = 0;
for (var i = _blobs.Count - 1; i >= 0; --i)
{
if (_firstPosition[i] <= position)
{
blob = _blobs[i];
firstPosition = _firstPosition[i];
blobSize = (i == _blobs.Count - 1 ? _lastKnownPosition : _firstPosition[i + 1]) - firstPosition;
break;
}
}
if (blob == null)
{
// Since _firstPosition[0] == 0, this means the position is negative
throw new ArgumentOutOfRangeException("Invalid position:" + position, "position");
}
// STEP 3: READ RAW DATA
// =====================
var startPos = position - firstPosition;
maxBytes = Math.Min(maxBytes, Math.Min(_buffer.Length, blobSize - startPos));
if (maxBytes == 0)
{
return(new DriverReadResult(_lastKnownPosition, new RawEvent[0]));
}
var length = await ReadRangeAsync(blob, startPos, maxBytes, cancel);
// STEP 4: PARSE DATA
// ==================
var events = new List <RawEvent>();
var readBytes = 0L;
using (var ms = new MemoryStream(_buffer, 0, length))
using (var reader = new BinaryReader(ms))
{
while (true)
{
try
{
events.Add(EventFormat.Read(reader));
// Only update after finishing a full read !
readBytes = ms.Position;
}
catch (EndOfStreamException)
{
break;
}
catch (InvalidDataException e)
{
throw new InvalidDataException($"{e.Message} at {position + readBytes}");
}
}
}
return(new DriverReadResult(position + readBytes, events));
}
public async Task <DriverWriteResult> WriteAsync(long position, IEnumerable <RawEvent> events, CancellationToken cancel = new CancellationToken())
{
// Caller knows something we don't? Refresh!
if (position > _lastKnownPosition)
{
_lastKnownPosition = await RefreshCache(cancel);
}
// Quick early-out with no Azure request involved.
if (position < _lastKnownPosition)
{
return(new DriverWriteResult(_lastKnownPosition, false));
}
// This should only happen very rarely, but it still needs to be done.
if (_blobs.Count == 0)
{
await CreateLastBlob(cancel);
}
// Generate appended payload
byte[] payload;
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
foreach (var e in events)
{
EventFormat.Write(writer, e);
}
payload = stream.ToArray();
}
// Nothing to write, but still check position
if (payload.Length == 0)
{
_lastKnownPosition = await RefreshCache(cancel);
return(new DriverWriteResult(_lastKnownPosition, position == _lastKnownPosition));
}
// Attempt write to last blob.
bool collision;
try
{
var offset = _firstPosition[_blobs.Count - 1];
await _blobs[_blobs.Count - 1].AppendTransactionalAsync(payload, position - offset, cancel);
_lastKnownPosition = position + payload.Length;
return(new DriverWriteResult(_lastKnownPosition, true));
}
catch (StorageException e)
{
if (!e.IsCollision() && !e.IsMaxReached())
{
throw;
}
collision = e.IsCollision();
}
// Collision means we do not have the proper _lastKnownPosition, so refresh
// the cache and return a failure.
if (collision)
{
return(new DriverWriteResult(await RefreshCache(cancel), false));
}
// Too many appends can be solved by creating a new blob and appending to it.
// The append code is similar but subtly different, so we just rewrite it
// below.
await CreateLastBlob(cancel);
try
{
await _blobs[_blobs.Count - 1].AppendTransactionalAsync(payload, 0, cancel);
_lastKnownPosition = position + payload.Length;
return(new DriverWriteResult(_lastKnownPosition, true));
}
catch (StorageException e)
{
// Only a collision can stop us here, no max-appends-reached should
// happen when appending at position 0.
if (!e.IsCollision())
{
throw;
}
}
// Collision here means someone else already created the new blob and appended
// to it, so refresh everything and report failure.
return(new DriverWriteResult(await RefreshCache(cancel), false));
}
