public virtual async Task OpenAsync(CancellationToken cancellationToken = default)
{
if (Open)
{
throw new MessageStreamOpenException("MessageStream already open");
}
Logger?.LogTrace("Opening message stream.");
closeCts = new CancellationTokenSource();
ReadStats.Reset();
WriteStats.Reset();
try
{
await duplexMessageStream.OpenAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
Cleanup();
Logger?.LogError(ex, "Error opening message stream.");
throw new MessageStreamOpenException("Error opening duplex message stream", ex);
}
Open = true;
Logger?.LogTrace("Opened message stream.");
}
public void SizeTriggersTest()
{
RunTest(TestCloseChunk).Wait();
async Task TestCloseChunk(string fname)
{
var opt = new WriterOptions()
{
CloseChunk = new Triggers()
{
Size = 0
},
FlushToOS = new Triggers()
{
Size = 0
},
DisposeFlushToDisk = false,
};
using (var writer = new Writer(fname, opt)) {
long written = 0;
while (true)
{
try {
await writer.WriteAsync(new Event <long>(new DateTime(written + 1, DateTimeKind.Utc), written + 1));
++written;
break;
} catch (TimeSeriesWriteException e) {
Assert.IsInstanceOfType(e.InnerException, typeof(InjectedWriteException));
if (e.Result == TimeSeriesWriteResult.RecordsBuffered)
{
++written;
}
else
{
Assert.AreEqual(TimeSeriesWriteResult.RecordsDropped, e.Result);
}
}
}
Assert.IsTrue(written > 0);
while (true)
{
ReadStats stats = await ReadAllAfter(fname, 0, 1, FileState.Expanding);
if (stats.Total >= written)
{
Assert.AreEqual(1, stats.First);
Assert.AreEqual(written, stats.Last);
Assert.AreEqual(written, stats.Total);
break;
}
await Task.Delay(TimeSpan.FromMilliseconds(1));
}
}
}
}
static async Task <ReadStats> ReadAllAfter(string fname, long after, long bufRecs, FileState state)
{
using (var reader = new Reader(fname)) {
var stats = new ReadStats();
long lastLen = -1;
IAsyncEnumerable <IDecodedChunk <Event <long> > > chunks =
reader.ReadAfter(new DateTime(after, DateTimeKind.Utc));
using (IAsyncEnumerator <IDecodedChunk <Event <long> > > iter = chunks.GetAsyncEnumerator()) {
while (await Do(() => iter.MoveNextAsync(CancellationToken.None)))
{
Event <long>[] events = iter.Current.ToArray();
for (int i = 0; i != events.Length; ++i)
{
Assert.IsTrue(events[i].Value > 0);
Assert.AreEqual(events[i].Timestamp.Ticks, events[i].Value);
if (i != 0)
{
Assert.AreEqual(events[i - 1].Value + 1, events[i].Value);
}
}
Assert.AreNotEqual(0, events.Length);
Assert.IsTrue(events.Length <= bufRecs);
Assert.IsTrue(events[0].Value > stats.Last);
Assert.IsTrue((events[0].Value - 1) % bufRecs == 0);
if (stats.First == 0)
{
stats.First = events.First().Value;
}
else
{
Assert.AreEqual(bufRecs, lastLen);
if (!state.HasFlag(FileState.Expanding))
{
Assert.IsTrue(events.First().Value > after);
}
}
stats.Last = events.Last().Value;
stats.Total += events.Length;
lastLen = events.Length;
}
}
return(stats);
}
async Task <T> Do <T>(Func <Task <T> > action)
{
while (true)
{
try {
return(await action.Invoke());
} catch (InjectedReadException) {
}
}
}
}
public void Save(ReadStats readStats)
{
var date = GetCurrent();
date.ToProgress = readStats.Progress;
if (date.FromProgress < 0)
{
date.FromProgress = readStats.Progress;
}
date.PageTurns++;
date.Seconds += readStats.Seconds;
date.Words += readStats.Words;
}
static async Task VerifyFile(string fname, long n, long bufRecs, FileState state)
{
Debug.Assert(n >= 0);
Debug.Assert(bufRecs > 0);
var pos = new[] {
0, 1, 2, 3, 4, 5,
bufRecs - 2, bufRecs - 1, bufRecs, bufRecs + 1, bufRecs + 2,
2 * bufRecs - 2, 2 * bufRecs - 1, 2 * bufRecs, 2 * bufRecs + 1, 2 * bufRecs + 2,
n - bufRecs - 2, n - bufRecs - 1, n - bufRecs, n - bufRecs + 1, n - bufRecs + 2,
n / 2 - 2, n / 2 - 1, n / 2, n / 2 + 1, n / 2 + 2,
n - 2, n - 1, n, n + 1, n + 2
};
foreach (long after in pos.Where(p => p >= 0).Distinct())
{
ReadStats stats = await ReadAllAfter(fname, after, bufRecs, state);
Assert.IsTrue(stats.Last <= n);
if (!state.HasFlag(FileState.Corrupted) && stats.Total > 0)
{
long start = Math.Max(0, Math.Min(n, after) - 1) / bufRecs * bufRecs + 1;
Assert.IsTrue(stats.First <= start);
if (!state.HasFlag(FileState.Truncated))
{
Assert.AreEqual(start, stats.First);
}
Assert.AreEqual(stats.Total, stats.Last - stats.First + 1);
}
if (state == FileState.Pristine)
{
Assert.AreEqual(n, stats.Last);
if (n > 0 && after <= 1)
{
Assert.AreEqual(n, stats.Total);
Assert.AreEqual(1, stats.First);
}
}
}
}
public void TimeTriggersTest()
{
RunTest((string fname) => Test(fname, new WriterOptions()
{
CloseChunk = new Triggers()
{
Age = TimeSpan.Zero,
AgeRetry = TimeSpan.Zero,
},
FlushToOS = new Triggers()
{
Size = 0,
},
DisposeFlushToDisk = false,
})).Wait();
RunTest((string fname) => Test(fname, new WriterOptions()
{
CloseChunk = new Triggers()
{
Age = TimeSpan.Zero,
AgeRetry = TimeSpan.Zero,
},
FlushToOS = new Triggers()
{
Age = TimeSpan.Zero,
AgeRetry = TimeSpan.Zero,
},
DisposeFlushToDisk = false,
})).Wait();
RunTest((string fname) => Test(fname, new WriterOptions()
{
CloseChunk = new Triggers()
{
Age = TimeSpan.Zero,
AgeRetry = TimeSpan.Zero,
},
FlushToDisk = new Triggers()
{
Age = TimeSpan.Zero,
AgeRetry = TimeSpan.Zero,
},
DisposeFlushToDisk = false,
})).Wait();
async Task Test(string fname, WriterOptions opt)
{
using (var writer = new Writer(fname, opt)) {
await writer.WriteAsync(new Event <long>(new DateTime(1, DateTimeKind.Utc), 1));
while (true)
{
ReadStats stats = await ReadAllAfter(fname, 0, 1, FileState.Expanding);
if (stats.Total > 0)
{
Assert.AreEqual(1, stats.Total);
Assert.AreEqual(1, stats.First);
Assert.AreEqual(1, stats.Last);
break;
}
await Task.Delay(TimeSpan.FromMilliseconds(1));
}
}
}
}
public virtual async ValueTask <MessageReadResult <T> > ReadAsync()
{
DateTime timeReceived = DateTime.UtcNow;
bool partialMessage = false;
// Try to read one full message.
try
{
T message = default;
SequencePosition read = default;
var closeToken = closeCts?.Token ?? default;
ReadResult result = await duplexMessageStream.ReadAsync(closeToken).ConfigureAwait(false);
var buffer = result.Buffer;
while (!Deserializer.Deserialize(in buffer, out read, out message))
{
// This case means we read a partial message, so try to read the rest
if (!result.IsCompleted)
{
duplexMessageStream.AdvanceReaderTo(read, result.Buffer.End);
ReadStats.IncrBytesRead(result.Buffer.Length);
result = await duplexMessageStream.ReadAsync(closeToken).ConfigureAwait(false);
buffer = result.Buffer;
}
// We didn't have enough data in the buffer, and the reader is closed so we can't read anymore, so mark it as a partial message.
else
{
partialMessage = true;
break;
}
}
// Track the time it took to parse
DateTime parsedTimeUtc = DateTime.UtcNow;
if (!partialMessage)
{
// not sure how expensive this is
var slicedBuffer = result.Buffer.Slice(result.Buffer.Start, read);
ReadStats.IncMessagesRead();
ReadStats.IncrBytesRead(slicedBuffer.Length);
try
{
ReadStats.IncMessagesIncomingBufferProcessing(1);
await ProcessIncomingBufferAsync(message, slicedBuffer).ConfigureAwait(false);
ReadStats.DecMessagesIncomingBufferProcessing(1);
}
catch (Exception ex)
{
ReadStats.DecMessagesIncomingBufferProcessing(1);
Logger?.LogError(ex, "Error processing incoming message buffer.");
}
}
if (!partialMessage)
{
duplexMessageStream.AdvanceReaderTo(read);
}
DateTime parsedTime = DateTime.UtcNow;
return(new MessageReadResult <T>
{
// if the stream is completed, we can still try to read more messages, so we use the partialMessage field to indicate that.
IsCompleted = result.IsCompleted && partialMessage,
Error = false,
Exception = null,
Result = message,
ReadResult = !partialMessage,
ReceivedTimeUtc = timeReceived,
ParsedTimeUtc = parsedTimeUtc
});
}
catch (Exception ex)
{
Logger?.LogError(ex, "Error reading message from duplex message stream.");
return(new MessageReadResult <T>
{
IsCompleted = duplexMessageStream.ReadCompleted,
Error = true,
Exception = ex,
Result = default,
private void Cleanup()
{
closeCts.Dispose();
ReadStats.Reset();
WriteStats.Reset();
}
private void MessagesOnReadStats(object sender, ReadStats e)
{
_bookshelfBook.ReadStats.Save(e);
}
