public async Task <CopyResult> CopyNextBatch(CancellationToken token)
{
var maxPos = await SourcePos
.ReadAsync(token)
.ConfigureAwait(false);
var localPos = TargetPos.ReadPositionVolatile();
var availableAmount = maxPos - localPos;
if (availableAmount <= 0)
{
// we don't have anything to write
return(new CopyResult(0, maxPos, localPos, localPos));
}
var amountToLoad = Math.Min(availableAmount, AmountToLoadMax);
using (var mem = _streamManager.GetStream("fetcher"))
{
await _sourceReader.DownloadRangeToStreamAsync(mem, localPos, (int)amountToLoad)
.ConfigureAwait(false);
mem.Seek(0, SeekOrigin.Begin);
_targetWriter.Save(mem, localPos);
var position = localPos + mem.Length;
TargetPos.Update(position);
return(new CopyResult(mem.Length, maxPos, localPos, position));
}
}
public async Task <DirectReadResult> ReadAll(CancellationToken token, long startingFrom, int maxCount, MessageHandler handler)
{
//var convertedLocalPos = pos[0];
//var currentRemotePosition = pos[1];
var watch = Stopwatch.StartNew();
var maxPos = await _remotePos.ReadAsync(token)
.ConfigureAwait(false);
var result = new DirectReadResult
{
StartingPosition = startingFrom,
CurrentPosition = startingFrom,
MaxPosition = maxPos,
};
if (maxPos <= startingFrom)
{
// we don't have anything to write
return(result);
}
var availableAmount = maxPos - startingFrom;
if (availableAmount <= 0)
{
// we don't have anything to write
return(result);
}
var amountToLoad = Math.Min(availableAmount, AmountToLoadMax);
using (var mem = _streamManager.GetStream("fetcher"))
{
await _remote.DownloadRangeToStreamAsync(mem, startingFrom, (int)amountToLoad)
.ConfigureAwait(false);
// cool, we've got some data back
mem.Seek(0, SeekOrigin.Begin);
var usedBytes = 0L;
var pages = new List <MessageWithId>();
using (var bin = new BinaryReader(mem))
{
MessageWithId msg;
while (TryRead(bin, out msg))
{
var hasMorePagesToRead = HasMore(msg);
if (false == hasMorePagesToRead && pages.Count == 0)
{
// fast path, we can save message directly without merging
handler(msg, msg.Id.GetOffset(), maxPos);
//WriteMessage(msg);
usedBytes = mem.Position;
result.ReadRecords += 1;
continue;
}
pages.Add(msg);
if (hasMorePagesToRead)
{
continue;
}
var total = pages.Sum(m => m.Value.Length);
using (var sub = _streamManager.GetStream("chase-1", total))
{
foreach (var page in pages)
{
sub.Write(page.Value, 0, page.Value.Length);
}
sub.Seek(0, SeekOrigin.Begin);
var last = pages.Last();
var final = new MessageWithId(last.Id, last.Attributes, last.Key, sub.ToArray(), 0);
handler(final, last.Id.GetOffset(), maxPos);
usedBytes = mem.Position;
result.ReadRecords += 1;
}
pages.Clear();
}
}
result.CurrentPosition = result.StartingPosition + usedBytes;
result.Elapsed = watch.Elapsed;
return(result);
}
}