public async Task <CopyResult> CopyNextBatch(CancellationToken token) { var maxPos = await SourcePos .ReadAsync(token) .ConfigureAwait(false); var localPos = TargetPos.ReadPositionVolatile(); var availableAmount = maxPos - localPos; if (availableAmount <= 0) { // we don't have anything to write return(new CopyResult(0, maxPos, localPos, localPos)); } var amountToLoad = Math.Min(availableAmount, AmountToLoadMax); using (var mem = _streamManager.GetStream("fetcher")) { await _sourceReader.DownloadRangeToStreamAsync(mem, localPos, (int)amountToLoad) .ConfigureAwait(false); mem.Seek(0, SeekOrigin.Begin); _targetWriter.Save(mem, localPos); var position = localPos + mem.Length; TargetPos.Update(position); return(new CopyResult(mem.Length, maxPos, localPos, position)); } }
public async Task <DirectReadResult> ReadAll(CancellationToken token, long startingFrom, int maxCount, MessageHandler handler) { //var convertedLocalPos = pos[0]; //var currentRemotePosition = pos[1]; var watch = Stopwatch.StartNew(); var maxPos = await _remotePos.ReadAsync(token) .ConfigureAwait(false); var result = new DirectReadResult { StartingPosition = startingFrom, CurrentPosition = startingFrom, MaxPosition = maxPos, }; if (maxPos <= startingFrom) { // we don't have anything to write return(result); } var availableAmount = maxPos - startingFrom; if (availableAmount <= 0) { // we don't have anything to write return(result); } var amountToLoad = Math.Min(availableAmount, AmountToLoadMax); using (var mem = _streamManager.GetStream("fetcher")) { await _remote.DownloadRangeToStreamAsync(mem, startingFrom, (int)amountToLoad) .ConfigureAwait(false); // cool, we've got some data back mem.Seek(0, SeekOrigin.Begin); var usedBytes = 0L; var pages = new List <MessageWithId>(); using (var bin = new BinaryReader(mem)) { MessageWithId msg; while (TryRead(bin, out msg)) { var hasMorePagesToRead = HasMore(msg); if (false == hasMorePagesToRead && pages.Count == 0) { // fast path, we can save message directly without merging handler(msg, msg.Id.GetOffset(), maxPos); //WriteMessage(msg); usedBytes = mem.Position; result.ReadRecords += 1; continue; } pages.Add(msg); if (hasMorePagesToRead) { continue; } var total = pages.Sum(m => m.Value.Length); using (var sub = _streamManager.GetStream("chase-1", total)) { foreach (var page in pages) { sub.Write(page.Value, 0, page.Value.Length); } sub.Seek(0, SeekOrigin.Begin); var last = pages.Last(); var final = new MessageWithId(last.Id, last.Attributes, last.Key, sub.ToArray(), 0); handler(final, last.Id.GetOffset(), maxPos); usedBytes = mem.Position; result.ReadRecords += 1; } pages.Clear(); } } result.CurrentPosition = result.StartingPosition + usedBytes; result.Elapsed = watch.Elapsed; return(result); } }