public static async ReusableTask WriteToFilesAsync(this IPieceWriter writer, ITorrentData manager, BlockInfo request, byte[] buffer)
{
var count = request.RequestLength;
var torrentOffset = request.ToByteOffset(manager.PieceLength);
if (torrentOffset < 0 || torrentOffset + count > manager.Size)
{
throw new ArgumentOutOfRangeException(nameof(request));
}
int totalWritten = 0;
var files = manager.Files;
int i = files.FindFileByOffset(torrentOffset);
var offset = torrentOffset - files[i].OffsetInTorrent;
while (totalWritten < count)
{
int fileToWrite = (int)Math.Min(files[i].Length - offset, count - totalWritten);
fileToWrite = Math.Min(fileToWrite, Piece.BlockSize);
await writer.WriteAsync(files[i], offset, buffer, totalWritten, fileToWrite);
offset += fileToWrite;
totalWritten += fileToWrite;
if (offset >= files[i].Length)
{
offset = 0;
i++;
}
}
}
public ReusableTask <bool> ReadFromCacheAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer)
{
if (torrent == null)
{
throw new ArgumentNullException(nameof(torrent));
}
if (CachedBlocks.TryGetValue(torrent, out List <CachedBlock> blocks))
{
for (int i = 0; i < blocks.Count; i++)
{
var cached = blocks[i];
if (cached.Block != block)
{
continue;
}
cached.Buffer.CopyTo(buffer);
if (!cached.Flushing)
{
blocks[i] = cached.SetFlushing();
FlushBlockAsync(torrent, blocks, cached);
}
Interlocked.Add(ref cacheHits, block.RequestLength);
ReadFromCache?.Invoke(this, block);
return(ReusableTask.FromResult(true));
}
}
return(ReusableTask.FromResult(false));
}
static void IgnoreTorrentFiles(ITorrentData torrent)
{
foreach (var file in torrent.Files)
{
file.Priority = Priority.DoNotDownload;
}
}
public static int BytesPerPiece(this ITorrentData self, int pieceIndex)
{
if (pieceIndex < self.PieceCount() - 1)
{
return(self.PieceLength);
}
return((int)(self.Size - self.PieceIndexToByteOffset(pieceIndex)));
}
public BufferedIO(ITorrentData manager, long offset, byte[] buffer, int count, ReusableTaskCompletionSource <bool> tcs)
{
this.manager = manager;
this.offset = offset;
this.buffer = buffer;
this.count = count;
this.tcs = tcs;
}
public BufferedIO(ITorrentData manager, BlockInfo request, byte[] buffer, bool preferSkipCache, ReusableTaskCompletionSource <bool> tcs)
{
this.manager = manager;
this.request = request;
this.buffer = buffer;
this.preferSkipCache = preferSkipCache;
this.tcs = tcs;
}
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests)
{
TorrentData = torrentData;
this.requests.Clear();
foreach (Piece p in requests)
{
this.requests.Add(p);
}
}
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests)
{
this.bitfield = bitfield;
this.endgameSelector = new BitField(bitfield.Length);
this.torrentData = torrentData;
inEndgame = false;
TryEnableEndgame();
ActivePicker.Initialise(bitfield, torrentData, requests);
}
public async ReusableTask <bool> ReadAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer)
{
if (await ReadFromCacheAsync(torrent, block, buffer))
{
return(true);
}
Interlocked.Add(ref cacheMisses, block.RequestLength);
return(await ReadFromFilesAsync(torrent, block, buffer).ConfigureAwait(false) == block.RequestLength);
}
async void FlushBlockAsync(ITorrentData torrent, List <CachedBlock> blocks, CachedBlock cached)
{
// FIXME: How do we handle failures from this?
using (cached.BufferReleaser) {
await WriteToFilesAsync(torrent, cached.Block, cached.Buffer);
Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength);
blocks.Remove(cached);
}
}
public async ReusableTask WriteAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer, bool preferSkipCache)
{
if (preferSkipCache || Capacity < block.RequestLength)
{
await WriteToFilesAsync(torrent, block, buffer);
}
else
{
if (!CachedBlocks.TryGetValue(torrent, out List <CachedBlock> blocks))
{
CachedBlocks[torrent] = blocks = new List <CachedBlock> ();
}
if (CacheUsed > (Capacity - block.RequestLength))
{
var firstFlushable = FindFirstFlushable(blocks);
if (firstFlushable < 0)
{
await WriteToFilesAsync(torrent, block, buffer);
return;
}
else
{
var cached = blocks[firstFlushable];
blocks[firstFlushable] = cached.SetFlushing();
using (cached.BufferReleaser)
await WriteToFilesAsync(torrent, cached.Block, cached.Buffer);
Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength);
blocks.Remove(cached);
}
}
CachedBlock?cache = null;
for (int i = 0; i < blocks.Count && !cache.HasValue; i++)
{
if (blocks[i].Block == block)
{
cache = blocks[i];
}
}
if (!cache.HasValue)
{
var releaser = BufferPool.Rent(block.RequestLength, out Memory <byte> memory);
cache = new CachedBlock(block, releaser, memory);
blocks.Add(cache.Value);
Interlocked.Add(ref cacheUsed, block.RequestLength);
}
buffer.CopyTo(cache.Value.Buffer);
WrittenToCache?.Invoke(this, block);
}
}
public static int BlocksPerPiece(this ITorrentData self, int pieceIndex)
{
if (pieceIndex < self.PieceCount() - 1)
{
return(self.PieceLength / Piece.BlockSize);
}
var remainder = self.Size - self.PieceIndexToByteOffset(pieceIndex);
return((int)((remainder + Piece.BlockSize - 1) / Piece.BlockSize));
}
internal async Task MoveFilesAsync(ITorrentData manager, string newRoot, bool overwrite)
{
await IOLoop;
foreach (TorrentFile file in manager.Files)
{
string newPath = Path.Combine(newRoot, file.Path);
Writer.Move(file, newPath, overwrite);
file.FullPath = newPath;
}
}
internal async Task CloseFilesAsync(ITorrentData manager)
{
await IOLoop;
// Process all pending reads/writes then close any open streams
ProcessBufferedIO(true);
foreach (var file in manager.Files)
{
Writer.Close(file);
}
}
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests)
{
this.bitfield = bitfield;
this.endgameSelector = new BitField(bitfield.Length);
this.torrentData = torrentData;
inEndgame = false;
// Always initialize both pickers, but we should only give the active requests to the Standard picker.
// We should never *default* to endgame mode, we should always start in regular mode and enter endgame
// mode after we fail to pick a piece.
standard.Initialise(bitfield, torrentData, requests);
endgame.Initialise(bitfield, torrentData, Enumerable.Empty <Piece> ());
}
internal async Task <bool> CheckAnyFilesExistAsync(ITorrentData manager)
{
await IOLoop;
for (int i = 0; i < manager.Files.Count; i++)
{
if (await Writer.ExistsAsync(manager.Files[i]).ConfigureAwait(false))
{
return(true);
}
}
return(false);
}
internal async Task <bool> CheckAnyFilesExistAsync(ITorrentData manager)
{
await IOLoop;
for (int i = 0; i < manager.Files.Length; i++)
{
if (Writer.Exists(manager.Files[i]))
{
return(true);
}
}
return(false);
}
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields)
{
TorrentData = torrentData;
IPiecePicker picker = new StandardPicker();
picker = new RandomisedPicker(picker);
picker = new RarestFirstPicker(picker);
picker = new PriorityPicker(picker);
Picker = IgnoringPicker.Wrap(picker, ignoringBitfields);
Picker.Initialise(torrentData);
}
internal void ChangePicker(PiecePicker picker, BitField bitfield, ITorrentData data)
{
if (UnhashedPieces.Length != bitfield.Length)
{
UnhashedPieces = new BitField(bitfield.Length);
}
picker = new IgnoringPicker(bitfield, picker);
picker = new IgnoringPicker(UnhashedPieces, picker);
IEnumerable <Piece> pieces = Picker == null ? new List <Piece>() : Picker.ExportActiveRequests();
picker.Initialise(bitfield, data, pieces);
Picker = picker;
}
public override void Initialise(ITorrentData torrentData)
{
base.Initialise(torrentData);
allPrioritisedPieces = new MutableBitField(torrentData.PieceCount());
temp = new MutableBitField(torrentData.PieceCount());
files.Clear();
for (int i = 0; i < torrentData.Files.Count; i++)
{
files.Add(new Files(torrentData.Files[i]));
}
BuildSelectors();
}
internal async ReusableTask <byte[]> GetHashAsync(ITorrentData manager, int pieceIndex)
{
if (GetHashAsyncOverride != null)
{
return(GetHashAsyncOverride(manager, pieceIndex));
}
await IOLoop;
if (IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash))
{
// Immediately remove it from the dictionary so another thread writing data to using `WriteAsync` can't try to use it
IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex));
using var lockReleaser = await incrementalHash.Locker.EnterAsync();
// We request the blocks for most pieces sequentially, and most (all?) torrent clients
// will process requests in the order they have been received. This means we can optimise
// hashing a received piece by hashing each block as it arrives. If blocks arrive out of order then
// we'll compute the final hash by reading the data from disk.
if (incrementalHash.NextOffsetToHash == manager.BytesPerPiece(pieceIndex))
{
byte[] result = incrementalHash.Hasher.Hash;
IncrementalHashCache.Enqueue(incrementalHash);
return(result);
}
}
else
{
// If we have no partial hash data for this piece we could be doing a full
// hash check, so let's create a IncrementalHashData for our piece!
incrementalHash = IncrementalHashCache.Dequeue();
}
// We can store up to 4MB of pieces in an in-memory queue so that, when we're rate limited
// we can process the queue in-order. When we try to hash a piece we need to make sure
// that in-memory cache is written to the PieceWriter before we try to Read the data back
// to hash it.
if (WriteQueue.Count > 0)
{
await WaitForPendingWrites();
}
using var releaser = await incrementalHash.Locker.EnterAsync();
// Note that 'startOffset' may not be the very start of the piece if we have a partial hash.
int startOffset = incrementalHash.NextOffsetToHash;
int endOffset = manager.BytesPerPiece(pieceIndex);
using (BufferPool.Rent(Piece.BlockSize, out byte[] hashBuffer)) {
public void Setup()
{
var pieceLength = Constants.BlockSize * 8;
var files = TorrentFileInfo.Create(pieceLength, ("Relative/Path.txt", Constants.BlockSize * 5, "Full/Path/Relative/Path.txt"));
torrent = new TorrentData {
Files = files,
PieceLength = pieceLength,
Size = files.Single().Length,
};
writer = new MemoryWriter();
cache = new MemoryCache(new MemoryPool(), Constants.BlockSize * 4, writer);
}
public void Setup()
{
buffer = new byte[100000];
offset = 2362;
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = 0xff;
}
torrentData = new TestTorrentData {
PieceLength = 16 * Piece.BlockSize,
Size = 40 * 16 * Piece.BlockSize,
};
}
internal async ReusableTask WriteAsync(ITorrentData manager, long offset, byte[] buffer, int count)
{
if (count < 1)
{
throw new ArgumentOutOfRangeException(nameof(count), $"Count must be greater than zero, but was {count}.");
}
Interlocked.Add(ref pendingWrites, count);
await IOLoop;
int pieceIndex = (int)(offset / manager.PieceLength);
long pieceStart = (long)pieceIndex * manager.PieceLength;
long pieceEnd = pieceStart + manager.PieceLength;
if (!IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash) && offset == pieceStart)
{
incrementalHash = IncrementalHashes[ValueTuple.Create(manager, pieceIndex)] = IncrementalHashCache.Dequeue();
incrementalHash.NextOffsetToHash = (long)manager.PieceLength * pieceIndex;
}
if (incrementalHash != null)
{
// Incremental hashing does not perform proper bounds checking to ensure
// that pieces are correctly incrementally hashed even if 'count' is greater
// than the PieceLength. This should never happen under normal operation, but
// unit tests do it for convenience sometimes. Keep things safe by cancelling
// incremental hashing if that occurs.
if ((incrementalHash.NextOffsetToHash + count) > pieceEnd)
{
IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex));
}
else if (incrementalHash.NextOffsetToHash == offset)
{
incrementalHash.Hasher.TransformBlock(buffer, 0, count, buffer, 0);
incrementalHash.NextOffsetToHash += count;
}
}
if (WriteLimiter.TryProcess(count))
{
Interlocked.Add(ref pendingWrites, -count);
Write(manager, offset, buffer, count);
}
else
{
var tcs = new ReusableTaskCompletionSource <bool> ();
WriteQueue.Enqueue(new BufferedIO(manager, offset, buffer, count, tcs));
await tcs.Task;
}
}
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests)
{
base.Initialise(bitfield, torrentData, requests);
AllSamePriority = file => file.Priority == files[0].Priority;
allPrioritisedPieces = new BitField(bitfield.Length);
temp = new BitField(bitfield.Length);
files.Clear();
for (int i = 0; i < torrentData.Files.Length; i++)
{
files.Add(new Files(torrentData.Files[i], torrentData.Files[i].GetSelector(bitfield.Length)));
}
BuildSelectors();
}
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields)
{
IgnorableBitfields = ignoringBitfields;
TorrentData = torrentData;
Temp = new MutableBitField(TorrentData.PieceCount());
IPiecePicker picker = new StandardPicker();
picker = new RandomisedPicker(picker);
picker = new RarestFirstPicker(picker);
Picker = new PriorityPicker(picker);
Picker.Initialise(torrentData);
}
public void Setup()
{
buffer = new byte[100000];
offset = 2362;
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = 0xff;
}
torrentData = new TestTorrentData {
Files = new List <ITorrentFileInfo> (),
PieceLength = 16 * Constants.BlockSize,
Size = 40 * 16 * Constants.BlockSize,
};
}
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields)
{
TorrentData = torrentData;
var standardPicker = new StandardPicker();
HighPriorityPicker = IgnoringPicker.Wrap(new PriorityPicker(standardPicker), ignoringBitfields);
LowPriorityPicker = new RandomisedPicker(standardPicker);
LowPriorityPicker = new RarestFirstPicker(LowPriorityPicker);
LowPriorityPicker = new PriorityPicker(LowPriorityPicker);
LowPriorityPicker = IgnoringPicker.Wrap(LowPriorityPicker, ignoringBitfields);
LowPriorityPicker.Initialise(torrentData);
HighPriorityPicker.Initialise(torrentData);
}
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests)
{
// 'Requests' should contain a list of all the pieces we need to complete
pieces = new List <Piece>(requests);
TorrentData = torrentData;
foreach (Piece piece in pieces)
{
for (int i = 0; i < piece.BlockCount; i++)
{
if (piece.Blocks[i].RequestedOff != null && !piece.Blocks[i].Received)
{
this.requests.Add(new Request(piece.Blocks[i].RequestedOff, piece.Blocks[i]));
}
}
}
}
internal async ReusableTask <byte[]> GetHashAsync(ITorrentData manager, int pieceIndex)
{
if (GetHashAsyncOverride != null)
{
return(GetHashAsyncOverride(manager, pieceIndex));
}
await IOLoop;
if (IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash))
{
// We request the blocks for most pieces sequentially, and most (all?) torrent clients
// will process requests in the order they have been received. This means we can optimise
// hashing a received piece by hashing each block as it arrives. If blocks arrive out of order then
// we'll compute the final hash by reading the data from disk.
if (incrementalHash.NextOffsetToHash == (long)manager.PieceLength * (pieceIndex + 1) ||
incrementalHash.NextOffsetToHash == manager.Size)
{
incrementalHash.Hasher.TransformFinalBlock(Array.Empty <byte> (), 0, 0);
byte[] result = incrementalHash.Hasher.Hash;
IncrementalHashCache.Enqueue(incrementalHash);
IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex));
return(result);
}
}
else
{
// If we have no partial hash data for this piece we could be doing a full
// hash check, so let's create a IncrementalHashData for our piece!
incrementalHash = IncrementalHashCache.Dequeue();
incrementalHash.NextOffsetToHash = (long)manager.PieceLength * pieceIndex;
}
// We can store up to 4MB of pieces in an in-memory queue so that, when we're rate limited
// we can process the queue in-order. When we try to hash a piece we need to make sure
// that in-memory cache is written to the PieceWriter before we try to Read the data back
// to hash it.
if (WriteQueue.Count > 0)
{
await WaitForPendingWrites();
}
// Note that 'startOffset' may not be the very start of the piece if we have a partial hash.
long startOffset = incrementalHash.NextOffsetToHash;
long endOffset = Math.Min((long)manager.PieceLength * (pieceIndex + 1), manager.Size);
using (ClientEngine.BufferPool.Rent(Piece.BlockSize, out byte[] hashBuffer)) {
