public void Setup() { int pieceLength = 16 * Constants.BlockSize; int pieces = 40; int size = pieces * pieceLength; bitfield = new BitField(pieces); torrentData = TestTorrentManagerInfo.Create( files: TorrentFileInfo.Create(pieceLength, ("Test", size, "Full/Path/Test")), pieceLength: pieceLength, size: size ); checker = new PiecePickerFilterChecker(); picker = new RarestFirstPicker(checker); picker.Initialise(torrentData); peer = PeerId.CreateNull(pieces); peer.BitField.SetAll(true); peers = new List <PeerId> (); for (int i = 0; i < 5; i++) { peers.Add(PeerId.CreateNull(pieces)); } }
public void SetAll(ITorrentManagerInfo info, Priority priority) { foreach (var file in info.Files) { Set(file, priority); } }
public static async ReusableTask WriteToFilesAsync(this IPieceWriter writer, ITorrentManagerInfo manager, BlockInfo request, Memory <byte> buffer) { var count = request.RequestLength; var torrentOffset = manager.TorrentInfo !.PieceIndexToByteOffset(request.PieceIndex) + request.StartOffset; if (torrentOffset < 0 || torrentOffset + count > manager.TorrentInfo !.Size) { throw new ArgumentOutOfRangeException(nameof(request)); } int totalWritten = 0; var files = manager.Files; int i = files.FindFileByOffset(torrentOffset); var offset = torrentOffset - files[i].OffsetInTorrent; while (totalWritten < count) { int fileToWrite = (int)Math.Min(files[i].Length - offset, count - totalWritten); fileToWrite = Math.Min(fileToWrite, Constants.BlockSize); await writer.WriteAsync(files[i], offset, buffer.Slice(totalWritten, fileToWrite)); offset += fileToWrite; totalWritten += fileToWrite; if (offset >= files[i].Length) { offset = 0; i++; } } }
public ReusableTask <bool> ReadFromCacheAsync(ITorrentManagerInfo torrent, BlockInfo block, Memory <byte> buffer) { if (torrent == null) { throw new ArgumentNullException(nameof(torrent)); } if (CachedBlocks.TryGetValue(torrent, out List <CachedBlock>?blocks)) { for (int i = 0; i < blocks.Count; i++) { var cached = blocks[i]; if (cached.Block != block) { continue; } cached.Buffer.CopyTo(buffer); if (!cached.Flushing) { blocks[i] = cached.SetFlushing(); FlushBlockAsync(torrent, blocks, cached); } Interlocked.Add(ref cacheHits, block.RequestLength); ReadFromCache?.Invoke(this, block); return(ReusableTask.FromResult(true)); } } return(ReusableTask.FromResult(false)); }
public void PrepareForFirstUse(ITorrentManagerInfo manager, int pieceIndex) { Manager = manager; PieceIndex = pieceIndex; if (UseV2) { BlockHashesReleaser = MemoryPool.Default.Rent(Manager.TorrentInfo !.BlocksPerPiece(pieceIndex) * 32, out Memory <byte> hashes); BlockHashes = hashes; } }
public void Setup() { var pieceLength = Constants.BlockSize * 8; var files = TorrentFileInfo.Create(pieceLength, ("Relative/Path.txt", Constants.BlockSize * 5, "Full/Path/Relative/Path.txt")); torrent = TestTorrentManagerInfo.Create(pieceLength: pieceLength, size: files.Single().Length, files: files); writer = new MemoryWriter(); cache = new MemoryCache(new MemoryPool(), Constants.BlockSize * 4, writer); }
public async ReusableTask <bool> ReadAsync(ITorrentManagerInfo torrent, BlockInfo block, Memory <byte> buffer) { if (await ReadFromCacheAsync(torrent, block, buffer)) { return(true); } Interlocked.Add(ref cacheMisses, block.RequestLength); return(await ReadFromFilesAsync(torrent, block, buffer).ConfigureAwait(false) == block.RequestLength); }
async void FlushBlockAsync(ITorrentManagerInfo torrent, List <CachedBlock> blocks, CachedBlock cached) { // FIXME: How do we handle failures from this? using (cached.BufferReleaser) { await WriteToFilesAsync(torrent, cached.Block, cached.Buffer); Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength); blocks.Remove(cached); } }
public async ReusableTask WriteAsync(ITorrentManagerInfo torrent, BlockInfo block, Memory <byte> buffer, bool preferSkipCache) { if (preferSkipCache || Capacity < block.RequestLength) { await WriteToFilesAsync(torrent, block, buffer); } else { if (!CachedBlocks.TryGetValue(torrent, out List <CachedBlock>?blocks)) { CachedBlocks[torrent] = blocks = new List <CachedBlock> (); } if (CacheUsed > (Capacity - block.RequestLength)) { var firstFlushable = FindFirstFlushable(blocks); if (firstFlushable < 0) { await WriteToFilesAsync(torrent, block, buffer); return; } else { var cached = blocks[firstFlushable]; blocks[firstFlushable] = cached.SetFlushing(); using (cached.BufferReleaser) await WriteToFilesAsync(torrent, cached.Block, cached.Buffer); Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength); blocks.Remove(cached); } } CachedBlock?cache = null; for (int i = 0; i < blocks.Count && !cache.HasValue; i++) { if (blocks[i].Block == block) { cache = blocks[i]; } } if (!cache.HasValue) { var releaser = BufferPool.Rent(block.RequestLength, out Memory <byte> memory); cache = new CachedBlock(block, releaser, memory); blocks.Add(cache.Value); Interlocked.Add(ref cacheUsed, block.RequestLength); } buffer.CopyTo(cache.Value.Buffer); WrittenToCache?.Invoke(this, block); } }
internal async Task <bool> CheckAnyFilesExistAsync(ITorrentManagerInfo manager) { await IOLoop; for (int i = 0; i < manager.Files.Count; i++) { if (await Cache.Writer.ExistsAsync(manager.Files[i]).ConfigureAwait(false)) { return(true); } } return(false); }
public void Setup() { buffer = new byte[100000]; offset = 2362; for (int i = 0; i < buffer.Length; i++) { buffer[i] = 0xff; } torrentData = TestTorrentManagerInfo.Create( pieceLength: 16 * Constants.BlockSize, size: 40 * 16 * Constants.BlockSize ); }
public void Setup() { singleFile = CreateSingleFile(); singleBitfield = new BitField(singleFile.TorrentInfo.PieceCount()).SetAll(true); singlePeer = PeerId.CreateNull(singleBitfield.Length); multiFile = CreateMultiFile(); multiBitfield = new BitField(multiFile.TorrentInfo.PieceCount()).SetAll(true); multiPeer = PeerId.CreateNull(multiBitfield.Length); checker = new PiecePickerFilterChecker(); picker = new PriorityPicker(checker); peers = new List <PeerId> (); }
public override void Initialise(ITorrentManagerInfo torrentData) { base.Initialise(torrentData); allPrioritisedPieces = new BitField(torrentData.TorrentInfo !.PieceCount()); temp = new BitField(torrentData.TorrentInfo !.PieceCount()); files.Clear(); for (int i = 0; i < torrentData.Files.Count; i++) { files.Add(new Files(torrentData.Files[i])); } BuildSelectors(); }
public void Initialise(ITorrentManagerInfo torrentData, IReadOnlyList <ReadOnlyBitField> ignoringBitfields) { TorrentData = torrentData.TorrentInfo !; IgnoringBitfields = ignoringBitfields; Temp = new BitField(TorrentData.PieceCount()); var standardPicker = new StandardPicker(); HighPriorityPicker = IgnoringPicker.Wrap(new PriorityPicker(standardPicker), ignoringBitfields); LowPriorityPicker = new RandomisedPicker(standardPicker); LowPriorityPicker = new RarestFirstPicker(LowPriorityPicker); LowPriorityPicker = new PriorityPicker(LowPriorityPicker); LowPriorityPicker = IgnoringPicker.Wrap(LowPriorityPicker, ignoringBitfields); LowPriorityPicker.Initialise(torrentData); HighPriorityPicker.Initialise(torrentData); }
public void Setup() { var pieceLength = Constants.BlockSize * 2; var files = TorrentFileInfo.Create(pieceLength, Constants.BlockSize / 2, Constants.BlockSize, Constants.BlockSize + Constants.BlockSize / 2, Constants.BlockSize * 2 + Constants.BlockSize / 2 ); buffer = new byte[Constants.BlockSize]; data = TestTorrentManagerInfo.Create( files: files, size: files.Sum(f => f.Length), pieceLength: pieceLength ); writer = new ExceptionWriter(); diskManager = new DiskManager(new EngineSettings(), Factories.Default, writer); }
static bool IsHybrid(this ITorrentManagerInfo self) => self.InfoHashes.IsHybrid;
ReusableTask WriteToFilesAsync(ITorrentManagerInfo torrent, BlockInfo block, Memory <byte> buffer) { WrittenThroughCache?.Invoke(this, block); return(Writer.WriteToFilesAsync(torrent, block, buffer)); }
ReusableTask <int> ReadFromFilesAsync(ITorrentManagerInfo torrent, BlockInfo block, Memory <byte> buffer) { ReadThroughCache?.Invoke(this, block); return(Writer.ReadFromFilesAsync(torrent, block, buffer)); }
internal async ReusableTask <bool> GetHashAsync(ITorrentManagerInfo manager, int pieceIndex, PieceHash dest) { if (GetHashAsyncOverride != null) { return(await GetHashAsyncOverride(manager, pieceIndex, dest)); } await IOLoop; if (IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData? incrementalHash)) { // Immediately remove it from the dictionary so another thread writing data to using `WriteAsync` can't try to use it IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex)); using var lockReleaser = await incrementalHash.Locker.EnterAsync(); // We request the blocks for most pieces sequentially, and most (all?) torrent clients // will process requests in the order they have been received. This means we can optimise // hashing a received piece by hashing each block as it arrives. If blocks arrive out of order then // we'll compute the final hash by reading the data from disk. if (incrementalHash.NextOffsetToHash == manager.TorrentInfo !.BytesPerPiece(pieceIndex)) { if (!incrementalHash.TryGetHashAndReset(dest)) { throw new NotSupportedException("Could not generate SHA1 hash for this piece"); } IncrementalHashCache.Enqueue(incrementalHash); return(true); } } else { // If we have no partial hash data for this piece we could be doing a full // hash check, so let's create a IncrementalHashData for our piece! incrementalHash = IncrementalHashCache.Dequeue(); incrementalHash.PrepareForFirstUse(manager, pieceIndex); } // We can store up to 4MB of pieces in an in-memory queue so that, when we're rate limited // we can process the queue in-order. When we try to hash a piece we need to make sure // that in-memory cache is written to the PieceWriter before we try to Read the data back // to hash it. if (WriteQueue.Count > 0) { await WaitForPendingWrites(); } using var releaser = await incrementalHash.Locker.EnterAsync(); // Note that 'startOffset' may not be the very start of the piece if we have a partial hash. int startOffset = incrementalHash.NextOffsetToHash; int endOffset = manager.TorrentInfo !.BytesPerPiece(pieceIndex); using (BufferPool.Rent(Constants.BlockSize, out Memory <byte> hashBuffer)) { try { while (startOffset != endOffset) { int count = Math.Min(Constants.BlockSize, endOffset - startOffset); if (!await ReadAsync(manager, new BlockInfo(pieceIndex, startOffset, count), hashBuffer).ConfigureAwait(false)) { return(false); } startOffset += count; incrementalHash.AppendData(hashBuffer.Slice(0, count)); } return(incrementalHash.TryGetHashAndReset(dest)); } finally { await IOLoop; IncrementalHashCache.Enqueue(incrementalHash); IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex)); } } }
public override void Initialise(ITorrentManagerInfo torrentData) { base.Initialise(torrentData); rarest.Clear(); spares.Clear(); }
public static async ReusableTask <int> ReadFromFilesAsync(this IPieceWriter writer, ITorrentManagerInfo manager, BlockInfo request, Memory <byte> buffer) { var count = request.RequestLength; var offset = manager.TorrentInfo !.PieceIndexToByteOffset(request.PieceIndex) + request.StartOffset; if (count < 1) { throw new ArgumentOutOfRangeException(nameof(count), $"Count must be greater than zero, but was {count}."); } if (offset < 0 || offset + count > manager.TorrentInfo !.Size) { throw new ArgumentOutOfRangeException(nameof(offset)); } int totalRead = 0; var files = manager.Files; int i = manager.Files.FindFileByOffset(offset); offset -= files[i].OffsetInTorrent; while (totalRead < count) { int fileToRead = (int)Math.Min(files[i].Length - offset, count - totalRead); fileToRead = Math.Min(fileToRead, Constants.BlockSize); if (fileToRead != await writer.ReadAsync(files[i], offset, buffer.Slice(totalRead, fileToRead))) { return(totalRead); } offset += fileToRead; totalRead += fileToRead; if (offset >= files[i].Length) { offset = 0; i++; } } return(totalRead); }