public static async ReusableTask WriteToFilesAsync(this IPieceWriter writer, ITorrentData manager, BlockInfo request, byte[] buffer) { var count = request.RequestLength; var torrentOffset = request.ToByteOffset(manager.PieceLength); if (torrentOffset < 0 || torrentOffset + count > manager.Size) { throw new ArgumentOutOfRangeException(nameof(request)); } int totalWritten = 0; var files = manager.Files; int i = files.FindFileByOffset(torrentOffset); var offset = torrentOffset - files[i].OffsetInTorrent; while (totalWritten < count) { int fileToWrite = (int)Math.Min(files[i].Length - offset, count - totalWritten); fileToWrite = Math.Min(fileToWrite, Piece.BlockSize); await writer.WriteAsync(files[i], offset, buffer, totalWritten, fileToWrite); offset += fileToWrite; totalWritten += fileToWrite; if (offset >= files[i].Length) { offset = 0; i++; } } }
public ReusableTask <bool> ReadFromCacheAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer) { if (torrent == null) { throw new ArgumentNullException(nameof(torrent)); } if (CachedBlocks.TryGetValue(torrent, out List <CachedBlock> blocks)) { for (int i = 0; i < blocks.Count; i++) { var cached = blocks[i]; if (cached.Block != block) { continue; } cached.Buffer.CopyTo(buffer); if (!cached.Flushing) { blocks[i] = cached.SetFlushing(); FlushBlockAsync(torrent, blocks, cached); } Interlocked.Add(ref cacheHits, block.RequestLength); ReadFromCache?.Invoke(this, block); return(ReusableTask.FromResult(true)); } } return(ReusableTask.FromResult(false)); }
static void IgnoreTorrentFiles(ITorrentData torrent) { foreach (var file in torrent.Files) { file.Priority = Priority.DoNotDownload; } }
public static int BytesPerPiece(this ITorrentData self, int pieceIndex) { if (pieceIndex < self.PieceCount() - 1) { return(self.PieceLength); } return((int)(self.Size - self.PieceIndexToByteOffset(pieceIndex))); }
public BufferedIO(ITorrentData manager, long offset, byte[] buffer, int count, ReusableTaskCompletionSource <bool> tcs) { this.manager = manager; this.offset = offset; this.buffer = buffer; this.count = count; this.tcs = tcs; }
public BufferedIO(ITorrentData manager, BlockInfo request, byte[] buffer, bool preferSkipCache, ReusableTaskCompletionSource <bool> tcs) { this.manager = manager; this.request = request; this.buffer = buffer; this.preferSkipCache = preferSkipCache; this.tcs = tcs; }
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests) { TorrentData = torrentData; this.requests.Clear(); foreach (Piece p in requests) { this.requests.Add(p); } }
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests) { this.bitfield = bitfield; this.endgameSelector = new BitField(bitfield.Length); this.torrentData = torrentData; inEndgame = false; TryEnableEndgame(); ActivePicker.Initialise(bitfield, torrentData, requests); }
public async ReusableTask <bool> ReadAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer) { if (await ReadFromCacheAsync(torrent, block, buffer)) { return(true); } Interlocked.Add(ref cacheMisses, block.RequestLength); return(await ReadFromFilesAsync(torrent, block, buffer).ConfigureAwait(false) == block.RequestLength); }
async void FlushBlockAsync(ITorrentData torrent, List <CachedBlock> blocks, CachedBlock cached) { // FIXME: How do we handle failures from this? using (cached.BufferReleaser) { await WriteToFilesAsync(torrent, cached.Block, cached.Buffer); Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength); blocks.Remove(cached); } }
public async ReusableTask WriteAsync(ITorrentData torrent, BlockInfo block, Memory <byte> buffer, bool preferSkipCache) { if (preferSkipCache || Capacity < block.RequestLength) { await WriteToFilesAsync(torrent, block, buffer); } else { if (!CachedBlocks.TryGetValue(torrent, out List <CachedBlock> blocks)) { CachedBlocks[torrent] = blocks = new List <CachedBlock> (); } if (CacheUsed > (Capacity - block.RequestLength)) { var firstFlushable = FindFirstFlushable(blocks); if (firstFlushable < 0) { await WriteToFilesAsync(torrent, block, buffer); return; } else { var cached = blocks[firstFlushable]; blocks[firstFlushable] = cached.SetFlushing(); using (cached.BufferReleaser) await WriteToFilesAsync(torrent, cached.Block, cached.Buffer); Interlocked.Add(ref cacheUsed, -cached.Block.RequestLength); blocks.Remove(cached); } } CachedBlock?cache = null; for (int i = 0; i < blocks.Count && !cache.HasValue; i++) { if (blocks[i].Block == block) { cache = blocks[i]; } } if (!cache.HasValue) { var releaser = BufferPool.Rent(block.RequestLength, out Memory <byte> memory); cache = new CachedBlock(block, releaser, memory); blocks.Add(cache.Value); Interlocked.Add(ref cacheUsed, block.RequestLength); } buffer.CopyTo(cache.Value.Buffer); WrittenToCache?.Invoke(this, block); } }
public static int BlocksPerPiece(this ITorrentData self, int pieceIndex) { if (pieceIndex < self.PieceCount() - 1) { return(self.PieceLength / Piece.BlockSize); } var remainder = self.Size - self.PieceIndexToByteOffset(pieceIndex); return((int)((remainder + Piece.BlockSize - 1) / Piece.BlockSize)); }
internal async Task MoveFilesAsync(ITorrentData manager, string newRoot, bool overwrite) { await IOLoop; foreach (TorrentFile file in manager.Files) { string newPath = Path.Combine(newRoot, file.Path); Writer.Move(file, newPath, overwrite); file.FullPath = newPath; } }
internal async Task CloseFilesAsync(ITorrentData manager) { await IOLoop; // Process all pending reads/writes then close any open streams ProcessBufferedIO(true); foreach (var file in manager.Files) { Writer.Close(file); } }
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests) { this.bitfield = bitfield; this.endgameSelector = new BitField(bitfield.Length); this.torrentData = torrentData; inEndgame = false; // Always initialize both pickers, but we should only give the active requests to the Standard picker. // We should never *default* to endgame mode, we should always start in regular mode and enter endgame // mode after we fail to pick a piece. standard.Initialise(bitfield, torrentData, requests); endgame.Initialise(bitfield, torrentData, Enumerable.Empty <Piece> ()); }
internal async Task <bool> CheckAnyFilesExistAsync(ITorrentData manager) { await IOLoop; for (int i = 0; i < manager.Files.Count; i++) { if (await Writer.ExistsAsync(manager.Files[i]).ConfigureAwait(false)) { return(true); } } return(false); }
internal async Task <bool> CheckAnyFilesExistAsync(ITorrentData manager) { await IOLoop; for (int i = 0; i < manager.Files.Length; i++) { if (Writer.Exists(manager.Files[i])) { return(true); } } return(false); }
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields) { TorrentData = torrentData; IPiecePicker picker = new StandardPicker(); picker = new RandomisedPicker(picker); picker = new RarestFirstPicker(picker); picker = new PriorityPicker(picker); Picker = IgnoringPicker.Wrap(picker, ignoringBitfields); Picker.Initialise(torrentData); }
internal void ChangePicker(PiecePicker picker, BitField bitfield, ITorrentData data) { if (UnhashedPieces.Length != bitfield.Length) { UnhashedPieces = new BitField(bitfield.Length); } picker = new IgnoringPicker(bitfield, picker); picker = new IgnoringPicker(UnhashedPieces, picker); IEnumerable <Piece> pieces = Picker == null ? new List <Piece>() : Picker.ExportActiveRequests(); picker.Initialise(bitfield, data, pieces); Picker = picker; }
public override void Initialise(ITorrentData torrentData) { base.Initialise(torrentData); allPrioritisedPieces = new MutableBitField(torrentData.PieceCount()); temp = new MutableBitField(torrentData.PieceCount()); files.Clear(); for (int i = 0; i < torrentData.Files.Count; i++) { files.Add(new Files(torrentData.Files[i])); } BuildSelectors(); }
internal async ReusableTask <byte[]> GetHashAsync(ITorrentData manager, int pieceIndex) { if (GetHashAsyncOverride != null) { return(GetHashAsyncOverride(manager, pieceIndex)); } await IOLoop; if (IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash)) { // Immediately remove it from the dictionary so another thread writing data to using `WriteAsync` can't try to use it IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex)); using var lockReleaser = await incrementalHash.Locker.EnterAsync(); // We request the blocks for most pieces sequentially, and most (all?) torrent clients // will process requests in the order they have been received. This means we can optimise // hashing a received piece by hashing each block as it arrives. If blocks arrive out of order then // we'll compute the final hash by reading the data from disk. if (incrementalHash.NextOffsetToHash == manager.BytesPerPiece(pieceIndex)) { byte[] result = incrementalHash.Hasher.Hash; IncrementalHashCache.Enqueue(incrementalHash); return(result); } } else { // If we have no partial hash data for this piece we could be doing a full // hash check, so let's create a IncrementalHashData for our piece! incrementalHash = IncrementalHashCache.Dequeue(); } // We can store up to 4MB of pieces in an in-memory queue so that, when we're rate limited // we can process the queue in-order. When we try to hash a piece we need to make sure // that in-memory cache is written to the PieceWriter before we try to Read the data back // to hash it. if (WriteQueue.Count > 0) { await WaitForPendingWrites(); } using var releaser = await incrementalHash.Locker.EnterAsync(); // Note that 'startOffset' may not be the very start of the piece if we have a partial hash. int startOffset = incrementalHash.NextOffsetToHash; int endOffset = manager.BytesPerPiece(pieceIndex); using (BufferPool.Rent(Piece.BlockSize, out byte[] hashBuffer)) {
public void Setup() { var pieceLength = Constants.BlockSize * 8; var files = TorrentFileInfo.Create(pieceLength, ("Relative/Path.txt", Constants.BlockSize * 5, "Full/Path/Relative/Path.txt")); torrent = new TorrentData { Files = files, PieceLength = pieceLength, Size = files.Single().Length, }; writer = new MemoryWriter(); cache = new MemoryCache(new MemoryPool(), Constants.BlockSize * 4, writer); }
public void Setup() { buffer = new byte[100000]; offset = 2362; for (int i = 0; i < buffer.Length; i++) { buffer[i] = 0xff; } torrentData = new TestTorrentData { PieceLength = 16 * Piece.BlockSize, Size = 40 * 16 * Piece.BlockSize, }; }
internal async ReusableTask WriteAsync(ITorrentData manager, long offset, byte[] buffer, int count) { if (count < 1) { throw new ArgumentOutOfRangeException(nameof(count), $"Count must be greater than zero, but was {count}."); } Interlocked.Add(ref pendingWrites, count); await IOLoop; int pieceIndex = (int)(offset / manager.PieceLength); long pieceStart = (long)pieceIndex * manager.PieceLength; long pieceEnd = pieceStart + manager.PieceLength; if (!IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash) && offset == pieceStart) { incrementalHash = IncrementalHashes[ValueTuple.Create(manager, pieceIndex)] = IncrementalHashCache.Dequeue(); incrementalHash.NextOffsetToHash = (long)manager.PieceLength * pieceIndex; } if (incrementalHash != null) { // Incremental hashing does not perform proper bounds checking to ensure // that pieces are correctly incrementally hashed even if 'count' is greater // than the PieceLength. This should never happen under normal operation, but // unit tests do it for convenience sometimes. Keep things safe by cancelling // incremental hashing if that occurs. if ((incrementalHash.NextOffsetToHash + count) > pieceEnd) { IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex)); } else if (incrementalHash.NextOffsetToHash == offset) { incrementalHash.Hasher.TransformBlock(buffer, 0, count, buffer, 0); incrementalHash.NextOffsetToHash += count; } } if (WriteLimiter.TryProcess(count)) { Interlocked.Add(ref pendingWrites, -count); Write(manager, offset, buffer, count); } else { var tcs = new ReusableTaskCompletionSource <bool> (); WriteQueue.Enqueue(new BufferedIO(manager, offset, buffer, count, tcs)); await tcs.Task; } }
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests) { base.Initialise(bitfield, torrentData, requests); AllSamePriority = file => file.Priority == files[0].Priority; allPrioritisedPieces = new BitField(bitfield.Length); temp = new BitField(bitfield.Length); files.Clear(); for (int i = 0; i < torrentData.Files.Length; i++) { files.Add(new Files(torrentData.Files[i], torrentData.Files[i].GetSelector(bitfield.Length))); } BuildSelectors(); }
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields) { IgnorableBitfields = ignoringBitfields; TorrentData = torrentData; Temp = new MutableBitField(TorrentData.PieceCount()); IPiecePicker picker = new StandardPicker(); picker = new RandomisedPicker(picker); picker = new RarestFirstPicker(picker); Picker = new PriorityPicker(picker); Picker.Initialise(torrentData); }
public void Setup() { buffer = new byte[100000]; offset = 2362; for (int i = 0; i < buffer.Length; i++) { buffer[i] = 0xff; } torrentData = new TestTorrentData { Files = new List <ITorrentFileInfo> (), PieceLength = 16 * Constants.BlockSize, Size = 40 * 16 * Constants.BlockSize, }; }
public void Initialise(ITorrentData torrentData, IReadOnlyList <BitField> ignoringBitfields) { TorrentData = torrentData; var standardPicker = new StandardPicker(); HighPriorityPicker = IgnoringPicker.Wrap(new PriorityPicker(standardPicker), ignoringBitfields); LowPriorityPicker = new RandomisedPicker(standardPicker); LowPriorityPicker = new RarestFirstPicker(LowPriorityPicker); LowPriorityPicker = new PriorityPicker(LowPriorityPicker); LowPriorityPicker = IgnoringPicker.Wrap(LowPriorityPicker, ignoringBitfields); LowPriorityPicker.Initialise(torrentData); HighPriorityPicker.Initialise(torrentData); }
public override void Initialise(BitField bitfield, ITorrentData torrentData, IEnumerable <Piece> requests) { // 'Requests' should contain a list of all the pieces we need to complete pieces = new List <Piece>(requests); TorrentData = torrentData; foreach (Piece piece in pieces) { for (int i = 0; i < piece.BlockCount; i++) { if (piece.Blocks[i].RequestedOff != null && !piece.Blocks[i].Received) { this.requests.Add(new Request(piece.Blocks[i].RequestedOff, piece.Blocks[i])); } } } }
internal async ReusableTask <byte[]> GetHashAsync(ITorrentData manager, int pieceIndex) { if (GetHashAsyncOverride != null) { return(GetHashAsyncOverride(manager, pieceIndex)); } await IOLoop; if (IncrementalHashes.TryGetValue(ValueTuple.Create(manager, pieceIndex), out IncrementalHashData incrementalHash)) { // We request the blocks for most pieces sequentially, and most (all?) torrent clients // will process requests in the order they have been received. This means we can optimise // hashing a received piece by hashing each block as it arrives. If blocks arrive out of order then // we'll compute the final hash by reading the data from disk. if (incrementalHash.NextOffsetToHash == (long)manager.PieceLength * (pieceIndex + 1) || incrementalHash.NextOffsetToHash == manager.Size) { incrementalHash.Hasher.TransformFinalBlock(Array.Empty <byte> (), 0, 0); byte[] result = incrementalHash.Hasher.Hash; IncrementalHashCache.Enqueue(incrementalHash); IncrementalHashes.Remove(ValueTuple.Create(manager, pieceIndex)); return(result); } } else { // If we have no partial hash data for this piece we could be doing a full // hash check, so let's create a IncrementalHashData for our piece! incrementalHash = IncrementalHashCache.Dequeue(); incrementalHash.NextOffsetToHash = (long)manager.PieceLength * pieceIndex; } // We can store up to 4MB of pieces in an in-memory queue so that, when we're rate limited // we can process the queue in-order. When we try to hash a piece we need to make sure // that in-memory cache is written to the PieceWriter before we try to Read the data back // to hash it. if (WriteQueue.Count > 0) { await WaitForPendingWrites(); } // Note that 'startOffset' may not be the very start of the piece if we have a partial hash. long startOffset = incrementalHash.NextOffsetToHash; long endOffset = Math.Min((long)manager.PieceLength * (pieceIndex + 1), manager.Size); using (ClientEngine.BufferPool.Rent(Piece.BlockSize, out byte[] hashBuffer)) {