TestRig(string savePath, int piecelength, IPieceWriter writer, string[][] trackers, TorrentFile[] files, bool metadataMode)
{
this.files = files;
this.savePath = savePath;
this.piecelength = piecelength;
this.tier = trackers;
MetadataMode = metadataMode;
var cacheDir = Path.Combine(Path.GetDirectoryName(typeof(TestRig).Assembly.Location), "test_cache_dir");
PeerListenerFactory.Creator = endpoint => new CustomListener();
LocalPeerDiscoveryFactory.Creator = port => new ManualLocalPeerListener();
Dht.Listeners.DhtListenerFactory.Creator = endpoint => new Dht.Listeners.NullDhtListener();
Engine = new ClientEngine(EngineSettingsBuilder.CreateForTests(
allowLocalPeerDiscovery: true,
dhtPort: 12345,
cacheDirectory: cacheDir,
listenPort: 12345
));
if (Directory.Exists(Engine.Settings.MetadataCacheDirectory))
{
Directory.Delete(Engine.Settings.MetadataCacheDirectory, true);
}
Engine.DiskManager.ChangePieceWriter(writer);
Writer = writer;
RecreateManager().Wait();
MetadataPath = Path.Combine(Engine.Settings.MetadataCacheDirectory, $"{Engine.Torrents.Single ().InfoHash.ToHex ()}.torrent");
}
internal BEncodedDictionary Create(string name, List <TorrentFile> files, CancellationToken token)
{
if (PieceLength == 0)
{
PieceLength = RecommendedPieceSize(files);
}
BEncodedDictionary torrent = BEncodedValue.Clone(Metadata);
BEncodedDictionary info = (BEncodedDictionary)torrent ["info"];
info ["name"] = (BEncodedString)name;
AddCommonStuff(torrent);
using (IPieceWriter reader = CreateReader()) {
info ["pieces"] = (BEncodedString)CalcPiecesHash(files, reader, token);
if (files.Count == 1 && files [0].Path == name)
{
CreateSingleFileTorrent(torrent, files, reader, name);
}
else
{
CreateMultiFileTorrent(torrent, files, reader, name);
}
}
return(torrent);
}
TestRig(string savePath, int piecelength, IPieceWriter writer, string[][] trackers, TorrentFile[] files, bool metadataMode)
{
this.files = files;
this.savePath = savePath;
this.piecelength = piecelength;
this.tier = trackers;
MetadataMode = metadataMode;
var cacheDir = Path.Combine(Path.GetDirectoryName(typeof(TestRig).Assembly.Location), "test_cache_dir");
var factories = Factories.Default
.WithDhtCreator(() => new ManualDhtEngine())
.WithDhtListenerCreator(port => new NullDhtListener())
.WithLocalPeerDiscoveryCreator(() => new ManualLocalPeerListener())
.WithPeerConnectionListenerCreator(endpoint => new CustomListener())
.WithTrackerCreator("custom", uri => new Tracker(new CustomTracker(uri)))
;
Engine = new ClientEngine(EngineSettingsBuilder.CreateForTests(
allowLocalPeerDiscovery: true,
dhtEndPoint: new IPEndPoint(IPAddress.Any, 12345),
cacheDirectory: cacheDir,
listenEndPoint: new IPEndPoint(IPAddress.Any, 12345)
), factories);
if (Directory.Exists(Engine.Settings.MetadataCacheDirectory))
{
Directory.Delete(Engine.Settings.MetadataCacheDirectory, true);
}
Engine.DiskManager.SetWriterAsync(writer).GetAwaiter().GetResult();
Writer = writer;
RecreateManager().Wait();
MetadataPath = Path.Combine(Engine.Settings.MetadataCacheDirectory, $"{Engine.Torrents.Single ().InfoHash.ToHex ()}.torrent");
}
public static async ReusableTask WriteToFilesAsync(this IPieceWriter writer, ITorrentData manager, BlockInfo request, byte[] buffer)
{
var count = request.RequestLength;
var torrentOffset = request.ToByteOffset(manager.PieceLength);
if (torrentOffset < 0 || torrentOffset + count > manager.Size)
{
throw new ArgumentOutOfRangeException(nameof(request));
}
int totalWritten = 0;
var files = manager.Files;
int i = files.FindFileByOffset(torrentOffset);
var offset = torrentOffset - files[i].OffsetInTorrent;
while (totalWritten < count)
{
int fileToWrite = (int)Math.Min(files[i].Length - offset, count - totalWritten);
fileToWrite = Math.Min(fileToWrite, Piece.BlockSize);
await writer.WriteAsync(files[i], offset, buffer, totalWritten, fileToWrite);
offset += fileToWrite;
totalWritten += fileToWrite;
if (offset >= files[i].Length)
{
offset = 0;
i++;
}
}
}
internal DiskManager(EngineSettings settings, IPieceWriter writer = null)
{
ReadLimiter = new RateLimiter();
ReadMonitor = new SpeedMonitor();
ReadQueue = new Queue <BufferedIO> ();
WriteLimiter = new RateLimiter();
WriteMonitor = new SpeedMonitor();
WriteQueue = new Queue <BufferedIO> ();
UpdateTimer = ValueStopwatch.StartNew();
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
// If we pass in an IPieceWriter it should be used *instead* of these.
// However we still create these so the properties are non-null.
DiskWriter = new DiskWriter(settings.MaximumOpenFiles)
{
ReadMonitor = ReadMonitor,
WriteMonitor = WriteMonitor,
};
MemoryWriter = new MemoryWriter(DiskWriter, settings.DiskCacheBytes);
Writer = writer ?? MemoryWriter;
}
public MemoryWriter (IPieceWriter writer, int capacity)
{
if (capacity < 0)
throw new ArgumentOutOfRangeException (nameof (capacity));
CachedBlocks = new List<CachedBlock> ();
Capacity = capacity;
Writer = writer ?? throw new ArgumentNullException (nameof (writer));
}
async Task ChangePieceWriterAsync (IPieceWriter writer)
{
writer = writer ?? throw new ArgumentNullException (nameof (writer));
await MainLoop;
if (IsRunning)
throw new InvalidOperationException ("You must stop all active downloads before changing the piece writer used to write data to disk.");
await DiskManager.SetWriterAsync (writer);
}
public MemoryCache(MemoryPool bufferPool, long capacity, IPieceWriter writer)
{
if (capacity < 0)
{
throw new ArgumentOutOfRangeException(nameof(capacity));
}
BufferPool = bufferPool;
Capacity = capacity;
Writer = writer ?? throw new ArgumentNullException(nameof(writer));
CachedBlocks = new Dictionary <ITorrentData, List <CachedBlock> > ();
}
internal MemoryCache(int capacity, IPieceWriter writer)
{
if (capacity < 0)
{
throw new ArgumentOutOfRangeException(nameof(capacity));
}
Capacity = capacity;
Writer = writer ?? throw new ArgumentNullException(nameof(writer));
CachedBlocks = new Dictionary <ITorrentData, List <CachedBlock> > ();
ReadMonitor = new SpeedMonitor();
WriteMonitor = new SpeedMonitor();
}
public ClientEngine(EngineSettings settings, IPeerListener listener, IPieceWriter writer)
{
Check.Settings(settings);
Check.Listener(listener);
Check.Writer(writer);
// This is just a sanity check to make sure the ReusableTasks.dll assembly is
// loadable.
GC.KeepAlive(ReusableTasks.ReusableTask.CompletedTask);
PeerId = GeneratePeerId();
Listener = listener ?? throw new ArgumentNullException(nameof(listener));
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
allTorrents = new List <TorrentManager> ();
publicTorrents = new List <TorrentManager> ();
Torrents = new ReadOnlyCollection <TorrentManager> (publicTorrents);
DiskManager = new DiskManager(Settings, writer);
ConnectionManager = new ConnectionManager(PeerId, Settings, DiskManager);
DhtEngine = new NullDhtEngine();
listenManager = new ListenManager(this);
PortForwarder = new MonoNatPortForwarder();
MainLoop.QueueTimeout(TimeSpan.FromMilliseconds(TickLength), delegate {
if (IsRunning && !Disposed)
{
LogicTick();
}
return(!Disposed);
});
downloadLimiter = new RateLimiter();
downloadLimiters = new RateLimiterGroup {
new DiskWriterLimiter(DiskManager),
downloadLimiter,
};
uploadLimiter = new RateLimiter();
uploadLimiters = new RateLimiterGroup {
uploadLimiter
};
listenManager.Register(listener);
if (SupportsLocalPeerDiscovery)
{
RegisterLocalPeerDiscovery(new LocalPeerDiscovery(Settings));
}
}
public ClientEngine(EngineSettings settings, IPieceWriter writer)
{
Check.Settings(settings);
// This is just a sanity check to make sure the ReusableTasks.dll assembly is
// loadable.
GC.KeepAlive(ReusableTasks.ReusableTask.CompletedTask);
PeerId = GeneratePeerId();
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
allTorrents = new List <TorrentManager> ();
publicTorrents = new List <TorrentManager> ();
Torrents = new ReadOnlyCollection <TorrentManager> (publicTorrents);
DiskManager = new DiskManager(Settings, writer);
ConnectionManager = new ConnectionManager(PeerId, Settings, DiskManager);
listenManager = new ListenManager(this);
PortForwarder = new MonoNatPortForwarder();
MainLoop.QueueTimeout(TimeSpan.FromMilliseconds(TickLength), delegate {
if (IsRunning && !Disposed)
{
LogicTick();
}
return(!Disposed);
});
downloadLimiter = new RateLimiter();
downloadLimiters = new RateLimiterGroup {
new DiskWriterLimiter(DiskManager),
downloadLimiter,
};
uploadLimiter = new RateLimiter();
uploadLimiters = new RateLimiterGroup {
uploadLimiter
};
Listener = PeerListenerFactory.CreateTcp(settings.ListenPort);
listenManager.SetListener(Listener);
DhtListener = DhtListenerFactory.CreateUdp(settings.DhtPort);
DhtEngine = settings.DhtPort == -1 ? new NullDhtEngine() : DhtEngineFactory.Create(DhtListener);
DhtEngine.StateChanged += DhtEngineStateChanged;
DhtEngine.PeersFound += DhtEnginePeersFound;
RegisterLocalPeerDiscovery(settings.AllowLocalPeerDiscovery && settings.ListenPort > 0 ? LocalPeerDiscoveryFactory.Create(settings.ListenPort) : null);
}
internal DiskManager(EngineSettings settings, IPieceWriter writer)
{
ReadLimiter = new RateLimiter();
ReadMonitor = new SpeedMonitor();
ReadQueue = new Queue <BufferedIO> ();
WriteLimiter = new RateLimiter();
WriteMonitor = new SpeedMonitor();
WriteQueue = new Queue <BufferedIO> ();
UpdateTimer = ValueStopwatch.StartNew();
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
Writer = writer ?? throw new ArgumentNullException(nameof(writer));
}
internal DiskManager(EngineSettings settings, IPieceWriter writer = null)
{
ReadLimiter = new RateLimiter();
ReadQueue = new Queue <BufferedIO> ();
WriteLimiter = new RateLimiter();
WriteQueue = new Queue <BufferedIO> ();
UpdateTimer = ValueStopwatch.StartNew();
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
writer ??= new DiskWriter(settings.MaximumOpenFiles);
Cache = new MemoryCache(settings.DiskCacheBytes, writer);
}
TestRig(string savePath, int piecelength, IPieceWriter writer, string[][] trackers, TorrentFile[] files, bool metadataMode)
{
this.files = files;
this.savePath = savePath;
this.piecelength = piecelength;
this.tier = trackers;
MetadataMode = metadataMode;
MetadataPath = "metadataSave.torrent";
PeerListenerFactory.Creator = endpoint => new CustomListener();
LocalPeerDiscoveryFactory.Creator = port => new ManualLocalPeerListener();
Dht.Listeners.DhtListenerFactory.Creator = endpoint => new Dht.Listeners.NullDhtListener();
Engine = new ClientEngine(new EngineSettingsBuilder {
ListenPort = 12345
}.ToSettings());
Engine.DiskManager.ChangePieceWriter(writer);
Writer = writer;
RecreateManager().Wait();
}
internal DiskManager(EngineSettings settings, Factories factories, IPieceWriter writer = null)
{
ReadLimiter = new RateLimiter();
ReadQueue = new Queue <BufferedIO> ();
WriteLimiter = new RateLimiter();
WriteQueue = new Queue <BufferedIO> ();
UpdateTimer = ValueStopwatch.StartNew();
Factories = factories ?? throw new ArgumentNullException(nameof(factories));
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
writer ??= factories.CreatePieceWriter(settings.MaximumOpenFiles);
Cache = factories.CreateBlockCache(writer, settings.DiskCacheBytes, BufferPool);
Cache.ReadThroughCache += (o, e) => WriterReadMonitor.AddDelta(e.RequestLength);
Cache.WrittenThroughCache += (o, e) => WriterWriteMonitor.AddDelta(e.RequestLength);
IncrementalHashCache = new Cache <IncrementalHashData> (() => new IncrementalHashData(IncrementalHash.CreateHash(HashAlgorithmName.SHA1)));
}
public ClientEngine(EngineSettings settings, IPeerListener listener, IPieceWriter writer)
{
Check.Settings(settings);
Check.Listener(listener);
Check.Writer(writer);
PeerId = GeneratePeerId();
Listener = listener ?? throw new ArgumentNullException(nameof(listener));
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
torrents = new List <TorrentManager>();
Torrents = new ReadOnlyCollection <TorrentManager> (torrents);
DiskManager = new DiskManager(Settings, writer);
ConnectionManager = new ConnectionManager(PeerId, Settings, DiskManager);
DhtEngine = new NullDhtEngine();
listenManager = new ListenManager(this);
MainLoop.QueueTimeout(TimeSpan.FromMilliseconds(TickLength), delegate {
if (IsRunning && !Disposed)
{
LogicTick();
}
return(!Disposed);
});
downloadLimiter = new RateLimiter();
downloadLimiters = new RateLimiterGroup {
new DiskWriterLimiter(DiskManager),
downloadLimiter,
};
uploadLimiter = new RateLimiter();
uploadLimiters = new RateLimiterGroup {
uploadLimiter
};
listenManager.Register(listener);
if (SupportsLocalPeerDiscovery)
{
RegisterLocalPeerDiscovery(new LocalPeerDiscovery(Settings));
}
}
public static async ReusableTask <int> ReadFromFilesAsync(this IPieceWriter writer, ITorrentData manager, BlockInfo request, byte[] buffer)
{
var count = request.RequestLength;
var offset = request.ToByteOffset(manager.PieceLength);
if (count < 1)
{
throw new ArgumentOutOfRangeException(nameof(count), $"Count must be greater than zero, but was {count}.");
}
if (offset < 0 || offset + count > manager.Size)
{
throw new ArgumentOutOfRangeException(nameof(offset));
}
int totalRead = 0;
var files = manager.Files;
int i = manager.Files.FindFileByOffset(offset);
offset -= files[i].OffsetInTorrent;
while (totalRead < count)
{
int fileToRead = (int)Math.Min(files[i].Length - offset, count - totalRead);
fileToRead = Math.Min(fileToRead, Piece.BlockSize);
if (fileToRead != await writer.ReadAsync(files[i], offset, buffer, totalRead, fileToRead))
{
return(totalRead);
}
offset += fileToRead;
totalRead += fileToRead;
if (offset >= files[i].Length)
{
offset = 0;
i++;
}
}
return(totalRead);
}
public MonotorrentDataAccessor(IPieceWriter writer)
{
_writer = writer;
}
internal static TorrentManager CreateMultiFileManager(TorrentFile[] files, int pieceLength, IPieceWriter writer = null)
{
return(CreateMultiFile(files, pieceLength, writer: writer).Manager);
}
internal static TorrentManager CreateMultiFileManager(long[] fileSizes, int pieceLength, IPieceWriter writer = null)
{
var files = TorrentFile.Create(pieceLength, fileSizes).ToArray();
return(CreateMultiFileManager(files, pieceLength, writer));
}
async Task <byte[]> CalcPiecesHash(int startPiece, long totalBytesToRead, Synchronizer synchronizer, List <TorrentFile> files, CancellationToken token)
{
// One buffer will be filled and will be passed to the hashing method.
// One buffer will be filled and will be waiting to be hashed.
// One buffer will be empty and will be filled from the disk.
// Aaaannd one extra buffer for good luck!
var emptyBuffers = new AsyncProducerConsumerQueue <byte[]> (4);
// Make this buffer one element larger so it can fit the placeholder which indicates a file has been completely read.
var filledBuffers = new AsyncProducerConsumerQueue <(byte[], int, TorrentFile)> (emptyBuffers.Capacity + 1);
// This is the IPieceWriter which we'll use to get our filestream. Each thread gets it's own writer.
using IPieceWriter writer = CreateReader();
// Read from the disk in 256kB chunks, instead of 16kB, as a performance optimisation.
// As the capacity is set to 4, this means we'll have 1 megabyte of buffers to handle.
for (int i = 0; i < emptyBuffers.Capacity; i++)
{
await emptyBuffers.EnqueueAsync(new byte[256 * 1024], token);
}
token.ThrowIfCancellationRequested();
using CancellationTokenRegistration cancellation = token.Register(() => {
emptyBuffers.CompleteAdding();
filledBuffers.CompleteAdding();
});
// We're going to do single-threaded reading from disk, which (unfortunately) means we're (more or less) restricted
// to single threaded hashing too as it's unlikely we'll have sufficient data in our buffers to do any better.
Task readAllTask = ReadAllDataAsync(startPiece * PieceLength, totalBytesToRead, synchronizer, files, writer, emptyBuffers, filledBuffers, token);
Task <byte[]> hashAllTask = HashAllDataAsync(totalBytesToRead, emptyBuffers, filledBuffers, token);
Task firstCompleted = null;
try {
// We first call 'WhenAny' so that if an exception is thrown in one of the tasks, execution will continue
// and we can kill the producer/consumer queues.
firstCompleted = await Task.WhenAny(readAllTask, hashAllTask);
// If the first completed task has faulted, force the exception to be thrown.
await firstCompleted;
} catch {
// We got an exception from the first or second task, so bail out now!
emptyBuffers.CompleteAdding();
filledBuffers.CompleteAdding();
}
try {
// If there is no exception from the first completed task, just wait for the second one.
await Task.WhenAll(readAllTask, hashAllTask);
} catch {
token.ThrowIfCancellationRequested();
if (firstCompleted != null)
{
await firstCompleted;
}
throw;
}
return(await hashAllTask);
}
TestRig(string savePath, int piecelength, IPieceWriter writer, string[][] trackers, TorrentFile[] files)
: this(savePath, piecelength, writer, trackers, files, false)
{
}
void CreateMultiFileTorrent(BEncodedDictionary dictionary, List <TorrentFile> mappings, IPieceWriter writer, string name)
{
BEncodedDictionary info = (BEncodedDictionary)dictionary ["info"];
List <BEncodedValue> files = mappings.ConvertAll <BEncodedValue> (ToFileInfoDict);
info.Add("files", new BEncodedList(files));
}
public MemoryWriter(IPieceWriter writer)
: this(writer, 2 * 1024 * 1024)
{
}
async Task ReadAllDataAsync(long startOffset, long totalBytesToRead, Synchronizer synchronizer, List <TorrentFile> files, IPieceWriter writer, AsyncProducerConsumerQueue <byte[]> emptyBuffers, AsyncProducerConsumerQueue <(byte[], int, TorrentFile)> filledBuffers, CancellationToken token)
public ClientEngine(EngineSettings settings, IPieceWriter writer)
: this(settings, new PeerListener(new IPEndPoint(IPAddress.Any, settings.ListenPort)), writer)
{
}
internal static TestRig CreateMultiFile(TorrentFile[] files, int pieceLength, IPieceWriter writer = null)
{
return(new TestRig("", pieceLength, writer ?? StandardWriter(), StandardTrackers(), files));
}
void CreateSingleFileTorrent(BEncodedDictionary dictionary, List <TorrentFile> mappings, IPieceWriter writer, string name)
{
BEncodedDictionary infoDict = (BEncodedDictionary)dictionary ["info"];
infoDict.Add("length", new BEncodedNumber(mappings [0].Length));
if (mappings [0].MD5 != null)
{
infoDict ["md5sum"] = (BEncodedString)mappings [0].MD5;
}
}
public ReusableTask SetWriterAsync(IPieceWriter writer)
{
Writer = writer;
return(ReusableTask.CompletedTask);
}
internal async ReusableTask SetWriterAsync(IPieceWriter writer) => await Cache.SetWriterAsync(writer);