public _MultiWorkerIter(WorkerPool worker_pool, Func <NDArrayList, NDArrayList> batchify_fn,
BatchSampler batch_sampler,
bool pin_memory = false, int pin_device_id = 0, WorkerFn worker_fn = null,
int prefetch = 0, Dataset <NDArray> dataset = null, DataLoader data_loader = null)
{
_worker_pool = worker_pool;
_batchify_fn = batchify_fn;
_batch_sampler = batch_sampler;
_data_buffer = new Dictionary <int, NDArrayList>();
_rcvd_idx = 0;
_sent_idx = 0;
_iter = _batch_sampler.GetEnumerator();
_worker_fn = worker_fn;
_pin_memory = pin_memory;
_pin_device_id = pin_device_id;
_dataset = dataset;
_data_loader = data_loader;
foreach (var item in Enumerable.Range(0, prefetch))
{
PushNext();
}
}
public _RandomTransformMultiWorkerIter(Func <NDArray, NDArray>[] transform_fns, int interval, WorkerPool worker_pool, Func <NDArrayList, NDArrayList> batchify_fn, BatchSampler batch_sampler, bool pin_memory = false, int pin_device_id = 0, WorkerFn worker_fn = null, int prefetch = 0) : base(worker_pool, batchify_fn, batch_sampler, pin_memory, pin_device_id, worker_fn, prefetch)
{
throw new NotImplementedException();
}
public _MultiWorkerIterV1(int num_workers, Dataset <NDArray> dataset, Func <NDArrayList, NDArrayList> batchify_fn,
BatchSampler batch_sampler, bool pin_memory = false, int pin_device_id = 0, WorkerFn worker_fn = null)
{
if (num_workers == 0)
{
throw new Exception($"_MultiWorkerIter is not for {num_workers} workers");
}
_num_workers = num_workers;
_dataset = dataset;
_batchify_fn = batchify_fn;
_batch_sampler = batch_sampler;
_key_queue = new Queue <int>();
_data_queue = new Queue <NDArray>();
_data_buffer = new Dictionary <int, NDArrayList>();
_data_buffer_lock = new object();
_rcvd_idx = 0;
_sent_idx = 0;
_iter = _batch_sampler.GetEnumerator();
_shutdown = false;
_workers = new List <Thread>();
for (var i = 0; i < _num_workers; i++)
{
var t = new Thread(obj => { worker_fn(_dataset, _key_queue, _data_queue, _batchify_fn); });
t.IsBackground = true;
t.Start();
_workers.Add(t);
}
_fetcher = new Thread(obj =>
{
DataLoader.FetcherLoopV1(_data_queue, _data_buffer, pin_memory, pin_device_id, _data_buffer_lock);
});
_fetcher.IsBackground = true;
_fetcher.Start();
for (var i = 0; i < 2 * _num_workers; i++)
{
PushNext();
}
}
