public _MultiWorkerIter(WorkerPool worker_pool, Func <NDArrayList, NDArrayList> batchify_fn, BatchSampler batch_sampler, bool pin_memory = false, int pin_device_id = 0, WorkerFn worker_fn = null, int prefetch = 0, Dataset <NDArray> dataset = null, DataLoader data_loader = null) { _worker_pool = worker_pool; _batchify_fn = batchify_fn; _batch_sampler = batch_sampler; _data_buffer = new Dictionary <int, NDArrayList>(); _rcvd_idx = 0; _sent_idx = 0; _iter = _batch_sampler.GetEnumerator(); _worker_fn = worker_fn; _pin_memory = pin_memory; _pin_device_id = pin_device_id; _dataset = dataset; _data_loader = data_loader; foreach (var item in Enumerable.Range(0, prefetch)) { PushNext(); } }
public DataLoader(Dataset <NDArray> dataset, int?batch_size = null, bool shuffle = false, Sampler sampler = null, string last_batch = null, BatchSampler batch_sampler = null, Func <NDArrayList, NDArrayList> batchify_fn = null, int num_workers = 0, bool pin_memory = false, int pin_device_id = 0, int?prefetch = null, bool thread_pool = false) { _dataset = dataset; _pin_memory = pin_memory; _pin_device_id = pin_device_id; _thread_pool = thread_pool; if (batch_sampler == null) { if (!batch_size.HasValue) { throw new Exception("batch_size must be specified unless " + "batch_sampler is specified"); } if (sampler == null) { if (shuffle) { sampler = new RandomSampler(dataset.Length); } else { sampler = new SequentialSampler(dataset.Length); } } else if (shuffle) { throw new Exception("shuffle must not be specified if sampler is specified"); } batch_sampler = new BatchSampler(sampler, batch_size.Value, !string.IsNullOrWhiteSpace(last_batch) ? last_batch : "keep"); } _batch_sampler = batch_sampler; _num_workers = num_workers >= 0 ? num_workers : 0; _prefetch = Math.Max(0, prefetch.HasValue ? prefetch.Value : 2 * _num_workers); if (_num_workers > 0) { if (thread_pool) { _worker_pool = new WorkerPool(_num_workers); } else { _worker_pool = new WorkerPool(_num_workers, WorkerInitializer, dataset); } ThreadPool.SetMinThreads(_worker_pool.NumThreads, _worker_pool.NumThreads); } if (batchify_fn == null) { if (_num_workers > 0) { _batchify_fn = DefaultBatchifyFn; } else { _batchify_fn = DefaultBatchifyFn; } } else { _batchify_fn = batchify_fn; } }