private List <Matrix <TType> > BatchSamples(List <TInput> samples, int batchCount, int batchSize, IProgressWriter progressWriter) { var result = new List <Matrix <TType> >(); var tracker = new ProgressTracker(batchSize); for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++) { if (tracker.ShouldReport(sampleIdx)) { progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching"); } var matrix = Matrix <TType> .Build.Dense(_size, batchSize); for (int col = 0; col < batchSize; col++) { var input = samples[sampleIdx + col * batchCount]; for (int i = 0; i < _size; i++) { matrix[i, col] = _mapper(input, i); } } result.Add(matrix); } progressWriter?.ItemComplete(); return(result); }
/// <summary> /// Downloads a file to <see cref="downloadFileName"/> and extracts contents using <see cref="extractor"/>. /// If the file being downloaded already exists, it won't be redownloaded. /// </summary> /// <param name="url">Url to download the file from.</param> /// <param name="downloadFileName">Path to download the file to.</param> /// <param name="extractor">Action to extract file with.</param> /// <param name="keepDownloaded">Keep downloaded file. If false, file is removed after exctraction.</param> /// <returns>True if suceeded, false otherwise.</returns> public bool DownloadAndExtract(string url, string downloadFileName, Action <ZipFile> extractor, bool keepDownloaded = false) { string fileName = Path.GetFileName(downloadFileName); if (!File.Exists(downloadFileName)) { bool complete = false; var client = new WebClient(); client.DownloadProgressChanged += (sender, args) => { if (!complete) { _progressWriter?.SetItemProgress(args.BytesReceived, args.TotalBytesToReceive, $"Downloading {fileName}"); } }; try { client.DownloadFileTaskAsync(url, downloadFileName).Wait(); } catch (Exception e) { _progressWriter?.Message($"Failed to download {fileName}: {e.Message}"); return(false); } complete = true; _progressWriter?.ItemComplete(); } try { using (var zip = ZipFile.Read(downloadFileName)) { extractor(zip); } _progressWriter?.Message($"Extracted {fileName}."); } catch (Exception e) { _progressWriter?.Message($"Failed to extract {fileName}: {e.Message}"); return(false); } finally { if (!keepDownloaded) { File.Delete(downloadFileName); } } return(true); }
private List <Sample <T> > BatchSamples(List <LinearSample <T> > samples, int batchCount, int batchSize, IProgressWriter progressWriter) { var result = new List <Sample <T> >(); int inputSize = samples[0].Input.Length; int targetSize = samples[0].Target.Length; var tracker = new ProgressTracker(batchSize); for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++) { if (tracker.ShouldReport(sampleIdx)) { progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching"); } var sample = new Sample <T>(inputSize, targetSize, batchSize); for (int col = 0; col < batchSize; col++) { var input = samples[sampleIdx + col * batchCount]; for (int i = 0; i < input.Input.Length; i++) { sample.Input[i, col] = input.Input[i]; } for (int i = 0; i < input.Target.Length; i++) { sample.Target[i, col] = input.Target[i]; } } result.Add(sample); } progressWriter?.ItemComplete(); return(result); }