private List <Matrix <TType> > BatchSamples(List <TInput> samples, int batchCount, int batchSize, IProgressWriter progressWriter)
{
var result = new List <Matrix <TType> >();
var tracker = new ProgressTracker(batchSize);
for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++)
{
if (tracker.ShouldReport(sampleIdx))
{
progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching");
}
var matrix = Matrix <TType> .Build.Dense(_size, batchSize);
for (int col = 0; col < batchSize; col++)
{
var input = samples[sampleIdx + col * batchCount];
for (int i = 0; i < _size; i++)
{
matrix[i, col] = _mapper(input, i);
}
}
result.Add(matrix);
}
progressWriter?.ItemComplete();
return(result);
}
/// <summary>
/// Downloads a file to <see cref="downloadFileName"/> and extracts contents using <see cref="extractor"/>.
/// If the file being downloaded already exists, it won't be redownloaded.
/// </summary>
/// <param name="url">Url to download the file from.</param>
/// <param name="downloadFileName">Path to download the file to.</param>
/// <param name="extractor">Action to extract file with.</param>
/// <param name="keepDownloaded">Keep downloaded file. If false, file is removed after exctraction.</param>
/// <returns>True if suceeded, false otherwise.</returns>
public bool DownloadAndExtract(string url, string downloadFileName, Action <ZipFile> extractor, bool keepDownloaded = false)
{
string fileName = Path.GetFileName(downloadFileName);
if (!File.Exists(downloadFileName))
{
bool complete = false;
var client = new WebClient();
client.DownloadProgressChanged += (sender, args) =>
{
if (!complete)
{
_progressWriter?.SetItemProgress(args.BytesReceived, args.TotalBytesToReceive, $"Downloading {fileName}");
}
};
try
{
client.DownloadFileTaskAsync(url, downloadFileName).Wait();
}
catch (Exception e)
{
_progressWriter?.Message($"Failed to download {fileName}: {e.Message}");
return(false);
}
complete = true;
_progressWriter?.ItemComplete();
}
try
{
using (var zip = ZipFile.Read(downloadFileName))
{
extractor(zip);
}
_progressWriter?.Message($"Extracted {fileName}.");
}
catch (Exception e)
{
_progressWriter?.Message($"Failed to extract {fileName}: {e.Message}");
return(false);
}
finally
{
if (!keepDownloaded)
{
File.Delete(downloadFileName);
}
}
return(true);
}
private List <Sample <T> > BatchSamples(List <LinearSample <T> > samples, int batchCount, int batchSize, IProgressWriter progressWriter)
{
var result = new List <Sample <T> >();
int inputSize = samples[0].Input.Length;
int targetSize = samples[0].Target.Length;
var tracker = new ProgressTracker(batchSize);
for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++)
{
if (tracker.ShouldReport(sampleIdx))
{
progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching");
}
var sample = new Sample <T>(inputSize, targetSize, batchSize);
for (int col = 0; col < batchSize; col++)
{
var input = samples[sampleIdx + col * batchCount];
for (int i = 0; i < input.Input.Length; i++)
{
sample.Input[i, col] = input.Input[i];
}
for (int i = 0; i < input.Target.Length; i++)
{
sample.Target[i, col] = input.Target[i];
}
}
result.Add(sample);
}
progressWriter?.ItemComplete();
return(result);
}