Example #1
0
        private List <Matrix <TType> > BatchSamples(List <TInput> samples, int batchCount, int batchSize, IProgressWriter progressWriter)
        {
            var result = new List <Matrix <TType> >();

            var tracker = new ProgressTracker(batchSize);

            for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++)
            {
                if (tracker.ShouldReport(sampleIdx))
                {
                    progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching");
                }

                var matrix = Matrix <TType> .Build.Dense(_size, batchSize);

                for (int col = 0; col < batchSize; col++)
                {
                    var input = samples[sampleIdx + col * batchCount];

                    for (int i = 0; i < _size; i++)
                    {
                        matrix[i, col] = _mapper(input, i);
                    }
                }

                result.Add(matrix);
            }

            progressWriter?.ItemComplete();

            return(result);
        }
Example #2
0
        /// <summary>
        /// Downloads a file to <see cref="downloadFileName"/> and extracts contents using <see cref="extractor"/>.
        /// If the file being downloaded already exists, it won't be redownloaded.
        /// </summary>
        /// <param name="url">Url to download the file from.</param>
        /// <param name="downloadFileName">Path to download the file to.</param>
        /// <param name="extractor">Action to extract file with.</param>
        /// <param name="keepDownloaded">Keep downloaded file. If false, file is removed after exctraction.</param>
        /// <returns>True if suceeded, false otherwise.</returns>
        public bool DownloadAndExtract(string url, string downloadFileName, Action <ZipFile> extractor, bool keepDownloaded = false)
        {
            string fileName = Path.GetFileName(downloadFileName);

            if (!File.Exists(downloadFileName))
            {
                bool complete = false;
                var  client   = new WebClient();
                client.DownloadProgressChanged += (sender, args) =>
                {
                    if (!complete)
                    {
                        _progressWriter?.SetItemProgress(args.BytesReceived, args.TotalBytesToReceive, $"Downloading {fileName}");
                    }
                };

                try
                {
                    client.DownloadFileTaskAsync(url, downloadFileName).Wait();
                }
                catch (Exception e)
                {
                    _progressWriter?.Message($"Failed to download {fileName}: {e.Message}");
                    return(false);
                }

                complete = true;
                _progressWriter?.ItemComplete();
            }

            try
            {
                using (var zip = ZipFile.Read(downloadFileName))
                {
                    extractor(zip);
                }

                _progressWriter?.Message($"Extracted {fileName}.");
            }
            catch (Exception e)
            {
                _progressWriter?.Message($"Failed to extract {fileName}: {e.Message}");
                return(false);
            }
            finally
            {
                if (!keepDownloaded)
                {
                    File.Delete(downloadFileName);
                }
            }

            return(true);
        }
Example #3
0
        private List <Sample <T> > BatchSamples(List <LinearSample <T> > samples, int batchCount, int batchSize, IProgressWriter progressWriter)
        {
            var result     = new List <Sample <T> >();
            int inputSize  = samples[0].Input.Length;
            int targetSize = samples[0].Target.Length;

            var tracker = new ProgressTracker(batchSize);

            for (int sampleIdx = 0; sampleIdx < batchCount; sampleIdx++)
            {
                if (tracker.ShouldReport(sampleIdx))
                {
                    progressWriter?.SetItemProgress(sampleIdx, batchCount, "Batching");
                }

                var sample = new Sample <T>(inputSize, targetSize, batchSize);
                for (int col = 0; col < batchSize; col++)
                {
                    var input = samples[sampleIdx + col * batchCount];

                    for (int i = 0; i < input.Input.Length; i++)
                    {
                        sample.Input[i, col] = input.Input[i];
                    }

                    for (int i = 0; i < input.Target.Length; i++)
                    {
                        sample.Target[i, col] = input.Target[i];
                    }
                }

                result.Add(sample);
            }

            progressWriter?.ItemComplete();

            return(result);
        }