bool IPackageLoader.TryParsePackage(Stream s, string filename, FS context, out IReadOnlyPackage package) { if (!filename.EndsWith(".bag", StringComparison.InvariantCultureIgnoreCase)) { package = null; return(false); } // A bag file is always accompanied with an .idx counterpart // For example: audio.bag requires the audio.idx file var indexFilename = Path.ChangeExtension(filename, ".idx"); List <IdxEntry> entries = null; try { // Build the index and dispose the stream, it is no longer needed after this using (var indexStream = context.Open(indexFilename)) entries = new IdxReader(indexStream).Entries; } catch { package = null; return(false); } package = new BagFile(s, entries, filename); return(true); }
public void ConstructorTest() { Accord.Math.Random.Generator.Seed = 0; string mnistPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "mnist", "train-images-idx3-ubyte.gz"); IdxReader idxReader = new IdxReader(mnistPath); double[][] X = idxReader.ReadToEndAsVectors <double>(); Assert.AreEqual(X.Length, 60000); Assert.AreEqual(X[59999].Length, 784); // Perform the initial dimensionality reduction using PCA //var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2); //pca.Learn(X); //pca.Save(@"pca_v3_1.bin"); string pcaPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "mnist", "pca_mnist_v3_1.bin"); var pca = Serializer.Load <PrincipalComponentAnalysis>(pcaPath); X = pca.Transform(X); TSNE tSNE = new TSNE(); var Y = tSNE.Transform(X); Assert.Fail(); }
public void Execute() { MftReader mft = MftReader.Create(_spec.GameDirectory.DirectoryPath); MftContent mftContent = mft.ReadContent(); IdxReader idx = new IdxReader(mftContent); IdxContent idxContent = idx.ReadContent(); TXEncoding.TryReadFromExecutable(_spec.GameDirectory.ExecutablePath); using (DbExtractor extractor = new DbExtractor(mftContent.Version, _spec.OutputDirectory)) { extractor.Rename = _spec.Rename; extractor.Convert = _spec.Convert; foreach (var group in idxContent) { DbPackage package = group.Key; foreach (IdxEntry entry in group) { extractor.Enqueue(package, entry); } } } Console.WriteLine("Extracted!"); }
public BagFile(string filename, int priority) { bagFilename = filename; bagFilePriority = priority; // A bag file is always accompanied with an .idx counterpart // For example: audio.bag requires the audio.idx file var indexFilename = Path.ChangeExtension(filename, ".idx"); s = GlobalFileSystem.Open(filename); // Build the index and dispose the stream, it is no longer needed after this List <IdxEntry> entries; using (var indexStream = GlobalFileSystem.Open(indexFilename)) { var reader = new IdxReader(indexStream); entries = reader.Entries; } index = entries.ToDictionaryWithConflictLog(x => x.Hash, "{0} (bag format)".F(filename), null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)); }
public void ConstructorTest() { Accord.Math.Random.Generator.Seed = 0; IdxReader idxReader = new IdxReader(Resources.train_images_idx3_ubyte); double[][] X = idxReader.ReadToEndAsVectors <double>(); Assert.AreEqual(X.Length, 60000); Assert.AreEqual(X[59999].Length, 784); // Perform the initial dimensionality reduction using PCA //var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2); //pca.Learn(X); //pca.Save(@"pca_v3_1.bin"); var pca = Serializer.Load <PrincipalComponentAnalysis>(Properties.Resources.pca_mnist_v3_1); X = pca.Transform(X); TSNE tSNE = new TSNE(); var Y = tSNE.Transform(X); Assert.Fail(); }
public void ReadSampleTest3() { MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte); IdxReader reader = new IdxReader(file, compressed: true); var samples = reader.ReadToEndAsVectors<byte>(); for (int i = 0; i < samples.Length; i++) Assert.AreEqual(28 * 28, samples[i].Length); }
public void ReadSampleTest3() { MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte); IdxReader reader = new IdxReader(file, compressed: true); var samples = reader.ReadToEndAsVectors <byte>(); for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28 * 28, samples[i].Length); } }
public void ReadSampleTest2() { MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte); IdxReader reader = new IdxReader(file, compressed: true); var samples = reader.ReadToEndAsMatrices<byte>(); for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28, samples[i].GetLength(0)); Assert.AreEqual(28, samples[i].GetLength(1)); } }
public void ReadSampleTest2() { string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz"); IdxReader reader = new IdxReader(fileName, compressed: true); var samples = reader.ReadToEndAsMatrices <byte>(); for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28, samples[i].GetLength(0)); Assert.AreEqual(28, samples[i].GetLength(1)); } }
public void ReadSampleTest3() { string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz"); var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read); IdxReader reader = new IdxReader(stream, compressed: true); var samples = reader.ReadToEndAsVectors <byte>(); for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28 * 28, samples[i].Length); } }
private void LoadGenericIdxDataButton_Click(object sender, RoutedEventArgs e) { LoadIdxDataWindow ldw = new LoadIdxDataWindow { Title = "Load generic IDX data" }; if (ldw.ShowDialog().GetValueOrDefault()) { LoadIdxDataWindowModel dc = ldw.DataContext as LoadIdxDataWindowModel; IdxReader dataReader = new IdxReader(dc.TestingData); IdxReader labelReader = new IdxReader(dc.TestingLabels); Type dataType = GetTypeFromIdxType(dataReader.DataType); Type labelType = GetTypeFromIdxType(labelReader.DataType); Type containerType = typeof(LabeledDataContainer <,>).MakeGenericType(dataType, labelType); dataContainer = Activator.CreateInstance(containerType) as ILabeledDataContainer; LoadData(dc.TrainingData, dc.TrainingLabels, dc.TestingData, dc.TestingLabels); model.TrainerType = "Generic trainer"; } }
public void ReadSampleTest() { MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte); IdxReader reader = new IdxReader(file, compressed: true); Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType); Assert.AreEqual(10000, reader.Samples); var samples = new byte[reader.Samples][,]; for (int i = 0; i < samples.Length; i++) samples[i] = reader.ReadMatrix<byte>(); for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28, samples[i].GetLength(0)); Assert.AreEqual(28, samples[i].GetLength(1)); } }
public void LoadTrainingData(string dataFile, string labelFile) { LoadData(dataFile, labelFile, out trainingSet, out trainingLabels); //read input output neuron count if (autoReadDataSize) { IdxReader dataReader = new IdxReader(dataFile); IdxReader labelsReader = new IdxReader(labelFile); inputDataSize = 1; foreach (int dimSize in dataReader.Dimensions) { inputDataSize *= dimSize; } outputDataSize = 1; foreach (int dimSize in labelsReader.Dimensions) { outputDataSize *= dimSize; } } }
public BagFile(FileSystem context, string filename) { Name = filename; // A bag file is always accompanied with an .idx counterpart // For example: audio.bag requires the audio.idx file var indexFilename = Path.ChangeExtension(filename, ".idx"); // Build the index and dispose the stream, it is no longer needed after this List <IdxEntry> entries; using (var indexStream = context.Open(indexFilename)) entries = new IdxReader(indexStream).Entries; index = entries.ToDictionaryWithConflictLog(x => x.Filename, "{0} (bag format)".F(filename), null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)); s = context.Open(filename); }
public void ReadSampleTest() { string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz"); IdxReader reader = new IdxReader(fileName, compressed: true); Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType); Assert.AreEqual(10000, reader.Samples); var samples = new byte[reader.Samples][, ]; for (int i = 0; i < samples.Length; i++) { samples[i] = reader.ReadMatrix <byte>(); } for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28, samples[i].GetLength(0)); Assert.AreEqual(28, samples[i].GetLength(1)); } }
public void ReadSampleTest() { MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte); IdxReader reader = new IdxReader(file, compressed: true); Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType); Assert.AreEqual(10000, reader.Samples); var samples = new byte[reader.Samples][, ]; for (int i = 0; i < samples.Length; i++) { samples[i] = reader.ReadMatrix <byte>(); } for (int i = 0; i < samples.Length; i++) { Assert.AreEqual(28, samples[i].GetLength(0)); Assert.AreEqual(28, samples[i].GetLength(1)); } }
public void ConstructorTest() { Accord.Math.Random.Generator.Seed = 0; IdxReader idxReader = new IdxReader(Resources.train_images_idx3_ubyte); double[][] X = idxReader.ReadToEndAsVectors<double>(); Assert.AreEqual(X.Length, 60000); Assert.AreEqual(X[59999].Length, 784); // Perform the initial dimensionality reduction using PCA //var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2); //pca.Learn(X); //pca.Save(@"pca_v3_1.bin"); var pca = Serializer.Load<PrincipalComponentAnalysis>(Properties.Resources.pca_mnist_v3_1); X = pca.Transform(X); TSNE tSNE = new TSNE(); var Y = tSNE.Transform(X); Assert.Fail(); }
public static DataTable Load(Stream stream, TableFormat format) { switch (format) { case TableFormat.SerializedXml: { XmlSerializer serializer = new XmlSerializer(typeof(DataTable)); return (DataTable)serializer.Deserialize(stream); } case TableFormat.SerializedBin: { BinaryFormatter serializer = new BinaryFormatter(); return (DataTable)serializer.Deserialize(stream); } case TableFormat.OctaveMatFile: { MatReader reader = new MatReader(stream); return reader.Fields.First().Value.GetValue<double[,]>().ToTable(); } case TableFormat.OpenDocument: { ExcelReader reader = new ExcelReader(stream, true); string ws = reader.GetWorksheetList().First(); return reader.GetWorksheet(ws); } case TableFormat.OlderExcel: { ExcelReader reader = new ExcelReader(stream, false); string ws = reader.GetWorksheetList().First(); return reader.GetWorksheet(ws); } case TableFormat.Csv: { CsvReader reader = new CsvReader(stream, true); return reader.ToTable(); } case TableFormat.Tsv: { CsvReader reader = new CsvReader(stream, true, '\t'); return reader.ToTable(); } case TableFormat.LibSVM: { SparseReader reader = new SparseReader(stream); return reader.ToTable(); } case TableFormat.Idx: { IdxReader reader = new IdxReader(stream); return reader.ReadToEndAsVectors().ToTable(); } case TableFormat.CSharp: throw new NotSupportedException(); } }
protected V[][] LoadSingleFile <V>(string dataFile) { IdxReader dataReader = new IdxReader(dataFile, isDataCompressed); return(dataReader.GetSamples <V>()); }