bool IPackageLoader.TryParsePackage(Stream s, string filename, FS context, out IReadOnlyPackage package)
{
if (!filename.EndsWith(".bag", StringComparison.InvariantCultureIgnoreCase))
{
package = null;
return(false);
}
// A bag file is always accompanied with an .idx counterpart
// For example: audio.bag requires the audio.idx file
var indexFilename = Path.ChangeExtension(filename, ".idx");
List <IdxEntry> entries = null;
try
{
// Build the index and dispose the stream, it is no longer needed after this
using (var indexStream = context.Open(indexFilename))
entries = new IdxReader(indexStream).Entries;
}
catch
{
package = null;
return(false);
}
package = new BagFile(s, entries, filename);
return(true);
}
public void ConstructorTest()
{
Accord.Math.Random.Generator.Seed = 0;
string mnistPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "mnist", "train-images-idx3-ubyte.gz");
IdxReader idxReader = new IdxReader(mnistPath);
double[][] X = idxReader.ReadToEndAsVectors <double>();
Assert.AreEqual(X.Length, 60000);
Assert.AreEqual(X[59999].Length, 784);
// Perform the initial dimensionality reduction using PCA
//var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2);
//pca.Learn(X);
//pca.Save(@"pca_v3_1.bin");
string pcaPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "mnist", "pca_mnist_v3_1.bin");
var pca = Serializer.Load <PrincipalComponentAnalysis>(pcaPath);
X = pca.Transform(X);
TSNE tSNE = new TSNE();
var Y = tSNE.Transform(X);
Assert.Fail();
}
public void Execute()
{
MftReader mft = MftReader.Create(_spec.GameDirectory.DirectoryPath);
MftContent mftContent = mft.ReadContent();
IdxReader idx = new IdxReader(mftContent);
IdxContent idxContent = idx.ReadContent();
TXEncoding.TryReadFromExecutable(_spec.GameDirectory.ExecutablePath);
using (DbExtractor extractor = new DbExtractor(mftContent.Version, _spec.OutputDirectory))
{
extractor.Rename = _spec.Rename;
extractor.Convert = _spec.Convert;
foreach (var group in idxContent)
{
DbPackage package = group.Key;
foreach (IdxEntry entry in group)
{
extractor.Enqueue(package, entry);
}
}
}
Console.WriteLine("Extracted!");
}
public BagFile(string filename, int priority)
{
bagFilename = filename;
bagFilePriority = priority;
// A bag file is always accompanied with an .idx counterpart
// For example: audio.bag requires the audio.idx file
var indexFilename = Path.ChangeExtension(filename, ".idx");
s = GlobalFileSystem.Open(filename);
// Build the index and dispose the stream, it is no longer needed after this
List <IdxEntry> entries;
using (var indexStream = GlobalFileSystem.Open(indexFilename))
{
var reader = new IdxReader(indexStream);
entries = reader.Entries;
}
index = entries.ToDictionaryWithConflictLog(x => x.Hash,
"{0} (bag format)".F(filename),
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length));
}
public void ConstructorTest()
{
Accord.Math.Random.Generator.Seed = 0;
IdxReader idxReader = new IdxReader(Resources.train_images_idx3_ubyte);
double[][] X = idxReader.ReadToEndAsVectors <double>();
Assert.AreEqual(X.Length, 60000);
Assert.AreEqual(X[59999].Length, 784);
// Perform the initial dimensionality reduction using PCA
//var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2);
//pca.Learn(X);
//pca.Save(@"pca_v3_1.bin");
var pca = Serializer.Load <PrincipalComponentAnalysis>(Properties.Resources.pca_mnist_v3_1);
X = pca.Transform(X);
TSNE tSNE = new TSNE();
var Y = tSNE.Transform(X);
Assert.Fail();
}
public void ReadSampleTest3()
{
MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte);
IdxReader reader = new IdxReader(file, compressed: true);
var samples = reader.ReadToEndAsVectors<byte>();
for (int i = 0; i < samples.Length; i++)
Assert.AreEqual(28 * 28, samples[i].Length);
}
public void ReadSampleTest3()
{
MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte);
IdxReader reader = new IdxReader(file, compressed: true);
var samples = reader.ReadToEndAsVectors <byte>();
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28 * 28, samples[i].Length);
}
}
public void ReadSampleTest2()
{
MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte);
IdxReader reader = new IdxReader(file, compressed: true);
var samples = reader.ReadToEndAsMatrices<byte>();
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28, samples[i].GetLength(0));
Assert.AreEqual(28, samples[i].GetLength(1));
}
}
public void ReadSampleTest2()
{
string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz");
IdxReader reader = new IdxReader(fileName, compressed: true);
var samples = reader.ReadToEndAsMatrices <byte>();
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28, samples[i].GetLength(0));
Assert.AreEqual(28, samples[i].GetLength(1));
}
}
public void ReadSampleTest3()
{
string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz");
var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
IdxReader reader = new IdxReader(stream, compressed: true);
var samples = reader.ReadToEndAsVectors <byte>();
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28 * 28, samples[i].Length);
}
}
private void LoadGenericIdxDataButton_Click(object sender, RoutedEventArgs e)
{
LoadIdxDataWindow ldw = new LoadIdxDataWindow {
Title = "Load generic IDX data"
};
if (ldw.ShowDialog().GetValueOrDefault())
{
LoadIdxDataWindowModel dc = ldw.DataContext as LoadIdxDataWindowModel;
IdxReader dataReader = new IdxReader(dc.TestingData);
IdxReader labelReader = new IdxReader(dc.TestingLabels);
Type dataType = GetTypeFromIdxType(dataReader.DataType);
Type labelType = GetTypeFromIdxType(labelReader.DataType);
Type containerType = typeof(LabeledDataContainer <,>).MakeGenericType(dataType, labelType);
dataContainer = Activator.CreateInstance(containerType) as ILabeledDataContainer;
LoadData(dc.TrainingData, dc.TrainingLabels, dc.TestingData, dc.TestingLabels);
model.TrainerType = "Generic trainer";
}
}
public void ReadSampleTest()
{
MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte);
IdxReader reader = new IdxReader(file, compressed: true);
Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType);
Assert.AreEqual(10000, reader.Samples);
var samples = new byte[reader.Samples][,];
for (int i = 0; i < samples.Length; i++)
samples[i] = reader.ReadMatrix<byte>();
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28, samples[i].GetLength(0));
Assert.AreEqual(28, samples[i].GetLength(1));
}
}
public void LoadTrainingData(string dataFile, string labelFile)
{
LoadData(dataFile, labelFile, out trainingSet, out trainingLabels);
//read input output neuron count
if (autoReadDataSize)
{
IdxReader dataReader = new IdxReader(dataFile);
IdxReader labelsReader = new IdxReader(labelFile);
inputDataSize = 1;
foreach (int dimSize in dataReader.Dimensions)
{
inputDataSize *= dimSize;
}
outputDataSize = 1;
foreach (int dimSize in labelsReader.Dimensions)
{
outputDataSize *= dimSize;
}
}
}
public BagFile(FileSystem context, string filename)
{
Name = filename;
// A bag file is always accompanied with an .idx counterpart
// For example: audio.bag requires the audio.idx file
var indexFilename = Path.ChangeExtension(filename, ".idx");
// Build the index and dispose the stream, it is no longer needed after this
List <IdxEntry> entries;
using (var indexStream = context.Open(indexFilename))
entries = new IdxReader(indexStream).Entries;
index = entries.ToDictionaryWithConflictLog(x => x.Filename,
"{0} (bag format)".F(filename),
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length));
s = context.Open(filename);
}
public void ReadSampleTest()
{
string fileName = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "t10k-images-idx3-ubyte.gz");
IdxReader reader = new IdxReader(fileName, compressed: true);
Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType);
Assert.AreEqual(10000, reader.Samples);
var samples = new byte[reader.Samples][, ];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = reader.ReadMatrix <byte>();
}
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28, samples[i].GetLength(0));
Assert.AreEqual(28, samples[i].GetLength(1));
}
}
public void ReadSampleTest()
{
MemoryStream file = new MemoryStream(Resources.t10k_images_idx3_ubyte);
IdxReader reader = new IdxReader(file, compressed: true);
Assert.AreEqual(IdxDataType.UnsignedByte, reader.DataType);
Assert.AreEqual(10000, reader.Samples);
var samples = new byte[reader.Samples][, ];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = reader.ReadMatrix <byte>();
}
for (int i = 0; i < samples.Length; i++)
{
Assert.AreEqual(28, samples[i].GetLength(0));
Assert.AreEqual(28, samples[i].GetLength(1));
}
}
public void ConstructorTest()
{
Accord.Math.Random.Generator.Seed = 0;
IdxReader idxReader = new IdxReader(Resources.train_images_idx3_ubyte);
double[][] X = idxReader.ReadToEndAsVectors<double>();
Assert.AreEqual(X.Length, 60000);
Assert.AreEqual(X[59999].Length, 784);
// Perform the initial dimensionality reduction using PCA
//var pca = new PrincipalComponentAnalysis(numberOfOutputs: 2);
//pca.Learn(X);
//pca.Save(@"pca_v3_1.bin");
var pca = Serializer.Load<PrincipalComponentAnalysis>(Properties.Resources.pca_mnist_v3_1);
X = pca.Transform(X);
TSNE tSNE = new TSNE();
var Y = tSNE.Transform(X);
Assert.Fail();
}
public static DataTable Load(Stream stream, TableFormat format)
{
switch (format)
{
case TableFormat.SerializedXml:
{
XmlSerializer serializer = new XmlSerializer(typeof(DataTable));
return (DataTable)serializer.Deserialize(stream);
}
case TableFormat.SerializedBin:
{
BinaryFormatter serializer = new BinaryFormatter();
return (DataTable)serializer.Deserialize(stream);
}
case TableFormat.OctaveMatFile:
{
MatReader reader = new MatReader(stream);
return reader.Fields.First().Value.GetValue<double[,]>().ToTable();
}
case TableFormat.OpenDocument:
{
ExcelReader reader = new ExcelReader(stream, true);
string ws = reader.GetWorksheetList().First();
return reader.GetWorksheet(ws);
}
case TableFormat.OlderExcel:
{
ExcelReader reader = new ExcelReader(stream, false);
string ws = reader.GetWorksheetList().First();
return reader.GetWorksheet(ws);
}
case TableFormat.Csv:
{
CsvReader reader = new CsvReader(stream, true);
return reader.ToTable();
}
case TableFormat.Tsv:
{
CsvReader reader = new CsvReader(stream, true, '\t');
return reader.ToTable();
}
case TableFormat.LibSVM:
{
SparseReader reader = new SparseReader(stream);
return reader.ToTable();
}
case TableFormat.Idx:
{
IdxReader reader = new IdxReader(stream);
return reader.ReadToEndAsVectors().ToTable();
}
case TableFormat.CSharp:
throw new NotSupportedException();
}
}
protected V[][] LoadSingleFile <V>(string dataFile)
{
IdxReader dataReader = new IdxReader(dataFile, isDataCompressed);
return(dataReader.GetSamples <V>());
}