public void TestPutRawImage(bool bSaveImagesToFile)
{
DatasetFactory factory = new DatasetFactory();
factory.DeleteSources("Test123");
int nSrcId = factory.AddSource("Test123", 1, 10, 10, false, 0, bSaveImagesToFile);
factory.Open(nSrcId, 10);
byte[] rgBytes = new byte[10 * 10];
for (int i = 0; i < 20; i++)
{
rgBytes[i] = (byte)i;
SimpleDatum sd = new SimpleDatum(false, 1, 10, 10, i, DateTime.MinValue, rgBytes.ToList(), null, 0, false, i);
factory.PutRawImageCache(i, sd);
}
factory.ClearImageCash(true);
List <RawImage> rgImg = factory.GetRawImagesAt(0, 20);
for (int i = 0; i < rgImg.Count; i++)
{
SimpleDatum sd = factory.LoadDatum(rgImg[i]);
bool bEncoded = false;
byte[] rgData = sd.GetByteData(out bEncoded);
for (int j = 0; j < 100; j++)
{
if (j <= i)
{
Assert.AreEqual(rgData[j], j);
}
else
{
Assert.AreEqual(rgData[j], 0);
}
}
}
factory.DeleteSources("Test123");
factory.Close();
}
private void loadFile(string strImagesFile, string strLabelsFile, string strSourceName)
{
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
reportProgress(0, 0, " loading " + strImagesFile + "...");
BinaryFile image_file = new app.BinaryFile(strImagesFile);
BinaryFile label_file = new app.BinaryFile(strLabelsFile);
Log log = new Log("MNIST");
log.OnWriteLine += Log_OnWriteLine;
try
{
// Verify the files
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
throw new Exception("Incorrect image file magic.");
}
if (magicLbl != 2049)
{
throw new Exception("Incorrect label file magic.");
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
throw new Exception("The number of items must be equal to the number of labels!");
}
// Add the data source to the database.
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
int nChannels = 1; // black and white
int nSrcId = m_factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true);
m_factory.Open(nSrcId);
m_factory.DeleteSourceData();
// Storing to database;
byte[] rgLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, null, null, 0, false, -1);
reportProgress(0, (int)num_items, " loading a total of " + num_items.ToString() + " items.");
reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
for (int i = 0; i < num_items; i++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, (int)num_items, " loading data...");
sw.Restart();
}
datum.SetData(rgPixels.ToList(), (int)rgLabel[0]);
m_factory.PutRawImageCache(i, datum);
rgImg.Add(new SimpleDatum(datum));
}
m_factory.ClearImageCash(true);
m_factory.UpdateSourceCounts();
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
reportProgress((int)num_items, (int)num_items, " loading completed.");
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
}
private void loadFile(string strImagesFile, string strSourceName, int nTotal, ref int nIdx)
{
Stopwatch sw = new Stopwatch();
int nStart = nIdx;
reportProgress(nIdx, nTotal, " Source: " + strSourceName);
reportProgress(nIdx, nTotal, " loading " + strImagesFile + "...");
FileStream fs = null;
try
{
fs = new FileStream(strImagesFile, FileMode.Open, FileAccess.Read);
using (BinaryReader br = new BinaryReader(fs))
{
fs = null;
int nSrcId = m_factory.AddSource(strSourceName, 3, 32, 32, false, 0, true);
m_factory.Open(nSrcId);
if (nIdx == 0)
{
m_factory.DeleteSourceData();
}
sw.Start();
for (int i = 0; i < 10000; i++)
{
int nLabel = (int)br.ReadByte();
byte[] rgImgBytes = br.ReadBytes(3072);
Bitmap img = createImage(rgImgBytes);
Datum d = ImageData.GetImageData(img, 3, false, nLabel);
m_factory.PutRawImageCache(nIdx, d);
m_rgImg.Add(new SimpleDatum(d));
nIdx++;
if (sw.ElapsedMilliseconds > 1000)
{
reportProgress(nStart + i, nTotal, "loading " + strImagesFile + " " + i.ToString("N0") + " of 10,000...");
sw.Restart();
}
}
m_factory.ClearImageCash(true);
if (nIdx == nTotal)
{
m_factory.UpdateSourceCounts();
}
}
}
finally
{
if (fs != null)
{
fs.Dispose();
}
}
}
