/// <summary>
/// Set the image mean.
/// </summary>
/// <param name="d">Specifies the image mean.</param>
/// <param name="bSave">Optionally, specifies whether or not to save the image mean in the database (default = false).</param>
public void SetImageMean(SimpleDatum d, bool bSave = false)
{
m_imgMean = d;
if (bSave)
{
m_factory.SaveImageMean(d, true);
}
}
/// <summary>
/// Saves the image mean in a SimpleDatum to the database.
/// </summary>
/// <param name="nSrcId">Specifies the ID of the data source to use.</param>
/// <param name="sd">Specifies the image mean data.</param>
/// <param name="bUpdate">Specifies whether or not to update the mean image.</param>
/// <returns>If saved successfully, this method returns <i>true</i>, otherwise <i>false</i> is returned.</returns>
public bool SaveImageMean(int nSrcId, SimpleDatum sd, bool bUpdate)
{
if (m_TestingImages.SourceID != nSrcId &&
m_TrainingImages.SourceID != nSrcId)
{
return(false);
}
return(m_factory.SaveImageMean(sd, bUpdate, nSrcId));
}
public void LoadDatabase()
{
int nIdx = 0;
int nTotal = 50000;
reportProgress(nIdx, 0, "Loading database...");
Log log = new Log("MNIST");
log.OnWriteLine += Log_OnWriteLine;
DatasetFactory factory = new DatasetFactory();
loadFile(m_param.DataBatchFile1, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile2, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile3, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile4, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile5, "CIFAR-10.training", nTotal, ref nIdx);
SourceDescriptor srcTrain = factory.LoadSource("CIFAR-10.training");
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID);
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 10000;
loadFile(m_param.TestBatchFile, "CIFAR-10.testing", nTotal, ref nIdx);
SourceDescriptor srcTest = factory.LoadSource("CIFAR-10.testing");
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID);
DatasetDescriptor ds = new DatasetDescriptor(0, "CIFAR-10", null, null, srcTrain, srcTest, "CIFAR-10", "CiFar-10 Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
private void loadFile(string strImagesFile, string strLabelsFile, string strSourceName)
{
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
reportProgress(0, 0, " loading " + strImagesFile + "...");
BinaryFile image_file = new app.BinaryFile(strImagesFile);
BinaryFile label_file = new app.BinaryFile(strLabelsFile);
Log log = new Log("MNIST");
log.OnWriteLine += Log_OnWriteLine;
try
{
// Verify the files
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
throw new Exception("Incorrect image file magic.");
}
if (magicLbl != 2049)
{
throw new Exception("Incorrect label file magic.");
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
throw new Exception("The number of items must be equal to the number of labels!");
}
// Add the data source to the database.
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
int nChannels = 1; // black and white
int nSrcId = m_factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true);
m_factory.Open(nSrcId);
m_factory.DeleteSourceData();
// Storing to database;
byte[] rgLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, null, null, 0, false, -1);
reportProgress(0, (int)num_items, " loading a total of " + num_items.ToString() + " items.");
reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
for (int i = 0; i < num_items; i++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, (int)num_items, " loading data...");
sw.Restart();
}
datum.SetData(rgPixels.ToList(), (int)rgLabel[0]);
m_factory.PutRawImageCache(i, datum);
rgImg.Add(new SimpleDatum(datum));
}
m_factory.ClearImageCashe(true);
m_factory.UpdateSourceCounts();
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
reportProgress((int)num_items, (int)num_items, " loading completed.");
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
}
/// <summary>
/// Create the dataset and load it into the database.
/// </summary>
/// <param name="nCreatorID">Specifies the creator ID.</param>
/// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns>
public bool LoadDatabase(int nCreatorID = 0)
{
try
{
int nIdx = 0;
int nTotal = 50000;
reportProgress(nIdx, 0, "Loading database " + dataset_name + "...");
DatasetFactory factory = new DatasetFactory();
string strTrainSrc = dataset_name + ".training";
int nSrcId = factory.GetSourceID(strTrainSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.DataBatchFile1, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile2, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile3, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile4, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile5, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc);
m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID);
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 10000;
string strTestSrc = dataset_name + ".testing";
nSrcId = factory.GetSourceID(strTestSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.TestBatchFile, strTestSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
SourceDescriptor srcTest = factory.LoadSource(strTestSrc);
m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID);
DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
return(true);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
}
private bool loadFile(DatasetFactory factory, List <Tuple <byte[], int> > rgData, int nC, int nH, int nW, string strSourceName, string strExportPath)
{
if (strExportPath != null)
{
strExportPath += strSourceName;
if (!Directory.Exists(strExportPath))
{
Directory.CreateDirectory(strExportPath);
}
}
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
try
{
if (factory != null)
{
int nSrcId = factory.AddSource(strSourceName, nC, nW, nH, false, 0, true);
factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log);
factory.DeleteSourceData();
}
// Storing to database;
int nLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nC, nW, nH, -1, DateTime.MinValue, new List <byte>(), 0, false, -1);
string strAction = (m_param.ExportToFile) ? "exporing" : "loading";
reportProgress(0, rgData.Count, " " + strAction + " a total of " + rgData.Count.ToString() + " items.");
reportProgress(0, rgData.Count, " (with rows: " + nH.ToString() + ", cols: " + nW.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
FileStream fsFileDesc = null;
StreamWriter swFileDesc = null;
if (m_param.ExportToFile)
{
string strFile = strExportPath + "\\file_list.txt";
fsFileDesc = File.OpenWrite(strFile);
swFileDesc = new StreamWriter(fsFileDesc);
}
for (int i = 0; i < rgData.Count; i++)
{
rgPixels = rgData[i].Item1;
nLabel = rgData[i].Item2;
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, rgData.Count, " " + strAction + " data...");
sw.Restart();
}
datum.SetData(rgPixels, nLabel);
if (factory != null)
{
factory.PutRawImageCache(i, datum, 5);
}
else if (strExportPath != null)
{
saveToFile(strExportPath, i, datum, swFileDesc);
}
rgImg.Add(new SimpleDatum(datum));
if (m_evtCancel.WaitOne(0))
{
return(false);
}
}
if (swFileDesc != null)
{
swFileDesc.Flush();
swFileDesc.Close();
swFileDesc.Dispose();
fsFileDesc.Close();
fsFileDesc.Dispose();
}
if (factory != null)
{
factory.ClearImageCache(true);
factory.UpdateSourceCounts();
factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
}
reportProgress(rgData.Count, rgData.Count, " " + strAction + " completed.");
}
finally
{
}
return(true);
}
private bool loadFile(DatasetFactory factory, string strImagesFile, string strLabelsFile, string strSourceName, string strExportPath)
{
if (strExportPath != null)
{
strExportPath += strSourceName;
if (!Directory.Exists(strExportPath))
{
Directory.CreateDirectory(strExportPath);
}
}
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
reportProgress(0, 0, " loading " + strImagesFile + "...");
BinaryFile image_file = new BinaryFile(strImagesFile);
BinaryFile label_file = new BinaryFile(strLabelsFile);
try
{
// Verify the files
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
throw new Exception("Incorrect image file magic.");
}
if (magicLbl != 2049)
{
throw new Exception("Incorrect label file magic.");
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
throw new Exception("The number of items must be equal to the number of labels!");
}
// Add the data source to the database.
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
int nChannels = 1; // black and white
if (factory != null)
{
int nSrcId = factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true);
factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log);
factory.DeleteSourceData();
}
// Storing to database;
byte[] rgLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, new List <byte>(), 0, false, -1);
string strAction = (m_param.ExportToFile) ? "exporing" : "loading";
reportProgress(0, (int)num_items, " " + strAction + " a total of " + num_items.ToString() + " items.");
reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
FileStream fsFileDesc = null;
StreamWriter swFileDesc = null;
if (m_param.ExportToFile)
{
string strFile = strExportPath + "\\file_list.txt";
fsFileDesc = File.OpenWrite(strFile);
swFileDesc = new StreamWriter(fsFileDesc);
}
for (int i = 0; i < num_items; i++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, (int)num_items, " " + strAction + " data...");
sw.Restart();
}
datum.SetData(rgPixels.ToList(), (int)rgLabel[0]);
if (factory != null)
{
factory.PutRawImageCache(i, datum);
}
else if (strExportPath != null)
{
saveToFile(strExportPath, i, datum, swFileDesc);
}
rgImg.Add(new SimpleDatum(datum));
if (m_evtCancel.WaitOne(0))
{
return(false);
}
}
if (swFileDesc != null)
{
swFileDesc.Flush();
swFileDesc.Close();
swFileDesc.Dispose();
fsFileDesc.Close();
fsFileDesc.Dispose();
}
if (factory != null)
{
factory.ClearImageCashe(true);
factory.UpdateSourceCounts();
factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
}
reportProgress((int)num_items, (int)num_items, " " + strAction + " completed.");
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
return(true);
}
