private List <Tuple <byte[], int> > loadFile(string strImagesFile, string strLabelsFile, string strExportPath)
{
if (!Directory.Exists(strExportPath))
{
Directory.CreateDirectory(strExportPath);
}
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " loading " + strImagesFile + "...");
BinaryFile image_file = new BinaryFile(strImagesFile);
BinaryFile label_file = new BinaryFile(strLabelsFile);
List <Tuple <byte[], int> > rgData = new List <Tuple <byte[], int> >();
try
{
// Verify the files
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
throw new Exception("Incorrect image file magic.");
}
if (magicLbl != 2049)
{
throw new Exception("Incorrect label file magic.");
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
throw new Exception("The number of items must be equal to the number of labels!");
}
// Add the data source to the database.
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
m_nHeight = (int)rows;
m_nWidth = (int)cols;
// Storing to database;
byte[] rgLabel;
byte[] rgPixels;
string strAction = "loading";
reportProgress(0, (int)num_items, " " + strAction + " a total of " + num_items.ToString() + " items.");
reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")");
sw.Start();
for (int i = 0; i < num_items; i++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
rgData.Add(new Tuple <byte[], int>(rgPixels, rgLabel[0]));
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, (int)num_items, " " + strAction + " data...");
sw.Restart();
}
}
reportProgress((int)num_items, (int)num_items, " " + strAction + " completed.");
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
return(rgData);
}
private bool loadFile(DatasetFactory factory, string strImagesFile, string strLabelsFile, string strSourceName, string strExportPath)
{
if (strExportPath != null)
{
strExportPath += strSourceName;
if (!Directory.Exists(strExportPath))
{
Directory.CreateDirectory(strExportPath);
}
}
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
reportProgress(0, 0, " loading " + strImagesFile + "...");
BinaryFile image_file = new BinaryFile(strImagesFile);
BinaryFile label_file = new BinaryFile(strLabelsFile);
try
{
// Verify the files
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
throw new Exception("Incorrect image file magic.");
}
if (magicLbl != 2049)
{
throw new Exception("Incorrect label file magic.");
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
throw new Exception("The number of items must be equal to the number of labels!");
}
// Add the data source to the database.
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
int nChannels = 1; // black and white
if (factory != null)
{
int nSrcId = factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true);
factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log);
factory.DeleteSourceData();
}
// Storing to database;
byte[] rgLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, new List <byte>(), 0, false, -1);
string strAction = (m_param.ExportToFile) ? "exporing" : "loading";
reportProgress(0, (int)num_items, " " + strAction + " a total of " + num_items.ToString() + " items.");
reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
FileStream fsFileDesc = null;
StreamWriter swFileDesc = null;
if (m_param.ExportToFile)
{
string strFile = strExportPath + "\\file_list.txt";
fsFileDesc = File.OpenWrite(strFile);
swFileDesc = new StreamWriter(fsFileDesc);
}
for (int i = 0; i < num_items; i++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, (int)num_items, " " + strAction + " data...");
sw.Restart();
}
datum.SetData(rgPixels.ToList(), (int)rgLabel[0]);
if (factory != null)
{
factory.PutRawImageCache(i, datum);
}
else if (strExportPath != null)
{
saveToFile(strExportPath, i, datum, swFileDesc);
}
rgImg.Add(new SimpleDatum(datum));
if (m_evtCancel.WaitOne(0))
{
return(false);
}
}
if (swFileDesc != null)
{
swFileDesc.Flush();
swFileDesc.Close();
swFileDesc.Dispose();
fsFileDesc.Close();
fsFileDesc.Dispose();
}
if (factory != null)
{
factory.ClearImageCashe(true);
factory.UpdateSourceCounts();
factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
}
reportProgress((int)num_items, (int)num_items, " " + strAction + " completed.");
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
return(true);
}