bool loadFile(Log log, string strName, string strFile, DatasetFactory factory, int nTotal, bool bCreateMeanImage, ref int nIdx)
{
Stopwatch sw = new Stopwatch();
log.WriteLine("Loading '" + strName + "' into '" + factory.OpenSource.Name + "'...");
sw.Start();
FileStream fs = null;
try
{
fs = new FileStream(strFile, FileMode.Open, FileAccess.Read);
using (BinaryReader br = new BinaryReader(fs))
{
fs = null;
for (int i = 0; i < 10000; i++)
{
int nLabel = (int)br.ReadByte();
byte[] rgImgBytes = br.ReadBytes(3072);
Bitmap img = createImage(rgImgBytes);
if (m_bCancel)
{
return(false);
}
Datum d = ImageData.GetImageDataD(img, 3, false, nLabel);
factory.PutRawImageCache(nIdx, d);
nIdx++;
if (bCreateMeanImage)
{
m_rgImages.Add(new SimpleDatum(d));
}
if (sw.ElapsedMilliseconds > 1000)
{
sw.Stop();
log.Progress = (double)nIdx / (double)nTotal;
log.WriteLine("Processing " + nIdx.ToString() + " of " + nTotal.ToString());
sw.Restart();
}
}
factory.ClearImageCache(true);
}
}
finally
{
if (fs != null)
{
fs.Dispose();
}
}
return(true);
}
bool loadData(Log log, DatasetFactory factory, CsvParser parser, List <DataItem> rgData, List <int> rgDim, bool bTraining, bool bCreateImageMean)
{
Stopwatch sw = new Stopwatch();
log.WriteLine("Loading data into '" + factory.OpenSource.Name + "'...");
sw.Start();
m_rgImages.Clear();
int nSize = rgDim[1] / m_schema.CellSize;
for (int i = 0; i < rgData.Count; i++)
{
int nLabel = rgData[i].Label;
Bitmap bmp = createImage(rgData[i].Data, nSize, m_schema.CellSize, m_schema.BinaryValues);
if (m_bCancel)
{
return(false);
}
Datum d = ImageData.GetImageDataD(bmp, rgDim[0], false, nLabel);
factory.PutRawImageCache(i, d);
if (bCreateImageMean)
{
m_rgImages.Add(new SimpleDatum(d));
}
if (sw.ElapsedMilliseconds > 1000)
{
sw.Stop();
log.Progress = (double)i / (double)rgData.Count;
log.WriteLine("Processing " + i.ToString() + " of " + rgData.Count.ToString());
sw.Restart();
factory.ClearImageCache(true);
}
}
factory.ClearImageCache(true);
return(true);
}
private bool loadFile(string strImagesFile, string strSourceName, int nExtractTotal, ref int nExtractIdx, int nTotal, ref int nIdx, Log log, bool bExtractFiles, Dictionary <string, int> rgNameToLabel, List <Tuple <int, string, Size> > rgFileSizes)
{
Stopwatch sw = new Stopwatch();
reportProgress(nIdx, nTotal, " Source: " + strSourceName);
reportProgress(nIdx, nTotal, " loading " + strImagesFile + "...");
FileStream fs = null;
try
{
int nSrcId = m_factory.AddSource(strSourceName, 3, -1, -1, false);
addLabels(nSrcId, rgNameToLabel);
m_factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, log);
int nPos = strImagesFile.ToLower().LastIndexOf(".tar");
string strPath = strImagesFile.Substring(0, nPos);
if (!Directory.Exists(strPath))
{
Directory.CreateDirectory(strPath);
}
if (bExtractFiles)
{
log.Progress = (double)nIdx / nExtractTotal;
log.WriteLine("Extracting files from '" + strImagesFile + "'...");
if ((nExtractIdx = TarFile.ExtractTar(strImagesFile, strPath, m_evtCancel, log, nExtractTotal, nExtractIdx)) == 0)
{
log.WriteLine("Aborted.");
return(false);
}
}
// Load the annotations.
SimpleDatum.ANNOTATION_TYPE type = SimpleDatum.ANNOTATION_TYPE.BBOX;
int nResizeHeight = 0;
int nResizeWidth = 0;
// Create the training database images.
// Create the master list file.
List <Tuple <string, string> > rgFiles = createFileList(log, strPath);
sw.Start();
for (int i = 0; i < rgFiles.Count; i++)
{
SimpleDatum datum = loadDatum(log, rgFiles[i].Item1, rgFiles[i].Item2, nResizeHeight, nResizeWidth, type, rgNameToLabel);
m_factory.PutRawImageCache(nIdx, datum);
nIdx++;
if (m_evtCancel.WaitOne(0))
{
log.WriteLine("Aborted.");
return(false);
}
if (sw.Elapsed.TotalMilliseconds > 1000)
{
log.Progress = (double)nIdx / nTotal;
log.WriteLine("Loading file " + i.ToString() + " of " + rgFiles.Count.ToString() + "...");
sw.Restart();
}
rgFileSizes.Add(new Tuple <int, string, Size>(nIdx, rgFiles[i].Item1, new Size(datum.Width, datum.Height)));
}
m_factory.ClearImageCache(true);
m_factory.Close();
}
finally
{
if (fs != null)
{
fs.Dispose();
}
}
return(true);
}
private bool loadFile(string strImagesFile, string strSourceName, int nTotal, ref int nIdx, Log log)
{
Stopwatch sw = new Stopwatch();
int nStart = nIdx;
reportProgress(nIdx, nTotal, " Source: " + strSourceName);
reportProgress(nIdx, nTotal, " loading " + strImagesFile + "...");
FileStream fs = null;
try
{
fs = new FileStream(strImagesFile, FileMode.Open, FileAccess.Read);
using (BinaryReader br = new BinaryReader(fs))
{
fs = null;
int nSrcId = m_factory.AddSource(strSourceName, 3, 32, 32, false, 0, true);
m_factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, log);
if (nIdx == 0)
{
m_factory.DeleteSourceData();
}
sw.Start();
for (int i = 0; i < 10000; i++)
{
int nLabel = (int)br.ReadByte();
byte[] rgImgBytes = br.ReadBytes(3072);
Bitmap img = createImage(rgImgBytes);
Datum d = ImageData.GetImageDataD(img, 3, false, nLabel);
m_factory.PutRawImageCache(nIdx, d, 5);
m_rgImg.Add(new SimpleDatum(d));
nIdx++;
if (sw.ElapsedMilliseconds > 1000)
{
reportProgress(nStart + i, nTotal, "loading " + strImagesFile + " " + i.ToString("N0") + " of 10,000...");
sw.Restart();
}
if (m_evtCancel.WaitOne(0))
{
return(false);
}
}
m_factory.ClearImageCache(true);
if (nIdx == nTotal)
{
m_factory.UpdateSourceCounts();
}
}
}
finally
{
if (fs != null)
{
fs.Dispose();
}
}
return(true);
}
private bool loadFile(DatasetFactory factory, List <Tuple <byte[], int> > rgData, int nC, int nH, int nW, string strSourceName, string strExportPath)
{
if (strExportPath != null)
{
strExportPath += strSourceName;
if (!Directory.Exists(strExportPath))
{
Directory.CreateDirectory(strExportPath);
}
}
Stopwatch sw = new Stopwatch();
reportProgress(0, 0, " Source: " + strSourceName);
try
{
if (factory != null)
{
int nSrcId = factory.AddSource(strSourceName, nC, nW, nH, false, 0, true);
factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log);
factory.DeleteSourceData();
}
// Storing to database;
int nLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nC, nW, nH, -1, DateTime.MinValue, new List <byte>(), 0, false, -1);
string strAction = (m_param.ExportToFile) ? "exporing" : "loading";
reportProgress(0, rgData.Count, " " + strAction + " a total of " + rgData.Count.ToString() + " items.");
reportProgress(0, rgData.Count, " (with rows: " + nH.ToString() + ", cols: " + nW.ToString() + ")");
sw.Start();
List <SimpleDatum> rgImg = new List <SimpleDatum>();
FileStream fsFileDesc = null;
StreamWriter swFileDesc = null;
if (m_param.ExportToFile)
{
string strFile = strExportPath + "\\file_list.txt";
fsFileDesc = File.OpenWrite(strFile);
swFileDesc = new StreamWriter(fsFileDesc);
}
for (int i = 0; i < rgData.Count; i++)
{
rgPixels = rgData[i].Item1;
nLabel = rgData[i].Item2;
if (sw.Elapsed.TotalMilliseconds > 1000)
{
reportProgress(i, rgData.Count, " " + strAction + " data...");
sw.Restart();
}
datum.SetData(rgPixels, nLabel);
if (factory != null)
{
factory.PutRawImageCache(i, datum, 5);
}
else if (strExportPath != null)
{
saveToFile(strExportPath, i, datum, swFileDesc);
}
rgImg.Add(new SimpleDatum(datum));
if (m_evtCancel.WaitOne(0))
{
return(false);
}
}
if (swFileDesc != null)
{
swFileDesc.Flush();
swFileDesc.Close();
swFileDesc.Dispose();
fsFileDesc.Close();
fsFileDesc.Dispose();
}
if (factory != null)
{
factory.ClearImageCache(true);
factory.UpdateSourceCounts();
factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true);
}
reportProgress(rgData.Count, rgData.Count, " " + strAction + " completed.");
}
finally
{
}
return(true);
}
public uint ConvertData(string strImageFile, string strLabelFile, string strDBPath, string strDBPathMean, bool bCreateImgMean, bool bGetItemCountOnly = false, int nChannels = 1)
{
string strExt;
List <SimpleDatum> rgImg = new List <SimpleDatum>();
strExt = Path.GetExtension(strImageFile).ToLower();
if (strExt == ".gz")
{
m_log.WriteLine("Unpacking '" + strImageFile + "'...");
strImageFile = expandFile(strImageFile);
}
strExt = Path.GetExtension(strLabelFile).ToLower();
if (strExt == ".gz")
{
m_log.WriteLine("Unpacking '" + strLabelFile + "'...");
strLabelFile = expandFile(strLabelFile);
}
BinaryFile image_file = new BinaryFile(strImageFile);
BinaryFile label_file = new BinaryFile(strLabelFile);
try
{
uint magicImg = image_file.ReadUInt32();
uint magicLbl = label_file.ReadUInt32();
if (magicImg != 2051)
{
if (m_log != null)
{
m_log.FAIL("Incorrect image file magic.");
}
if (OnLoadError != null)
{
OnLoadError(this, new LoadErrorArgs("Incorrect image file magic."));
}
}
if (magicLbl != 2049)
{
if (m_log != null)
{
m_log.FAIL("Incorrect label file magic.");
}
if (OnLoadError != null)
{
OnLoadError(this, new LoadErrorArgs("Incorrect label file magic."));
}
}
uint num_items = image_file.ReadUInt32();
uint num_labels = label_file.ReadUInt32();
if (num_items != num_labels)
{
if (m_log != null)
{
m_log.FAIL("The number of items must equal the number of labels.");
}
throw new Exception("The number of items must equal the number of labels." + Environment.NewLine + " Label File: '" + strLabelFile + Environment.NewLine + " Image File: '" + strImageFile + "'.");
}
if (bGetItemCountOnly)
{
return(num_items);
}
uint rows = image_file.ReadUInt32();
uint cols = image_file.ReadUInt32();
int nSrcId = m_factory.AddSource(strDBPath, nChannels, (int)cols, (int)rows, false, 0, true);
m_factory.Open(nSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.
m_factory.DeleteSourceData();
// Storing to db
byte[] rgLabel;
byte[] rgPixels;
Datum datum = new Datum(false, nChannels, (int)cols, (int)rows);
if (m_log != null)
{
m_log.WriteHeader("LOADING " + strDBPath + " items.");
m_log.WriteLine("A total of " + num_items.ToString() + " items.");
m_log.WriteLine("Rows: " + rows.ToString() + " Cols: " + cols.ToString());
}
if (OnLoadStart != null)
{
OnLoadStart(this, new LoadStartArgs((int)num_items));
}
for (int item_id = 0; item_id < num_items; item_id++)
{
rgPixels = image_file.ReadBytes((int)(rows * cols));
rgLabel = label_file.ReadBytes(1);
List <byte> rgData = new List <byte>(rgPixels);
if (nChannels == 3)
{
rgData.AddRange(new List <byte>(rgPixels));
rgData.AddRange(new List <byte>(rgPixels));
}
datum.SetData(rgData, (int)rgLabel[0]);
if (m_bmpTargetOverlay != null)
{
datum = createTargetOverlay(datum);
}
m_factory.PutRawImageCache(item_id, datum);
if (bCreateImgMean)
{
rgImg.Add(new SimpleDatum(datum));
}
if ((item_id % 1000) == 0)
{
if (m_log != null)
{
m_log.WriteLine("Loaded " + item_id.ToString("N") + " items...");
m_log.Progress = (double)item_id / (double)num_items;
}
if (OnLoadProgress != null)
{
LoadArgs args = new LoadArgs(item_id);
OnLoadProgress(this, args);
if (args.Cancel)
{
break;
}
}
}
}
m_factory.ClearImageCache(true);
m_factory.UpdateSourceCounts();
if (bCreateImgMean)
{
if (strDBPath != strDBPathMean)
{
m_factory.CopyImageMean(strDBPathMean, strDBPath);
}
else
{
m_log.WriteLine("Creating image mean...");
SimpleDatum dMean = SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) });
m_factory.PutRawImageMean(dMean, true);
}
}
if (OnLoadProgress != null)
{
LoadArgs args = new LoadArgs((int)num_items);
OnLoadProgress(this, args);
}
return(num_items);
}
finally
{
image_file.Dispose();
label_file.Dispose();
}
}
