/// <summary>
/// Initialize the DatasetEx by loading the training and testing data sources into memory.
/// </summary>
/// <param name="ds">Specifies the dataset to load.</param>
/// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
/// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
/// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
/// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
/// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
/// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
/// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param>
/// <returns>Upon loading the dataset <i>true</i> is returned, otherwise on failure or abort <i>false</i> is returned.</returns>
public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, bool bSkipMeanCheck = false)
{
lock (m_syncObj)
{
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0)
{
throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method.");
}
SimpleDatum imgMean = null;
if (ds != null)
{
m_ds = ds;
}
if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1)
{
log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped.");
bSkipMeanCheck = true;
}
m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false, bSkipMeanCheck);
if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount)
{
m_nLastTrainingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false, bSkipMeanCheck);
if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount)
{
m_nLastTestingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
return(true);
}
}
/// <summary>
/// Initialize the DatasetEx by loading the training and testing data sources into memory.
/// </summary>
/// <param name="ds">Specifies the dataset to load.</param>
/// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
/// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
/// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
/// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
/// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
/// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
/// <returns></returns>
public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0)
{
lock (m_syncObj)
{
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0)
{
throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method.");
}
SimpleDatum imgMean = null;
if (ds != null)
{
m_ds = ds;
}
m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false);
if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount)
{
m_nLastTrainingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false);
if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount)
{
m_nLastTestingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
return(true);
}
}
/// <summary>
/// Releases all resources used.
/// </summary>
/// <param name="bDisposing">Set to <i>true</i> when called by Dispose().</param>
protected virtual void Dispose(bool bDisposing)
{
m_ds = null;
if (m_TestingImages != null)
{
m_TestingImages.Dispose();
m_TestingImages = null;
}
if (m_TrainingImages != null)
{
m_TrainingImages.Dispose();
m_TrainingImages = null;
}
if (m_factory != null)
{
m_factory.Dispose();
m_factory = null;
}
}
/// <summary>
/// Searches for the ImageSet with a given data source name.
/// </summary>
/// <param name="strSource">Specifies the name of the data source.</param>
/// <returns>If found, the ImageSet is returned, otherwise an Exception is thrown.</returns>
public ImageSet FindImageset(string strSource)
{
lock (m_syncObj)
{
if (m_lastImgSet != null && m_lastImgSet.SourceName == strSource)
{
return(m_lastImgSet);
}
foreach (DatasetEx ds in m_rgDatasets)
{
ImageSet imgSet = ds.Find(strSource);
if (imgSet != null)
{
m_lastImgSet = imgSet;
return(imgSet);
}
}
throw new Exception("Could not find source with Name = " + strSource + "!");
}
}
/// <summary>
/// Searches for the ImageSet with a given data source ID.
/// </summary>
/// <param name="nSourceID">Specifies the ID of the data source.</param>
/// <returns>If found, the ImageSet is returned, otherwise an Exception is thrown.</returns>
public ImageSet FindImageset(int nSourceID)
{
lock (m_syncObj)
{
if (m_lastImgSet != null && m_lastImgSet.SourceID == nSourceID)
{
return(m_lastImgSet);
}
foreach (DatasetEx ds in m_rgDatasets)
{
ImageSet imgSet = ds.Find(nSourceID);
if (imgSet != null)
{
m_lastImgSet = imgSet;
return(imgSet);
}
}
throw new Exception("Could not find source with ID = " + nSourceID.ToString() + "!");
}
}
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false)
{
try
{
RawImageMean imgMeanRaw = null;
m_factory.Open(src);
nLastImageIdx = nImageDbLoadLimitStartIdx;
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL)
{
if (imgMean == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
if (imgMeanRaw == null)
{
if (log != null)
{
log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString());
}
loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
}
}
}
int nCount = src.ImageCount;
if (nCount == 0)
{
if (log != null)
{
log.WriteLine("WARNING: Could not find any images with " + strType + " Source = '" + src.Name + "'. If this is a training dataset, you will need to enable the 'UseTrainingSrcForTesting' setting.");
}
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
if (log != null)
{
log.WriteLine("Because there are no images in this set, the image loading method has been changed to LOAD_ON_DEMAND for this dataset.");
}
loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND;
}
}
ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit);
if (log != null && nCount > 0)
{
log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images.");
}
if (OnCalculateImageMean != null)
{
imgset.OnCalculateImageMean += OnCalculateImageMean;
}
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
bool bDataIsReal = src.IsRealData;
int nBatchSize = 20000;
Stopwatch sw = new Stopwatch();
int nImageSize = src.ImageHeight * src.ImageWidth;
if (nImageSize > 60000)
{
nBatchSize = 5000;
}
else if (nBatchSize > 20000)
{
nBatchSize = 7500;
}
else if (nImageSize > 3000)
{
nBatchSize = 10000;
}
if (nImageDbLoadLimit <= 0)
{
nImageDbLoadLimit = nCount;
}
List <int> rgIdx = (nCount == 0) ? new List <int>() : getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit);
int nIdx = 0;
sw.Start();
while (nIdx < rgIdx.Count)
{
int nImageIdx = rgIdx[nIdx];
int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize);
List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount);
for (int j = 0; j < rgImg.Count; j++)
{
SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH);
imgset.Add(nIdx + j, sd1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
if (log != null)
{
double dfPct = (double)(nIdx + j) / (double)nCount;
log.Progress = dfPct;
log.WriteLine("image loading at " + dfPct.ToString("P") + "...");
}
sw.Restart();
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(null);
}
}
}
nIdx += rgImg.Count;
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount)
{
log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset.");
break;
}
}
if (log != null)
{
log.Progress = 0;
}
if (rgIdx.Count > 0)
{
nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1;
}
}
else if (bLoadNext)
{
nLastImageIdx += nImageDbLoadLimit;
}
if (imgMean == null)
{
if (imgMeanRaw == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
}
if (imgMeanRaw != null)
{
imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH);
}
else
{
if (log != null)
{
log.WriteLine("Calculating mean...");
}
imgMean = imgset.GetImageMean(log, rgAbort);
m_factory.PutRawImageMean(imgMean, true);
}
}
if (imgMean != null)
{
imgset.SetImageMean(imgMean);
}
imgset.CompleteLoad(nLastImageIdx);
return(imgset);
}
finally
{
m_factory.Close();
}
}
/// <summary>
/// Resets the last image set used to <i>null</i>, thus clearing it.
/// </summary>
public void Reset()
{
m_lastImgSet = null;
}
