/// <summary> /// Initialize the DatasetEx by loading the training and testing data sources into memory. /// </summary> /// <param name="ds">Specifies the dataset to load.</param> /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param> /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param> /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param> /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param> /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param> /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param> /// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param> /// <returns>Upon loading the dataset <i>true</i> is returned, otherwise on failure or abort <i>false</i> is returned.</returns> public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, bool bSkipMeanCheck = false) { lock (m_syncObj) { if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0) { throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method."); } SimpleDatum imgMean = null; if (ds != null) { m_ds = ds; } if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1) { log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped."); bSkipMeanCheck = true; } m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false, bSkipMeanCheck); if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount) { m_nLastTrainingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false, bSkipMeanCheck); if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount) { m_nLastTestingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } return(true); } }
/// <summary> /// Initialize the DatasetEx by loading the training and testing data sources into memory. /// </summary> /// <param name="ds">Specifies the dataset to load.</param> /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param> /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param> /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param> /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param> /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param> /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param> /// <returns></returns> public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0) { lock (m_syncObj) { if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0) { throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method."); } SimpleDatum imgMean = null; if (ds != null) { m_ds = ds; } m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false); if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount) { m_nLastTrainingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false); if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount) { m_nLastTestingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } return(true); } }
/// <summary> /// Releases all resources used. /// </summary> /// <param name="bDisposing">Set to <i>true</i> when called by Dispose().</param> protected virtual void Dispose(bool bDisposing) { m_ds = null; if (m_TestingImages != null) { m_TestingImages.Dispose(); m_TestingImages = null; } if (m_TrainingImages != null) { m_TrainingImages.Dispose(); m_TrainingImages = null; } if (m_factory != null) { m_factory.Dispose(); m_factory = null; } }
/// <summary> /// Searches for the ImageSet with a given data source name. /// </summary> /// <param name="strSource">Specifies the name of the data source.</param> /// <returns>If found, the ImageSet is returned, otherwise an Exception is thrown.</returns> public ImageSet FindImageset(string strSource) { lock (m_syncObj) { if (m_lastImgSet != null && m_lastImgSet.SourceName == strSource) { return(m_lastImgSet); } foreach (DatasetEx ds in m_rgDatasets) { ImageSet imgSet = ds.Find(strSource); if (imgSet != null) { m_lastImgSet = imgSet; return(imgSet); } } throw new Exception("Could not find source with Name = " + strSource + "!"); } }
/// <summary> /// Searches for the ImageSet with a given data source ID. /// </summary> /// <param name="nSourceID">Specifies the ID of the data source.</param> /// <returns>If found, the ImageSet is returned, otherwise an Exception is thrown.</returns> public ImageSet FindImageset(int nSourceID) { lock (m_syncObj) { if (m_lastImgSet != null && m_lastImgSet.SourceID == nSourceID) { return(m_lastImgSet); } foreach (DatasetEx ds in m_rgDatasets) { ImageSet imgSet = ds.Find(nSourceID); if (imgSet != null) { m_lastImgSet = imgSet; return(imgSet); } } throw new Exception("Could not find source with ID = " + nSourceID.ToString() + "!"); } }
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false) { try { RawImageMean imgMeanRaw = null; m_factory.Open(src); nLastImageIdx = nImageDbLoadLimitStartIdx; if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL) { if (imgMean == null) { imgMeanRaw = m_factory.GetRawImageMean(); if (imgMeanRaw == null) { if (log != null) { log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString()); } loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL; } } } int nCount = src.ImageCount; if (nCount == 0) { if (log != null) { log.WriteLine("WARNING: Could not find any images with " + strType + " Source = '" + src.Name + "'. If this is a training dataset, you will need to enable the 'UseTrainingSrcForTesting' setting."); } if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { if (log != null) { log.WriteLine("Because there are no images in this set, the image loading method has been changed to LOAD_ON_DEMAND for this dataset."); } loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND; } } ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit); if (log != null && nCount > 0) { log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images."); } if (OnCalculateImageMean != null) { imgset.OnCalculateImageMean += OnCalculateImageMean; } if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { bool bDataIsReal = src.IsRealData; int nBatchSize = 20000; Stopwatch sw = new Stopwatch(); int nImageSize = src.ImageHeight * src.ImageWidth; if (nImageSize > 60000) { nBatchSize = 5000; } else if (nBatchSize > 20000) { nBatchSize = 7500; } else if (nImageSize > 3000) { nBatchSize = 10000; } if (nImageDbLoadLimit <= 0) { nImageDbLoadLimit = nCount; } List <int> rgIdx = (nCount == 0) ? new List <int>() : getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit); int nIdx = 0; sw.Start(); while (nIdx < rgIdx.Count) { int nImageIdx = rgIdx[nIdx]; int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize); List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount); for (int j = 0; j < rgImg.Count; j++) { SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH); imgset.Add(nIdx + j, sd1); if (sw.Elapsed.TotalMilliseconds > 1000) { if (log != null) { double dfPct = (double)(nIdx + j) / (double)nCount; log.Progress = dfPct; log.WriteLine("image loading at " + dfPct.ToString("P") + "..."); } sw.Restart(); if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(null); } } } nIdx += rgImg.Count; if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount) { log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset."); break; } } if (log != null) { log.Progress = 0; } if (rgIdx.Count > 0) { nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1; } } else if (bLoadNext) { nLastImageIdx += nImageDbLoadLimit; } if (imgMean == null) { if (imgMeanRaw == null) { imgMeanRaw = m_factory.GetRawImageMean(); } if (imgMeanRaw != null) { imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH); } else { if (log != null) { log.WriteLine("Calculating mean..."); } imgMean = imgset.GetImageMean(log, rgAbort); m_factory.PutRawImageMean(imgMean, true); } } if (imgMean != null) { imgset.SetImageMean(imgMean); } imgset.CompleteLoad(nLastImageIdx); return(imgset); } finally { m_factory.Close(); } }
/// <summary> /// Resets the last image set used to <i>null</i>, thus clearing it. /// </summary> public void Reset() { m_lastImgSet = null; }