public void TestPutRawImage(bool bSaveImagesToFile) { DatasetFactory factory = new DatasetFactory(); factory.DeleteSources("Test123"); int nSrcId = factory.AddSource("Test123", 1, 10, 10, false, 0, bSaveImagesToFile); factory.Open(nSrcId, 10); byte[] rgBytes = new byte[10 * 10]; for (int i = 0; i < 20; i++) { rgBytes[i] = (byte)i; SimpleDatum sd = new SimpleDatum(false, 1, 10, 10, i, DateTime.MinValue, rgBytes.ToList(), null, 0, false, i); factory.PutRawImageCache(i, sd); } factory.ClearImageCash(true); List <RawImage> rgImg = factory.GetRawImagesAt(0, 20); for (int i = 0; i < rgImg.Count; i++) { SimpleDatum sd = factory.LoadDatum(rgImg[i]); bool bEncoded = false; byte[] rgData = sd.GetByteData(out bEncoded); for (int j = 0; j < 100; j++) { if (j <= i) { Assert.AreEqual(rgData[j], j); } else { Assert.AreEqual(rgData[j], 0); } } } factory.DeleteSources("Test123"); factory.Close(); }
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false) { try { RawImageMean imgMeanRaw = null; m_factory.Open(src); nLastImageIdx = nImageDbLoadLimitStartIdx; if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL) { if (imgMean == null) { imgMeanRaw = m_factory.GetRawImageMean(); if (imgMeanRaw == null) { if (log != null) { log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString()); } loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL; } } } int nCount = src.ImageCount; if (nCount == 0) { throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'."); } if (log != null) { log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images."); } ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit); if (OnCalculateImageMean != null) { imgset.OnCalculateImageMean += OnCalculateImageMean; } if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { bool bDataIsReal = src.IsRealData; int nBatchSize = 20000; Stopwatch sw = new Stopwatch(); int nImageSize = src.ImageHeight * src.ImageWidth; if (nImageSize > 60000) { nBatchSize = 5000; } else if (nBatchSize > 20000) { nBatchSize = 7500; } else if (nImageSize > 3000) { nBatchSize = 10000; } if (nImageDbLoadLimit <= 0) { nImageDbLoadLimit = nCount; } List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit); int nIdx = 0; sw.Start(); while (nIdx < rgIdx.Count) { int nImageIdx = rgIdx[nIdx]; int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize); List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount); for (int j = 0; j < rgImg.Count; j++) { SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH); imgset.Add(nIdx + j, sd1); if (sw.Elapsed.TotalMilliseconds > 1000) { if (log != null) { double dfPct = (double)(nIdx + j) / (double)nCount; log.Progress = dfPct; log.WriteLine("image loading at " + dfPct.ToString("P") + "..."); } sw.Restart(); if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(null); } } } nIdx += rgImg.Count; if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount) { log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset."); break; } } if (log != null) { log.Progress = 0; } if (rgIdx.Count > 0) { nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1; } } else if (bLoadNext) { nLastImageIdx += nImageDbLoadLimit; } if (imgMean == null) { if (imgMeanRaw == null) { imgMeanRaw = m_factory.GetRawImageMean(); } if (imgMeanRaw != null) { imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH); } else { if (log != null) { log.WriteLine("Calculating mean..."); } imgMean = imgset.GetImageMean(log, rgAbort); m_factory.PutRawImageMean(imgMean, true); } } if (imgMean != null) { imgset.SetImageMean(imgMean); } imgset.CompleteLoad(nLastImageIdx); return(imgset); } finally { m_factory.Close(); } }
/// <summary> /// The dataLoadThread is responsible for loading the data source images in the background. /// </summary> private void dataLoadThread() { m_evtRunning.Set(); DatasetFactory factory = new DatasetFactory(m_factory); int? nNextIdx = m_loadSequence.GetNext(); Stopwatch sw = new Stopwatch(); if (m_refreshManager != null) { m_refreshManager.Reset(); } try { sw.Start(); List <int> rgIdxBatch = new List <int>(); int nBatchSize = getBatchSize(m_src); if (m_nLoadedCount > 0) { throw new Exception("The loaded count is > 0!"); } factory.Open(m_src); m_log.WriteLine(m_src.Name + " loading " + m_loadSequence.Count.ToString("N0") + " items..."); while (nNextIdx.HasValue || rgIdxBatch.Count > 0) { if (nNextIdx.HasValue) { rgIdxBatch.Add(nNextIdx.Value); } if (rgIdxBatch.Count >= nBatchSize || !nNextIdx.HasValue) { List <RawImage> rgImg; if (m_refreshManager == null) { rgImg = factory.GetRawImagesAt(rgIdxBatch[0], rgIdxBatch.Count); } else { rgImg = factory.GetRawImagesAt(rgIdxBatch, m_evtCancel); } if (rgImg == null) { break; } for (int j = 0; j < rgImg.Count; j++) { SimpleDatum sd = factory.LoadDatum(rgImg[j]); if (m_refreshManager != null) { m_refreshManager.AddLoaded(sd); } m_rgImages[m_nLoadedCount] = sd; m_nLoadedCount++; if (sw.Elapsed.TotalMilliseconds > 1000) { if (m_log != null && !m_bSilent) { double dfPct = m_nLoadedCount / (double)m_rgImages.Length; m_log.Progress = dfPct; m_log.WriteLine("Loading '" + m_src.Name + "' at " + dfPct.ToString("P") + " (" + m_nLoadedCount.ToString("N0") + " of " + m_rgImages.Length.ToString("N0") + ")..."); } int nWait = WaitHandle.WaitAny(m_rgAbort.ToArray(), 0); if (nWait != WaitHandle.WaitTimeout) { return; } sw.Restart(); } } rgIdxBatch = new List <int>(); } nNextIdx = m_loadSequence.GetNext(); } if (rgIdxBatch.Count > 0) { m_log.FAIL("Not all images were loaded!"); } } finally { factory.Close(); factory.Dispose(); m_evtRunning.Reset(); m_evtDone.Set(); } }