public void TestPutRawImage(bool bSaveImagesToFile)
{
DatasetFactory factory = new DatasetFactory();
factory.DeleteSources("Test123");
int nSrcId = factory.AddSource("Test123", 1, 10, 10, false, 0, bSaveImagesToFile);
factory.Open(nSrcId, 10);
byte[] rgBytes = new byte[10 * 10];
for (int i = 0; i < 20; i++)
{
rgBytes[i] = (byte)i;
SimpleDatum sd = new SimpleDatum(false, 1, 10, 10, i, DateTime.MinValue, rgBytes.ToList(), null, 0, false, i);
factory.PutRawImageCache(i, sd);
}
factory.ClearImageCash(true);
List <RawImage> rgImg = factory.GetRawImagesAt(0, 20);
for (int i = 0; i < rgImg.Count; i++)
{
SimpleDatum sd = factory.LoadDatum(rgImg[i]);
bool bEncoded = false;
byte[] rgData = sd.GetByteData(out bEncoded);
for (int j = 0; j < 100; j++)
{
if (j <= i)
{
Assert.AreEqual(rgData[j], j);
}
else
{
Assert.AreEqual(rgData[j], 0);
}
}
}
factory.DeleteSources("Test123");
factory.Close();
}
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false)
{
try
{
RawImageMean imgMeanRaw = null;
m_factory.Open(src);
nLastImageIdx = nImageDbLoadLimitStartIdx;
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL)
{
if (imgMean == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
if (imgMeanRaw == null)
{
if (log != null)
{
log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString());
}
loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
}
}
}
int nCount = src.ImageCount;
if (nCount == 0)
{
throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'.");
}
if (log != null)
{
log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images.");
}
ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit);
if (OnCalculateImageMean != null)
{
imgset.OnCalculateImageMean += OnCalculateImageMean;
}
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
bool bDataIsReal = src.IsRealData;
int nBatchSize = 20000;
Stopwatch sw = new Stopwatch();
int nImageSize = src.ImageHeight * src.ImageWidth;
if (nImageSize > 60000)
{
nBatchSize = 5000;
}
else if (nBatchSize > 20000)
{
nBatchSize = 7500;
}
else if (nImageSize > 3000)
{
nBatchSize = 10000;
}
if (nImageDbLoadLimit <= 0)
{
nImageDbLoadLimit = nCount;
}
List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit);
int nIdx = 0;
sw.Start();
while (nIdx < rgIdx.Count)
{
int nImageIdx = rgIdx[nIdx];
int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize);
List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount);
for (int j = 0; j < rgImg.Count; j++)
{
SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH);
imgset.Add(nIdx + j, sd1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
if (log != null)
{
double dfPct = (double)(nIdx + j) / (double)nCount;
log.Progress = dfPct;
log.WriteLine("image loading at " + dfPct.ToString("P") + "...");
}
sw.Restart();
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(null);
}
}
}
nIdx += rgImg.Count;
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount)
{
log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset.");
break;
}
}
if (log != null)
{
log.Progress = 0;
}
if (rgIdx.Count > 0)
{
nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1;
}
}
else if (bLoadNext)
{
nLastImageIdx += nImageDbLoadLimit;
}
if (imgMean == null)
{
if (imgMeanRaw == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
}
if (imgMeanRaw != null)
{
imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH);
}
else
{
if (log != null)
{
log.WriteLine("Calculating mean...");
}
imgMean = imgset.GetImageMean(log, rgAbort);
m_factory.PutRawImageMean(imgMean, true);
}
}
if (imgMean != null)
{
imgset.SetImageMean(imgMean);
}
imgset.CompleteLoad(nLastImageIdx);
return(imgset);
}
finally
{
m_factory.Close();
}
}
/// <summary>
/// The dataLoadThread is responsible for loading the data source images in the background.
/// </summary>
private void dataLoadThread()
{
m_evtRunning.Set();
DatasetFactory factory = new DatasetFactory(m_factory);
int? nNextIdx = m_loadSequence.GetNext();
Stopwatch sw = new Stopwatch();
if (m_refreshManager != null)
{
m_refreshManager.Reset();
}
try
{
sw.Start();
List <int> rgIdxBatch = new List <int>();
int nBatchSize = getBatchSize(m_src);
if (m_nLoadedCount > 0)
{
throw new Exception("The loaded count is > 0!");
}
factory.Open(m_src);
m_log.WriteLine(m_src.Name + " loading " + m_loadSequence.Count.ToString("N0") + " items...");
while (nNextIdx.HasValue || rgIdxBatch.Count > 0)
{
if (nNextIdx.HasValue)
{
rgIdxBatch.Add(nNextIdx.Value);
}
if (rgIdxBatch.Count >= nBatchSize || !nNextIdx.HasValue)
{
List <RawImage> rgImg;
if (m_refreshManager == null)
{
rgImg = factory.GetRawImagesAt(rgIdxBatch[0], rgIdxBatch.Count);
}
else
{
rgImg = factory.GetRawImagesAt(rgIdxBatch, m_evtCancel);
}
if (rgImg == null)
{
break;
}
for (int j = 0; j < rgImg.Count; j++)
{
SimpleDatum sd = factory.LoadDatum(rgImg[j]);
if (m_refreshManager != null)
{
m_refreshManager.AddLoaded(sd);
}
m_rgImages[m_nLoadedCount] = sd;
m_nLoadedCount++;
if (sw.Elapsed.TotalMilliseconds > 1000)
{
if (m_log != null && !m_bSilent)
{
double dfPct = m_nLoadedCount / (double)m_rgImages.Length;
m_log.Progress = dfPct;
m_log.WriteLine("Loading '" + m_src.Name + "' at " + dfPct.ToString("P") + " (" + m_nLoadedCount.ToString("N0") + " of " + m_rgImages.Length.ToString("N0") + ")...");
}
int nWait = WaitHandle.WaitAny(m_rgAbort.ToArray(), 0);
if (nWait != WaitHandle.WaitTimeout)
{
return;
}
sw.Restart();
}
}
rgIdxBatch = new List <int>();
}
nNextIdx = m_loadSequence.GetNext();
}
if (rgIdxBatch.Count > 0)
{
m_log.FAIL("Not all images were loaded!");
}
}
finally
{
factory.Close();
factory.Dispose();
m_evtRunning.Reset();
m_evtDone.Set();
}
}
