/// <summary> /// The ImageSet constructor. /// </summary> /// <param name="factory">Specifies the DatasetFactory.</param> /// <param name="src">Specifies the data source.</param> /// <param name="loadMethod">Specifies the method to use when loading the images.</param> /// <param name="nLoadLimit">Specifies the image load limit.</param> public ImageSet(DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { m_factory = new DatasetFactory(factory); m_factory.Open(src.ID); m_loadMethod = loadMethod; m_nLoadLimit = nLoadLimit; m_src = new SourceDescriptor(src); m_imgMean = null; m_rgImages = new SimpleDatum[m_src.ImageCount]; foreach (LabelDescriptor label in src.Labels) { if (label.ImageCount > 0) { m_rgLabelSet.Add(new LabelSet(label)); } } }
public void TestLoadSecondaryDataset(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { Log log = new Log("Test secondary dataset"); log.EnableTrace = true; IXImageDatabase db = new MyCaffeImageDatabase(log); Stopwatch sw = new Stopwatch(); string strDs = "MNIST"; string strDs2 = "CIFAR-10"; string str; SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; sw.Start(); db.InitializeWithDsName(settings, strDs); str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); sw.Restart(); db.LoadDatasetByName(strDs2); str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs2 + " Initialization Time: " + str + " ms."); sw.Restart(); db.CleanUp(); str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine("Cleanup Time: " + str + " ms."); sw.Restart(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine("Dispose Time: " + str + " ms."); }
/// <summary> /// The SettingsCaffe constructor used during deserialization. /// </summary> /// <param name="info">Specifies the serialization information.</param> /// <param name="context">Specifies the serialization context.</param> public SettingsCaffe(SerializationInfo info, StreamingContext context) { m_bEnableLabelBalancing = getBool(info, "bEnableLabelBalancing", m_bEnableLabelBalancing); m_bEnableLabelBoosting = getBool(info, "bEnableLabelBoosting", m_bEnableLabelBoosting); m_bEnableRandomInputSelection = getBool(info, "bEnableRandomInputSelection", m_bEnableRandomInputSelection); m_bEnablePairInputSelection = getBool(info, "bEnablePairInputSelection", m_bEnablePairInputSelection); m_bUseTrainingSourceForTesting = getBool(info, "bUseTrainingSourceForTesting", m_bUseTrainingSourceForTesting); m_dfSuperBoostProbability = getDouble(info, "dfSuperBoostProbability", m_dfSuperBoostProbability); m_nMaximumIterationOverride = getInt(info, "nMaximumIterationOverride", m_nMaximumIterationOverride); m_nTestingIterationOverride = getInt(info, "nTestingIterationOverride", m_nTestingIterationOverride); m_strDefaultModelGroup = info.GetString("strDefaultModelGroup"); m_strGpuIds = getString(info, "strGpuIds", m_strGpuIds); m_nMaskAllButLastColumns = getInt(info, "nMaskAllButLastColumns", m_nMaskAllButLastColumns); m_imageDbLoadMethod = (IMAGEDB_LOAD_METHOD)getInt(info, "ImageDbLoadMethod", (int)m_imageDbLoadMethod); m_nImageDbLoadLimit = getInt(info, "ImageDbLoadLimit", m_nImageDbLoadLimit); m_bImageDbLoadDataCriteria = getBool(info, "ImageDbLoadDataCriteria", m_bImageDbLoadDataCriteria); m_bImageDbLoadDebugData = getBool(info, "ImageDbLoadDebugData", m_bImageDbLoadDebugData); m_snapshotWeightUpdateMethod = (SNAPSHOT_WEIGHT_UPDATE_METHOD)getInt(info, "SnapshotWeightUpdateMethod", (int)m_snapshotWeightUpdateMethod); m_snapshotLoadMethod = (SNAPSHOT_LOAD_METHOD)getInt(info, "SnapshotLoadMethod", (int)m_snapshotLoadMethod); }
/// <summary> /// The SettingsCaffe constructor that copies another SettingsCaffe instance. /// </summary> public SettingsCaffe(SettingsCaffe s) { m_nMaskAllButLastColumns = s.m_nMaskAllButLastColumns; m_bEnableLabelBalancing = s.m_bEnableLabelBalancing; m_bEnableLabelBoosting = s.m_bEnableLabelBoosting; m_bEnableRandomInputSelection = s.m_bEnableRandomInputSelection; m_bEnablePairInputSelection = s.m_bEnablePairInputSelection; m_bUseTrainingSourceForTesting = s.m_bUseTrainingSourceForTesting; m_dfSuperBoostProbability = s.m_dfSuperBoostProbability; m_nMaximumIterationOverride = s.m_nMaximumIterationOverride; m_nTestingIterationOverride = s.m_nTestingIterationOverride; m_strDefaultModelGroup = s.m_strDefaultModelGroup; m_strGpuIds = s.m_strGpuIds; m_imageDbLoadMethod = s.m_imageDbLoadMethod; m_nImageDbLoadLimit = s.m_nImageDbLoadLimit; m_bImageDbLoadDataCriteria = s.m_bImageDbLoadDataCriteria; m_bImageDbLoadDebugData = s.m_bImageDbLoadDebugData; m_snapshotWeightUpdateMethod = s.m_snapshotWeightUpdateMethod; m_snapshotLoadMethod = s.m_snapshotLoadMethod; }
/// <summary> /// The SettingsCaffe constructor that copies another SettingsCaffe instance. /// </summary> public SettingsCaffe(SettingsCaffe s) { m_imgDbVersion = s.m_imgDbVersion; m_bEnableLabelBalancing = s.m_bEnableLabelBalancing; m_bEnableLabelBoosting = s.m_bEnableLabelBoosting; m_bEnableRandomInputSelection = s.m_bEnableRandomInputSelection; m_bEnablePairInputSelection = s.m_bEnablePairInputSelection; m_bUseTrainingSourceForTesting = s.m_bUseTrainingSourceForTesting; m_dfSuperBoostProbability = s.m_dfSuperBoostProbability; m_nMaximumIterationOverride = s.m_nMaximumIterationOverride; m_nTestingIterationOverride = s.m_nTestingIterationOverride; m_strDefaultModelGroup = s.m_strDefaultModelGroup; m_strGpuIds = s.m_strGpuIds; m_imageDbLoadMethod = s.m_imageDbLoadMethod; m_nImageDbLoadLimit = s.m_nImageDbLoadLimit; m_nAutoRefreshScheduledUpdateInMs = s.m_nAutoRefreshScheduledUpdateInMs; m_dfAutoRefreshScheduledReplacementPct = s.m_dfAutoRefreshScheduledReplacementPct; m_bImageDbLoadDataCriteria = s.m_bImageDbLoadDataCriteria; m_bImageDbLoadDebugData = s.m_bImageDbLoadDebugData; m_snapshotWeightUpdateMethod = s.m_snapshotWeightUpdateMethod; m_snapshotLoadMethod = s.m_snapshotLoadMethod; m_bSkipMeanCheck = s.m_bSkipMeanCheck; }
/// <summary> /// The SettingsCaffe constructor used during deserialization. /// </summary> /// <param name="info">Specifies the serialization information.</param> /// <param name="context">Specifies the serialization context.</param> public SettingsCaffe(SerializationInfo info, StreamingContext context) { m_imgDbVersion = (IMGDB_VERSION)getInt(info, "ImageDbVersion", (int)m_imgDbVersion); m_bEnableLabelBalancing = getBool(info, "bEnableLabelBalancing", m_bEnableLabelBalancing); m_bEnableLabelBoosting = getBool(info, "bEnableLabelBoosting", m_bEnableLabelBoosting); m_bEnableRandomInputSelection = getBool(info, "bEnableRandomInputSelection", m_bEnableRandomInputSelection); m_bEnablePairInputSelection = getBool(info, "bEnablePairInputSelection", m_bEnablePairInputSelection); m_bUseTrainingSourceForTesting = getBool(info, "bUseTrainingSourceForTesting", m_bUseTrainingSourceForTesting); m_dfSuperBoostProbability = getDouble(info, "dfSuperBoostProbability", m_dfSuperBoostProbability); m_nMaximumIterationOverride = getInt(info, "nMaximumIterationOverride", m_nMaximumIterationOverride); m_nTestingIterationOverride = getInt(info, "nTestingIterationOverride", m_nTestingIterationOverride); m_strDefaultModelGroup = info.GetString("strDefaultModelGroup"); m_strGpuIds = getString(info, "strGpuIds", m_strGpuIds); m_imageDbLoadMethod = (IMAGEDB_LOAD_METHOD)getInt(info, "ImageDbLoadMethod", (int)m_imageDbLoadMethod); m_nImageDbLoadLimit = getInt(info, "ImageDbLoadLimit", m_nImageDbLoadLimit); m_nAutoRefreshScheduledUpdateInMs = getInt(info, "ImageDbAutoRefreshScheduledUpdateInMs", m_nAutoRefreshScheduledUpdateInMs); m_dfAutoRefreshScheduledReplacementPct = getDouble(info, "ImageDbAutoRefreshReplacementPct", m_dfAutoRefreshScheduledReplacementPct); m_bImageDbLoadDataCriteria = getBool(info, "ImageDbLoadDataCriteria", m_bImageDbLoadDataCriteria); m_bImageDbLoadDebugData = getBool(info, "ImageDbLoadDebugData", m_bImageDbLoadDebugData); m_snapshotWeightUpdateMethod = (SNAPSHOT_WEIGHT_UPDATE_METHOD)getInt(info, "SnapshotWeightUpdateMethod", (int)m_snapshotWeightUpdateMethod); m_snapshotLoadMethod = (SNAPSHOT_LOAD_METHOD)getInt(info, "SnapshotLoadMethod", (int)m_snapshotLoadMethod); m_bSkipMeanCheck = getBool(info, "SkipMeanCheck", m_bSkipMeanCheck); }
/// <summary> /// Initialize the DatasetEx by loading the training and testing data sources into memory. /// </summary> /// <param name="ds">Specifies the dataset to load.</param> /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param> /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param> /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param> /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param> /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param> /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param> /// <returns></returns> public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0) { lock (m_syncObj) { if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0) { throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method."); } SimpleDatum imgMean = null; if (ds != null) { m_ds = ds; } m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false); if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount) { m_nLastTrainingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false); if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount) { m_nLastTestingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } return(true); } }
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false) { try { RawImageMean imgMeanRaw = null; m_factory.Open(src); nLastImageIdx = nImageDbLoadLimitStartIdx; if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL) { if (imgMean == null) { imgMeanRaw = m_factory.GetRawImageMean(); if (imgMeanRaw == null) { if (log != null) { log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString()); } loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL; } } } int nCount = src.ImageCount; if (nCount == 0) { throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'."); } if (log != null) { log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images."); } ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit); if (OnCalculateImageMean != null) { imgset.OnCalculateImageMean += OnCalculateImageMean; } if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { bool bDataIsReal = src.IsRealData; int nBatchSize = 20000; Stopwatch sw = new Stopwatch(); int nImageSize = src.ImageHeight * src.ImageWidth; if (nImageSize > 60000) { nBatchSize = 5000; } else if (nBatchSize > 20000) { nBatchSize = 7500; } else if (nImageSize > 3000) { nBatchSize = 10000; } if (nImageDbLoadLimit <= 0) { nImageDbLoadLimit = nCount; } List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit); int nIdx = 0; sw.Start(); while (nIdx < rgIdx.Count) { int nImageIdx = rgIdx[nIdx]; int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize); List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount); for (int j = 0; j < rgImg.Count; j++) { SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH); imgset.Add(nIdx + j, sd1); if (sw.Elapsed.TotalMilliseconds > 1000) { if (log != null) { double dfPct = (double)(nIdx + j) / (double)nCount; log.Progress = dfPct; log.WriteLine("image loading at " + dfPct.ToString("P") + "..."); } sw.Restart(); if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(null); } } } nIdx += rgImg.Count; if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount) { log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset."); break; } } if (log != null) { log.Progress = 0; } if (rgIdx.Count > 0) { nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1; } } else if (bLoadNext) { nLastImageIdx += nImageDbLoadLimit; } if (imgMean == null) { if (imgMeanRaw == null) { imgMeanRaw = m_factory.GetRawImageMean(); } if (imgMeanRaw != null) { imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH); } else { if (log != null) { log.WriteLine("Calculating mean..."); } imgMean = imgset.GetImageMean(log, rgAbort); m_factory.PutRawImageMean(imgMean, true); } } if (imgMean != null) { imgset.SetImageMean(imgMean); } imgset.CompleteLoad(nLastImageIdx); return(imgset); } finally { m_factory.Close(); } }
/// <summary> /// Initialize the DatasetEx by loading the training and testing data sources into memory. /// </summary> /// <param name="ds">Specifies the dataset to load.</param> /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param> /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param> /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param> /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param> /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param> /// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param> /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param> /// <param name="nImageDbAutoRefreshScheduledUpdateInMs">Optionally, specifies the scheduled refresh update period in ms (default = 0).</param> /// <param name="dfImageDbAutoRefreshScheduledReplacementPct">Optionally, specifies the scheduled refresh replacement percent (default = 0).</param> /// <returns>Upon loading the dataset a handle to the default QueryState is returned, or 0 on cancel.</returns> public long Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, bool bSkipMeanCheck = false, int nImageDbLoadLimit = 0, int nImageDbAutoRefreshScheduledUpdateInMs = 0, double dfImageDbAutoRefreshScheduledReplacementPct = 0) { lock (m_syncObj) { m_log = log; if (ds != null) { m_ds = ds; } if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1) { log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped."); bSkipMeanCheck = true; } bool bSilentLoad = (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false; m_TrainingImages = new ImageSet2(ImageSet2.TYPE.TRAIN, log, m_factory, m_ds.TrainingSource, loadMethod, m_random, rgAbort); m_TrainingImages.OnCalculateImageMean += OnCalculateImageMean; QueryState qsTraining = m_TrainingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit); SimpleDatum sdMean = null; if (!bSkipMeanCheck) { bool bQueryOnly = false; if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE) { bQueryOnly = true; } sdMean = m_TrainingImages.GetImageMean(log, rgAbort, bQueryOnly); } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(0); } m_TestingImages = new ImageSet2(ImageSet2.TYPE.TEST, log, m_factory, m_ds.TestingSource, loadMethod, m_random, rgAbort); m_TestingImages.OnCalculateImageMean += OnCalculateImageMean; QueryState qsTesting = m_TestingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit); if (!bSkipMeanCheck) { bool bSave = true; if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE) { bSave = false; } m_TestingImages.SetImageMean(sdMean, bSave); } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(0); } if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbAutoRefreshScheduledUpdateInMs > 0 && dfImageDbAutoRefreshScheduledReplacementPct > 0) { StartAutomaticRefreshSchedule(true, true, nImageDbAutoRefreshScheduledUpdateInMs, dfImageDbAutoRefreshScheduledReplacementPct); } m_lDefaultQueryState = m_queryStates.CreateNewState(qsTraining, qsTesting); return(m_lDefaultQueryState); } }
/// <summary> /// Get the image with a specific image index. /// </summary> /// <param name="nIdx">Specifies the image index.</param> /// <param name="bLoadDataCriteria">Specifies whether or not to load the data criteria along with the image.</param> /// <param name="bLoadDebugData">Specifies whether or not to load the debug data with the image.</param> /// <param name="loadMethod">Specifies the image loading method used.</param> /// <returns>If found, the image is returned.</returns> public SimpleDatum GetImage(int nIdx, bool bLoadDataCriteria, bool bLoadDebugData, IMAGEDB_LOAD_METHOD loadMethod) { SimpleDatum sd = m_rgImages[nIdx]; if (sd == null) { if (m_refreshManager != null) { while (nIdx > 0 && m_rgImages[nIdx] == null) { nIdx--; } sd = m_rgImages[nIdx]; if (sd == null) { throw new Exception("No images should be null when using LoadLimit loading!"); } } else { if (!m_evtRunning.WaitOne(0) && (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND && loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_NOCACHE)) { Load((loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false); } sd = directLoadImage(nIdx); if (sd == null) { throw new Exception("The image is still null yet should have loaded!"); } if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { m_rgImages[nIdx] = sd; } } } // Double check that the conditional data has loaded (if needed). if (bLoadDataCriteria || bLoadDebugData) { m_factory.LoadRawData(sd, bLoadDataCriteria, bLoadDebugData); } return(sd); }
public void TestLoadLimitNextSequential(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { DatasetFactory df = new DatasetFactory(); int nDs = df.GetDatasetID(strDs); if (nDs == 0) { throw new Exception("The dataset '" + strDs + "' does not exist - you need to load it."); } SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); Dictionary <int, List <SimpleDatum> > rgFirst = new Dictionary <int, List <SimpleDatum> >(); int nTotal = ds.TrainingSource.ImageCount; int nCount = 0; double dfTotalMs = 0; while (nCount < nTotal) { for (int i = 0; i < nLoadLimit; i++) { sw.Reset(); sw.Start(); SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d1.Index)) { rg.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rg[d1.Index].Add(d1); } if (nCount == 0) { if (!rgFirst.Keys.Contains(d1.Index)) { rgFirst.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rgFirst[d1.Index].Add(d1); } } } db.LoadNextSet(null); nCount += nLoadLimit; } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify that all items have been queried Assert.AreEqual(nTotal, rg.Count); Dictionary <int, List <SimpleDatum> > rgWrapAround = new Dictionary <int, List <SimpleDatum> >(); for (int i = 0; i < nLoadLimit; i++) { SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); if (!rgWrapAround.Keys.Contains(d1.Index)) { rgWrapAround.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rgWrapAround[d1.Index].Add(d1); } } // Verify that the reads wrap around to the start. Assert.AreEqual(rgWrapAround.Count, rgFirst.Count); List <KeyValuePair <int, List <SimpleDatum> > > rg1 = new List <KeyValuePair <int, List <SimpleDatum> > >(); List <KeyValuePair <int, List <SimpleDatum> > > rg2 = new List <KeyValuePair <int, List <SimpleDatum> > >(); foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgWrapAround) { rg1.Add(kv); } foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgFirst) { rg2.Add(kv); } for (int i = 0; i < rg1.Count; i++) { Assert.AreEqual(rg1[i].Key, rg2[i].Key); Assert.AreEqual(rg1[i].Value.Count, rg2[i].Value.Count); for (int j = 0; j < rg1[i].Value.Count; j++) { Assert.AreEqual(rg1[i].Value[j].Label, rg2[i].Value[j].Label); } } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQuerySequential4(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); int nCount = 100; double dfTotalMs = 0; List <int> rgIdx = new List <int>(); for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, i); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } rgIdx.Add(d.Index); } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify sequential selection. int nIdx = 0; rgIdx.Sort(); foreach (KeyValuePair <int, List <SimpleDatum> > kv in rg) { int nIdx1 = rgIdx[nIdx]; Assert.AreEqual(kv.Value.Count, (nLoadLimit == 0) ? 1 : nLoadLimit); Assert.AreEqual(rg[nIdx1][0].Index, (nLoadLimit == 0) ? nIdx1 : nIdx1 % nLoadLimit); nIdx++; } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQuerySequential(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { DatasetFactory df = new DatasetFactory(); int nDs = df.GetDatasetID(strDs); if (nDs == 0) { throw new Exception("The dataset '" + strDs + "' does not exist - you need to load it."); } SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); int nCount = 100; double dfTotalMs = 0; for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify sequential selection, so all indexes should be the same. foreach (KeyValuePair <int, List <SimpleDatum> > kv in rg) { Assert.AreEqual(kv.Value.Count, nCount); } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQueryRandom2(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; foreach (string strDs in rgDs) { IXImageDatabase db = new MyCaffeImageDatabase(); SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); Dictionary <int, int> rgCounts = new Dictionary <int, int>(); int nCount = 10000; double dfTotalMs = 0; int nCount1 = 0; double dfTotalMs1 = 0; Stopwatch swTimer = new Stopwatch(); swTimer.Start(); // Randomly query each image and count up the number if times a given label is hit. for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.RANDOM); sw.Stop(); dfTotalMs += sw.ElapsedMilliseconds; dfTotalMs1 += sw.ElapsedMilliseconds; nCount1++; if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } if (!rgCounts.Keys.Contains(d.Label)) { rgCounts.Add(d.Label, 1); } else { rgCounts[d.Label]++; } if (swTimer.Elapsed.TotalMilliseconds > 2000) { double dfPct = (double)i / (double)nCount; Trace.WriteLine("(" + dfPct.ToString("P") + ") ave time = " + (dfTotalMs1 / nCount1).ToString("N3") + " ms."); dfTotalMs1 = 0; nCount1 = 0; swTimer.Restart(); } } // Total the label counts and calculate the average and stddev. List <KeyValuePair <int, int> > rgCountsNoLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList(); Trace.WriteLine("NO LABEL BALANCING COUNTS"); CalculationArray ca = new CalculationArray(); foreach (KeyValuePair <int, int> kv in rgCountsNoLabelBalancing) { ca.Add(kv.Value); Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0")); } double dfAve = ca.Average; double dfStdDev1 = ca.CalculateStandardDeviation(dfAve); Trace.WriteLine("Average = " + dfAve.ToString()); Trace.WriteLine("StdDev = " + dfStdDev1.ToString()); // Load the labels by first selecting the label randomly and then the image randomly from the label set. rg = new Dictionary <int, List <SimpleDatum> >(); rgCounts = new Dictionary <int, int>(); for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.RANDOM, IMGDB_IMAGE_SELECTION_METHOD.RANDOM); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } if (!rgCounts.Keys.Contains(d.Label)) { rgCounts.Add(d.Label, 1); } else { rgCounts[d.Label]++; } } // Total the balanced label counts and calculate the average and stddev. List <KeyValuePair <int, int> > rgCountsLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList(); Trace.WriteLine("LABEL BALANCING COUNTS"); ca = new CalculationArray(); foreach (KeyValuePair <int, int> kv in rgCountsLabelBalancing) { ca.Add(kv.Value); Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0")); } dfAve = ca.Average; double dfStdDev2 = ca.CalculateStandardDeviation(dfAve); Trace.WriteLine("Average = " + dfAve.ToString()); Trace.WriteLine("StdDev = " + dfStdDev2.ToString()); Assert.AreEqual(true, dfStdDev2 < dfStdDev1 * 1.5); str = (dfTotalMs / (double)(nCount * 2)).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } } }
/// <summary> /// Initialize the DatasetEx by loading the training and testing data sources into memory. /// </summary> /// <param name="ds">Specifies the dataset to load.</param> /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param> /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param> /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param> /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param> /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param> /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param> /// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param> /// <returns>Upon loading the dataset <i>true</i> is returned, otherwise on failure or abort <i>false</i> is returned.</returns> public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, bool bSkipMeanCheck = false) { lock (m_syncObj) { if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0) { throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method."); } SimpleDatum imgMean = null; if (ds != null) { m_ds = ds; } if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1) { log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped."); bSkipMeanCheck = true; } m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false, bSkipMeanCheck); if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount) { m_nLastTrainingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false, bSkipMeanCheck); if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount) { m_nLastTestingImageIdx = 0; } if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(false); } return(true); } }
/// <summary> /// The ImageSet2 constructor. /// </summary> /// <param name="type">Specifies the type of data source managed.</param> /// <param name="log">Specifies the output log.</param> /// <param name="factory">Specifies the data factory used to access the database data.</param> /// <param name="src">Specifies the data source descriptor.</param> /// <param name="loadMethod">Specifies the load method used to load the data.</param> /// <param name="random">Specifies the random number generator.</param> /// <param name="rgAbort">Specifies the cancellation handles.</param> public ImageSet2(TYPE type, Log log, DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, CryptoRandom random, WaitHandle[] rgAbort) : base(factory, src) { m_type = type; m_log = log; m_loadMethod = loadMethod; m_random = random; m_rgAbort.Add(m_evtCancel); if (rgAbort.Length > 0) { m_rgAbort.AddRange(rgAbort); } }
public void TestQueryPair(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); int nCount = 100; double dfTotalMs = 0; for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); SimpleDatum d2 = db.QueryImage(ds.TrainingSource.ID, i, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.PAIR); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d1.Index)) { rg.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rg[d1.Index].Add(d1); } if (nLoadLimit > 0) { Assert.AreEqual(true, d1.Index == d2.Index - 1 || d1.Index == nLoadLimit - 1 && d2.Index == 0); } else { Assert.AreEqual(d1.Index, d2.Index - 1); } } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify that all labels are hit. if (nLoadLimit > 0) { Assert.AreEqual(rg.Count, nLoadLimit); } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }