public void TestLoadLimitNextSequential(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { DatasetFactory df = new DatasetFactory(); int nDs = df.GetDatasetID(strDs); if (nDs == 0) { throw new Exception("The dataset '" + strDs + "' does not exist - you need to load it."); } SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); Dictionary <int, List <SimpleDatum> > rgFirst = new Dictionary <int, List <SimpleDatum> >(); int nTotal = ds.TrainingSource.ImageCount; int nCount = 0; double dfTotalMs = 0; while (nCount < nTotal) { for (int i = 0; i < nLoadLimit; i++) { sw.Reset(); sw.Start(); SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d1.Index)) { rg.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rg[d1.Index].Add(d1); } if (nCount == 0) { if (!rgFirst.Keys.Contains(d1.Index)) { rgFirst.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rgFirst[d1.Index].Add(d1); } } } db.LoadNextSet(null); nCount += nLoadLimit; } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify that all items have been queried Assert.AreEqual(nTotal, rg.Count); Dictionary <int, List <SimpleDatum> > rgWrapAround = new Dictionary <int, List <SimpleDatum> >(); for (int i = 0; i < nLoadLimit; i++) { SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); if (!rgWrapAround.Keys.Contains(d1.Index)) { rgWrapAround.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rgWrapAround[d1.Index].Add(d1); } } // Verify that the reads wrap around to the start. Assert.AreEqual(rgWrapAround.Count, rgFirst.Count); List <KeyValuePair <int, List <SimpleDatum> > > rg1 = new List <KeyValuePair <int, List <SimpleDatum> > >(); List <KeyValuePair <int, List <SimpleDatum> > > rg2 = new List <KeyValuePair <int, List <SimpleDatum> > >(); foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgWrapAround) { rg1.Add(kv); } foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgFirst) { rg2.Add(kv); } for (int i = 0; i < rg1.Count; i++) { Assert.AreEqual(rg1[i].Key, rg2[i].Key); Assert.AreEqual(rg1[i].Value.Count, rg2[i].Value.Count); for (int j = 0; j < rg1[i].Value.Count; j++) { Assert.AreEqual(rg1[i].Value[j].Label, rg2[i].Value[j].Label); } } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQuerySequential4(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); int nCount = 100; double dfTotalMs = 0; List <int> rgIdx = new List <int>(); for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, i); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } rgIdx.Add(d.Index); } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify sequential selection. int nIdx = 0; rgIdx.Sort(); foreach (KeyValuePair <int, List <SimpleDatum> > kv in rg) { int nIdx1 = rgIdx[nIdx]; Assert.AreEqual(kv.Value.Count, (nLoadLimit == 0) ? 1 : nLoadLimit); Assert.AreEqual(rg[nIdx1][0].Index, (nLoadLimit == 0) ? nIdx1 : nIdx1 % nLoadLimit); nIdx++; } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQueryPair(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; IXImageDatabase db = new MyCaffeImageDatabase(); foreach (string strDs in rgDs) { SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); int nCount = 100; double dfTotalMs = 0; for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i); SimpleDatum d2 = db.QueryImage(ds.TrainingSource.ID, i, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.PAIR); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d1.Index)) { rg.Add(d1.Index, new List <SimpleDatum>() { d1 }); } else { rg[d1.Index].Add(d1); } if (nLoadLimit > 0) { Assert.AreEqual(true, d1.Index == d2.Index - 1 || d1.Index == nLoadLimit - 1 && d2.Index == 0); } else { Assert.AreEqual(d1.Index, d2.Index - 1); } } str = (dfTotalMs / (double)nCount).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); // Verify that all labels are hit. if (nLoadLimit > 0) { Assert.AreEqual(rg.Count, nLoadLimit); } } db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } }
public void TestQueryRandom2(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { List <string> rgDs = new List <string>() { "MNIST", "CIFAR-10", "MNIST" }; foreach (string strDs in rgDs) { IXImageDatabase db = new MyCaffeImageDatabase(); SettingsCaffe settings = new SettingsCaffe(); settings.ImageDbLoadMethod = loadMethod; settings.ImageDbLoadLimit = nLoadLimit; Stopwatch sw = new Stopwatch(); sw.Start(); db.InitializeWithDsName(settings, strDs); db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE); string str = sw.ElapsedMilliseconds.ToString(); Trace.WriteLine(strDs + " Initialization Time: " + str + " ms."); DatasetDescriptor ds = db.GetDatasetByName(strDs); Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >(); Dictionary <int, int> rgCounts = new Dictionary <int, int>(); int nCount = 10000; double dfTotalMs = 0; int nCount1 = 0; double dfTotalMs1 = 0; Stopwatch swTimer = new Stopwatch(); swTimer.Start(); // Randomly query each image and count up the number if times a given label is hit. for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.RANDOM); sw.Stop(); dfTotalMs += sw.ElapsedMilliseconds; dfTotalMs1 += sw.ElapsedMilliseconds; nCount1++; if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } if (!rgCounts.Keys.Contains(d.Label)) { rgCounts.Add(d.Label, 1); } else { rgCounts[d.Label]++; } if (swTimer.Elapsed.TotalMilliseconds > 2000) { double dfPct = (double)i / (double)nCount; Trace.WriteLine("(" + dfPct.ToString("P") + ") ave time = " + (dfTotalMs1 / nCount1).ToString("N3") + " ms."); dfTotalMs1 = 0; nCount1 = 0; swTimer.Restart(); } } // Total the label counts and calculate the average and stddev. List <KeyValuePair <int, int> > rgCountsNoLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList(); Trace.WriteLine("NO LABEL BALANCING COUNTS"); CalculationArray ca = new CalculationArray(); foreach (KeyValuePair <int, int> kv in rgCountsNoLabelBalancing) { ca.Add(kv.Value); Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0")); } double dfAve = ca.Average; double dfStdDev1 = ca.CalculateStandardDeviation(dfAve); Trace.WriteLine("Average = " + dfAve.ToString()); Trace.WriteLine("StdDev = " + dfStdDev1.ToString()); // Load the labels by first selecting the label randomly and then the image randomly from the label set. rg = new Dictionary <int, List <SimpleDatum> >(); rgCounts = new Dictionary <int, int>(); for (int i = 0; i < nCount; i++) { sw.Reset(); sw.Start(); SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.RANDOM, IMGDB_IMAGE_SELECTION_METHOD.RANDOM); dfTotalMs += sw.ElapsedMilliseconds; sw.Stop(); if (!rg.Keys.Contains(d.Index)) { rg.Add(d.Index, new List <SimpleDatum>() { d }); } else { rg[d.Index].Add(d); } if (!rgCounts.Keys.Contains(d.Label)) { rgCounts.Add(d.Label, 1); } else { rgCounts[d.Label]++; } } // Total the balanced label counts and calculate the average and stddev. List <KeyValuePair <int, int> > rgCountsLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList(); Trace.WriteLine("LABEL BALANCING COUNTS"); ca = new CalculationArray(); foreach (KeyValuePair <int, int> kv in rgCountsLabelBalancing) { ca.Add(kv.Value); Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0")); } dfAve = ca.Average; double dfStdDev2 = ca.CalculateStandardDeviation(dfAve); Trace.WriteLine("Average = " + dfAve.ToString()); Trace.WriteLine("StdDev = " + dfStdDev2.ToString()); Assert.AreEqual(true, dfStdDev2 < dfStdDev1 * 1.5); str = (dfTotalMs / (double)(nCount * 2)).ToString(); Trace.WriteLine("Average Query Time: " + str + " ms."); db.CleanUp(); IDisposable idisp = db as IDisposable; if (idisp != null) { idisp.Dispose(); } } }