/// <summary>
/// The ImageSet constructor.
/// </summary>
/// <param name="factory">Specifies the DatasetFactory.</param>
/// <param name="src">Specifies the data source.</param>
/// <param name="loadMethod">Specifies the method to use when loading the images.</param>
/// <param name="nLoadLimit">Specifies the image load limit.</param>
public ImageSet(DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
m_factory = new DatasetFactory(factory);
m_factory.Open(src.ID);
m_loadMethod = loadMethod;
m_nLoadLimit = nLoadLimit;
m_src = new SourceDescriptor(src);
m_imgMean = null;
m_rgImages = new SimpleDatum[m_src.ImageCount];
foreach (LabelDescriptor label in src.Labels)
{
if (label.ImageCount > 0)
{
m_rgLabelSet.Add(new LabelSet(label));
}
}
}
public void TestLoadSecondaryDataset(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
Log log = new Log("Test secondary dataset");
log.EnableTrace = true;
IXImageDatabase db = new MyCaffeImageDatabase(log);
Stopwatch sw = new Stopwatch();
string strDs = "MNIST";
string strDs2 = "CIFAR-10";
string str;
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
sw.Start();
db.InitializeWithDsName(settings, strDs);
str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
sw.Restart();
db.LoadDatasetByName(strDs2);
str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs2 + " Initialization Time: " + str + " ms.");
sw.Restart();
db.CleanUp();
str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine("Cleanup Time: " + str + " ms.");
sw.Restart();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine("Dispose Time: " + str + " ms.");
}
/// <summary>
/// The SettingsCaffe constructor used during deserialization.
/// </summary>
/// <param name="info">Specifies the serialization information.</param>
/// <param name="context">Specifies the serialization context.</param>
public SettingsCaffe(SerializationInfo info, StreamingContext context)
{
m_bEnableLabelBalancing = getBool(info, "bEnableLabelBalancing", m_bEnableLabelBalancing);
m_bEnableLabelBoosting = getBool(info, "bEnableLabelBoosting", m_bEnableLabelBoosting);
m_bEnableRandomInputSelection = getBool(info, "bEnableRandomInputSelection", m_bEnableRandomInputSelection);
m_bEnablePairInputSelection = getBool(info, "bEnablePairInputSelection", m_bEnablePairInputSelection);
m_bUseTrainingSourceForTesting = getBool(info, "bUseTrainingSourceForTesting", m_bUseTrainingSourceForTesting);
m_dfSuperBoostProbability = getDouble(info, "dfSuperBoostProbability", m_dfSuperBoostProbability);
m_nMaximumIterationOverride = getInt(info, "nMaximumIterationOverride", m_nMaximumIterationOverride);
m_nTestingIterationOverride = getInt(info, "nTestingIterationOverride", m_nTestingIterationOverride);
m_strDefaultModelGroup = info.GetString("strDefaultModelGroup");
m_strGpuIds = getString(info, "strGpuIds", m_strGpuIds);
m_nMaskAllButLastColumns = getInt(info, "nMaskAllButLastColumns", m_nMaskAllButLastColumns);
m_imageDbLoadMethod = (IMAGEDB_LOAD_METHOD)getInt(info, "ImageDbLoadMethod", (int)m_imageDbLoadMethod);
m_nImageDbLoadLimit = getInt(info, "ImageDbLoadLimit", m_nImageDbLoadLimit);
m_bImageDbLoadDataCriteria = getBool(info, "ImageDbLoadDataCriteria", m_bImageDbLoadDataCriteria);
m_bImageDbLoadDebugData = getBool(info, "ImageDbLoadDebugData", m_bImageDbLoadDebugData);
m_snapshotWeightUpdateMethod = (SNAPSHOT_WEIGHT_UPDATE_METHOD)getInt(info, "SnapshotWeightUpdateMethod", (int)m_snapshotWeightUpdateMethod);
m_snapshotLoadMethod = (SNAPSHOT_LOAD_METHOD)getInt(info, "SnapshotLoadMethod", (int)m_snapshotLoadMethod);
}
/// <summary>
/// The SettingsCaffe constructor that copies another SettingsCaffe instance.
/// </summary>
public SettingsCaffe(SettingsCaffe s)
{
m_nMaskAllButLastColumns = s.m_nMaskAllButLastColumns;
m_bEnableLabelBalancing = s.m_bEnableLabelBalancing;
m_bEnableLabelBoosting = s.m_bEnableLabelBoosting;
m_bEnableRandomInputSelection = s.m_bEnableRandomInputSelection;
m_bEnablePairInputSelection = s.m_bEnablePairInputSelection;
m_bUseTrainingSourceForTesting = s.m_bUseTrainingSourceForTesting;
m_dfSuperBoostProbability = s.m_dfSuperBoostProbability;
m_nMaximumIterationOverride = s.m_nMaximumIterationOverride;
m_nTestingIterationOverride = s.m_nTestingIterationOverride;
m_strDefaultModelGroup = s.m_strDefaultModelGroup;
m_strGpuIds = s.m_strGpuIds;
m_imageDbLoadMethod = s.m_imageDbLoadMethod;
m_nImageDbLoadLimit = s.m_nImageDbLoadLimit;
m_bImageDbLoadDataCriteria = s.m_bImageDbLoadDataCriteria;
m_bImageDbLoadDebugData = s.m_bImageDbLoadDebugData;
m_snapshotWeightUpdateMethod = s.m_snapshotWeightUpdateMethod;
m_snapshotLoadMethod = s.m_snapshotLoadMethod;
}
/// <summary>
/// The SettingsCaffe constructor that copies another SettingsCaffe instance.
/// </summary>
public SettingsCaffe(SettingsCaffe s)
{
m_imgDbVersion = s.m_imgDbVersion;
m_bEnableLabelBalancing = s.m_bEnableLabelBalancing;
m_bEnableLabelBoosting = s.m_bEnableLabelBoosting;
m_bEnableRandomInputSelection = s.m_bEnableRandomInputSelection;
m_bEnablePairInputSelection = s.m_bEnablePairInputSelection;
m_bUseTrainingSourceForTesting = s.m_bUseTrainingSourceForTesting;
m_dfSuperBoostProbability = s.m_dfSuperBoostProbability;
m_nMaximumIterationOverride = s.m_nMaximumIterationOverride;
m_nTestingIterationOverride = s.m_nTestingIterationOverride;
m_strDefaultModelGroup = s.m_strDefaultModelGroup;
m_strGpuIds = s.m_strGpuIds;
m_imageDbLoadMethod = s.m_imageDbLoadMethod;
m_nImageDbLoadLimit = s.m_nImageDbLoadLimit;
m_nAutoRefreshScheduledUpdateInMs = s.m_nAutoRefreshScheduledUpdateInMs;
m_dfAutoRefreshScheduledReplacementPct = s.m_dfAutoRefreshScheduledReplacementPct;
m_bImageDbLoadDataCriteria = s.m_bImageDbLoadDataCriteria;
m_bImageDbLoadDebugData = s.m_bImageDbLoadDebugData;
m_snapshotWeightUpdateMethod = s.m_snapshotWeightUpdateMethod;
m_snapshotLoadMethod = s.m_snapshotLoadMethod;
m_bSkipMeanCheck = s.m_bSkipMeanCheck;
}
/// <summary>
/// The SettingsCaffe constructor used during deserialization.
/// </summary>
/// <param name="info">Specifies the serialization information.</param>
/// <param name="context">Specifies the serialization context.</param>
public SettingsCaffe(SerializationInfo info, StreamingContext context)
{
m_imgDbVersion = (IMGDB_VERSION)getInt(info, "ImageDbVersion", (int)m_imgDbVersion);
m_bEnableLabelBalancing = getBool(info, "bEnableLabelBalancing", m_bEnableLabelBalancing);
m_bEnableLabelBoosting = getBool(info, "bEnableLabelBoosting", m_bEnableLabelBoosting);
m_bEnableRandomInputSelection = getBool(info, "bEnableRandomInputSelection", m_bEnableRandomInputSelection);
m_bEnablePairInputSelection = getBool(info, "bEnablePairInputSelection", m_bEnablePairInputSelection);
m_bUseTrainingSourceForTesting = getBool(info, "bUseTrainingSourceForTesting", m_bUseTrainingSourceForTesting);
m_dfSuperBoostProbability = getDouble(info, "dfSuperBoostProbability", m_dfSuperBoostProbability);
m_nMaximumIterationOverride = getInt(info, "nMaximumIterationOverride", m_nMaximumIterationOverride);
m_nTestingIterationOverride = getInt(info, "nTestingIterationOverride", m_nTestingIterationOverride);
m_strDefaultModelGroup = info.GetString("strDefaultModelGroup");
m_strGpuIds = getString(info, "strGpuIds", m_strGpuIds);
m_imageDbLoadMethod = (IMAGEDB_LOAD_METHOD)getInt(info, "ImageDbLoadMethod", (int)m_imageDbLoadMethod);
m_nImageDbLoadLimit = getInt(info, "ImageDbLoadLimit", m_nImageDbLoadLimit);
m_nAutoRefreshScheduledUpdateInMs = getInt(info, "ImageDbAutoRefreshScheduledUpdateInMs", m_nAutoRefreshScheduledUpdateInMs);
m_dfAutoRefreshScheduledReplacementPct = getDouble(info, "ImageDbAutoRefreshReplacementPct", m_dfAutoRefreshScheduledReplacementPct);
m_bImageDbLoadDataCriteria = getBool(info, "ImageDbLoadDataCriteria", m_bImageDbLoadDataCriteria);
m_bImageDbLoadDebugData = getBool(info, "ImageDbLoadDebugData", m_bImageDbLoadDebugData);
m_snapshotWeightUpdateMethod = (SNAPSHOT_WEIGHT_UPDATE_METHOD)getInt(info, "SnapshotWeightUpdateMethod", (int)m_snapshotWeightUpdateMethod);
m_snapshotLoadMethod = (SNAPSHOT_LOAD_METHOD)getInt(info, "SnapshotLoadMethod", (int)m_snapshotLoadMethod);
m_bSkipMeanCheck = getBool(info, "SkipMeanCheck", m_bSkipMeanCheck);
}
/// <summary>
/// Initialize the DatasetEx by loading the training and testing data sources into memory.
/// </summary>
/// <param name="ds">Specifies the dataset to load.</param>
/// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
/// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
/// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
/// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
/// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
/// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
/// <returns></returns>
public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0)
{
lock (m_syncObj)
{
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0)
{
throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method.");
}
SimpleDatum imgMean = null;
if (ds != null)
{
m_ds = ds;
}
m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false);
if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount)
{
m_nLastTrainingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false);
if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount)
{
m_nLastTestingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
return(true);
}
}
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false)
{
try
{
RawImageMean imgMeanRaw = null;
m_factory.Open(src);
nLastImageIdx = nImageDbLoadLimitStartIdx;
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL)
{
if (imgMean == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
if (imgMeanRaw == null)
{
if (log != null)
{
log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString());
}
loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
}
}
}
int nCount = src.ImageCount;
if (nCount == 0)
{
throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'.");
}
if (log != null)
{
log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images.");
}
ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit);
if (OnCalculateImageMean != null)
{
imgset.OnCalculateImageMean += OnCalculateImageMean;
}
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
bool bDataIsReal = src.IsRealData;
int nBatchSize = 20000;
Stopwatch sw = new Stopwatch();
int nImageSize = src.ImageHeight * src.ImageWidth;
if (nImageSize > 60000)
{
nBatchSize = 5000;
}
else if (nBatchSize > 20000)
{
nBatchSize = 7500;
}
else if (nImageSize > 3000)
{
nBatchSize = 10000;
}
if (nImageDbLoadLimit <= 0)
{
nImageDbLoadLimit = nCount;
}
List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit);
int nIdx = 0;
sw.Start();
while (nIdx < rgIdx.Count)
{
int nImageIdx = rgIdx[nIdx];
int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize);
List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount);
for (int j = 0; j < rgImg.Count; j++)
{
SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH);
imgset.Add(nIdx + j, sd1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
if (log != null)
{
double dfPct = (double)(nIdx + j) / (double)nCount;
log.Progress = dfPct;
log.WriteLine("image loading at " + dfPct.ToString("P") + "...");
}
sw.Restart();
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(null);
}
}
}
nIdx += rgImg.Count;
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount)
{
log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset.");
break;
}
}
if (log != null)
{
log.Progress = 0;
}
if (rgIdx.Count > 0)
{
nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1;
}
}
else if (bLoadNext)
{
nLastImageIdx += nImageDbLoadLimit;
}
if (imgMean == null)
{
if (imgMeanRaw == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
}
if (imgMeanRaw != null)
{
imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH);
}
else
{
if (log != null)
{
log.WriteLine("Calculating mean...");
}
imgMean = imgset.GetImageMean(log, rgAbort);
m_factory.PutRawImageMean(imgMean, true);
}
}
if (imgMean != null)
{
imgset.SetImageMean(imgMean);
}
imgset.CompleteLoad(nLastImageIdx);
return(imgset);
}
finally
{
m_factory.Close();
}
}
/// <summary>
/// Initialize the DatasetEx by loading the training and testing data sources into memory.
/// </summary>
/// <param name="ds">Specifies the dataset to load.</param>
/// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
/// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
/// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
/// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
/// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
/// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param>
/// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
/// <param name="nImageDbAutoRefreshScheduledUpdateInMs">Optionally, specifies the scheduled refresh update period in ms (default = 0).</param>
/// <param name="dfImageDbAutoRefreshScheduledReplacementPct">Optionally, specifies the scheduled refresh replacement percent (default = 0).</param>
/// <returns>Upon loading the dataset a handle to the default QueryState is returned, or 0 on cancel.</returns>
public long Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, bool bSkipMeanCheck = false, int nImageDbLoadLimit = 0, int nImageDbAutoRefreshScheduledUpdateInMs = 0, double dfImageDbAutoRefreshScheduledReplacementPct = 0)
{
lock (m_syncObj)
{
m_log = log;
if (ds != null)
{
m_ds = ds;
}
if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1)
{
log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped.");
bSkipMeanCheck = true;
}
bool bSilentLoad = (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false;
m_TrainingImages = new ImageSet2(ImageSet2.TYPE.TRAIN, log, m_factory, m_ds.TrainingSource, loadMethod, m_random, rgAbort);
m_TrainingImages.OnCalculateImageMean += OnCalculateImageMean;
QueryState qsTraining = m_TrainingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit);
SimpleDatum sdMean = null;
if (!bSkipMeanCheck)
{
bool bQueryOnly = false;
if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE)
{
bQueryOnly = true;
}
sdMean = m_TrainingImages.GetImageMean(log, rgAbort, bQueryOnly);
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(0);
}
m_TestingImages = new ImageSet2(ImageSet2.TYPE.TEST, log, m_factory, m_ds.TestingSource, loadMethod, m_random, rgAbort);
m_TestingImages.OnCalculateImageMean += OnCalculateImageMean;
QueryState qsTesting = m_TestingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit);
if (!bSkipMeanCheck)
{
bool bSave = true;
if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE)
{
bSave = false;
}
m_TestingImages.SetImageMean(sdMean, bSave);
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(0);
}
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbAutoRefreshScheduledUpdateInMs > 0 && dfImageDbAutoRefreshScheduledReplacementPct > 0)
{
StartAutomaticRefreshSchedule(true, true, nImageDbAutoRefreshScheduledUpdateInMs, dfImageDbAutoRefreshScheduledReplacementPct);
}
m_lDefaultQueryState = m_queryStates.CreateNewState(qsTraining, qsTesting);
return(m_lDefaultQueryState);
}
}
/// <summary>
/// Get the image with a specific image index.
/// </summary>
/// <param name="nIdx">Specifies the image index.</param>
/// <param name="bLoadDataCriteria">Specifies whether or not to load the data criteria along with the image.</param>
/// <param name="bLoadDebugData">Specifies whether or not to load the debug data with the image.</param>
/// <param name="loadMethod">Specifies the image loading method used.</param>
/// <returns>If found, the image is returned.</returns>
public SimpleDatum GetImage(int nIdx, bool bLoadDataCriteria, bool bLoadDebugData, IMAGEDB_LOAD_METHOD loadMethod)
{
SimpleDatum sd = m_rgImages[nIdx];
if (sd == null)
{
if (m_refreshManager != null)
{
while (nIdx > 0 && m_rgImages[nIdx] == null)
{
nIdx--;
}
sd = m_rgImages[nIdx];
if (sd == null)
{
throw new Exception("No images should be null when using LoadLimit loading!");
}
}
else
{
if (!m_evtRunning.WaitOne(0) && (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND && loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_NOCACHE))
{
Load((loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false);
}
sd = directLoadImage(nIdx);
if (sd == null)
{
throw new Exception("The image is still null yet should have loaded!");
}
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
m_rgImages[nIdx] = sd;
}
}
}
// Double check that the conditional data has loaded (if needed).
if (bLoadDataCriteria || bLoadDebugData)
{
m_factory.LoadRawData(sd, bLoadDataCriteria, bLoadDebugData);
}
return(sd);
}
public void TestLoadLimitNextSequential(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
List <string> rgDs = new List <string>()
{
"MNIST", "CIFAR-10", "MNIST"
};
IXImageDatabase db = new MyCaffeImageDatabase();
foreach (string strDs in rgDs)
{
DatasetFactory df = new DatasetFactory();
int nDs = df.GetDatasetID(strDs);
if (nDs == 0)
{
throw new Exception("The dataset '" + strDs + "' does not exist - you need to load it.");
}
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
Stopwatch sw = new Stopwatch();
sw.Start();
db.InitializeWithDsName(settings, strDs);
db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE);
string str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
DatasetDescriptor ds = db.GetDatasetByName(strDs);
Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >();
Dictionary <int, List <SimpleDatum> > rgFirst = new Dictionary <int, List <SimpleDatum> >();
int nTotal = ds.TrainingSource.ImageCount;
int nCount = 0;
double dfTotalMs = 0;
while (nCount < nTotal)
{
for (int i = 0; i < nLoadLimit; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i);
dfTotalMs += sw.ElapsedMilliseconds;
sw.Stop();
if (!rg.Keys.Contains(d1.Index))
{
rg.Add(d1.Index, new List <SimpleDatum>()
{
d1
});
}
else
{
rg[d1.Index].Add(d1);
}
if (nCount == 0)
{
if (!rgFirst.Keys.Contains(d1.Index))
{
rgFirst.Add(d1.Index, new List <SimpleDatum>()
{
d1
});
}
else
{
rgFirst[d1.Index].Add(d1);
}
}
}
db.LoadNextSet(null);
nCount += nLoadLimit;
}
str = (dfTotalMs / (double)nCount).ToString();
Trace.WriteLine("Average Query Time: " + str + " ms.");
// Verify that all items have been queried
Assert.AreEqual(nTotal, rg.Count);
Dictionary <int, List <SimpleDatum> > rgWrapAround = new Dictionary <int, List <SimpleDatum> >();
for (int i = 0; i < nLoadLimit; i++)
{
SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i);
if (!rgWrapAround.Keys.Contains(d1.Index))
{
rgWrapAround.Add(d1.Index, new List <SimpleDatum>()
{
d1
});
}
else
{
rgWrapAround[d1.Index].Add(d1);
}
}
// Verify that the reads wrap around to the start.
Assert.AreEqual(rgWrapAround.Count, rgFirst.Count);
List <KeyValuePair <int, List <SimpleDatum> > > rg1 = new List <KeyValuePair <int, List <SimpleDatum> > >();
List <KeyValuePair <int, List <SimpleDatum> > > rg2 = new List <KeyValuePair <int, List <SimpleDatum> > >();
foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgWrapAround)
{
rg1.Add(kv);
}
foreach (KeyValuePair <int, List <SimpleDatum> > kv in rgFirst)
{
rg2.Add(kv);
}
for (int i = 0; i < rg1.Count; i++)
{
Assert.AreEqual(rg1[i].Key, rg2[i].Key);
Assert.AreEqual(rg1[i].Value.Count, rg2[i].Value.Count);
for (int j = 0; j < rg1[i].Value.Count; j++)
{
Assert.AreEqual(rg1[i].Value[j].Label, rg2[i].Value[j].Label);
}
}
}
db.CleanUp();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
}
public void TestQuerySequential4(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
List <string> rgDs = new List <string>()
{
"MNIST", "CIFAR-10", "MNIST"
};
IXImageDatabase db = new MyCaffeImageDatabase();
foreach (string strDs in rgDs)
{
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
Stopwatch sw = new Stopwatch();
sw.Start();
db.InitializeWithDsName(settings, strDs);
db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE);
string str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
DatasetDescriptor ds = db.GetDatasetByName(strDs);
Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >();
int nCount = 100;
double dfTotalMs = 0;
List <int> rgIdx = new List <int>();
for (int i = 0; i < nCount; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, i);
dfTotalMs += sw.ElapsedMilliseconds;
sw.Stop();
if (!rg.Keys.Contains(d.Index))
{
rg.Add(d.Index, new List <SimpleDatum>()
{
d
});
}
else
{
rg[d.Index].Add(d);
}
rgIdx.Add(d.Index);
}
str = (dfTotalMs / (double)nCount).ToString();
Trace.WriteLine("Average Query Time: " + str + " ms.");
// Verify sequential selection.
int nIdx = 0;
rgIdx.Sort();
foreach (KeyValuePair <int, List <SimpleDatum> > kv in rg)
{
int nIdx1 = rgIdx[nIdx];
Assert.AreEqual(kv.Value.Count, (nLoadLimit == 0) ? 1 : nLoadLimit);
Assert.AreEqual(rg[nIdx1][0].Index, (nLoadLimit == 0) ? nIdx1 : nIdx1 % nLoadLimit);
nIdx++;
}
}
db.CleanUp();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
}
public void TestQuerySequential(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
List <string> rgDs = new List <string>()
{
"MNIST", "CIFAR-10", "MNIST"
};
IXImageDatabase db = new MyCaffeImageDatabase();
foreach (string strDs in rgDs)
{
DatasetFactory df = new DatasetFactory();
int nDs = df.GetDatasetID(strDs);
if (nDs == 0)
{
throw new Exception("The dataset '" + strDs + "' does not exist - you need to load it.");
}
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
Stopwatch sw = new Stopwatch();
sw.Start();
db.InitializeWithDsName(settings, strDs);
string str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
DatasetDescriptor ds = db.GetDatasetByName(strDs);
Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >();
int nCount = 100;
double dfTotalMs = 0;
for (int i = 0; i < nCount; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE);
dfTotalMs += sw.ElapsedMilliseconds;
sw.Stop();
if (!rg.Keys.Contains(d.Index))
{
rg.Add(d.Index, new List <SimpleDatum>()
{
d
});
}
else
{
rg[d.Index].Add(d);
}
}
str = (dfTotalMs / (double)nCount).ToString();
Trace.WriteLine("Average Query Time: " + str + " ms.");
// Verify sequential selection, so all indexes should be the same.
foreach (KeyValuePair <int, List <SimpleDatum> > kv in rg)
{
Assert.AreEqual(kv.Value.Count, nCount);
}
}
db.CleanUp();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
}
public void TestQueryRandom2(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
List <string> rgDs = new List <string>()
{
"MNIST", "CIFAR-10", "MNIST"
};
foreach (string strDs in rgDs)
{
IXImageDatabase db = new MyCaffeImageDatabase();
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
Stopwatch sw = new Stopwatch();
sw.Start();
db.InitializeWithDsName(settings, strDs);
db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE);
string str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
DatasetDescriptor ds = db.GetDatasetByName(strDs);
Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >();
Dictionary <int, int> rgCounts = new Dictionary <int, int>();
int nCount = 10000;
double dfTotalMs = 0;
int nCount1 = 0;
double dfTotalMs1 = 0;
Stopwatch swTimer = new Stopwatch();
swTimer.Start();
// Randomly query each image and count up the number if times a given label is hit.
for (int i = 0; i < nCount; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.RANDOM);
sw.Stop();
dfTotalMs += sw.ElapsedMilliseconds;
dfTotalMs1 += sw.ElapsedMilliseconds;
nCount1++;
if (!rg.Keys.Contains(d.Index))
{
rg.Add(d.Index, new List <SimpleDatum>()
{
d
});
}
else
{
rg[d.Index].Add(d);
}
if (!rgCounts.Keys.Contains(d.Label))
{
rgCounts.Add(d.Label, 1);
}
else
{
rgCounts[d.Label]++;
}
if (swTimer.Elapsed.TotalMilliseconds > 2000)
{
double dfPct = (double)i / (double)nCount;
Trace.WriteLine("(" + dfPct.ToString("P") + ") ave time = " + (dfTotalMs1 / nCount1).ToString("N3") + " ms.");
dfTotalMs1 = 0;
nCount1 = 0;
swTimer.Restart();
}
}
// Total the label counts and calculate the average and stddev.
List <KeyValuePair <int, int> > rgCountsNoLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList();
Trace.WriteLine("NO LABEL BALANCING COUNTS");
CalculationArray ca = new CalculationArray();
foreach (KeyValuePair <int, int> kv in rgCountsNoLabelBalancing)
{
ca.Add(kv.Value);
Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0"));
}
double dfAve = ca.Average;
double dfStdDev1 = ca.CalculateStandardDeviation(dfAve);
Trace.WriteLine("Average = " + dfAve.ToString());
Trace.WriteLine("StdDev = " + dfStdDev1.ToString());
// Load the labels by first selecting the label randomly and then the image randomly from the label set.
rg = new Dictionary <int, List <SimpleDatum> >();
rgCounts = new Dictionary <int, int>();
for (int i = 0; i < nCount; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d = db.QueryImage(ds.TrainingSource.ID, 0, IMGDB_LABEL_SELECTION_METHOD.RANDOM, IMGDB_IMAGE_SELECTION_METHOD.RANDOM);
dfTotalMs += sw.ElapsedMilliseconds;
sw.Stop();
if (!rg.Keys.Contains(d.Index))
{
rg.Add(d.Index, new List <SimpleDatum>()
{
d
});
}
else
{
rg[d.Index].Add(d);
}
if (!rgCounts.Keys.Contains(d.Label))
{
rgCounts.Add(d.Label, 1);
}
else
{
rgCounts[d.Label]++;
}
}
// Total the balanced label counts and calculate the average and stddev.
List <KeyValuePair <int, int> > rgCountsLabelBalancing = rgCounts.OrderBy(p => p.Key).ToList();
Trace.WriteLine("LABEL BALANCING COUNTS");
ca = new CalculationArray();
foreach (KeyValuePair <int, int> kv in rgCountsLabelBalancing)
{
ca.Add(kv.Value);
Trace.WriteLine(kv.Key + " -> " + kv.Value.ToString("N0"));
}
dfAve = ca.Average;
double dfStdDev2 = ca.CalculateStandardDeviation(dfAve);
Trace.WriteLine("Average = " + dfAve.ToString());
Trace.WriteLine("StdDev = " + dfStdDev2.ToString());
Assert.AreEqual(true, dfStdDev2 < dfStdDev1 * 1.5);
str = (dfTotalMs / (double)(nCount * 2)).ToString();
Trace.WriteLine("Average Query Time: " + str + " ms.");
db.CleanUp();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
}
}
/// <summary>
/// Initialize the DatasetEx by loading the training and testing data sources into memory.
/// </summary>
/// <param name="ds">Specifies the dataset to load.</param>
/// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
/// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
/// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
/// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
/// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
/// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
/// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param>
/// <returns>Upon loading the dataset <i>true</i> is returned, otherwise on failure or abort <i>false</i> is returned.</returns>
public bool Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, bool bSkipMeanCheck = false)
{
lock (m_syncObj)
{
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0)
{
throw new Exception("Currently the load-limit only works with the LOAD_ALLL image loading method.");
}
SimpleDatum imgMean = null;
if (ds != null)
{
m_ds = ds;
}
if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1)
{
log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images. The mean check will be skipped.");
bSkipMeanCheck = true;
}
m_TrainingImages = loadImageset("Training", m_ds.TrainingSource, rgAbort, ref imgMean, out m_nLastTrainingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTrainingImageIdx, (ds == null) ? true : false, bSkipMeanCheck);
if (m_nLastTrainingImageIdx >= m_ds.TrainingSource.ImageCount)
{
m_nLastTrainingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
m_TestingImages = loadImageset("Testing", m_ds.TestingSource, rgAbort, ref imgMean, out m_nLastTestingImageIdx, nPadW, nPadH, log, loadMethod, nImageDbLoadLimit, m_nLastTestingImageIdx, (ds == null) ? true : false, bSkipMeanCheck);
if (m_nLastTestingImageIdx >= m_ds.TestingSource.ImageCount)
{
m_nLastTestingImageIdx = 0;
}
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(false);
}
return(true);
}
}
/// <summary>
/// The ImageSet2 constructor.
/// </summary>
/// <param name="type">Specifies the type of data source managed.</param>
/// <param name="log">Specifies the output log.</param>
/// <param name="factory">Specifies the data factory used to access the database data.</param>
/// <param name="src">Specifies the data source descriptor.</param>
/// <param name="loadMethod">Specifies the load method used to load the data.</param>
/// <param name="random">Specifies the random number generator.</param>
/// <param name="rgAbort">Specifies the cancellation handles.</param>
public ImageSet2(TYPE type, Log log, DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, CryptoRandom random, WaitHandle[] rgAbort)
: base(factory, src)
{
m_type = type;
m_log = log;
m_loadMethod = loadMethod;
m_random = random;
m_rgAbort.Add(m_evtCancel);
if (rgAbort.Length > 0)
{
m_rgAbort.AddRange(rgAbort);
}
}
public void TestQueryPair(IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
List <string> rgDs = new List <string>()
{
"MNIST", "CIFAR-10", "MNIST"
};
IXImageDatabase db = new MyCaffeImageDatabase();
foreach (string strDs in rgDs)
{
SettingsCaffe settings = new SettingsCaffe();
settings.ImageDbLoadMethod = loadMethod;
settings.ImageDbLoadLimit = nLoadLimit;
Stopwatch sw = new Stopwatch();
sw.Start();
db.InitializeWithDsName(settings, strDs);
db.SetSelectionMethod(IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.NONE);
string str = sw.ElapsedMilliseconds.ToString();
Trace.WriteLine(strDs + " Initialization Time: " + str + " ms.");
DatasetDescriptor ds = db.GetDatasetByName(strDs);
Dictionary <int, List <SimpleDatum> > rg = new Dictionary <int, List <SimpleDatum> >();
int nCount = 100;
double dfTotalMs = 0;
for (int i = 0; i < nCount; i++)
{
sw.Reset();
sw.Start();
SimpleDatum d1 = db.QueryImage(ds.TrainingSource.ID, i);
SimpleDatum d2 = db.QueryImage(ds.TrainingSource.ID, i, IMGDB_LABEL_SELECTION_METHOD.NONE, IMGDB_IMAGE_SELECTION_METHOD.PAIR);
dfTotalMs += sw.ElapsedMilliseconds;
sw.Stop();
if (!rg.Keys.Contains(d1.Index))
{
rg.Add(d1.Index, new List <SimpleDatum>()
{
d1
});
}
else
{
rg[d1.Index].Add(d1);
}
if (nLoadLimit > 0)
{
Assert.AreEqual(true, d1.Index == d2.Index - 1 || d1.Index == nLoadLimit - 1 && d2.Index == 0);
}
else
{
Assert.AreEqual(d1.Index, d2.Index - 1);
}
}
str = (dfTotalMs / (double)nCount).ToString();
Trace.WriteLine("Average Query Time: " + str + " ms.");
// Verify that all labels are hit.
if (nLoadLimit > 0)
{
Assert.AreEqual(rg.Count, nLoadLimit);
}
}
db.CleanUp();
IDisposable idisp = db as IDisposable;
if (idisp != null)
{
idisp.Dispose();
}
}
