/// <summary>
/// The constructor.
/// </summary>
/// <param name="random">Specifies the CryptoRandom to use for random selection.</param>
/// <param name="log">Specifies the output log.</param>
/// <param name="src">Specifies the data source that holds the data on the database.</param>
/// <param name="factory">Specifies the data factory used to access the database data.</param>
/// <param name="rgAbort">Specifies the cancel handles.</param>
/// <param name="nMaxLoadCount">Optionally, specifies to automaticall start the image refresh which only applies when the number of images loaded into memory is less than the actual number of images (default = false).</param>
public MasterList(CryptoRandom random, Log log, SourceDescriptor src, DatasetFactory factory, List <WaitHandle> rgAbort, int nMaxLoadCount = 0)
{
m_random = random;
m_log = log;
m_src = src;
m_factory = factory;
m_rgAbort.Add(m_evtCancel);
if (rgAbort.Count > 0)
{
m_rgAbort.AddRange(rgAbort);
}
m_imgMean = m_factory.LoadImageMean(m_src.ID);
m_nLoadCount = nMaxLoadCount;
if (m_nLoadCount == 0 || m_nLoadCount > m_src.ImageCount)
{
m_nLoadCount = m_src.ImageCount;
}
m_rgImages = new SimpleDatum[m_nLoadCount];
m_nLoadedCount = 0;
if (m_nLoadCount < m_src.ImageCount)
{
m_refreshManager = new RefreshManager(random, m_src, m_factory);
}
}
private int getBatchSize(SourceDescriptor src)
{
int nBatchSize = 20000;
int nImageSize = src.ImageHeight * src.ImageWidth * src.ImageChannels;
if (nImageSize > 60000)
{
nBatchSize = 5000;
}
else if (nImageSize > 20000)
{
nBatchSize = 7500;
}
else if (nImageSize > 3000)
{
nBatchSize = 10000;
}
if (nBatchSize > m_nLoadCount)
{
nBatchSize = m_nLoadCount;
}
return(nBatchSize);
}
/// <summary>
/// The BaseDataLayer constructor.
/// </summary>
/// <param name="cuda">Specifies the CudaDnn connection to Cuda.</param>
/// <param name="log">Specifies the Log for output.</param>
/// <param name="p">Specifies the LayerParameter</param>
/// <param name="db">Specifies the external database to use.</param>
public BaseDataLayer(CudaDnn <T> cuda, Log log, LayerParameter p, IXImageDatabase db)
: base(cuda, log, p)
{
if (db != null)
{
m_imgdb = db;
if (p.type == LayerParameter.LayerType.DATA ||
p.type == LayerParameter.LayerType.TRIPLET_DATA)
{
m_src = m_imgdb.GetSourceByName(p.data_param.source);
}
else if (p.type == LayerParameter.LayerType.BATCHDATA)
{
m_src = m_imgdb.GetSourceByName(p.batch_data_param.source);
}
if (p.transform_param.use_imagedb_mean)
{
if (db != null)
{
m_imgMean = db.GetImageMean(m_src.ID);
}
else
{
m_log.WriteLine("WARNING: The image database is NULL, and therefore no mean image can not be acquired.");
}
}
}
}
public void LoadDatabase()
{
int nIdx = 0;
int nTotal = 0;
reportProgress(nIdx, nTotal, "Unpacking files...");
string strTrainImagesBin = expandFile(m_param.TrainImagesFile);
string strTrainLabelsBin = expandFile(m_param.TrainLabelsFile);
string strTestImagesBin = expandFile(m_param.TestImagesFile);
string strTestLabelsBin = expandFile(m_param.TestLabelsFile);
reportProgress(nIdx, nTotal, "Loading database...");
loadFile(strTrainImagesBin, strTrainLabelsBin, "MNIST.training");
loadFile(strTestImagesBin, strTestLabelsBin, "MNIST.testing");
DatasetFactory factory = new DatasetFactory();
SourceDescriptor srcTrain = factory.LoadSource("MNIST.training");
SourceDescriptor srcTest = factory.LoadSource("MNIST.testing");
DatasetDescriptor ds = new DatasetDescriptor(0, "MNIST", null, null, srcTrain, srcTest, "MNIST", "MNIST Character Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
/// <summary>
/// Returns the dataset descriptor of the dynamic dataset produced by the Gym.
/// </summary>
/// <param name="dt">Specifies the data-type to use.</param>
/// <param name="log">Optionally, specifies the output log to use (default = <i>null</i>).</param>
/// <returns>The dataset descriptor is returned.</returns>
public DatasetDescriptor GetDataset(DATA_TYPE dt, Log log = null)
{
int nH = 1;
int nW = 1;
int nC = 4;
if (dt == DATA_TYPE.DEFAULT)
{
dt = DATA_TYPE.VALUES;
}
if (dt == DATA_TYPE.BLOB)
{
nH = 156;
nW = 156;
nC = 3;
}
SourceDescriptor srcTrain = new SourceDescriptor((int)GYM_DS_ID.CARTPOLE, Name + ".training", nW, nH, nC, false, false);
SourceDescriptor srcTest = new SourceDescriptor((int)GYM_SRC_TEST_ID.CARTPOLE, Name + ".testing", nW, nH, nC, false, false);
DatasetDescriptor ds = new DatasetDescriptor((int)GYM_SRC_TRAIN_ID.CARTPOLE, Name, null, null, srcTrain, srcTest, "CartPoleGym", "CartPole Gym", null, GYM_TYPE.DYNAMIC);
m_dt = dt;
return(ds);
}
protected bool IsSourceItemNs(SourceDescriptor descr, string uri)
{
if (descr == null || uri == null)
{
return(false);
}
return(uri.StartsWith(descr.Ns + Separator));
}
/// <summary>
/// The Cursor constructor.
/// </summary>
/// <param name="db">Specifies the underlying database.</param>
/// <param name="strSrc">Specifies the name of the data source to use.</param>
public Cursor(IXImageDatabase db, string strSrc)
{
m_db = db;
SourceDescriptor src = m_db.GetSourceByName(strSrc);
m_nSrcID = src.ID;
m_nCount = src.ImageCount;
}
public void ReadXElement_NoProviderFields_ThrowsException()
{
SourceDescriptor descriptor = new SourceDescriptor();
descriptor.ReadXElement(XElement.Parse(@"
<Source>
<IDataProvider>
<SqlDataProvider connectionString=""connstr"" query=""query"" />
</IDataProvider>
</Source>"));
}
/// <summary>
/// Disposes the current object
/// </summary>
/// <param name="disposing">true to dispose both managed an unmanaged resources, false to dispose only managed resources</param>
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
Image.BeginAnimation(Image.SourceProperty, null);
Animation.Completed -= AnimationCompleted;
SourceDescriptor.RemoveValueChanged(Image, ImageSourceChanged);
Image.Source = null;
}
}
/// <summary>
/// The Cursor constructor.
/// </summary>
/// <param name="db">Specifies the underlying database.</param>
/// <param name="transformer">Specifies the data transformer used to transform the lables (when active).</param>
/// <param name="strSrc">Specifies the name of the data source to use.</param>
/// <param name="log">Optionally, specifies an output log used for diagnostic information if specified (default = null).</param>
public Cursor(IXImageDatabaseBase db, DataTransformer <T> transformer, string strSrc, Log log = null)
{
m_log = log;
m_db = db;
m_transformer = transformer;
SourceDescriptor src = m_db.GetSourceByName(strSrc);
m_strSrc = src.Name;
m_nSrcID = src.ID;
m_nCount = src.ImageCount;
}
public void ReadXElement_NoIDataProvider_ThrowsException()
{
SourceDescriptor descriptor = new SourceDescriptor();
descriptor.ReadXElement(XElement.Parse(@"
<Source>
<ProviderFields>
<ProviderField name=""fieldName1"" type=""System.Int32"" />
<ProviderField name=""fieldName2"" type=""System.Int32"" />
</ProviderFields>
</Source>"));
}
public void AppendIncludes(SourceDescriptor desc, bool Ext = false)
{
StringBuilder builder = Ext ? ExternalBuilder : Builder;
builder.AppendLine();
builder.AppendLine($"# Include directories for {desc.Name}");
foreach (var inc in desc.IncludeDirectories)
{
builder.AppendLine($"CFLAGS += -I{inc.FullPath}");
}
}
public override int GetHashCode()
{
unchecked {
var hashCode = EqualityComparer <T> .Default.GetHashCode(Value);
hashCode = (hashCode * 397) ^ (ValueDescriptor?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ (int)Source;
hashCode = (hashCode * 397) ^ (SourceDescriptor?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ (CustomExpression?.GetHashCode() ?? 0);
return(hashCode);
}
}
/// <summary>
/// The constructor used to copy another MasterIndexes and optionally specify a sorting for the indexes.
/// </summary>
/// <param name="idx">Specifies the MasterIndexes to copy.</param>
/// <param name="sort">Optionally, specifies a sorting to use on the indexes.</param>
public MasterIndexes(MasterIndexes idx, IMGDB_SORT sort)
{
m_sort = sort;
m_src = idx.m_src;
m_random = idx.m_random;
m_nLoadLimit = idx.m_nLoadLimit;
m_factory = new DatasetFactory(idx.m_factory);
m_index = idx.m_index.Clone(sort);
m_rgLabels = new LabelIndex(idx.m_rgLabels);
m_boosted = idx.m_boosted.Clone(sort);
m_rgLabelsBoosted = new LabelIndex(idx.m_rgLabelsBoosted);
}
/// <summary>
/// The ImageSet2 constructor.
/// </summary>
/// <param name="type">Specifies the type of data source managed.</param>
/// <param name="log">Specifies the output log.</param>
/// <param name="factory">Specifies the data factory used to access the database data.</param>
/// <param name="src">Specifies the data source descriptor.</param>
/// <param name="loadMethod">Specifies the load method used to load the data.</param>
/// <param name="random">Specifies the random number generator.</param>
/// <param name="rgAbort">Specifies the cancellation handles.</param>
public ImageSet2(TYPE type, Log log, DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, CryptoRandom random, WaitHandle[] rgAbort)
: base(factory, src)
{
m_type = type;
m_log = log;
m_loadMethod = loadMethod;
m_random = random;
m_rgAbort.Add(m_evtCancel);
if (rgAbort.Length > 0)
{
m_rgAbort.AddRange(rgAbort);
}
}
/// <summary>
/// The constructor.
/// </summary>
/// <param name="random">Specifies the random number generator.</param>
/// <param name="src">Specifies the data source.</param>
/// <param name="nLoadLimit">Optionally, specifies the load limit used which when set to a value > 0, limits queries to RANDOM image selection within the load limit count (default = 0).</param>
public MasterIndexes(CryptoRandom random, SourceDescriptor src, int nLoadLimit = 0)
{
m_random = random;
m_src = src;
m_factory.Open(src);
m_nLoadLimit = nLoadLimit;
if (m_nLoadLimit > src.ImageCount)
{
m_nLoadLimit = src.ImageCount;
}
m_rgImageIdx = m_factory.LoadImageIndexes(false);
load(m_rgImageIdx.Where(p => p != null).ToList());
}
public void AppendSourceFiles(SourceDescriptor desc, bool Ext = false)
{
StringBuilder builder = Ext ? ExternalBuilder : Builder;
if (desc.SourceFiles != null)
{
builder.AppendLine();
builder.AppendLine($"# Source files for {desc.Name}");
foreach (var src in desc.SourceFiles)
{
builder.AppendLine($"SRC += {src.FullPath}");
src.PreProcess();
}
}
}
public void ReadXElement_SetTwoProviderFields_ProviderFieldsHas2Items()
{
SourceDescriptor descriptor = new SourceDescriptor();
descriptor.ReadXElement(XElement.Parse(@"
<Source>
<ProviderFields>
<ProviderField name=""fieldName1"" type=""System.Int32"" />
<ProviderField name=""fieldName2"" type=""System.Int32"" />
</ProviderFields>
<IDataProvider>
<SqlDataProvider connectionString=""connstr"" query=""query"" />
</IDataProvider>
</Source>"));
Assert.IsTrue(descriptor.ProviderFields.Count == 2);
}
internal ImageAnimationController(Image image, ObjectAnimationUsingKeyFrames animation, bool autoStart)
{
Image = image;
Animation = animation;
Animation.Completed += AnimationCompleted;
Clock = Animation.CreateClock();
ClockController = Clock.Controller;
SourceDescriptor.AddValueChanged(image, ImageSourceChanged);
// ReSharper disable once PossibleNullReferenceException
ClockController.Pause();
Image.ApplyAnimationClock(Image.SourceProperty, Clock);
if (autoStart)
{
ClockController.Resume();
}
}
/// <summary>
/// The ImageSet constructor.
/// </summary>
/// <param name="factory">Specifies the DatasetFactory.</param>
/// <param name="src">Specifies the data source.</param>
/// <param name="loadMethod">Specifies the method to use when loading the images.</param>
/// <param name="nLoadLimit">Specifies the image load limit.</param>
public ImageSet(DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit)
{
m_factory = new DatasetFactory(factory);
m_factory.Open(src.ID);
m_loadMethod = loadMethod;
m_nLoadLimit = nLoadLimit;
m_src = new SourceDescriptor(src);
m_imgMean = null;
m_rgImages = new SimpleDatum[m_src.ImageCount];
foreach (LabelDescriptor label in src.Labels)
{
if (label.ImageCount > 0)
{
m_rgLabelSet.Add(new LabelSet(label));
}
}
}
protected object ExtractSourceId(SourceDescriptor descr, string uri)
{
var idStr = uri.Substring(descr.Ns.Length + Separator.Length);
if (descr.IdFieldType != null)
{
return(ConvertManager.ChangeType(idStr, descr.IdFieldType));
}
// may be this field is registered?
if (FieldNameSourceHash[descr].ContainsKey(descr.IdFieldName))
{
var fldDescr = FieldNameSourceHash[descr][descr.IdFieldName];
if (fldDescr.FieldType != null)
{
return(ConvertManager.ChangeType(idStr, fldDescr.FieldType));
}
}
return(idStr);
}
void AppendDataFiles(SourceDescriptor desc)
{
var d = desc.DataFiles;
if (d != null)
{
var DirName = Catalog.ResolvePath(Path.Combine("gen", desc.Name));
ExternalBuilder.AppendLine($"CFLAGS += -I{DirName}");
if (!Directory.Exists(DirName))
{
Directory.CreateDirectory(DirName);
}
foreach (var df in d)
{
string FilePath = Path.Combine(DirName, df.Name);
var hg = new HeaderGenerator(df);
hg.Export(FilePath);
}
}
}
public void LoadDatabase()
{
int nIdx = 0;
int nTotal = 50000;
reportProgress(nIdx, 0, "Loading database...");
Log log = new Log("MNIST");
log.OnWriteLine += Log_OnWriteLine;
DatasetFactory factory = new DatasetFactory();
loadFile(m_param.DataBatchFile1, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile2, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile3, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile4, "CIFAR-10.training", nTotal, ref nIdx);
loadFile(m_param.DataBatchFile5, "CIFAR-10.training", nTotal, ref nIdx);
SourceDescriptor srcTrain = factory.LoadSource("CIFAR-10.training");
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID);
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 10000;
loadFile(m_param.TestBatchFile, "CIFAR-10.testing", nTotal, ref nIdx);
SourceDescriptor srcTest = factory.LoadSource("CIFAR-10.testing");
m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID);
DatasetDescriptor ds = new DatasetDescriptor(0, "CIFAR-10", null, null, srcTrain, srcTest, "CIFAR-10", "CiFar-10 Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
public LabelIndex(LabelIndex idx)
{
m_strName = idx.m_strName + " copy";
m_randomLabel = idx.m_randomLabel;
m_randomData = idx.m_randomData;
m_src = idx.m_src;
m_bBoosted = idx.m_bBoosted;
m_rgIdx = new List <int>();
if (idx.m_rgLabels != null && idx.m_rgLabels.Length > 0)
{
m_rgLabels = new Index[idx.m_rgLabels.Length];
bool bFillLabelMap = false;
if (m_rgLabelToIdxMap == null || m_rgLabelToIdxMap.Count == 0 || m_rgIdxToLabelMap == null || m_rgIdxToLabelMap.Count == 0)
{
m_rgLabelToIdxMap = new Dictionary <int, int>();
m_rgIdxToLabelMap = new Dictionary <int, int>();
bFillLabelMap = true;
}
for (int i = 0; i < idx.m_rgLabels.Length; i++)
{
m_rgLabels[i] = idx.m_rgLabels[i].Clone();
if (m_rgLabels[i].Count > 0)
{
m_rgIdx.Add(i);
}
if (bFillLabelMap)
{
int nLabel = m_rgLabels[i].Label;
m_rgLabelToIdxMap[nLabel] = i;
m_rgIdxToLabelMap[i] = nLabel;
}
}
}
}
public LabelIndex(string strName, CryptoRandom randomLabel, CryptoRandom randomData, SourceDescriptor src, bool bBoosted, List <DbItem> rgItems)
{
m_strName = strName;
m_randomLabel = randomLabel;
m_randomData = randomData;
m_src = src;
m_bBoosted = bBoosted;
m_rgIdx = new List <int>();
m_rgLabelToIdxMap = new Dictionary <int, int>();
m_rgIdxToLabelMap = new Dictionary <int, int>();
List <LabelDescriptor> rgLabels = src.Labels.Where(p => p.ImageCount > 0).OrderBy(p => p.ActiveLabel).ToList();
if (rgLabels.Count > 0)
{
m_rgLabels = new Index[rgLabels.Count];
for (int i = 0; i < rgLabels.Count; i++)
{
int nLabel = rgLabels[i].ActiveLabel;
List <DbItem> rgLabelList = rgItems.Where(p => p.Label == nLabel).ToList();
if (i < rgLabels.Count - 1)
{
rgItems = rgItems.Where(p => p.Label != nLabel).ToList();
}
m_rgLabels[i] = new Index(strName + " label " + nLabel.ToString(), randomData, rgLabelList, nLabel, false);
if (rgLabelList.Count > 0)
{
m_rgIdx.Add(i);
}
m_rgLabelToIdxMap[nLabel] = i;
m_rgIdxToLabelMap[i] = nLabel;
}
}
}
/// <summary>
/// Returns the dataset descriptor of the dynamic dataset produced by the Gym.
/// </summary>
/// <param name="dt">Specifies the data-type to use.</param>
/// <param name="log">Optionally, specifies the output log to use (default = <i>null</i>).</param>
/// <returns>The dataset descriptor is returned.</returns>
public DatasetDescriptor GetDataset(DATA_TYPE dt, Log log = null)
{
if (dt == DATA_TYPE.DEFAULT)
{
dt = DATA_TYPE.BLOB;
}
if (dt != DATA_TYPE.BLOB)
{
if (log == null)
{
log = m_log;
}
if (log != null)
{
log.WriteLine("WARNING: This gym only supports the BLOB type, the datatype will be changed to BLOB.");
}
else
{
throw new Exception("This gym only supports the BLOB type.");
}
dt = DATA_TYPE.BLOB;
}
int nH = 80;
int nW = 80;
int nC = 1;
SourceDescriptor srcTrain = new SourceDescriptor((int)GYM_DS_ID.ATARI, Name + ".training", nW, nH, nC, false, false);
SourceDescriptor srcTest = new SourceDescriptor((int)GYM_SRC_TEST_ID.ATARI, Name + ".testing", nW, nH, nC, false, false);
DatasetDescriptor ds = new DatasetDescriptor((int)GYM_SRC_TRAIN_ID.ATARI, Name, null, null, srcTrain, srcTest, "AtariGym", "Atari Gym", null, GYM_TYPE.DYNAMIC);
m_dt = dt;
return(ds);
}
public RefreshManager(CryptoRandom random, SourceDescriptor src, DatasetFactory factory)
{
m_random = random;
m_rgItems = factory.LoadImageIndexes(false, true);
}
/// <summary>
/// Create the dataset and load it into the database.
/// </summary>
/// <param name="nCreatorID">Specifies the creator ID.</param>
/// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns>
public bool LoadDatabase(int nCreatorID = 0)
{
try
{
reportProgress(0, 0, "Loading " + dataset_name + " database...");
int nIdx = 0;
int nTotal = 5011 + 17125;
int nExtractIdx = 0;
int nExtractTotal = 10935 + 40178;
// Get the label map.
LabelMap labelMap = loadLabelMap();
Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true);
string strSrc = dataset_name + ".training";
int nSrcId = m_factory.GetSourceID(strSrc);
if (nSrcId > 0)
{
m_factory.DeleteSourceData(nSrcId);
}
List <Tuple <int, string, Size> > rgFileSizes = new List <Tuple <int, string, Size> >();
if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes))
{
return(false);
}
if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes))
{
return(false);
}
string strDir = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData) + "\\MyCaffe\\test_data\\data\\ssd\\VOC0712\\";
if (!Directory.Exists(strDir))
{
Directory.CreateDirectory(strDir);
}
saveFileSizes(rgFileSizes, strDir + "train_name_size.txt");
SourceDescriptor srcTrain = m_factory.LoadSource(strSrc);
rgFileSizes = new List <Tuple <int, string, Size> >();
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 4952;
nExtractIdx = 0;
nExtractTotal = 10347;
strSrc = dataset_name + ".testing";
nSrcId = m_factory.GetSourceID(strSrc);
if (nSrcId > 0)
{
m_factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes))
{
return(false);
}
saveFileSizes(rgFileSizes, strDir + "test_name_size.txt");
SourceDescriptor srcTest = m_factory.LoadSource(strSrc);
DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset");
m_factory.AddDataset(ds);
m_factory.UpdateDatasetCounts(ds.ID);
return(true);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
}
/// <summary>
/// The ImageSet constructor.
/// </summary>
/// <param name="factory">Specifies the DatasetFactory.</param>
/// <param name="src">Specifies the data source.</param>
public ImageSetBase(DatasetFactory factory, SourceDescriptor src)
{
m_src = new SourceDescriptor(src);
m_factory = new DatasetFactory(factory);
m_factory.Open(src.ID);
}
/// <summary>
/// Create the dataset and load it into the database.
/// </summary>
/// <param name="nCreatorID">Specifies the creator ID.</param>
/// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns>
public bool LoadDatabase(int nCreatorID = 0)
{
try
{
reportProgress(0, 0, "Loading " + dataset_name + " database...");
int nIdx = 0;
int nTotal = 5011 + 17125;
int nExtractIdx = 0;
int nExtractTotal = 10935 + 40178;
// Get the label map.
LabelMap labelMap = loadLabelMap();
Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true);
string strSrc = dataset_name + ".training";
int nSrcId = m_factory.GetSourceID(strSrc);
if (nSrcId > 0)
{
m_factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel))
{
return(false);
}
if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel))
{
return(false);
}
SourceDescriptor srcTrain = m_factory.LoadSource(strSrc);
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 4952;
nExtractIdx = 0;
nExtractTotal = 10347;
strSrc = dataset_name + ".testing";
nSrcId = m_factory.GetSourceID(strSrc);
if (nSrcId > 0)
{
m_factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel))
{
return(false);
}
SourceDescriptor srcTest = m_factory.LoadSource(strSrc);
DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset");
m_factory.AddDataset(ds);
m_factory.UpdateDatasetCounts(ds.ID);
return(true);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
}
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false)
{
try
{
RawImageMean imgMeanRaw = null;
m_factory.Open(src);
nLastImageIdx = nImageDbLoadLimitStartIdx;
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL)
{
if (imgMean == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
if (imgMeanRaw == null)
{
if (log != null)
{
log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString());
}
loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
}
}
}
int nCount = src.ImageCount;
if (nCount == 0)
{
throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'.");
}
if (log != null)
{
log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images.");
}
ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit);
if (OnCalculateImageMean != null)
{
imgset.OnCalculateImageMean += OnCalculateImageMean;
}
if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND)
{
bool bDataIsReal = src.IsRealData;
int nBatchSize = 20000;
Stopwatch sw = new Stopwatch();
int nImageSize = src.ImageHeight * src.ImageWidth;
if (nImageSize > 60000)
{
nBatchSize = 5000;
}
else if (nBatchSize > 20000)
{
nBatchSize = 7500;
}
else if (nImageSize > 3000)
{
nBatchSize = 10000;
}
if (nImageDbLoadLimit <= 0)
{
nImageDbLoadLimit = nCount;
}
List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit);
int nIdx = 0;
sw.Start();
while (nIdx < rgIdx.Count)
{
int nImageIdx = rgIdx[nIdx];
int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize);
List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount);
for (int j = 0; j < rgImg.Count; j++)
{
SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH);
imgset.Add(nIdx + j, sd1);
if (sw.Elapsed.TotalMilliseconds > 1000)
{
if (log != null)
{
double dfPct = (double)(nIdx + j) / (double)nCount;
log.Progress = dfPct;
log.WriteLine("image loading at " + dfPct.ToString("P") + "...");
}
sw.Restart();
if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
{
return(null);
}
}
}
nIdx += rgImg.Count;
if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount)
{
log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset.");
break;
}
}
if (log != null)
{
log.Progress = 0;
}
if (rgIdx.Count > 0)
{
nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1;
}
}
else if (bLoadNext)
{
nLastImageIdx += nImageDbLoadLimit;
}
if (imgMean == null)
{
if (imgMeanRaw == null)
{
imgMeanRaw = m_factory.GetRawImageMean();
}
if (imgMeanRaw != null)
{
imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH);
}
else
{
if (log != null)
{
log.WriteLine("Calculating mean...");
}
imgMean = imgset.GetImageMean(log, rgAbort);
m_factory.PutRawImageMean(imgMean, true);
}
}
if (imgMean != null)
{
imgset.SetImageMean(imgMean);
}
imgset.CompleteLoad(nLastImageIdx);
return(imgset);
}
finally
{
m_factory.Close();
}
}
public void ReadXElement_Empty_ThrowsArgumentNullException()
{
SourceDescriptor sd = new SourceDescriptor();
XElement element = null;
sd.ReadXElement(element);
}
/// <summary>
/// Create the dataset and load it into the database.
/// </summary>
/// <param name="nCreatorID">Specifies the creator ID.</param>
/// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns>
public bool LoadDatabase(int nCreatorID = 0)
{
int nIdx = 0;
int nTotal = 0;
try
{
List <Tuple <byte[], int> > rgTrainImg;
List <Tuple <byte[], int> > rgTestImg;
m_extractor.ExtractImages(out rgTrainImg, out rgTestImg);
reportProgress(nIdx, nTotal, "Loading " + dataset_name + " database...");
DatasetFactory factory = null;
string strExportFolder = null;
if (m_param.ExportToFile)
{
strExportFolder = m_param.ExportPath.TrimEnd('\\') + "\\";
if (!Directory.Exists(strExportFolder))
{
Directory.CreateDirectory(strExportFolder);
}
}
string strTrainSrc = "training";
if (!m_param.ExportToFile)
{
factory = new DatasetFactory();
strTrainSrc = dataset_name + "." + strTrainSrc;
int nSrcId = factory.GetSourceID(strTrainSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
}
if (!loadFile(factory, rgTrainImg, m_extractor.Channels, m_extractor.Height, m_extractor.Width, strTrainSrc, strExportFolder))
{
return(false);
}
string strTestSrc = "testing";
if (!m_param.ExportToFile)
{
strTestSrc = dataset_name + "." + strTestSrc;
int nSrcId = factory.GetSourceID(strTestSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
}
if (!loadFile(factory, rgTestImg, m_extractor.Channels, m_extractor.Height, m_extractor.Width, strTestSrc, strExportFolder))
{
return(false);
}
if (!m_param.ExportToFile)
{
SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc);
SourceDescriptor srcTest = factory.LoadSource(strTestSrc);
DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Character Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
}
return(true);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
}
public void ReadXElement_Empty_ThrowsSyncDocumentException()
{
SourceDescriptor sd = new SourceDescriptor();
XElement element = XElement.Parse("<NotSourceDescriptor />");
sd.ReadXElement(element);
}
/// <summary>
/// Create the dataset and load it into the database.
/// </summary>
/// <param name="nCreatorID">Specifies the creator ID.</param>
/// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns>
public bool LoadDatabase(int nCreatorID = 0)
{
try
{
int nIdx = 0;
int nTotal = 50000;
reportProgress(nIdx, 0, "Loading database " + dataset_name + "...");
DatasetFactory factory = new DatasetFactory();
string strTrainSrc = dataset_name + ".training";
int nSrcId = factory.GetSourceID(strTrainSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.DataBatchFile1, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile2, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile3, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile4, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
if (!loadFile(m_param.DataBatchFile5, strTrainSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc);
m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID);
m_rgImg = new List <SimpleDatum>();
nIdx = 0;
nTotal = 10000;
string strTestSrc = dataset_name + ".testing";
nSrcId = factory.GetSourceID(strTestSrc);
if (nSrcId != 0)
{
factory.DeleteSourceData(nSrcId);
}
if (!loadFile(m_param.TestBatchFile, strTestSrc, nTotal, ref nIdx, m_log))
{
return(false);
}
SourceDescriptor srcTest = factory.LoadSource(strTestSrc);
m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID);
DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset");
factory.AddDataset(ds);
factory.UpdateDatasetCounts(ds.ID);
return(true);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (OnCompleted != null)
{
OnCompleted(this, new EventArgs());
}
}
}
