/// <summary> /// The constructor. /// </summary> /// <param name="random">Specifies the CryptoRandom to use for random selection.</param> /// <param name="log">Specifies the output log.</param> /// <param name="src">Specifies the data source that holds the data on the database.</param> /// <param name="factory">Specifies the data factory used to access the database data.</param> /// <param name="rgAbort">Specifies the cancel handles.</param> /// <param name="nMaxLoadCount">Optionally, specifies to automaticall start the image refresh which only applies when the number of images loaded into memory is less than the actual number of images (default = false).</param> public MasterList(CryptoRandom random, Log log, SourceDescriptor src, DatasetFactory factory, List <WaitHandle> rgAbort, int nMaxLoadCount = 0) { m_random = random; m_log = log; m_src = src; m_factory = factory; m_rgAbort.Add(m_evtCancel); if (rgAbort.Count > 0) { m_rgAbort.AddRange(rgAbort); } m_imgMean = m_factory.LoadImageMean(m_src.ID); m_nLoadCount = nMaxLoadCount; if (m_nLoadCount == 0 || m_nLoadCount > m_src.ImageCount) { m_nLoadCount = m_src.ImageCount; } m_rgImages = new SimpleDatum[m_nLoadCount]; m_nLoadedCount = 0; if (m_nLoadCount < m_src.ImageCount) { m_refreshManager = new RefreshManager(random, m_src, m_factory); } }
private int getBatchSize(SourceDescriptor src) { int nBatchSize = 20000; int nImageSize = src.ImageHeight * src.ImageWidth * src.ImageChannels; if (nImageSize > 60000) { nBatchSize = 5000; } else if (nImageSize > 20000) { nBatchSize = 7500; } else if (nImageSize > 3000) { nBatchSize = 10000; } if (nBatchSize > m_nLoadCount) { nBatchSize = m_nLoadCount; } return(nBatchSize); }
/// <summary> /// The BaseDataLayer constructor. /// </summary> /// <param name="cuda">Specifies the CudaDnn connection to Cuda.</param> /// <param name="log">Specifies the Log for output.</param> /// <param name="p">Specifies the LayerParameter</param> /// <param name="db">Specifies the external database to use.</param> public BaseDataLayer(CudaDnn <T> cuda, Log log, LayerParameter p, IXImageDatabase db) : base(cuda, log, p) { if (db != null) { m_imgdb = db; if (p.type == LayerParameter.LayerType.DATA || p.type == LayerParameter.LayerType.TRIPLET_DATA) { m_src = m_imgdb.GetSourceByName(p.data_param.source); } else if (p.type == LayerParameter.LayerType.BATCHDATA) { m_src = m_imgdb.GetSourceByName(p.batch_data_param.source); } if (p.transform_param.use_imagedb_mean) { if (db != null) { m_imgMean = db.GetImageMean(m_src.ID); } else { m_log.WriteLine("WARNING: The image database is NULL, and therefore no mean image can not be acquired."); } } } }
public void LoadDatabase() { int nIdx = 0; int nTotal = 0; reportProgress(nIdx, nTotal, "Unpacking files..."); string strTrainImagesBin = expandFile(m_param.TrainImagesFile); string strTrainLabelsBin = expandFile(m_param.TrainLabelsFile); string strTestImagesBin = expandFile(m_param.TestImagesFile); string strTestLabelsBin = expandFile(m_param.TestLabelsFile); reportProgress(nIdx, nTotal, "Loading database..."); loadFile(strTrainImagesBin, strTrainLabelsBin, "MNIST.training"); loadFile(strTestImagesBin, strTestLabelsBin, "MNIST.testing"); DatasetFactory factory = new DatasetFactory(); SourceDescriptor srcTrain = factory.LoadSource("MNIST.training"); SourceDescriptor srcTest = factory.LoadSource("MNIST.testing"); DatasetDescriptor ds = new DatasetDescriptor(0, "MNIST", null, null, srcTrain, srcTest, "MNIST", "MNIST Character Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } }
/// <summary> /// Returns the dataset descriptor of the dynamic dataset produced by the Gym. /// </summary> /// <param name="dt">Specifies the data-type to use.</param> /// <param name="log">Optionally, specifies the output log to use (default = <i>null</i>).</param> /// <returns>The dataset descriptor is returned.</returns> public DatasetDescriptor GetDataset(DATA_TYPE dt, Log log = null) { int nH = 1; int nW = 1; int nC = 4; if (dt == DATA_TYPE.DEFAULT) { dt = DATA_TYPE.VALUES; } if (dt == DATA_TYPE.BLOB) { nH = 156; nW = 156; nC = 3; } SourceDescriptor srcTrain = new SourceDescriptor((int)GYM_DS_ID.CARTPOLE, Name + ".training", nW, nH, nC, false, false); SourceDescriptor srcTest = new SourceDescriptor((int)GYM_SRC_TEST_ID.CARTPOLE, Name + ".testing", nW, nH, nC, false, false); DatasetDescriptor ds = new DatasetDescriptor((int)GYM_SRC_TRAIN_ID.CARTPOLE, Name, null, null, srcTrain, srcTest, "CartPoleGym", "CartPole Gym", null, GYM_TYPE.DYNAMIC); m_dt = dt; return(ds); }
protected bool IsSourceItemNs(SourceDescriptor descr, string uri) { if (descr == null || uri == null) { return(false); } return(uri.StartsWith(descr.Ns + Separator)); }
/// <summary> /// The Cursor constructor. /// </summary> /// <param name="db">Specifies the underlying database.</param> /// <param name="strSrc">Specifies the name of the data source to use.</param> public Cursor(IXImageDatabase db, string strSrc) { m_db = db; SourceDescriptor src = m_db.GetSourceByName(strSrc); m_nSrcID = src.ID; m_nCount = src.ImageCount; }
public void ReadXElement_NoProviderFields_ThrowsException() { SourceDescriptor descriptor = new SourceDescriptor(); descriptor.ReadXElement(XElement.Parse(@" <Source> <IDataProvider> <SqlDataProvider connectionString=""connstr"" query=""query"" /> </IDataProvider> </Source>")); }
/// <summary> /// Disposes the current object /// </summary> /// <param name="disposing">true to dispose both managed an unmanaged resources, false to dispose only managed resources</param> protected virtual void Dispose(bool disposing) { if (disposing) { Image.BeginAnimation(Image.SourceProperty, null); Animation.Completed -= AnimationCompleted; SourceDescriptor.RemoveValueChanged(Image, ImageSourceChanged); Image.Source = null; } }
/// <summary> /// The Cursor constructor. /// </summary> /// <param name="db">Specifies the underlying database.</param> /// <param name="transformer">Specifies the data transformer used to transform the lables (when active).</param> /// <param name="strSrc">Specifies the name of the data source to use.</param> /// <param name="log">Optionally, specifies an output log used for diagnostic information if specified (default = null).</param> public Cursor(IXImageDatabaseBase db, DataTransformer <T> transformer, string strSrc, Log log = null) { m_log = log; m_db = db; m_transformer = transformer; SourceDescriptor src = m_db.GetSourceByName(strSrc); m_strSrc = src.Name; m_nSrcID = src.ID; m_nCount = src.ImageCount; }
public void ReadXElement_NoIDataProvider_ThrowsException() { SourceDescriptor descriptor = new SourceDescriptor(); descriptor.ReadXElement(XElement.Parse(@" <Source> <ProviderFields> <ProviderField name=""fieldName1"" type=""System.Int32"" /> <ProviderField name=""fieldName2"" type=""System.Int32"" /> </ProviderFields> </Source>")); }
public void AppendIncludes(SourceDescriptor desc, bool Ext = false) { StringBuilder builder = Ext ? ExternalBuilder : Builder; builder.AppendLine(); builder.AppendLine($"# Include directories for {desc.Name}"); foreach (var inc in desc.IncludeDirectories) { builder.AppendLine($"CFLAGS += -I{inc.FullPath}"); } }
public override int GetHashCode() { unchecked { var hashCode = EqualityComparer <T> .Default.GetHashCode(Value); hashCode = (hashCode * 397) ^ (ValueDescriptor?.GetHashCode() ?? 0); hashCode = (hashCode * 397) ^ (int)Source; hashCode = (hashCode * 397) ^ (SourceDescriptor?.GetHashCode() ?? 0); hashCode = (hashCode * 397) ^ (CustomExpression?.GetHashCode() ?? 0); return(hashCode); } }
/// <summary> /// The constructor used to copy another MasterIndexes and optionally specify a sorting for the indexes. /// </summary> /// <param name="idx">Specifies the MasterIndexes to copy.</param> /// <param name="sort">Optionally, specifies a sorting to use on the indexes.</param> public MasterIndexes(MasterIndexes idx, IMGDB_SORT sort) { m_sort = sort; m_src = idx.m_src; m_random = idx.m_random; m_nLoadLimit = idx.m_nLoadLimit; m_factory = new DatasetFactory(idx.m_factory); m_index = idx.m_index.Clone(sort); m_rgLabels = new LabelIndex(idx.m_rgLabels); m_boosted = idx.m_boosted.Clone(sort); m_rgLabelsBoosted = new LabelIndex(idx.m_rgLabelsBoosted); }
/// <summary> /// The ImageSet2 constructor. /// </summary> /// <param name="type">Specifies the type of data source managed.</param> /// <param name="log">Specifies the output log.</param> /// <param name="factory">Specifies the data factory used to access the database data.</param> /// <param name="src">Specifies the data source descriptor.</param> /// <param name="loadMethod">Specifies the load method used to load the data.</param> /// <param name="random">Specifies the random number generator.</param> /// <param name="rgAbort">Specifies the cancellation handles.</param> public ImageSet2(TYPE type, Log log, DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, CryptoRandom random, WaitHandle[] rgAbort) : base(factory, src) { m_type = type; m_log = log; m_loadMethod = loadMethod; m_random = random; m_rgAbort.Add(m_evtCancel); if (rgAbort.Length > 0) { m_rgAbort.AddRange(rgAbort); } }
/// <summary> /// The constructor. /// </summary> /// <param name="random">Specifies the random number generator.</param> /// <param name="src">Specifies the data source.</param> /// <param name="nLoadLimit">Optionally, specifies the load limit used which when set to a value > 0, limits queries to RANDOM image selection within the load limit count (default = 0).</param> public MasterIndexes(CryptoRandom random, SourceDescriptor src, int nLoadLimit = 0) { m_random = random; m_src = src; m_factory.Open(src); m_nLoadLimit = nLoadLimit; if (m_nLoadLimit > src.ImageCount) { m_nLoadLimit = src.ImageCount; } m_rgImageIdx = m_factory.LoadImageIndexes(false); load(m_rgImageIdx.Where(p => p != null).ToList()); }
public void AppendSourceFiles(SourceDescriptor desc, bool Ext = false) { StringBuilder builder = Ext ? ExternalBuilder : Builder; if (desc.SourceFiles != null) { builder.AppendLine(); builder.AppendLine($"# Source files for {desc.Name}"); foreach (var src in desc.SourceFiles) { builder.AppendLine($"SRC += {src.FullPath}"); src.PreProcess(); } } }
public void ReadXElement_SetTwoProviderFields_ProviderFieldsHas2Items() { SourceDescriptor descriptor = new SourceDescriptor(); descriptor.ReadXElement(XElement.Parse(@" <Source> <ProviderFields> <ProviderField name=""fieldName1"" type=""System.Int32"" /> <ProviderField name=""fieldName2"" type=""System.Int32"" /> </ProviderFields> <IDataProvider> <SqlDataProvider connectionString=""connstr"" query=""query"" /> </IDataProvider> </Source>")); Assert.IsTrue(descriptor.ProviderFields.Count == 2); }
internal ImageAnimationController(Image image, ObjectAnimationUsingKeyFrames animation, bool autoStart) { Image = image; Animation = animation; Animation.Completed += AnimationCompleted; Clock = Animation.CreateClock(); ClockController = Clock.Controller; SourceDescriptor.AddValueChanged(image, ImageSourceChanged); // ReSharper disable once PossibleNullReferenceException ClockController.Pause(); Image.ApplyAnimationClock(Image.SourceProperty, Clock); if (autoStart) { ClockController.Resume(); } }
/// <summary> /// The ImageSet constructor. /// </summary> /// <param name="factory">Specifies the DatasetFactory.</param> /// <param name="src">Specifies the data source.</param> /// <param name="loadMethod">Specifies the method to use when loading the images.</param> /// <param name="nLoadLimit">Specifies the image load limit.</param> public ImageSet(DatasetFactory factory, SourceDescriptor src, IMAGEDB_LOAD_METHOD loadMethod, int nLoadLimit) { m_factory = new DatasetFactory(factory); m_factory.Open(src.ID); m_loadMethod = loadMethod; m_nLoadLimit = nLoadLimit; m_src = new SourceDescriptor(src); m_imgMean = null; m_rgImages = new SimpleDatum[m_src.ImageCount]; foreach (LabelDescriptor label in src.Labels) { if (label.ImageCount > 0) { m_rgLabelSet.Add(new LabelSet(label)); } } }
protected object ExtractSourceId(SourceDescriptor descr, string uri) { var idStr = uri.Substring(descr.Ns.Length + Separator.Length); if (descr.IdFieldType != null) { return(ConvertManager.ChangeType(idStr, descr.IdFieldType)); } // may be this field is registered? if (FieldNameSourceHash[descr].ContainsKey(descr.IdFieldName)) { var fldDescr = FieldNameSourceHash[descr][descr.IdFieldName]; if (fldDescr.FieldType != null) { return(ConvertManager.ChangeType(idStr, fldDescr.FieldType)); } } return(idStr); }
void AppendDataFiles(SourceDescriptor desc) { var d = desc.DataFiles; if (d != null) { var DirName = Catalog.ResolvePath(Path.Combine("gen", desc.Name)); ExternalBuilder.AppendLine($"CFLAGS += -I{DirName}"); if (!Directory.Exists(DirName)) { Directory.CreateDirectory(DirName); } foreach (var df in d) { string FilePath = Path.Combine(DirName, df.Name); var hg = new HeaderGenerator(df); hg.Export(FilePath); } } }
public void LoadDatabase() { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database..."); Log log = new Log("MNIST"); log.OnWriteLine += Log_OnWriteLine; DatasetFactory factory = new DatasetFactory(); loadFile(m_param.DataBatchFile1, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile2, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile3, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile4, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile5, "CIFAR-10.training", nTotal, ref nIdx); SourceDescriptor srcTrain = factory.LoadSource("CIFAR-10.training"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; loadFile(m_param.TestBatchFile, "CIFAR-10.testing", nTotal, ref nIdx); SourceDescriptor srcTest = factory.LoadSource("CIFAR-10.testing"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(0, "CIFAR-10", null, null, srcTrain, srcTest, "CIFAR-10", "CiFar-10 Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } }
public LabelIndex(LabelIndex idx) { m_strName = idx.m_strName + " copy"; m_randomLabel = idx.m_randomLabel; m_randomData = idx.m_randomData; m_src = idx.m_src; m_bBoosted = idx.m_bBoosted; m_rgIdx = new List <int>(); if (idx.m_rgLabels != null && idx.m_rgLabels.Length > 0) { m_rgLabels = new Index[idx.m_rgLabels.Length]; bool bFillLabelMap = false; if (m_rgLabelToIdxMap == null || m_rgLabelToIdxMap.Count == 0 || m_rgIdxToLabelMap == null || m_rgIdxToLabelMap.Count == 0) { m_rgLabelToIdxMap = new Dictionary <int, int>(); m_rgIdxToLabelMap = new Dictionary <int, int>(); bFillLabelMap = true; } for (int i = 0; i < idx.m_rgLabels.Length; i++) { m_rgLabels[i] = idx.m_rgLabels[i].Clone(); if (m_rgLabels[i].Count > 0) { m_rgIdx.Add(i); } if (bFillLabelMap) { int nLabel = m_rgLabels[i].Label; m_rgLabelToIdxMap[nLabel] = i; m_rgIdxToLabelMap[i] = nLabel; } } } }
public LabelIndex(string strName, CryptoRandom randomLabel, CryptoRandom randomData, SourceDescriptor src, bool bBoosted, List <DbItem> rgItems) { m_strName = strName; m_randomLabel = randomLabel; m_randomData = randomData; m_src = src; m_bBoosted = bBoosted; m_rgIdx = new List <int>(); m_rgLabelToIdxMap = new Dictionary <int, int>(); m_rgIdxToLabelMap = new Dictionary <int, int>(); List <LabelDescriptor> rgLabels = src.Labels.Where(p => p.ImageCount > 0).OrderBy(p => p.ActiveLabel).ToList(); if (rgLabels.Count > 0) { m_rgLabels = new Index[rgLabels.Count]; for (int i = 0; i < rgLabels.Count; i++) { int nLabel = rgLabels[i].ActiveLabel; List <DbItem> rgLabelList = rgItems.Where(p => p.Label == nLabel).ToList(); if (i < rgLabels.Count - 1) { rgItems = rgItems.Where(p => p.Label != nLabel).ToList(); } m_rgLabels[i] = new Index(strName + " label " + nLabel.ToString(), randomData, rgLabelList, nLabel, false); if (rgLabelList.Count > 0) { m_rgIdx.Add(i); } m_rgLabelToIdxMap[nLabel] = i; m_rgIdxToLabelMap[i] = nLabel; } } }
/// <summary> /// Returns the dataset descriptor of the dynamic dataset produced by the Gym. /// </summary> /// <param name="dt">Specifies the data-type to use.</param> /// <param name="log">Optionally, specifies the output log to use (default = <i>null</i>).</param> /// <returns>The dataset descriptor is returned.</returns> public DatasetDescriptor GetDataset(DATA_TYPE dt, Log log = null) { if (dt == DATA_TYPE.DEFAULT) { dt = DATA_TYPE.BLOB; } if (dt != DATA_TYPE.BLOB) { if (log == null) { log = m_log; } if (log != null) { log.WriteLine("WARNING: This gym only supports the BLOB type, the datatype will be changed to BLOB."); } else { throw new Exception("This gym only supports the BLOB type."); } dt = DATA_TYPE.BLOB; } int nH = 80; int nW = 80; int nC = 1; SourceDescriptor srcTrain = new SourceDescriptor((int)GYM_DS_ID.ATARI, Name + ".training", nW, nH, nC, false, false); SourceDescriptor srcTest = new SourceDescriptor((int)GYM_SRC_TEST_ID.ATARI, Name + ".testing", nW, nH, nC, false, false); DatasetDescriptor ds = new DatasetDescriptor((int)GYM_SRC_TRAIN_ID.ATARI, Name, null, null, srcTrain, srcTest, "AtariGym", "Atari Gym", null, GYM_TYPE.DYNAMIC); m_dt = dt; return(ds); }
public RefreshManager(CryptoRandom random, SourceDescriptor src, DatasetFactory factory) { m_random = random; m_rgItems = factory.LoadImageIndexes(false, true); }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { reportProgress(0, 0, "Loading " + dataset_name + " database..."); int nIdx = 0; int nTotal = 5011 + 17125; int nExtractIdx = 0; int nExtractTotal = 10935 + 40178; // Get the label map. LabelMap labelMap = loadLabelMap(); Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true); string strSrc = dataset_name + ".training"; int nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } List <Tuple <int, string, Size> > rgFileSizes = new List <Tuple <int, string, Size> >(); if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } string strDir = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData) + "\\MyCaffe\\test_data\\data\\ssd\\VOC0712\\"; if (!Directory.Exists(strDir)) { Directory.CreateDirectory(strDir); } saveFileSizes(rgFileSizes, strDir + "train_name_size.txt"); SourceDescriptor srcTrain = m_factory.LoadSource(strSrc); rgFileSizes = new List <Tuple <int, string, Size> >(); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 4952; nExtractIdx = 0; nExtractTotal = 10347; strSrc = dataset_name + ".testing"; nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } saveFileSizes(rgFileSizes, strDir + "test_name_size.txt"); SourceDescriptor srcTest = m_factory.LoadSource(strSrc); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); m_factory.AddDataset(ds); m_factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
/// <summary> /// The ImageSet constructor. /// </summary> /// <param name="factory">Specifies the DatasetFactory.</param> /// <param name="src">Specifies the data source.</param> public ImageSetBase(DatasetFactory factory, SourceDescriptor src) { m_src = new SourceDescriptor(src); m_factory = new DatasetFactory(factory); m_factory.Open(src.ID); }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { reportProgress(0, 0, "Loading " + dataset_name + " database..."); int nIdx = 0; int nTotal = 5011 + 17125; int nExtractIdx = 0; int nExtractTotal = 10935 + 40178; // Get the label map. LabelMap labelMap = loadLabelMap(); Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true); string strSrc = dataset_name + ".training"; int nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } SourceDescriptor srcTrain = m_factory.LoadSource(strSrc); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 4952; nExtractIdx = 0; nExtractTotal = 10347; strSrc = dataset_name + ".testing"; nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } SourceDescriptor srcTest = m_factory.LoadSource(strSrc); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); m_factory.AddDataset(ds); m_factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
private ImageSet loadImageset(string strType, SourceDescriptor src, WaitHandle[] rgAbort, ref SimpleDatum imgMean, out int nLastImageIdx, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, int nImageDbLoadLimit = 0, int nImageDbLoadLimitStartIdx = 0, bool bLoadNext = false) { try { RawImageMean imgMeanRaw = null; m_factory.Open(src); nLastImageIdx = nImageDbLoadLimitStartIdx; if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ALL) { if (imgMean == null) { imgMeanRaw = m_factory.GetRawImageMean(); if (imgMeanRaw == null) { if (log != null) { log.WriteLine("WARNING: No image mean exists in the database, changing image database load from " + loadMethod.ToString() + " to " + IMAGEDB_LOAD_METHOD.LOAD_ALL.ToString()); } loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL; } } } int nCount = src.ImageCount; if (nCount == 0) { throw new Exception("Could not find any images with " + strType + " Source = '" + src.Name + "'."); } if (log != null) { log.WriteLine("Loading '" + src.Name + "' - " + nCount.ToString("N0") + " images."); } ImageSet imgset = new ImageSet(m_factory, src, loadMethod, nImageDbLoadLimit); if (OnCalculateImageMean != null) { imgset.OnCalculateImageMean += OnCalculateImageMean; } if (loadMethod != IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND) { bool bDataIsReal = src.IsRealData; int nBatchSize = 20000; Stopwatch sw = new Stopwatch(); int nImageSize = src.ImageHeight * src.ImageWidth; if (nImageSize > 60000) { nBatchSize = 5000; } else if (nBatchSize > 20000) { nBatchSize = 7500; } else if (nImageSize > 3000) { nBatchSize = 10000; } if (nImageDbLoadLimit <= 0) { nImageDbLoadLimit = nCount; } List <int> rgIdx = getIndexList(nImageDbLoadLimitStartIdx, nImageDbLoadLimit); int nIdx = 0; sw.Start(); while (nIdx < rgIdx.Count) { int nImageIdx = rgIdx[nIdx]; int nImageCount = Math.Min(rgIdx.Count - nIdx, nBatchSize); List <RawImage> rgImg = m_factory.GetRawImagesAt(nImageIdx, nImageCount); for (int j = 0; j < rgImg.Count; j++) { SimpleDatum sd1 = m_factory.LoadDatum(rgImg[j], nPadW, nPadH); imgset.Add(nIdx + j, sd1); if (sw.Elapsed.TotalMilliseconds > 1000) { if (log != null) { double dfPct = (double)(nIdx + j) / (double)nCount; log.Progress = dfPct; log.WriteLine("image loading at " + dfPct.ToString("P") + "..."); } sw.Restart(); if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout) { return(null); } } } nIdx += rgImg.Count; if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && rgImg.Count == 0 && nIdx < nCount) { log.WriteLine("WARNING: Loaded " + nIdx.ToString("N0") + " images, yet " + (nCount - nIdx).ToString("N0") + " images are unaccounted for. You may need to reindex the dataset."); break; } } if (log != null) { log.Progress = 0; } if (rgIdx.Count > 0) { nLastImageIdx = rgIdx[rgIdx.Count - 1] + 1; } } else if (bLoadNext) { nLastImageIdx += nImageDbLoadLimit; } if (imgMean == null) { if (imgMeanRaw == null) { imgMeanRaw = m_factory.GetRawImageMean(); } if (imgMeanRaw != null) { imgMean = m_factory.LoadDatum(imgMeanRaw, nPadW, nPadH); } else { if (log != null) { log.WriteLine("Calculating mean..."); } imgMean = imgset.GetImageMean(log, rgAbort); m_factory.PutRawImageMean(imgMean, true); } } if (imgMean != null) { imgset.SetImageMean(imgMean); } imgset.CompleteLoad(nLastImageIdx); return(imgset); } finally { m_factory.Close(); } }
public void ReadXElement_Empty_ThrowsArgumentNullException() { SourceDescriptor sd = new SourceDescriptor(); XElement element = null; sd.ReadXElement(element); }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { int nIdx = 0; int nTotal = 0; try { List <Tuple <byte[], int> > rgTrainImg; List <Tuple <byte[], int> > rgTestImg; m_extractor.ExtractImages(out rgTrainImg, out rgTestImg); reportProgress(nIdx, nTotal, "Loading " + dataset_name + " database..."); DatasetFactory factory = null; string strExportFolder = null; if (m_param.ExportToFile) { strExportFolder = m_param.ExportPath.TrimEnd('\\') + "\\"; if (!Directory.Exists(strExportFolder)) { Directory.CreateDirectory(strExportFolder); } } string strTrainSrc = "training"; if (!m_param.ExportToFile) { factory = new DatasetFactory(); strTrainSrc = dataset_name + "." + strTrainSrc; int nSrcId = factory.GetSourceID(strTrainSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } } if (!loadFile(factory, rgTrainImg, m_extractor.Channels, m_extractor.Height, m_extractor.Width, strTrainSrc, strExportFolder)) { return(false); } string strTestSrc = "testing"; if (!m_param.ExportToFile) { strTestSrc = dataset_name + "." + strTestSrc; int nSrcId = factory.GetSourceID(strTestSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } } if (!loadFile(factory, rgTestImg, m_extractor.Channels, m_extractor.Height, m_extractor.Width, strTestSrc, strExportFolder)) { return(false); } if (!m_param.ExportToFile) { SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc); SourceDescriptor srcTest = factory.LoadSource(strTestSrc); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Character Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); } return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
public void ReadXElement_Empty_ThrowsSyncDocumentException() { SourceDescriptor sd = new SourceDescriptor(); XElement element = XElement.Parse("<NotSourceDescriptor />"); sd.ReadXElement(element); }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database " + dataset_name + "..."); DatasetFactory factory = new DatasetFactory(); string strTrainSrc = dataset_name + ".training"; int nSrcId = factory.GetSourceID(strTrainSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFile1, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile2, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile3, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile4, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile5, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; string strTestSrc = dataset_name + ".testing"; nSrcId = factory.GetSourceID(strTestSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.TestBatchFile, strTestSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTest = factory.LoadSource(strTestSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }