public void LoadDatabase() { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database..."); Log log = new Log("MNIST"); log.OnWriteLine += Log_OnWriteLine; DatasetFactory factory = new DatasetFactory(); loadFile(m_param.DataBatchFile1, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile2, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile3, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile4, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile5, "CIFAR-10.training", nTotal, ref nIdx); SourceDescriptor srcTrain = factory.LoadSource("CIFAR-10.training"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; loadFile(m_param.TestBatchFile, "CIFAR-10.testing", nTotal, ref nIdx); SourceDescriptor srcTest = factory.LoadSource("CIFAR-10.testing"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(0, "CIFAR-10", null, null, srcTrain, srcTest, "CIFAR-10", "CiFar-10 Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { reportProgress(0, 0, "Loading " + dataset_name + " database..."); int nIdx = 0; int nTotal = 5011 + 17125; int nExtractIdx = 0; int nExtractTotal = 10935 + 40178; // Get the label map. LabelMap labelMap = loadLabelMap(); Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true); string strSrc = dataset_name + ".training"; int nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } SourceDescriptor srcTrain = m_factory.LoadSource(strSrc); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 4952; nExtractIdx = 0; nExtractTotal = 10347; strSrc = dataset_name + ".testing"; nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel)) { return(false); } SourceDescriptor srcTest = m_factory.LoadSource(strSrc); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); m_factory.AddDataset(ds); m_factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { reportProgress(0, 0, "Loading " + dataset_name + " database..."); int nIdx = 0; int nTotal = 5011 + 17125; int nExtractIdx = 0; int nExtractTotal = 10935 + 40178; // Get the label map. LabelMap labelMap = loadLabelMap(); Dictionary <string, int> rgNameToLabel = labelMap.MapToLabel(m_log, true); string strSrc = dataset_name + ".training"; int nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } List <Tuple <int, string, Size> > rgFileSizes = new List <Tuple <int, string, Size> >(); if (!loadFile(m_param.DataBatchFileTrain2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } if (!loadFile(m_param.DataBatchFileTrain2012, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } string strDir = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData) + "\\MyCaffe\\test_data\\data\\ssd\\VOC0712\\"; if (!Directory.Exists(strDir)) { Directory.CreateDirectory(strDir); } saveFileSizes(rgFileSizes, strDir + "train_name_size.txt"); SourceDescriptor srcTrain = m_factory.LoadSource(strSrc); rgFileSizes = new List <Tuple <int, string, Size> >(); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 4952; nExtractIdx = 0; nExtractTotal = 10347; strSrc = dataset_name + ".testing"; nSrcId = m_factory.GetSourceID(strSrc); if (nSrcId > 0) { m_factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFileTest2007, strSrc, nExtractTotal, ref nExtractIdx, nTotal, ref nIdx, m_log, m_param.ExtractFiles, rgNameToLabel, rgFileSizes)) { return(false); } saveFileSizes(rgFileSizes, strDir + "test_name_size.txt"); SourceDescriptor srcTest = m_factory.LoadSource(strSrc); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); m_factory.AddDataset(ds); m_factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database " + dataset_name + "..."); DatasetFactory factory = new DatasetFactory(); string strTrainSrc = dataset_name + ".training"; int nSrcId = factory.GetSourceID(strTrainSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFile1, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile2, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile3, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile4, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile5, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; string strTestSrc = dataset_name + ".testing"; nSrcId = factory.GetSourceID(strTestSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.TestBatchFile, strTestSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTest = factory.LoadSource(strTestSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }