/// <summary> /// Set the image mean. /// </summary> /// <param name="d">Specifies the image mean.</param> /// <param name="bSave">Optionally, specifies whether or not to save the image mean in the database (default = false).</param> public void SetImageMean(SimpleDatum d, bool bSave = false) { m_imgMean = d; if (bSave) { m_factory.SaveImageMean(d, true); } }
/// <summary> /// Saves the image mean in a SimpleDatum to the database. /// </summary> /// <param name="nSrcId">Specifies the ID of the data source to use.</param> /// <param name="sd">Specifies the image mean data.</param> /// <param name="bUpdate">Specifies whether or not to update the mean image.</param> /// <returns>If saved successfully, this method returns <i>true</i>, otherwise <i>false</i> is returned.</returns> public bool SaveImageMean(int nSrcId, SimpleDatum sd, bool bUpdate) { if (m_TestingImages.SourceID != nSrcId && m_TrainingImages.SourceID != nSrcId) { return(false); } return(m_factory.SaveImageMean(sd, bUpdate, nSrcId)); }
public void LoadDatabase() { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database..."); Log log = new Log("MNIST"); log.OnWriteLine += Log_OnWriteLine; DatasetFactory factory = new DatasetFactory(); loadFile(m_param.DataBatchFile1, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile2, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile3, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile4, "CIFAR-10.training", nTotal, ref nIdx); loadFile(m_param.DataBatchFile5, "CIFAR-10.training", nTotal, ref nIdx); SourceDescriptor srcTrain = factory.LoadSource("CIFAR-10.training"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; loadFile(m_param.TestBatchFile, "CIFAR-10.testing", nTotal, ref nIdx); SourceDescriptor srcTest = factory.LoadSource("CIFAR-10.testing"); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(0, "CIFAR-10", null, null, srcTrain, srcTest, "CIFAR-10", "CiFar-10 Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } }
private void loadFile(string strImagesFile, string strLabelsFile, string strSourceName) { Stopwatch sw = new Stopwatch(); reportProgress(0, 0, " Source: " + strSourceName); reportProgress(0, 0, " loading " + strImagesFile + "..."); BinaryFile image_file = new app.BinaryFile(strImagesFile); BinaryFile label_file = new app.BinaryFile(strLabelsFile); Log log = new Log("MNIST"); log.OnWriteLine += Log_OnWriteLine; try { // Verify the files uint magicImg = image_file.ReadUInt32(); uint magicLbl = label_file.ReadUInt32(); if (magicImg != 2051) { throw new Exception("Incorrect image file magic."); } if (magicLbl != 2049) { throw new Exception("Incorrect label file magic."); } uint num_items = image_file.ReadUInt32(); uint num_labels = label_file.ReadUInt32(); if (num_items != num_labels) { throw new Exception("The number of items must be equal to the number of labels!"); } // Add the data source to the database. uint rows = image_file.ReadUInt32(); uint cols = image_file.ReadUInt32(); int nChannels = 1; // black and white int nSrcId = m_factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true); m_factory.Open(nSrcId); m_factory.DeleteSourceData(); // Storing to database; byte[] rgLabel; byte[] rgPixels; Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, null, null, 0, false, -1); reportProgress(0, (int)num_items, " loading a total of " + num_items.ToString() + " items."); reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")"); sw.Start(); List <SimpleDatum> rgImg = new List <SimpleDatum>(); for (int i = 0; i < num_items; i++) { rgPixels = image_file.ReadBytes((int)(rows * cols)); rgLabel = label_file.ReadBytes(1); if (sw.Elapsed.TotalMilliseconds > 1000) { reportProgress(i, (int)num_items, " loading data..."); sw.Restart(); } datum.SetData(rgPixels.ToList(), (int)rgLabel[0]); m_factory.PutRawImageCache(i, datum); rgImg.Add(new SimpleDatum(datum)); } m_factory.ClearImageCashe(true); m_factory.UpdateSourceCounts(); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true); reportProgress((int)num_items, (int)num_items, " loading completed."); } finally { image_file.Dispose(); label_file.Dispose(); } }
/// <summary> /// Create the dataset and load it into the database. /// </summary> /// <param name="nCreatorID">Specifies the creator ID.</param> /// <returns>On successful creation, <i>true</i> is returned, otherwise <i>false</i> is returned on abort.</returns> public bool LoadDatabase(int nCreatorID = 0) { try { int nIdx = 0; int nTotal = 50000; reportProgress(nIdx, 0, "Loading database " + dataset_name + "..."); DatasetFactory factory = new DatasetFactory(); string strTrainSrc = dataset_name + ".training"; int nSrcId = factory.GetSourceID(strTrainSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.DataBatchFile1, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile2, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile3, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile4, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } if (!loadFile(m_param.DataBatchFile5, strTrainSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTrain = factory.LoadSource(strTrainSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTrain.ID); m_rgImg = new List <SimpleDatum>(); nIdx = 0; nTotal = 10000; string strTestSrc = dataset_name + ".testing"; nSrcId = factory.GetSourceID(strTestSrc); if (nSrcId != 0) { factory.DeleteSourceData(nSrcId); } if (!loadFile(m_param.TestBatchFile, strTestSrc, nTotal, ref nIdx, m_log)) { return(false); } SourceDescriptor srcTest = factory.LoadSource(strTestSrc); m_factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, m_rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true, srcTest.ID); DatasetDescriptor ds = new DatasetDescriptor(nCreatorID, dataset_name, null, null, srcTrain, srcTest, dataset_name, dataset_name + " Dataset"); factory.AddDataset(ds); factory.UpdateDatasetCounts(ds.ID); return(true); } catch (Exception excpt) { throw excpt; } finally { if (OnCompleted != null) { OnCompleted(this, new EventArgs()); } } }
private bool loadFile(DatasetFactory factory, List <Tuple <byte[], int> > rgData, int nC, int nH, int nW, string strSourceName, string strExportPath) { if (strExportPath != null) { strExportPath += strSourceName; if (!Directory.Exists(strExportPath)) { Directory.CreateDirectory(strExportPath); } } Stopwatch sw = new Stopwatch(); reportProgress(0, 0, " Source: " + strSourceName); try { if (factory != null) { int nSrcId = factory.AddSource(strSourceName, nC, nW, nH, false, 0, true); factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log); factory.DeleteSourceData(); } // Storing to database; int nLabel; byte[] rgPixels; Datum datum = new Datum(false, nC, nW, nH, -1, DateTime.MinValue, new List <byte>(), 0, false, -1); string strAction = (m_param.ExportToFile) ? "exporing" : "loading"; reportProgress(0, rgData.Count, " " + strAction + " a total of " + rgData.Count.ToString() + " items."); reportProgress(0, rgData.Count, " (with rows: " + nH.ToString() + ", cols: " + nW.ToString() + ")"); sw.Start(); List <SimpleDatum> rgImg = new List <SimpleDatum>(); FileStream fsFileDesc = null; StreamWriter swFileDesc = null; if (m_param.ExportToFile) { string strFile = strExportPath + "\\file_list.txt"; fsFileDesc = File.OpenWrite(strFile); swFileDesc = new StreamWriter(fsFileDesc); } for (int i = 0; i < rgData.Count; i++) { rgPixels = rgData[i].Item1; nLabel = rgData[i].Item2; if (sw.Elapsed.TotalMilliseconds > 1000) { reportProgress(i, rgData.Count, " " + strAction + " data..."); sw.Restart(); } datum.SetData(rgPixels, nLabel); if (factory != null) { factory.PutRawImageCache(i, datum, 5); } else if (strExportPath != null) { saveToFile(strExportPath, i, datum, swFileDesc); } rgImg.Add(new SimpleDatum(datum)); if (m_evtCancel.WaitOne(0)) { return(false); } } if (swFileDesc != null) { swFileDesc.Flush(); swFileDesc.Close(); swFileDesc.Dispose(); fsFileDesc.Close(); fsFileDesc.Dispose(); } if (factory != null) { factory.ClearImageCache(true); factory.UpdateSourceCounts(); factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true); } reportProgress(rgData.Count, rgData.Count, " " + strAction + " completed."); } finally { } return(true); }
private bool loadFile(DatasetFactory factory, string strImagesFile, string strLabelsFile, string strSourceName, string strExportPath) { if (strExportPath != null) { strExportPath += strSourceName; if (!Directory.Exists(strExportPath)) { Directory.CreateDirectory(strExportPath); } } Stopwatch sw = new Stopwatch(); reportProgress(0, 0, " Source: " + strSourceName); reportProgress(0, 0, " loading " + strImagesFile + "..."); BinaryFile image_file = new BinaryFile(strImagesFile); BinaryFile label_file = new BinaryFile(strLabelsFile); try { // Verify the files uint magicImg = image_file.ReadUInt32(); uint magicLbl = label_file.ReadUInt32(); if (magicImg != 2051) { throw new Exception("Incorrect image file magic."); } if (magicLbl != 2049) { throw new Exception("Incorrect label file magic."); } uint num_items = image_file.ReadUInt32(); uint num_labels = label_file.ReadUInt32(); if (num_items != num_labels) { throw new Exception("The number of items must be equal to the number of labels!"); } // Add the data source to the database. uint rows = image_file.ReadUInt32(); uint cols = image_file.ReadUInt32(); int nChannels = 1; // black and white if (factory != null) { int nSrcId = factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true); factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log); factory.DeleteSourceData(); } // Storing to database; byte[] rgLabel; byte[] rgPixels; Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, new List <byte>(), 0, false, -1); string strAction = (m_param.ExportToFile) ? "exporing" : "loading"; reportProgress(0, (int)num_items, " " + strAction + " a total of " + num_items.ToString() + " items."); reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")"); sw.Start(); List <SimpleDatum> rgImg = new List <SimpleDatum>(); FileStream fsFileDesc = null; StreamWriter swFileDesc = null; if (m_param.ExportToFile) { string strFile = strExportPath + "\\file_list.txt"; fsFileDesc = File.OpenWrite(strFile); swFileDesc = new StreamWriter(fsFileDesc); } for (int i = 0; i < num_items; i++) { rgPixels = image_file.ReadBytes((int)(rows * cols)); rgLabel = label_file.ReadBytes(1); if (sw.Elapsed.TotalMilliseconds > 1000) { reportProgress(i, (int)num_items, " " + strAction + " data..."); sw.Restart(); } datum.SetData(rgPixels.ToList(), (int)rgLabel[0]); if (factory != null) { factory.PutRawImageCache(i, datum); } else if (strExportPath != null) { saveToFile(strExportPath, i, datum, swFileDesc); } rgImg.Add(new SimpleDatum(datum)); if (m_evtCancel.WaitOne(0)) { return(false); } } if (swFileDesc != null) { swFileDesc.Flush(); swFileDesc.Close(); swFileDesc.Dispose(); fsFileDesc.Close(); fsFileDesc.Dispose(); } if (factory != null) { factory.ClearImageCashe(true); factory.UpdateSourceCounts(); factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true); } reportProgress((int)num_items, (int)num_items, " " + strAction + " completed."); } finally { image_file.Dispose(); label_file.Dispose(); } return(true); }