private void loadFile(string strImagesFile, string strLabelsFile, string strSourceName) { Stopwatch sw = new Stopwatch(); reportProgress(0, 0, " Source: " + strSourceName); reportProgress(0, 0, " loading " + strImagesFile + "..."); BinaryFile image_file = new app.BinaryFile(strImagesFile); BinaryFile label_file = new app.BinaryFile(strLabelsFile); Log log = new Log("MNIST"); log.OnWriteLine += Log_OnWriteLine; try { // Verify the files uint magicImg = image_file.ReadUInt32(); uint magicLbl = label_file.ReadUInt32(); if (magicImg != 2051) { throw new Exception("Incorrect image file magic."); } if (magicLbl != 2049) { throw new Exception("Incorrect label file magic."); } uint num_items = image_file.ReadUInt32(); uint num_labels = label_file.ReadUInt32(); if (num_items != num_labels) { throw new Exception("The number of items must be equal to the number of labels!"); } // Add the data source to the database. uint rows = image_file.ReadUInt32(); uint cols = image_file.ReadUInt32(); int nChannels = 1; // black and white int nSrcId = m_factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true); m_factory.Open(nSrcId); m_factory.DeleteSourceData(); // Storing to database; byte[] rgLabel; byte[] rgPixels; Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, null, null, 0, false, -1); reportProgress(0, (int)num_items, " loading a total of " + num_items.ToString() + " items."); reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")"); sw.Start(); List <SimpleDatum> rgImg = new List <SimpleDatum>(); for (int i = 0; i < num_items; i++) { rgPixels = image_file.ReadBytes((int)(rows * cols)); rgLabel = label_file.ReadBytes(1); if (sw.Elapsed.TotalMilliseconds > 1000) { reportProgress(i, (int)num_items, " loading data..."); sw.Restart(); } datum.SetData(rgPixels.ToList(), (int)rgLabel[0]); m_factory.PutRawImageCache(i, datum); rgImg.Add(new SimpleDatum(datum)); } m_factory.ClearImageCashe(true); m_factory.UpdateSourceCounts(); m_factory.SaveImageMean(SimpleDatum.CalculateMean(log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true); reportProgress((int)num_items, (int)num_items, " loading completed."); } finally { image_file.Dispose(); label_file.Dispose(); } }
private bool loadFile(string strImagesFile, string strSourceName, int nTotal, ref int nIdx, Log log) { Stopwatch sw = new Stopwatch(); int nStart = nIdx; reportProgress(nIdx, nTotal, " Source: " + strSourceName); reportProgress(nIdx, nTotal, " loading " + strImagesFile + "..."); FileStream fs = null; try { fs = new FileStream(strImagesFile, FileMode.Open, FileAccess.Read); using (BinaryReader br = new BinaryReader(fs)) { fs = null; int nSrcId = m_factory.AddSource(strSourceName, 3, 32, 32, false, 0, true); m_factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, log); if (nIdx == 0) { m_factory.DeleteSourceData(); } sw.Start(); for (int i = 0; i < 10000; i++) { int nLabel = (int)br.ReadByte(); byte[] rgImgBytes = br.ReadBytes(3072); Bitmap img = createImage(rgImgBytes); Datum d = ImageData.GetImageDataD(img, 3, false, nLabel); m_factory.PutRawImageCache(nIdx, d); m_rgImg.Add(new SimpleDatum(d)); nIdx++; if (sw.ElapsedMilliseconds > 1000) { reportProgress(nStart + i, nTotal, "loading " + strImagesFile + " " + i.ToString("N0") + " of 10,000..."); sw.Restart(); } if (m_evtCancel.WaitOne(0)) { return(false); } } m_factory.ClearImageCashe(true); if (nIdx == nTotal) { m_factory.UpdateSourceCounts(); } } } finally { if (fs != null) { fs.Dispose(); } } return(true); }
private bool loadFile(string strImagesFile, string strSourceName, int nExtractTotal, ref int nExtractIdx, int nTotal, ref int nIdx, Log log, bool bExtractFiles, Dictionary <string, int> rgNameToLabel) { Stopwatch sw = new Stopwatch(); reportProgress(nIdx, nTotal, " Source: " + strSourceName); reportProgress(nIdx, nTotal, " loading " + strImagesFile + "..."); FileStream fs = null; try { int nSrcId = m_factory.AddSource(strSourceName, 3, -1, -1, false); addLabels(nSrcId, rgNameToLabel); m_factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, log); int nPos = strImagesFile.ToLower().LastIndexOf(".tar"); string strPath = strImagesFile.Substring(0, nPos); if (!Directory.Exists(strPath)) { Directory.CreateDirectory(strPath); } if (bExtractFiles) { log.Progress = (double)nIdx / nExtractTotal; log.WriteLine("Extracting files from '" + strImagesFile + "'..."); if ((nExtractIdx = TarFile.ExtractTar(strImagesFile, strPath, m_evtCancel, log, nExtractTotal, nExtractIdx)) == 0) { log.WriteLine("Aborted."); return(false); } } // Load the annotations. SimpleDatum.ANNOTATION_TYPE type = SimpleDatum.ANNOTATION_TYPE.BBOX; int nResizeHeight = 0; int nResizeWidth = 0; // Create the training database images. // Create the master list file. List <Tuple <string, string> > rgFiles = createFileList(log, strPath); sw.Start(); for (int i = 0; i < rgFiles.Count; i++) { SimpleDatum datum = loadDatum(log, rgFiles[i].Item1, rgFiles[i].Item2, nResizeHeight, nResizeWidth, type, rgNameToLabel); m_factory.PutRawImageCache(nIdx, datum); nIdx++; if (m_evtCancel.WaitOne(0)) { log.WriteLine("Aborted."); return(false); } if (sw.Elapsed.TotalMilliseconds > 1000) { log.Progress = (double)nIdx / nTotal; log.WriteLine("Loading file " + i.ToString() + " of " + rgFiles.Count.ToString() + "..."); sw.Restart(); } } m_factory.ClearImageCashe(true); m_factory.Close(); } finally { if (fs != null) { fs.Dispose(); } } return(true); }
private bool loadFile(DatasetFactory factory, string strImagesFile, string strLabelsFile, string strSourceName, string strExportPath) { if (strExportPath != null) { strExportPath += strSourceName; if (!Directory.Exists(strExportPath)) { Directory.CreateDirectory(strExportPath); } } Stopwatch sw = new Stopwatch(); reportProgress(0, 0, " Source: " + strSourceName); reportProgress(0, 0, " loading " + strImagesFile + "..."); BinaryFile image_file = new BinaryFile(strImagesFile); BinaryFile label_file = new BinaryFile(strLabelsFile); try { // Verify the files uint magicImg = image_file.ReadUInt32(); uint magicLbl = label_file.ReadUInt32(); if (magicImg != 2051) { throw new Exception("Incorrect image file magic."); } if (magicLbl != 2049) { throw new Exception("Incorrect label file magic."); } uint num_items = image_file.ReadUInt32(); uint num_labels = label_file.ReadUInt32(); if (num_items != num_labels) { throw new Exception("The number of items must be equal to the number of labels!"); } // Add the data source to the database. uint rows = image_file.ReadUInt32(); uint cols = image_file.ReadUInt32(); int nChannels = 1; // black and white if (factory != null) { int nSrcId = factory.AddSource(strSourceName, nChannels, (int)cols, (int)rows, false, 0, true); factory.Open(nSrcId, 500, Database.FORCE_LOAD.NONE, m_log); factory.DeleteSourceData(); } // Storing to database; byte[] rgLabel; byte[] rgPixels; Datum datum = new Datum(false, nChannels, (int)cols, (int)rows, -1, DateTime.MinValue, new List <byte>(), 0, false, -1); string strAction = (m_param.ExportToFile) ? "exporing" : "loading"; reportProgress(0, (int)num_items, " " + strAction + " a total of " + num_items.ToString() + " items."); reportProgress(0, (int)num_items, " (with rows: " + rows.ToString() + ", cols: " + cols.ToString() + ")"); sw.Start(); List <SimpleDatum> rgImg = new List <SimpleDatum>(); FileStream fsFileDesc = null; StreamWriter swFileDesc = null; if (m_param.ExportToFile) { string strFile = strExportPath + "\\file_list.txt"; fsFileDesc = File.OpenWrite(strFile); swFileDesc = new StreamWriter(fsFileDesc); } for (int i = 0; i < num_items; i++) { rgPixels = image_file.ReadBytes((int)(rows * cols)); rgLabel = label_file.ReadBytes(1); if (sw.Elapsed.TotalMilliseconds > 1000) { reportProgress(i, (int)num_items, " " + strAction + " data..."); sw.Restart(); } datum.SetData(rgPixels.ToList(), (int)rgLabel[0]); if (factory != null) { factory.PutRawImageCache(i, datum); } else if (strExportPath != null) { saveToFile(strExportPath, i, datum, swFileDesc); } rgImg.Add(new SimpleDatum(datum)); if (m_evtCancel.WaitOne(0)) { return(false); } } if (swFileDesc != null) { swFileDesc.Flush(); swFileDesc.Close(); swFileDesc.Dispose(); fsFileDesc.Close(); fsFileDesc.Dispose(); } if (factory != null) { factory.ClearImageCashe(true); factory.UpdateSourceCounts(); factory.SaveImageMean(SimpleDatum.CalculateMean(m_log, rgImg.ToArray(), new WaitHandle[] { new ManualResetEvent(false) }), true); } reportProgress((int)num_items, (int)num_items, " " + strAction + " completed."); } finally { image_file.Dispose(); label_file.Dispose(); } return(true); }