Пример #1
0
        public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
        {
            string strCsvFile     = Properties.Settings.Default.CsvFile;
            string strDsName      = config.Name;
            string strTrainingSrc = config.Name + ".training";
            string strTestingSrc  = config.Name + ".testing";

            m_bCancel   = false;
            m_iprogress = progress;
            m_factory.DeleteSources(strTrainingSrc, strTestingSrc);

            Log log = new Log("CSV Dataset Creator");

            log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);

            try
            {
                //-----------------------------------------
                //  Load the schema that defines the layout
                //  of the CSV file.
                //-----------------------------------------

                m_schema = loadSchema(config.Settings);


                //-----------------------------------------
                // Load and parse the CSV file.
                //-----------------------------------------

                DataConfigSetting dsCsvFile = config.Settings.Find("CSV File");

                strCsvFile = dsCsvFile.Value.ToString();
                if (strCsvFile.Length == 0)
                {
                    throw new Exception("CSV data file name not specified!");
                }

                log.WriteLine("Loading the data file...");

                if (m_bCancel)
                {
                    return;
                }

                m_parser.Load(strCsvFile, m_schema);


                //-----------------------------------------
                // Split the data into training and testing
                // sets.
                //-----------------------------------------

                List <DataItem> rgTraining = new List <DataItem>();
                List <DataItem> rgTesting  = new List <DataItem>();

                DataConfigSetting dsPctTesting = config.Settings.Find("Testing Percentage");
                double            dfVal        = (double)dsPctTesting.Value;
                Random            random       = new Random();

                for (int i = 0; i < m_parser.Data.Count; i++)
                {
                    if (random.NextDouble() > dfVal)
                    {
                        rgTraining.Add(m_parser.Data[i]);
                    }
                    else
                    {
                        rgTesting.Add(m_parser.Data[i]);
                    }
                }

                Properties.Settings.Default.TestingPct = dfVal;


                //-----------------------------------------
                // Create the training data source.
                //-----------------------------------------

                int        nCellHorizCount = 0;
                List <int> rgDim           = getImageDim(m_parser, m_schema, out nCellHorizCount);
                int        nTrainSrcId     = m_factory.AddSource(strTrainingSrc, rgDim[0], rgDim[1], rgDim[2], false, 0);
                m_factory.Open(nTrainSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.

                log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
                m_factory.DeleteSourceData();

                if (!loadData(log, m_factory, m_parser, rgTraining, rgDim, true, true))
                {
                    return;
                }

                m_factory.UpdateSourceCounts();
                updateLabels(m_factory);

                log.WriteLine("Creating the image mean...");
                SimpleDatum dMean = SimpleDatum.CalculateMean(log, m_rgImages.ToArray(), new WaitHandle[] { new ManualResetEvent(false) });
                m_factory.PutRawImageMean(dMean, true);
                m_rgImages.Clear();

                m_factory.Close();


                //-----------------------------------------
                // Create the testing data source.
                //-----------------------------------------

                int nTestSrcId = m_factory.AddSource(strTestingSrc, rgDim[0], rgDim[1], rgDim[2], false, 0);
                m_factory.Open(nTestSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.

                log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
                m_factory.DeleteSourceData();

                if (!loadData(log, m_factory, m_parser, rgTesting, rgDim, false, false))
                {
                    return;
                }

                m_factory.UpdateSourceCounts();
                updateLabels(m_factory);
                m_factory.Close();


                //-----------------------------------------
                // Crate the data set.
                //-----------------------------------------

                log.WriteLine("Done loading training and testing data.");
                int nDatasetID = 0;

                using (DNNEntities entities = EntitiesConnection.CreateEntities())
                {
                    List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
                    List <Source> rgSrcTesting  = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();

                    if (rgSrcTraining.Count == 0)
                    {
                        throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
                    }

                    if (rgSrcTesting.Count == 0)
                    {
                        throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
                    }

                    DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
                    int    nSrcTestingCount  = rgSrcTesting[0].ImageCount.GetValueOrDefault();
                    int    nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
                    int    nSrcTotalCount    = nSrcTestingCount + nSrcTrainingCount;
                    double dfTestingPct      = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;

                    Dataset ds = new Dataset();
                    ds.ImageHeight      = rgSrcTraining[0].ImageHeight;
                    ds.ImageWidth       = rgSrcTraining[0].ImageWidth;
                    ds.Name             = strDsName;
                    ds.ImageEncoded     = rgSrcTesting[0].ImageEncoded;
                    ds.ImageChannels    = rgSrcTesting[0].ImageChannels;
                    ds.TestingPercent   = (decimal)dfTestingPct;
                    ds.TestingSourceID  = rgSrcTesting[0].ID;
                    ds.TestingTotal     = rgSrcTesting[0].ImageCount;
                    ds.TrainingSourceID = rgSrcTraining[0].ID;
                    ds.TrainingTotal    = rgSrcTraining[0].ImageCount;
                    ds.DatasetCreatorID = config.ID;
                    ds.DatasetGroupID   = 0;
                    ds.ModelGroupID     = 0;

                    entities.Datasets.Add(ds);
                    entities.SaveChanges();

                    nDatasetID = ds.ID;
                }

                m_factory.SetDatasetParameter(nDatasetID, "PixelSize", m_schema.CellSize.ToString());
                m_factory.SetDatasetParameter(nDatasetID, "AttributeCount", m_parser.DataDescriptions.Count.ToString());
                m_factory.SetDatasetParameter(nDatasetID, "AttributeCountHoriz", nCellHorizCount.ToString());
                m_factory.SetDatasetParameter(nDatasetID, "AttributeCountVert", nCellHorizCount.ToString());
            }
            catch (Exception excpt)
            {
                log.WriteLine("ERROR: " + excpt.Message);
            }
            finally
            {
                Properties.Settings.Default.CsvFile = strCsvFile;
                Properties.Settings.Default.Save();

                if (m_bCancel)
                {
                    log.WriteLine("ABORTED converting CSV data files.");
                }
                else
                {
                    log.WriteLine("Done converting CSV data files.");
                }

                if (m_bCancel)
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, 1, "ABORTED!", null, true));
                }
                else
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
                }
            }
        }
        public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
        {
            string strTrainingDataFile  = Properties.Settings.Default.TrainingDataFile;
            string strTrainingLabelFile = Properties.Settings.Default.TrainingLabelFile;
            string strTestingDataFile   = Properties.Settings.Default.TestingDataFile;
            string strTestingLabelFile  = Properties.Settings.Default.TestingLabelFile;
            string strTargetOverlayFile = Properties.Settings.Default.TargetOverlayFile;
            int    nChannels            = Properties.Settings.Default.Channels;

            m_bCancel = false;

            DataConfigSetting dsTargetOverlayFile = config.Settings.Find("Target Overlay File");

            strTargetOverlayFile = dsTargetOverlayFile.Value.ToString();
            bool bTargetOutput = false;

            if (!string.IsNullOrEmpty(strTargetOverlayFile) && File.Exists(strTargetOverlayFile))
            {
                bTargetOutput = true;
            }

            DataConfigSetting dsName    = config.Settings.Find("Output Dataset Name");
            string            strDsName = dsName.Value.ToString();

            if (bTargetOutput)
            {
                strDsName += "_Target";
            }
            else
            {
                strTargetOverlayFile = null;
            }

            string strTrainingSrc = strDsName + ".training";
            string strTestingSrc  = strDsName + ".testing";

            DataConfigSetting dsChannels = config.Settings.Find("Channels");

            if (int.TryParse(dsChannels.Value.ToString(), out nChannels))
            {
                if (nChannels != 1 && nChannels != 3)
                {
                    nChannels = 1;
                }
            }

            m_iprogress = progress;

            if (nChannels == 3)
            {
                strTrainingSrc += "." + "3_ch";
                strTestingSrc  += "." + "3_ch";
                strDsName      += "." + "3_ch";
            }

            m_factory.DeleteSources(strTrainingSrc, strTestingSrc);

            Log log = new Log("MNist Dataset Creator");

            log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);

            try
            {
                MgrMnistData mgr = new MgrMnistData(m_factory, log, strTargetOverlayFile);
                mgr.OnLoadError    += new EventHandler <LoadErrorArgs>(mgr_OnLoadError);
                mgr.OnLoadProgress += new EventHandler <LoadArgs>(mgr_OnLoadProgress);

                DataConfigSetting dsTestingDataFile   = config.Settings.Find("Testing Data File");
                DataConfigSetting dsTestingLabelFile  = config.Settings.Find("Testing Label File");
                DataConfigSetting dsTrainingDataFile  = config.Settings.Find("Training Data File");
                DataConfigSetting dsTrainingLabelFile = config.Settings.Find("Training Label File");

                strTrainingDataFile = dsTrainingDataFile.Value.ToString();
                if (strTrainingDataFile.Length == 0)
                {
                    throw new Exception("Training data file name not specified!");
                }

                strTrainingLabelFile = dsTrainingLabelFile.Value.ToString();
                if (strTrainingLabelFile.Length == 0)
                {
                    throw new Exception("Training label file name not specified!");
                }

                strTestingDataFile = dsTestingDataFile.Value.ToString();
                if (strTestingDataFile.Length == 0)
                {
                    throw new Exception("Testing data file name not specified!");
                }

                strTestingLabelFile = dsTestingLabelFile.Value.ToString();
                if (strTestingLabelFile.Length == 0)
                {
                    throw new Exception("Testing label file name not specified!");
                }

                log.WriteLine("Converting the training data files...");

                if (m_bCancel)
                {
                    return;
                }

                mgr.ConvertData(strTrainingDataFile, strTrainingLabelFile, strTrainingSrc, strTrainingSrc, true, false, nChannels);

                if (m_bCancel)
                {
                    return;
                }

                log.WriteLine("Converting the testing data files...");

                if (m_bCancel)
                {
                    return;
                }


                mgr.ConvertData(strTestingDataFile, strTestingLabelFile, strTestingSrc, strTrainingSrc, false, false, nChannels);

                if (m_bCancel)
                {
                    return;
                }

                using (DNNEntities entities = EntitiesConnection.CreateEntities())
                {
                    List <Dataset> rgDs          = entities.Datasets.Where(p => p.Name == strDsName).ToList();
                    List <Source>  rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
                    List <Source>  rgSrcTesting  = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();

                    if (rgSrcTraining.Count == 0)
                    {
                        throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
                    }

                    if (rgSrcTesting.Count == 0)
                    {
                        throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
                    }

                    int    nSrcTestingCount  = rgSrcTesting[0].ImageCount.GetValueOrDefault();
                    int    nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
                    int    nSrcTotalCount    = nSrcTestingCount + nSrcTrainingCount;
                    double dfTestingPct      = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;

                    Dataset ds = null;

                    if (rgDs.Count == 0)
                    {
                        ds      = new Dataset();
                        ds.Name = strDsName;
                    }
                    else
                    {
                        ds = rgDs[0];
                    }

                    ds.ImageEncoded     = rgSrcTesting[0].ImageEncoded;
                    ds.ImageChannels    = rgSrcTesting[0].ImageChannels;
                    ds.ImageHeight      = rgSrcTraining[0].ImageHeight;
                    ds.ImageWidth       = rgSrcTraining[0].ImageWidth;
                    ds.TestingPercent   = (decimal)dfTestingPct;
                    ds.TestingSourceID  = rgSrcTesting[0].ID;
                    ds.TestingTotal     = rgSrcTesting[0].ImageCount;
                    ds.TrainingSourceID = rgSrcTraining[0].ID;
                    ds.TrainingTotal    = rgSrcTraining[0].ImageCount;
                    ds.DatasetCreatorID = config.ID;
                    ds.DatasetGroupID   = 0;
                    ds.ModelGroupID     = 0;

                    if (rgDs.Count == 0)
                    {
                        entities.Datasets.Add(ds);
                    }

                    entities.SaveChanges();
                }
            }
            catch (Exception excpt)
            {
                log.WriteLine("ERROR: " + excpt.Message);
            }
            finally
            {
                if (m_bCancel)
                {
                    log.WriteLine("ABORTED converting MNIST data files.");
                }
                else
                {
                    log.WriteLine("Done converting MNIST data files.");
                }

                if (m_bCancel)
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "ABORTED!", null, true));
                }
                else
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
                }

                Properties.Settings.Default.TrainingDataFile  = strTrainingDataFile;
                Properties.Settings.Default.TrainingLabelFile = strTrainingLabelFile;
                Properties.Settings.Default.TestingDataFile   = strTestingDataFile;
                Properties.Settings.Default.TestingLabelFile  = strTestingLabelFile;
                Properties.Settings.Default.Channels          = nChannels;
                Properties.Settings.Default.TargetOverlayFile = strTargetOverlayFile;
                Properties.Settings.Default.Save();
            }
        }
Пример #3
0
        public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
        {
            string strTrainingFile1 = Properties.Settings.Default.vocTrainingFile1;
            string strTrainingFile2 = Properties.Settings.Default.vocTrainingFile2;
            string strTestingFile1  = Properties.Settings.Default.vocTestingFile1;
            bool   bExtract         = Properties.Settings.Default.ExpandFiles;

            m_evtCancel.Reset();

            DataConfigSetting dsTrainingFile1 = config.Settings.Find("Training Data File 2007");

            strTrainingFile1 = dsTrainingFile1.Value.ToString();

            DataConfigSetting dsTrainingFile2 = config.Settings.Find("Training Data File 2012");

            strTrainingFile2 = dsTrainingFile2.Value.ToString();

            DataConfigSetting dsTestingFile1 = config.Settings.Find("Testing Data File 2007");

            strTestingFile1 = dsTestingFile1.Value.ToString();

            DataConfigSetting dsExtract     = config.Settings.Find("Extract Data Files");
            OptionItem        extractOption = dsExtract.Value as OptionItem;

            bExtract = (extractOption.Index == 0) ? false : true;

            DataConfigSetting dsName    = config.Settings.Find("Output Dataset Name");
            string            strDsName = dsName.Value.ToString();

            string strTrainingSrc = strDsName + ".training";
            string strTestingSrc  = strDsName + ".testing";

            m_iprogress = progress;

            m_factory.DeleteSources(strTrainingSrc, strTestingSrc);

            Log log = new Log("VOC0712 Dataset Creator");

            log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);

            try
            {
                VOCDataParameters param  = new VOCDataParameters(strTrainingFile1, strTrainingFile2, strTestingFile1, bExtract);
                VOCDataLoader     loader = new VOCDataLoader(param, log, m_evtCancel);

                loader.OnProgress  += Loader_OnProgress;
                loader.OnError     += Loader_OnError;
                loader.OnCompleted += Loader_OnCompleted;

                if (!loader.LoadDatabase(config.ID))
                {
                    return;
                }

                using (DNNEntities entities = EntitiesConnection.CreateEntities())
                {
                    List <Dataset> rgDs          = entities.Datasets.Where(p => p.Name == strDsName).ToList();
                    List <Source>  rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
                    List <Source>  rgSrcTesting  = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();

                    if (rgSrcTraining.Count == 0)
                    {
                        throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
                    }

                    if (rgSrcTesting.Count == 0)
                    {
                        throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
                    }

                    int    nSrcTestingCount  = rgSrcTesting[0].ImageCount.GetValueOrDefault();
                    int    nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
                    int    nSrcTotalCount    = nSrcTestingCount + nSrcTrainingCount;
                    double dfTestingPct      = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;

                    Dataset ds = null;

                    if (rgDs.Count == 0)
                    {
                        ds      = new Dataset();
                        ds.Name = strDsName;
                    }
                    else
                    {
                        ds = rgDs[0];
                    }

                    ds.ImageEncoded     = rgSrcTesting[0].ImageEncoded;
                    ds.ImageChannels    = rgSrcTesting[0].ImageChannels;
                    ds.ImageHeight      = rgSrcTraining[0].ImageHeight;
                    ds.ImageWidth       = rgSrcTraining[0].ImageWidth;
                    ds.TestingPercent   = (decimal)dfTestingPct;
                    ds.TestingSourceID  = rgSrcTesting[0].ID;
                    ds.TestingTotal     = rgSrcTesting[0].ImageCount;
                    ds.TrainingSourceID = rgSrcTraining[0].ID;
                    ds.TrainingTotal    = rgSrcTraining[0].ImageCount;
                    ds.DatasetCreatorID = config.ID;
                    ds.DatasetGroupID   = 0;
                    ds.ModelGroupID     = 0;

                    if (rgDs.Count == 0)
                    {
                        entities.Datasets.Add(ds);
                    }

                    entities.SaveChanges();
                }
            }
            catch (Exception excpt)
            {
                log.WriteLine("ERROR: " + excpt.Message);
            }
            finally
            {
                if (m_evtCancel.WaitOne(0))
                {
                    log.WriteLine("ABORTED converting VOC0712 data files.");
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "ABORTED!", null, true));
                }
                else
                {
                    log.WriteLine("Done converting VOC0712 data files.");
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
                }

                Properties.Settings.Default.vocTestingFile1  = strTestingFile1;
                Properties.Settings.Default.vocTrainingFile1 = strTrainingFile1;
                Properties.Settings.Default.vocTrainingFile2 = strTrainingFile2;
                Properties.Settings.Default.ExpandFiles      = bExtract;
                Properties.Settings.Default.Save();
            }
        }
        public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
        {
            string strTrainingBatchFile1 = Properties.Settings.Default.TrainingDataFile1;
            string strTrainingBatchFile2 = Properties.Settings.Default.TrainingDataFile2;
            string strTrainingBatchFile3 = Properties.Settings.Default.TrainingDataFile3;
            string strTrainingBatchFile4 = Properties.Settings.Default.TrainingDataFile4;
            string strTrainingBatchFile5 = Properties.Settings.Default.TrainingDataFile5;
            string strTestingBatchFile   = Properties.Settings.Default.TestingDataFile;
            string strDsName             = config.Name;
            string strTrainingSrc        = config.Name + ".training";
            string strTestingSrc         = config.Name + ".testing";
            int    nIdx   = 0;
            int    nTotal = 50000;

            m_bCancel   = false;
            m_iprogress = progress;
            m_factory.DeleteSources(strTrainingSrc, strTestingSrc);

            Log log = new Log("CIFAR Dataset Creator");

            log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);

            try
            {
                DataConfigSetting dsTrainingDataFile1 = config.Settings.Find("Training Data File 1");
                DataConfigSetting dsTrainingDataFile2 = config.Settings.Find("Training Data File 2");
                DataConfigSetting dsTrainingDataFile3 = config.Settings.Find("Training Data File 3");
                DataConfigSetting dsTrainingDataFile4 = config.Settings.Find("Training Data File 4");
                DataConfigSetting dsTrainingDataFile5 = config.Settings.Find("Training Data File 5");
                DataConfigSetting dsTestingDataFile   = config.Settings.Find("Testing Data File");

                strTrainingBatchFile1 = dsTrainingDataFile1.Value.ToString();
                if (strTrainingBatchFile1.Length == 0)
                {
                    throw new Exception("Training data file #1 name not specified!");
                }

                strTrainingBatchFile2 = dsTrainingDataFile2.Value.ToString();
                if (strTrainingBatchFile2.Length == 0)
                {
                    throw new Exception("Training data file #2 name not specified!");
                }

                strTrainingBatchFile3 = dsTrainingDataFile3.Value.ToString();
                if (strTrainingBatchFile3.Length == 0)
                {
                    throw new Exception("Training data file #3 name not specified!");
                }

                strTrainingBatchFile4 = dsTrainingDataFile4.Value.ToString();
                if (strTrainingBatchFile4.Length == 0)
                {
                    throw new Exception("Training data file #4 name not specified!");
                }

                strTrainingBatchFile5 = dsTrainingDataFile5.Value.ToString();
                if (strTrainingBatchFile5.Length == 0)
                {
                    throw new Exception("Training data file #5 name not specified!");
                }

                strTestingBatchFile = dsTestingDataFile.Value.ToString();
                if (strTestingBatchFile.Length == 0)
                {
                    throw new Exception("Testing data file name not specified!");
                }

                log.WriteLine("Loading the data files...");

                if (m_bCancel)
                {
                    return;
                }

                int nTrainSrcId = m_factory.AddSource(strTrainingSrc, 3, 32, 32, false, 0);
                m_factory.Open(nTrainSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.

                log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
                m_factory.DeleteSourceData();

                if (!loadFile(log, dsTrainingDataFile1.Name, strTrainingBatchFile1, m_factory, nTotal, true, ref nIdx))
                {
                    return;
                }

                if (!loadFile(log, dsTrainingDataFile2.Name, strTrainingBatchFile2, m_factory, nTotal, true, ref nIdx))
                {
                    return;
                }

                if (!loadFile(log, dsTrainingDataFile3.Name, strTrainingBatchFile3, m_factory, nTotal, true, ref nIdx))
                {
                    return;
                }

                if (!loadFile(log, dsTrainingDataFile4.Name, strTrainingBatchFile4, m_factory, nTotal, true, ref nIdx))
                {
                    return;
                }

                if (!loadFile(log, dsTrainingDataFile5.Name, strTrainingBatchFile5, m_factory, nTotal, true, ref nIdx))
                {
                    return;
                }

                m_factory.UpdateSourceCounts();
                updateLabels(m_factory);

                log.WriteLine("Creating the image mean...");
                SimpleDatum dMean = SimpleDatum.CalculateMean(log, m_rgImages.ToArray(), new WaitHandle[] { new ManualResetEvent(false) });
                m_factory.PutRawImageMean(dMean, true);
                m_rgImages.Clear();

                m_factory.Close();

                int nTestSrcId = m_factory.AddSource(strTestingSrc, 3, 32, 32, false, 0);
                m_factory.Open(nTestSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.

                log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
                m_factory.DeleteSourceData();

                nIdx   = 0;
                nTotal = 10000;

                if (!loadFile(log, dsTestingDataFile.Name, strTestingBatchFile, m_factory, nTotal, false, ref nIdx))
                {
                    return;
                }

                m_factory.CopyImageMean(strTrainingSrc, strTestingSrc);

                m_factory.UpdateSourceCounts();
                updateLabels(m_factory);
                m_factory.Close();

                log.WriteLine("Done loading training and testing data.");

                using (DNNEntities entities = EntitiesConnection.CreateEntities())
                {
                    List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
                    List <Source> rgSrcTesting  = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();

                    if (rgSrcTraining.Count == 0)
                    {
                        throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
                    }

                    if (rgSrcTesting.Count == 0)
                    {
                        throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
                    }

                    DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
                    int    nSrcTestingCount  = rgSrcTesting[0].ImageCount.GetValueOrDefault();
                    int    nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
                    int    nSrcTotalCount    = nSrcTestingCount + nSrcTrainingCount;
                    double dfTestingPct      = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;

                    Dataset ds = new Dataset();
                    ds.ImageHeight      = rgSrcTraining[0].ImageHeight;
                    ds.ImageWidth       = rgSrcTraining[0].ImageWidth;
                    ds.Name             = strDsName;
                    ds.ImageEncoded     = rgSrcTesting[0].ImageEncoded;
                    ds.ImageChannels    = rgSrcTesting[0].ImageChannels;
                    ds.TestingPercent   = (decimal)dfTestingPct;
                    ds.TestingSourceID  = rgSrcTesting[0].ID;
                    ds.TestingTotal     = rgSrcTesting[0].ImageCount;
                    ds.TrainingSourceID = rgSrcTraining[0].ID;
                    ds.TrainingTotal    = rgSrcTraining[0].ImageCount;
                    ds.DatasetCreatorID = config.ID;
                    ds.DatasetGroupID   = 0;
                    ds.ModelGroupID     = 0;

                    entities.Datasets.Add(ds);
                    entities.SaveChanges();
                }
            }
            catch (Exception excpt)
            {
                log.WriteLine("ERROR: " + excpt.Message);
            }
            finally
            {
                Properties.Settings.Default.TrainingDataFile1 = strTrainingBatchFile1;
                Properties.Settings.Default.TrainingDataFile2 = strTrainingBatchFile2;
                Properties.Settings.Default.TrainingDataFile3 = strTrainingBatchFile3;
                Properties.Settings.Default.TrainingDataFile4 = strTrainingBatchFile4;
                Properties.Settings.Default.TrainingDataFile5 = strTrainingBatchFile5;
                Properties.Settings.Default.TestingDataFile   = strTestingBatchFile;
                Properties.Settings.Default.Save();

                if (m_bCancel)
                {
                    log.WriteLine("ABORTED converting CIFAR data files.");
                }
                else
                {
                    log.WriteLine("Done converting CIFAR data files.");
                }

                if (m_bCancel)
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(nIdx, nTotal, "ABORTED!", null, true));
                }
                else
                {
                    m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
                }
            }
        }