public async void LoadCatalog()
{
var catalogCommand = new CatalogCommand();
var catalog = await catalogCommand.GetCatalog();
MainModel.DatasetConfigurations.Clear();
foreach (var schema in catalog.Schemas)
{
var dataSetConfig = new DatasetConfiguration
{
Schema = schema,
Backend = "progressive",
ThrottleInMillis = MainModel.ThrottleInMillis,
SampleSize = MainModel.SampleSize
};
MainModel.DatasetConfigurations.Add(dataSetConfig);
}
if (MainModel.DatasetConfigurations.Any(ds => ds.Schema.DisplayName.ToLower().Contains(MainModel.StartDataset)))
{
LoadData(MainModel.DatasetConfigurations.First(ds => ds.Schema.DisplayName.ToLower().Contains(MainModel.StartDataset)));
}
else
{
LoadData(MainModel.DatasetConfigurations.First());
}
setupOperationTypeModels();
}
public void QueryConfiguration(DatasetConfiguration config)
{
string strTrainingFile1 = Properties.Settings.Default.vocTrainingFile1;
string strTrainingFile2 = Properties.Settings.Default.vocTrainingFile2;
string strTestingFile1 = Properties.Settings.Default.vocTestingFile1;
bool bExpandFiles = Properties.Settings.Default.ExpandFiles;
config.Settings.Add(new DataConfigSetting("Output Dataset Name", "VOC0712"));
config.Settings.Add(new DataConfigSetting("Testing Data File 2007", strTestingFile1, DataConfigSetting.TYPE.FILENAME, "tar"));
config.Settings.Add(new DataConfigSetting("Training Data File 2007", strTrainingFile1, DataConfigSetting.TYPE.FILENAME, "tar"));
config.Settings.Add(new DataConfigSetting("Training Data File 2012", strTrainingFile2, DataConfigSetting.TYPE.FILENAME, "tar"));
addList(config, "Extract Data Files", (bExpandFiles) ? BOOLEAN.True : BOOLEAN.False, BOOLEAN.True, BOOLEAN.False);
}
private void addList(DatasetConfiguration config, string strName, object objDefault, params object[] rgParam)
{
OptionItemList rgItems = new OptionItemList();
foreach (object obj in rgParam)
{
rgItems.Add(new OptionItem(obj.ToString(), (int)obj, null));
}
OptionItem item = new OptionItem(objDefault.ToString(), (int)objDefault, rgItems);
config.Settings.Add(new DataConfigSetting(strName, item, DataConfigSetting.TYPE.LIST));
}
public void QueryConfiguration(DatasetConfiguration config)
{
string strTrainingDataFile = Properties.Settings.Default.TrainingDataFile;
string strTrainingLabelFile = Properties.Settings.Default.TrainingLabelFile;
string strTestingDataFile = Properties.Settings.Default.TestingDataFile;
string strTestingLabelFile = Properties.Settings.Default.TestingLabelFile;
string strTargetOverlayFile = Properties.Settings.Default.TargetOverlayFile;
int nChannels = Properties.Settings.Default.Channels;
config.Settings.Add(new DataConfigSetting("Output Dataset Name", "MNIST"));
config.Settings.Add(new DataConfigSetting("Testing Data File", strTestingDataFile, DataConfigSetting.TYPE.FILENAME, "gz"));
config.Settings.Add(new DataConfigSetting("Testing Label File", strTestingLabelFile, DataConfigSetting.TYPE.FILENAME, "gz"));
config.Settings.Add(new DataConfigSetting("Training Data File", strTrainingDataFile, DataConfigSetting.TYPE.FILENAME, "gz"));
config.Settings.Add(new DataConfigSetting("Training Label File", strTrainingLabelFile, DataConfigSetting.TYPE.FILENAME, "gz"));
config.Settings.Add(new DataConfigSetting("Channels", nChannels.ToString(), DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Target Overlay File", strTargetOverlayFile, DataConfigSetting.TYPE.FILENAME, "png,jpg,bmp"));
}
public void QueryConfiguration(DatasetConfiguration config)
{
string strTrainingBatchFile1 = Properties.Settings.Default.TrainingDataFile1;
string strTrainingBatchFile2 = Properties.Settings.Default.TrainingDataFile2;
string strTrainingBatchFile3 = Properties.Settings.Default.TrainingDataFile3;
string strTrainingBatchFile4 = Properties.Settings.Default.TrainingDataFile4;
string strTrainingBatchFile5 = Properties.Settings.Default.TrainingDataFile5;
string strTestingBatchFile = Properties.Settings.Default.TestingDataFile;
config.Settings.Add(new DataConfigSetting("Output Dataset Name", "CIFAR-10"));
config.Settings.Add(new DataConfigSetting("Training Data File 1", strTrainingBatchFile1, DataConfigSetting.TYPE.FILENAME, "bin"));
config.Settings.Add(new DataConfigSetting("Training Data File 2", strTrainingBatchFile2, DataConfigSetting.TYPE.FILENAME, "bin"));
config.Settings.Add(new DataConfigSetting("Training Data File 3", strTrainingBatchFile3, DataConfigSetting.TYPE.FILENAME, "bin"));
config.Settings.Add(new DataConfigSetting("Training Data File 4", strTrainingBatchFile4, DataConfigSetting.TYPE.FILENAME, "bin"));
config.Settings.Add(new DataConfigSetting("Training Data File 5", strTrainingBatchFile5, DataConfigSetting.TYPE.FILENAME, "bin"));
config.Settings.Add(new DataConfigSetting("Testing Data File", strTestingBatchFile, DataConfigSetting.TYPE.FILENAME, "bin"));
}
public void LoadData(DatasetConfiguration datasetConfiguration)
{
if (MainModel.QueryExecuter != null)
{
MainModel.QueryExecuter.HaltAllJobs();
}
if (datasetConfiguration.Backend.ToLower() == "progressive")
{
MainModel.SchemaModel = new IDEASchemaModel();
MainModel.ThrottleInMillis = datasetConfiguration.ThrottleInMillis;
MainModel.SampleSize = datasetConfiguration.SampleSize;
MainModel.QueryExecuter = new IDEAQueryExecuter();
HypothesesViewController.Instance.ClearAllStatisticalComparison();
((IDEASchemaModel)MainModel.SchemaModel).RootOriginModel = new IDEAOriginModel(datasetConfiguration);
((IDEASchemaModel)MainModel.SchemaModel).RootOriginModel.LoadInputFields();
}
}
public void QueryConfiguration(DatasetConfiguration config)
{
string strCsvFile = Properties.Settings.Default.CsvFile;
string strTagCol = Properties.Settings.Default.TagCol; // A
string strDateCol = Properties.Settings.Default.DateCol; // B
string strDataStartCol = Properties.Settings.Default.DataStartCol; // C
string strDataEndCol = Properties.Settings.Default.DataEndCol; // AK
string strLabelStartCol = Properties.Settings.Default.LabelStartCol; // AL
string strLabelEndCol = Properties.Settings.Default.LabelEndCol; // AQ
string strExtraStartCol = Properties.Settings.Default.ExtraStartCol; // AR
string strExtraEndCol = Properties.Settings.Default.ExtraEndCol; // AS
int nDataDescRow = Properties.Settings.Default.DataDescRow; // 6
int nLabelStartRow = Properties.Settings.Default.LabelStartRow; // 7
int nLabelEndRow = Properties.Settings.Default.LabelEndRow; // 8
int nDataStartRow = Properties.Settings.Default.DataStartRow; // 10
bool bUseBinary = Properties.Settings.Default.UseBinary; // true
int nCellSize = Properties.Settings.Default.CellSize; // 5
double dfTestingPct = Properties.Settings.Default.TestingPct; // 0.2 (20%)
config.Settings.Add(new DataConfigSetting("Output Dataset Name", "CSV"));
config.Settings.Add(new DataConfigSetting("CSV File", strCsvFile, DataConfigSetting.TYPE.FILENAME, "csv"));
config.Settings.Add(new DataConfigSetting("Tag Col", strTagCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Date Col", strDateCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Data Start Col", strDataStartCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Data End Col", strDataEndCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Label Start Col", strLabelStartCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Label End Col", strLabelEndCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Extra Start Col", strExtraStartCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Extra End Col", strExtraEndCol, DataConfigSetting.TYPE.TEXT));
config.Settings.Add(new DataConfigSetting("Data Desc Row", nDataDescRow, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Label Start Row", nLabelStartRow, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Label End Row", nLabelEndRow, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Data Start Row", nDataStartRow, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Use Binary Values", (bUseBinary) ? 1 : 0, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Cell Size", nCellSize, DataConfigSetting.TYPE.INTEGER));
config.Settings.Add(new DataConfigSetting("Testing Percentage", dfTestingPct, DataConfigSetting.TYPE.REAL));
}
public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
{
string strCsvFile = Properties.Settings.Default.CsvFile;
string strDsName = config.Name;
string strTrainingSrc = config.Name + ".training";
string strTestingSrc = config.Name + ".testing";
m_bCancel = false;
m_iprogress = progress;
m_factory.DeleteSources(strTrainingSrc, strTestingSrc);
Log log = new Log("CSV Dataset Creator");
log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);
try
{
//-----------------------------------------
// Load the schema that defines the layout
// of the CSV file.
//-----------------------------------------
m_schema = loadSchema(config.Settings);
//-----------------------------------------
// Load and parse the CSV file.
//-----------------------------------------
DataConfigSetting dsCsvFile = config.Settings.Find("CSV File");
strCsvFile = dsCsvFile.Value.ToString();
if (strCsvFile.Length == 0)
{
throw new Exception("CSV data file name not specified!");
}
log.WriteLine("Loading the data file...");
if (m_bCancel)
{
return;
}
m_parser.Load(strCsvFile, m_schema);
//-----------------------------------------
// Split the data into training and testing
// sets.
//-----------------------------------------
List <DataItem> rgTraining = new List <DataItem>();
List <DataItem> rgTesting = new List <DataItem>();
DataConfigSetting dsPctTesting = config.Settings.Find("Testing Percentage");
double dfVal = (double)dsPctTesting.Value;
Random random = new Random();
for (int i = 0; i < m_parser.Data.Count; i++)
{
if (random.NextDouble() > dfVal)
{
rgTraining.Add(m_parser.Data[i]);
}
else
{
rgTesting.Add(m_parser.Data[i]);
}
}
Properties.Settings.Default.TestingPct = dfVal;
//-----------------------------------------
// Create the training data source.
//-----------------------------------------
int nCellHorizCount = 0;
List <int> rgDim = getImageDim(m_parser, m_schema, out nCellHorizCount);
int nTrainSrcId = m_factory.AddSource(strTrainingSrc, rgDim[0], rgDim[1], rgDim[2], false, 0);
m_factory.Open(nTrainSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.
log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
m_factory.DeleteSourceData();
if (!loadData(log, m_factory, m_parser, rgTraining, rgDim, true, true))
{
return;
}
m_factory.UpdateSourceCounts();
updateLabels(m_factory);
log.WriteLine("Creating the image mean...");
SimpleDatum dMean = SimpleDatum.CalculateMean(log, m_rgImages.ToArray(), new WaitHandle[] { new ManualResetEvent(false) });
m_factory.PutRawImageMean(dMean, true);
m_rgImages.Clear();
m_factory.Close();
//-----------------------------------------
// Create the testing data source.
//-----------------------------------------
int nTestSrcId = m_factory.AddSource(strTestingSrc, rgDim[0], rgDim[1], rgDim[2], false, 0);
m_factory.Open(nTestSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.
log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
m_factory.DeleteSourceData();
if (!loadData(log, m_factory, m_parser, rgTesting, rgDim, false, false))
{
return;
}
m_factory.UpdateSourceCounts();
updateLabels(m_factory);
m_factory.Close();
//-----------------------------------------
// Crate the data set.
//-----------------------------------------
log.WriteLine("Done loading training and testing data.");
int nDatasetID = 0;
using (DNNEntities entities = EntitiesConnection.CreateEntities())
{
List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
List <Source> rgSrcTesting = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();
if (rgSrcTraining.Count == 0)
{
throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
}
if (rgSrcTesting.Count == 0)
{
throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
}
DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
int nSrcTestingCount = rgSrcTesting[0].ImageCount.GetValueOrDefault();
int nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
int nSrcTotalCount = nSrcTestingCount + nSrcTrainingCount;
double dfTestingPct = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;
Dataset ds = new Dataset();
ds.ImageHeight = rgSrcTraining[0].ImageHeight;
ds.ImageWidth = rgSrcTraining[0].ImageWidth;
ds.Name = strDsName;
ds.ImageEncoded = rgSrcTesting[0].ImageEncoded;
ds.ImageChannels = rgSrcTesting[0].ImageChannels;
ds.TestingPercent = (decimal)dfTestingPct;
ds.TestingSourceID = rgSrcTesting[0].ID;
ds.TestingTotal = rgSrcTesting[0].ImageCount;
ds.TrainingSourceID = rgSrcTraining[0].ID;
ds.TrainingTotal = rgSrcTraining[0].ImageCount;
ds.DatasetCreatorID = config.ID;
ds.DatasetGroupID = 0;
ds.ModelGroupID = 0;
entities.Datasets.Add(ds);
entities.SaveChanges();
nDatasetID = ds.ID;
}
m_factory.SetDatasetParameter(nDatasetID, "PixelSize", m_schema.CellSize.ToString());
m_factory.SetDatasetParameter(nDatasetID, "AttributeCount", m_parser.DataDescriptions.Count.ToString());
m_factory.SetDatasetParameter(nDatasetID, "AttributeCountHoriz", nCellHorizCount.ToString());
m_factory.SetDatasetParameter(nDatasetID, "AttributeCountVert", nCellHorizCount.ToString());
}
catch (Exception excpt)
{
log.WriteLine("ERROR: " + excpt.Message);
}
finally
{
Properties.Settings.Default.CsvFile = strCsvFile;
Properties.Settings.Default.Save();
if (m_bCancel)
{
log.WriteLine("ABORTED converting CSV data files.");
}
else
{
log.WriteLine("Done converting CSV data files.");
}
if (m_bCancel)
{
m_iprogress.OnCompleted(new CreateProgressArgs(1, 1, "ABORTED!", null, true));
}
else
{
m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
}
}
}
public static async Task <bool> PublishReportDataFor(string reportUniqueId, string datasetName, List <DeviceMessage> deviceMessagelList, ServiceContext serviceContext, HttpClient httpClient, CancellationToken cancellationToken, IServiceEventSource serviceEventSource, int resampleSetsLimit = 0, int minMagnitudeAllowed = 1)
{
bool bRet = false;
int maxSendingSet = 9999;
ServiceEventSourceHelper serviceEventSourceHelper = new ServiceEventSourceHelper(serviceEventSource);
DatasetConfiguration datasetConfiguration = DatasetRegistry.GetDatasetConfiguration(datasetName);
int initialMessageCount = 0;
serviceEventSourceHelper.ServiceMessage(serviceContext, $"PublishReportDataFor - About to publish data for Report id[{reportUniqueId}] Dataset Name[{datasetName}] Number of messages [{deviceMessagelList.Count}]");
ReportStreamDataset reportStreamDataset = new ReportStreamDataset(reportUniqueId, datasetConfiguration, deviceMessagelList);
if (reportStreamDataset.Count > 0 && resampleSetsLimit > 0)
{
initialMessageCount = reportStreamDataset.Count;
int maxNumberOfMessagesToSend = (resampleSetsLimit * maxSendingSet);
ReportStreamDataset resampledReportStreamDataset = new ReportStreamDataset(reportUniqueId, datasetConfiguration);
if (reportStreamDataset.Count > maxNumberOfMessagesToSend)
{
float selectInterval = reportStreamDataset.Count / maxNumberOfMessagesToSend;
int selectedCount = 0;
int index = 0;
foreach (DeviceMessage message in reportStreamDataset.DeviceMessageList)
{
if (index >= (selectedCount * selectInterval))
{
selectedCount++;
resampledReportStreamDataset.AddMessage(message);
}
index++;
if (selectedCount == maxNumberOfMessagesToSend)
{
break;
}
}
reportStreamDataset = resampledReportStreamDataset;
}
}
serviceEventSourceHelper.ServiceMessage(serviceContext, $"PublishReportDataFor - Report id[{reportUniqueId}] Final number of rows [{reportStreamDataset.Count}] generated from messages [{initialMessageCount}] requested for Dataset [{datasetName}]");
if (reportStreamDataset.Count > 0)
{
int messageCounter = 0;
int messageSet = 1;
bool firstSet = true;
StringBuilder bodyContent = new StringBuilder();
bodyContent.Append("[");
bool firstItem = true;
foreach (DeviceMessage message in reportStreamDataset.DeviceMessageList)
{
if (firstItem)
{
firstItem = false;
}
else
{
bodyContent.Append(",");
}
bodyContent.Append(datasetConfiguration.GetMessageToPublish(message));
messageCounter++;
if (messageCounter == maxSendingSet)
{
if (!firstSet)
{
await Task.Delay(global::Iot.Common.Names.IntervalBetweenReportStreamingCalls);
}
bodyContent.Append("]");
await RESTHandler.ExecuteHttpPOST(datasetConfiguration.PostUrl, bodyContent.ToString(), httpClient, cancellationToken, serviceEventSource);
serviceEventSourceHelper.ServiceMessage(serviceContext, $"PublishReportDataFor - Sending set [{messageSet}] with number of rows [{messageCounter}] generated from messages [{reportStreamDataset.Count}] to dataset [{datasetName}]");
bodyContent.Clear();
bodyContent.Append("[");
firstItem = true;
messageCounter = 0;
messageSet++;
firstSet = false;
}
}
if (reportStreamDataset.Count > 0)
{
if (!firstSet)
{
await Task.Delay(global::Iot.Common.Names.IntervalBetweenReportStreamingCalls);
}
bodyContent.Append("]");
await RESTHandler.ExecuteHttpPOST(datasetConfiguration.PostUrl, bodyContent, httpClient, cancellationToken, serviceEventSource);
serviceEventSourceHelper.ServiceMessage(serviceContext, $"PublishReportDataFor - Sending set [{messageSet}] with number of rows [{messageCounter}] generated from messages [{reportStreamDataset.Count}] to dataset [{datasetName}]");
}
bRet = true;
}
else
{
serviceEventSourceHelper.ServiceMessage(serviceContext, $"Embed Report - No data found to publish for Dataset [{datasetName}]");
}
return(bRet);
}
public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
{
string strTrainingFile1 = Properties.Settings.Default.vocTrainingFile1;
string strTrainingFile2 = Properties.Settings.Default.vocTrainingFile2;
string strTestingFile1 = Properties.Settings.Default.vocTestingFile1;
bool bExtract = Properties.Settings.Default.ExpandFiles;
m_evtCancel.Reset();
DataConfigSetting dsTrainingFile1 = config.Settings.Find("Training Data File 2007");
strTrainingFile1 = dsTrainingFile1.Value.ToString();
DataConfigSetting dsTrainingFile2 = config.Settings.Find("Training Data File 2012");
strTrainingFile2 = dsTrainingFile2.Value.ToString();
DataConfigSetting dsTestingFile1 = config.Settings.Find("Testing Data File 2007");
strTestingFile1 = dsTestingFile1.Value.ToString();
DataConfigSetting dsExtract = config.Settings.Find("Extract Data Files");
OptionItem extractOption = dsExtract.Value as OptionItem;
bExtract = (extractOption.Index == 0) ? false : true;
DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
string strDsName = dsName.Value.ToString();
string strTrainingSrc = strDsName + ".training";
string strTestingSrc = strDsName + ".testing";
m_iprogress = progress;
m_factory.DeleteSources(strTrainingSrc, strTestingSrc);
Log log = new Log("VOC0712 Dataset Creator");
log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);
try
{
VOCDataParameters param = new VOCDataParameters(strTrainingFile1, strTrainingFile2, strTestingFile1, bExtract);
VOCDataLoader loader = new VOCDataLoader(param, log, m_evtCancel);
loader.OnProgress += Loader_OnProgress;
loader.OnError += Loader_OnError;
loader.OnCompleted += Loader_OnCompleted;
if (!loader.LoadDatabase(config.ID))
{
return;
}
using (DNNEntities entities = EntitiesConnection.CreateEntities())
{
List <Dataset> rgDs = entities.Datasets.Where(p => p.Name == strDsName).ToList();
List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
List <Source> rgSrcTesting = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();
if (rgSrcTraining.Count == 0)
{
throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
}
if (rgSrcTesting.Count == 0)
{
throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
}
int nSrcTestingCount = rgSrcTesting[0].ImageCount.GetValueOrDefault();
int nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
int nSrcTotalCount = nSrcTestingCount + nSrcTrainingCount;
double dfTestingPct = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;
Dataset ds = null;
if (rgDs.Count == 0)
{
ds = new Dataset();
ds.Name = strDsName;
}
else
{
ds = rgDs[0];
}
ds.ImageEncoded = rgSrcTesting[0].ImageEncoded;
ds.ImageChannels = rgSrcTesting[0].ImageChannels;
ds.ImageHeight = rgSrcTraining[0].ImageHeight;
ds.ImageWidth = rgSrcTraining[0].ImageWidth;
ds.TestingPercent = (decimal)dfTestingPct;
ds.TestingSourceID = rgSrcTesting[0].ID;
ds.TestingTotal = rgSrcTesting[0].ImageCount;
ds.TrainingSourceID = rgSrcTraining[0].ID;
ds.TrainingTotal = rgSrcTraining[0].ImageCount;
ds.DatasetCreatorID = config.ID;
ds.DatasetGroupID = 0;
ds.ModelGroupID = 0;
if (rgDs.Count == 0)
{
entities.Datasets.Add(ds);
}
entities.SaveChanges();
}
}
catch (Exception excpt)
{
log.WriteLine("ERROR: " + excpt.Message);
}
finally
{
if (m_evtCancel.WaitOne(0))
{
log.WriteLine("ABORTED converting VOC0712 data files.");
m_iprogress.OnCompleted(new CreateProgressArgs(1, "ABORTED!", null, true));
}
else
{
log.WriteLine("Done converting VOC0712 data files.");
m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
}
Properties.Settings.Default.vocTestingFile1 = strTestingFile1;
Properties.Settings.Default.vocTrainingFile1 = strTrainingFile1;
Properties.Settings.Default.vocTrainingFile2 = strTrainingFile2;
Properties.Settings.Default.ExpandFiles = bExtract;
Properties.Settings.Default.Save();
}
}
public ReportStreamDataset(string reportUniqueId, DatasetConfiguration datasetConfiguration, List <DeviceMessage> deviceMessagelList)
{
this.ReportUniqueId = reportUniqueId;
this.DatasetConfiguration = datasetConfiguration;
this.DeviceMessageList = deviceMessagelList;
}
public ReportStreamDataset(string reportUniqueId, DatasetConfiguration datasetConfiguration)
{
this.ReportUniqueId = reportUniqueId;
this.DatasetConfiguration = datasetConfiguration;
this.DeviceMessageList = new List <DeviceMessage>();
}
public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
{
string strTrainingDataFile = Properties.Settings.Default.TrainingDataFile;
string strTrainingLabelFile = Properties.Settings.Default.TrainingLabelFile;
string strTestingDataFile = Properties.Settings.Default.TestingDataFile;
string strTestingLabelFile = Properties.Settings.Default.TestingLabelFile;
string strTargetOverlayFile = Properties.Settings.Default.TargetOverlayFile;
int nChannels = Properties.Settings.Default.Channels;
m_bCancel = false;
DataConfigSetting dsTargetOverlayFile = config.Settings.Find("Target Overlay File");
strTargetOverlayFile = dsTargetOverlayFile.Value.ToString();
bool bTargetOutput = false;
if (!string.IsNullOrEmpty(strTargetOverlayFile) && File.Exists(strTargetOverlayFile))
{
bTargetOutput = true;
}
DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
string strDsName = dsName.Value.ToString();
if (bTargetOutput)
{
strDsName += "_Target";
}
else
{
strTargetOverlayFile = null;
}
string strTrainingSrc = strDsName + ".training";
string strTestingSrc = strDsName + ".testing";
DataConfigSetting dsChannels = config.Settings.Find("Channels");
if (int.TryParse(dsChannels.Value.ToString(), out nChannels))
{
if (nChannels != 1 && nChannels != 3)
{
nChannels = 1;
}
}
m_iprogress = progress;
if (nChannels == 3)
{
strTrainingSrc += "." + "3_ch";
strTestingSrc += "." + "3_ch";
strDsName += "." + "3_ch";
}
m_factory.DeleteSources(strTrainingSrc, strTestingSrc);
Log log = new Log("MNist Dataset Creator");
log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);
try
{
MgrMnistData mgr = new MgrMnistData(m_factory, log, strTargetOverlayFile);
mgr.OnLoadError += new EventHandler <LoadErrorArgs>(mgr_OnLoadError);
mgr.OnLoadProgress += new EventHandler <LoadArgs>(mgr_OnLoadProgress);
DataConfigSetting dsTestingDataFile = config.Settings.Find("Testing Data File");
DataConfigSetting dsTestingLabelFile = config.Settings.Find("Testing Label File");
DataConfigSetting dsTrainingDataFile = config.Settings.Find("Training Data File");
DataConfigSetting dsTrainingLabelFile = config.Settings.Find("Training Label File");
strTrainingDataFile = dsTrainingDataFile.Value.ToString();
if (strTrainingDataFile.Length == 0)
{
throw new Exception("Training data file name not specified!");
}
strTrainingLabelFile = dsTrainingLabelFile.Value.ToString();
if (strTrainingLabelFile.Length == 0)
{
throw new Exception("Training label file name not specified!");
}
strTestingDataFile = dsTestingDataFile.Value.ToString();
if (strTestingDataFile.Length == 0)
{
throw new Exception("Testing data file name not specified!");
}
strTestingLabelFile = dsTestingLabelFile.Value.ToString();
if (strTestingLabelFile.Length == 0)
{
throw new Exception("Testing label file name not specified!");
}
log.WriteLine("Converting the training data files...");
if (m_bCancel)
{
return;
}
mgr.ConvertData(strTrainingDataFile, strTrainingLabelFile, strTrainingSrc, strTrainingSrc, true, false, nChannels);
if (m_bCancel)
{
return;
}
log.WriteLine("Converting the testing data files...");
if (m_bCancel)
{
return;
}
mgr.ConvertData(strTestingDataFile, strTestingLabelFile, strTestingSrc, strTrainingSrc, false, false, nChannels);
if (m_bCancel)
{
return;
}
using (DNNEntities entities = EntitiesConnection.CreateEntities())
{
List <Dataset> rgDs = entities.Datasets.Where(p => p.Name == strDsName).ToList();
List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
List <Source> rgSrcTesting = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();
if (rgSrcTraining.Count == 0)
{
throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
}
if (rgSrcTesting.Count == 0)
{
throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
}
int nSrcTestingCount = rgSrcTesting[0].ImageCount.GetValueOrDefault();
int nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
int nSrcTotalCount = nSrcTestingCount + nSrcTrainingCount;
double dfTestingPct = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;
Dataset ds = null;
if (rgDs.Count == 0)
{
ds = new Dataset();
ds.Name = strDsName;
}
else
{
ds = rgDs[0];
}
ds.ImageEncoded = rgSrcTesting[0].ImageEncoded;
ds.ImageChannels = rgSrcTesting[0].ImageChannels;
ds.ImageHeight = rgSrcTraining[0].ImageHeight;
ds.ImageWidth = rgSrcTraining[0].ImageWidth;
ds.TestingPercent = (decimal)dfTestingPct;
ds.TestingSourceID = rgSrcTesting[0].ID;
ds.TestingTotal = rgSrcTesting[0].ImageCount;
ds.TrainingSourceID = rgSrcTraining[0].ID;
ds.TrainingTotal = rgSrcTraining[0].ImageCount;
ds.DatasetCreatorID = config.ID;
ds.DatasetGroupID = 0;
ds.ModelGroupID = 0;
if (rgDs.Count == 0)
{
entities.Datasets.Add(ds);
}
entities.SaveChanges();
}
}
catch (Exception excpt)
{
log.WriteLine("ERROR: " + excpt.Message);
}
finally
{
if (m_bCancel)
{
log.WriteLine("ABORTED converting MNIST data files.");
}
else
{
log.WriteLine("Done converting MNIST data files.");
}
if (m_bCancel)
{
m_iprogress.OnCompleted(new CreateProgressArgs(1, "ABORTED!", null, true));
}
else
{
m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
}
Properties.Settings.Default.TrainingDataFile = strTrainingDataFile;
Properties.Settings.Default.TrainingLabelFile = strTrainingLabelFile;
Properties.Settings.Default.TestingDataFile = strTestingDataFile;
Properties.Settings.Default.TestingLabelFile = strTestingLabelFile;
Properties.Settings.Default.Channels = nChannels;
Properties.Settings.Default.TargetOverlayFile = strTargetOverlayFile;
Properties.Settings.Default.Save();
}
}
public void Create(DatasetConfiguration config, IXDatasetCreatorProgress progress)
{
string strTrainingBatchFile1 = Properties.Settings.Default.TrainingDataFile1;
string strTrainingBatchFile2 = Properties.Settings.Default.TrainingDataFile2;
string strTrainingBatchFile3 = Properties.Settings.Default.TrainingDataFile3;
string strTrainingBatchFile4 = Properties.Settings.Default.TrainingDataFile4;
string strTrainingBatchFile5 = Properties.Settings.Default.TrainingDataFile5;
string strTestingBatchFile = Properties.Settings.Default.TestingDataFile;
string strDsName = config.Name;
string strTrainingSrc = config.Name + ".training";
string strTestingSrc = config.Name + ".testing";
int nIdx = 0;
int nTotal = 50000;
m_bCancel = false;
m_iprogress = progress;
m_factory.DeleteSources(strTrainingSrc, strTestingSrc);
Log log = new Log("CIFAR Dataset Creator");
log.OnWriteLine += new EventHandler <LogArg>(log_OnWriteLine);
try
{
DataConfigSetting dsTrainingDataFile1 = config.Settings.Find("Training Data File 1");
DataConfigSetting dsTrainingDataFile2 = config.Settings.Find("Training Data File 2");
DataConfigSetting dsTrainingDataFile3 = config.Settings.Find("Training Data File 3");
DataConfigSetting dsTrainingDataFile4 = config.Settings.Find("Training Data File 4");
DataConfigSetting dsTrainingDataFile5 = config.Settings.Find("Training Data File 5");
DataConfigSetting dsTestingDataFile = config.Settings.Find("Testing Data File");
strTrainingBatchFile1 = dsTrainingDataFile1.Value.ToString();
if (strTrainingBatchFile1.Length == 0)
{
throw new Exception("Training data file #1 name not specified!");
}
strTrainingBatchFile2 = dsTrainingDataFile2.Value.ToString();
if (strTrainingBatchFile2.Length == 0)
{
throw new Exception("Training data file #2 name not specified!");
}
strTrainingBatchFile3 = dsTrainingDataFile3.Value.ToString();
if (strTrainingBatchFile3.Length == 0)
{
throw new Exception("Training data file #3 name not specified!");
}
strTrainingBatchFile4 = dsTrainingDataFile4.Value.ToString();
if (strTrainingBatchFile4.Length == 0)
{
throw new Exception("Training data file #4 name not specified!");
}
strTrainingBatchFile5 = dsTrainingDataFile5.Value.ToString();
if (strTrainingBatchFile5.Length == 0)
{
throw new Exception("Training data file #5 name not specified!");
}
strTestingBatchFile = dsTestingDataFile.Value.ToString();
if (strTestingBatchFile.Length == 0)
{
throw new Exception("Testing data file name not specified!");
}
log.WriteLine("Loading the data files...");
if (m_bCancel)
{
return;
}
int nTrainSrcId = m_factory.AddSource(strTrainingSrc, 3, 32, 32, false, 0);
m_factory.Open(nTrainSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.
log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
m_factory.DeleteSourceData();
if (!loadFile(log, dsTrainingDataFile1.Name, strTrainingBatchFile1, m_factory, nTotal, true, ref nIdx))
{
return;
}
if (!loadFile(log, dsTrainingDataFile2.Name, strTrainingBatchFile2, m_factory, nTotal, true, ref nIdx))
{
return;
}
if (!loadFile(log, dsTrainingDataFile3.Name, strTrainingBatchFile3, m_factory, nTotal, true, ref nIdx))
{
return;
}
if (!loadFile(log, dsTrainingDataFile4.Name, strTrainingBatchFile4, m_factory, nTotal, true, ref nIdx))
{
return;
}
if (!loadFile(log, dsTrainingDataFile5.Name, strTrainingBatchFile5, m_factory, nTotal, true, ref nIdx))
{
return;
}
m_factory.UpdateSourceCounts();
updateLabels(m_factory);
log.WriteLine("Creating the image mean...");
SimpleDatum dMean = SimpleDatum.CalculateMean(log, m_rgImages.ToArray(), new WaitHandle[] { new ManualResetEvent(false) });
m_factory.PutRawImageMean(dMean, true);
m_rgImages.Clear();
m_factory.Close();
int nTestSrcId = m_factory.AddSource(strTestingSrc, 3, 32, 32, false, 0);
m_factory.Open(nTestSrcId, 500, Database.FORCE_LOAD.FROM_FILE); // use file based data.
log.WriteLine("Deleting existing data from '" + m_factory.OpenSource.Name + "'.");
m_factory.DeleteSourceData();
nIdx = 0;
nTotal = 10000;
if (!loadFile(log, dsTestingDataFile.Name, strTestingBatchFile, m_factory, nTotal, false, ref nIdx))
{
return;
}
m_factory.CopyImageMean(strTrainingSrc, strTestingSrc);
m_factory.UpdateSourceCounts();
updateLabels(m_factory);
m_factory.Close();
log.WriteLine("Done loading training and testing data.");
using (DNNEntities entities = EntitiesConnection.CreateEntities())
{
List <Source> rgSrcTraining = entities.Sources.Where(p => p.Name == strTrainingSrc).ToList();
List <Source> rgSrcTesting = entities.Sources.Where(p => p.Name == strTestingSrc).ToList();
if (rgSrcTraining.Count == 0)
{
throw new Exception("Could not find the training source '" + strTrainingSrc + "'.");
}
if (rgSrcTesting.Count == 0)
{
throw new Exception("Could not find the tesing source '" + strTestingSrc + "'.");
}
DataConfigSetting dsName = config.Settings.Find("Output Dataset Name");
int nSrcTestingCount = rgSrcTesting[0].ImageCount.GetValueOrDefault();
int nSrcTrainingCount = rgSrcTraining[0].ImageCount.GetValueOrDefault();
int nSrcTotalCount = nSrcTestingCount + nSrcTrainingCount;
double dfTestingPct = (nSrcTrainingCount == 0) ? 0.0 : nSrcTestingCount / (double)nSrcTotalCount;
Dataset ds = new Dataset();
ds.ImageHeight = rgSrcTraining[0].ImageHeight;
ds.ImageWidth = rgSrcTraining[0].ImageWidth;
ds.Name = strDsName;
ds.ImageEncoded = rgSrcTesting[0].ImageEncoded;
ds.ImageChannels = rgSrcTesting[0].ImageChannels;
ds.TestingPercent = (decimal)dfTestingPct;
ds.TestingSourceID = rgSrcTesting[0].ID;
ds.TestingTotal = rgSrcTesting[0].ImageCount;
ds.TrainingSourceID = rgSrcTraining[0].ID;
ds.TrainingTotal = rgSrcTraining[0].ImageCount;
ds.DatasetCreatorID = config.ID;
ds.DatasetGroupID = 0;
ds.ModelGroupID = 0;
entities.Datasets.Add(ds);
entities.SaveChanges();
}
}
catch (Exception excpt)
{
log.WriteLine("ERROR: " + excpt.Message);
}
finally
{
Properties.Settings.Default.TrainingDataFile1 = strTrainingBatchFile1;
Properties.Settings.Default.TrainingDataFile2 = strTrainingBatchFile2;
Properties.Settings.Default.TrainingDataFile3 = strTrainingBatchFile3;
Properties.Settings.Default.TrainingDataFile4 = strTrainingBatchFile4;
Properties.Settings.Default.TrainingDataFile5 = strTrainingBatchFile5;
Properties.Settings.Default.TestingDataFile = strTestingBatchFile;
Properties.Settings.Default.Save();
if (m_bCancel)
{
log.WriteLine("ABORTED converting CIFAR data files.");
}
else
{
log.WriteLine("Done converting CIFAR data files.");
}
if (m_bCancel)
{
m_iprogress.OnCompleted(new CreateProgressArgs(nIdx, nTotal, "ABORTED!", null, true));
}
else
{
m_iprogress.OnCompleted(new CreateProgressArgs(1, "COMPLETED."));
}
}
}
public IDEAOriginModel(DatasetConfiguration datasetConfiguration)
{
_datasetConfiguration = datasetConfiguration;
}
