} /* LoadTrainingModels */
private void LoadData()
{
loadingThreadRunning = true;
if (trainingModel != null)
{
BlockStart();
trainingModel = null;
BlockEnd();
}
if (groundTruth != null)
{
BlockStart();
groundTruth = null;
BlockEnd();
}
if (crossValidation)
{
GradeAgainstCrossValidationRun();
}
else
{
GradeAgainstGroundTruth();
}
if (!cancelProcessing)
{
displayConfusionMatrix = true;
}
loadingThreadRunning = false;
} /* LoadData */
} /* CreateFeatureDataFileThread */
void MakeSureDepthFieldIsIncluded(PicesFeatureVectorList data)
{
PicesDataBase dbConn = null;
PicesRunLog runLog = new PicesRunLog(dialogMsgQueue);
data.SortByImageFileName();
String curSipperFileName = "";
float curSipperFileDepth = 0.0f;
foreach (PicesFeatureVector fv in data)
{
if (fv.Depth == 0.0f)
{
if (dbConn == null)
{
dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog);
}
uint sacnLineNum = 0;
uint scanCol = 0;
String nextSipperFileName = "";
PicesFeatureVector.ParseImageFileName(fv.ExampleFileName, ref nextSipperFileName, ref sacnLineNum, ref scanCol);
if (nextSipperFileName != curSipperFileName)
{
curSipperFileDepth = 0.0f;
curSipperFileName = nextSipperFileName;
PicesSipperFile sf = dbConn.SipperFileRecLoad(curSipperFileName);
if (sf != null)
{
curSipperFileDepth = sf.Depth;
}
}
PicesInstrumentData id = dbConn.InstrumentDataGetByScanLine(curSipperFileName, sacnLineNum);
if ((id == null) || (id.Depth == 0.0))
{
fv.Depth = curSipperFileDepth;
}
else
{
fv.Depth = id.Depth;
}
}
}
if (dbConn != null)
{
dbConn.Close();
dbConn = null;
GC.Collect();
}
} /* MakeSureDepthFieldIsIncluded */
} /* SaveSampleImages */
private PicesFeatureVectorList FilterFeatureVectorList(PicesFeatureVectorList data,
ref uint numFilteredOut
)
{
numFilteredOut = 0;
PicesFeatureVectorList keepers = new PicesFeatureVectorList();
foreach (PicesFeatureVector fv in data)
{
bool keep = true;
if ((probMin > 0.0f) && (fv.Probability < probMin))
{
keep = false;
}
else if ((probMax < 1.0f) && (fv.Probability > probMax))
{
keep = false;
}
else if ((sizeMin > 0) && (fv.OrigSize < sizeMin))
{
keep = false;
}
else if ((sizeMax > 0) && (fv.OrigSize > sizeMax))
{
keep = false;
}
else if ((depthMin > 0) && (fv.Depth < depthMin))
{
keep = false;
}
else if ((depthMax > 0) && (fv.Depth > depthMax))
{
keep = false;
}
if (keep)
{
keepers.Add(fv);
}
else
{
numFilteredOut++;
}
}
return(keepers);
} /* FilterFeatureVectorList */
} /* FilterFeatureVectorList */
private uint CountImagesMissingFeatureData(PicesFeatureVectorList data)
{
uint imagesMissingFeatureData = 0;
foreach (PicesFeatureVector fv in data)
{
if (fv.FeatureDataMissing)
{
imagesMissingFeatureData++;
}
}
return(imagesMissingFeatureData);
} /* CountImagesMissingFeatureData */
private void BackGroundThread()
{
PicesDataBase.ThreadInit();
backGroundThreadRunning = true;
imagesLoaded = 0;
imagesFilteredOut = 0;
imagesMissingFeatureData = 0;
if (group != null)
{
featureData = SaveFeatureDataByGroup();
}
else
{
featureData = SaveFeatureDataBySipperFiles();
}
if (!cancelRequested)
{
backGroundStatus = "Saving Feature File";
String pathName = OSservices.GetPathPartOfFile(featureFileName);
if (!String.IsNullOrEmpty(pathName))
{
OSservices.CreateDirectoryPath(pathName);
}
if (IncludeSampleImages.Checked)
{
SaveSampleImages(featureData);
}
driver = new PicesFeatureFileIO(fileFormat);
if (driver != null)
{
if (NormalizeData.Checked)
{
driver.SaveFeatureFileNormalized(featureFileName, featureData, runLog);
}
else
{
driver.SaveFeatureFile(featureFileName, featureData, runLog);
}
}
}
PicesDataBase.ThreadEnd();
backgroundThreadCompleted = true;
backGroundThreadRunning = false;
} /* BackGroundThread */
} /* CreateTuningFeatureFiles */
void MakeSureDepthFieldIsIncluded(PicesFeatureVectorList data)
{
PicesDataBase dbConn = null;
PicesRunLog runLog = new PicesRunLog();
data.SortByImageFileName();
String curSipperFileName = "";
float curSipperFileDepth = 0.0f;
foreach (PicesFeatureVector fv in data)
{
if (fv.Depth == 0.0f)
{
String nextSipperFileName = PicesMethods.SipperFileNameFromImageFileName(fv.ExampleFileName);
if (nextSipperFileName != curSipperFileName)
{
curSipperFileDepth = 0.0f;
curSipperFileName = nextSipperFileName;
if (dbConn == null)
{
dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog);
}
PicesSipperFile sf = dbConn.SipperFileRecLoad(curSipperFileName);
if (sf != null)
{
curSipperFileDepth = sf.Depth;
}
}
if (fv.Depth == 0.0f)
{
fv.Depth = curSipperFileDepth;
}
}
}
if (dbConn != null)
{
dbConn.Close();
dbConn = null;
GC.Collect();
}
} /* MakeSureDepthFieldIsIncluded */
} /* SaveFeatureDataBySipperFiles */
private PicesFeatureVectorList SaveFeatureDataByGroup()
{
backGroundStatus = "Group [" + group.Name + "]";
PicesFeatureVectorList data = dbConn.FeatureDataForImageGroup(group, mlClass, classKeyToUse);
if (data != null)
{
imagesLoaded += (uint)data.Count;
uint numFilteredOut = 0;
data = FilterFeatureVectorList(data, ref numFilteredOut);
imagesFilteredOut += numFilteredOut;
imagesMissingFeatureData += CountImagesMissingFeatureData(data);
}
return(data);
} /* SaveFeatureDataByGroup */
} /* BackGroundThread */
private void SaveSampleImages(PicesFeatureVectorList featureData)
{
PicesClassList classes = featureData.ExtractListOfClasses();
if (classes == null)
{
return;
}
String sampleDir = OSservices.AddSlash(OSservices.GetPathPartOfFile(featureFileName)) +
OSservices.GetRootName(featureFileName) +
"_SampleImages";
sampleDir = OSservices.AddSlash(sampleDir);
OSservices.CreateDirectoryPath(sampleDir);
foreach (PicesClass pc in classes)
{
String classSampleDir = OSservices.AddSlash(sampleDir + pc.Name);
OSservices.CreateDirectoryPath(classSampleDir);
PicesFeatureVectorList examplesThisClass = featureData.ExtractExamplesForAGivenClass(pc);
examplesThisClass.RandomizeOrder();
int numWritten = 0;
foreach (PicesFeatureVector fv in examplesThisClass)
{
String imageRootName = OSservices.GetRootName(fv.ExampleFileName);
PicesRaster i = dbConn.ImageFullSizeFind(imageRootName);
if (i != null)
{
String imageFileName = classSampleDir + imageRootName + ".bmp";
i.Save(imageFileName);
++numWritten;
}
if (numWritten >= includeSampleImagesNumPerClass)
{
break;
}
}
}
} /* SaveSampleImages */
} /* CountImagesMissingFeatureData */
private PicesFeatureVectorList SaveFeatureDataBySipperFiles()
{
PicesFeatureVectorList data = new PicesFeatureVectorList();
String[] sipperFileNames = null;
if (String.IsNullOrEmpty(sipperFileName))
{
sipperFileNames = dbConn.SipperFileGetList(cruise, station, deployment);
}
else
{
sipperFileNames = new String[1];
sipperFileNames[0] = sipperFileName;
}
if (sipperFileNames != null)
{
for (int idx = 0; (idx < sipperFileNames.Length) && (!cancelRequested); idx++)
{
String curSipperFileName = sipperFileNames[idx];
backGroundStatus = "Loading[" + (idx + 1).ToString() + " of " + sipperFileNames.Length.ToString() + "] File[" + curSipperFileName + "]";
PicesFeatureVectorList dataThisSipperFile = dbConn.FeatureDataGetOneSipperFile(curSipperFileName, mlClass, classKeyToUse, false);
if (dataThisSipperFile != null)
{
imagesLoaded += (uint)dataThisSipperFile.Count;
uint numFilteredOut = 0;
dataThisSipperFile = FilterFeatureVectorList(dataThisSipperFile, ref numFilteredOut);
imagesFilteredOut += numFilteredOut;
imagesMissingFeatureData += CountImagesMissingFeatureData(dataThisSipperFile);
foreach (PicesFeatureVector fv in dataThisSipperFile)
{
data.Add(fv);
}
}
}
}
return(data);
} /* SaveFeatureDataBySipperFiles */
} /* GradeAgainstCrossValidationRun */
void CrossValidate(PicesFeatureVectorList testExamples,
PicesFeatureVectorList trainingExamples,
int foldNum
)
{
msgQueue.AddMsg("\n\n" + "Starting Fold[" + (foldNum + 1).ToString() + "] of Fold[" + numOfFolds + "]" + "\n");
if (trainingModel != null)
{
trainingModel = null;
}
msgQueue.AddMsg("Building Classifier");
try
{
trainingModel = new TrainingModel2(runLog, curSelModel);
trainingModel.BuildTrainingModel(trainingExamples);
}
catch (Exception e)
{
msgQueue.AddMsg("\n\n\n" + "Exception occur building training model" + "\n" + e.ToString() + "\n\n");
MessageBox.Show(this, "Exception Occurred Building Training Model." + "\n" + e.ToString(), "CrossValidate", MessageBoxButtons.OK);
trainingModel = null;
return;
}
if (cancelProcessing)
{
trainingModel = null;
return;
}
if (!trainingModel.Valid)
{
MessageBox.Show(this, "Training Model is Invalid.", "Ground Thruthing", MessageBoxButtons.OK);
}
else
{
msgQueue.AddMsg("\n" + "Predictions for fold[" + (foldNum + 1).ToString() + "]");
PicesClass knownClass = null;
foreach (PicesFeatureVector example in testExamples)
{
if (cancelProcessing)
{
break;
}
knownClass = example.MLClass;
PicesPrediction prediction = trainingModel.PredictClass(example);
if ((otherClass != null) && (prediction.MLClass == noAgreementClass))
{
prediction.MLClass = otherClass;
}
/**@todo Add code for "NoAgreement:" to be treeted as a Other Class when there is "NoAgreement". */
confusionMatrix.MakePrediction(knownClass, prediction.MLClass, prediction.Probability, example, trainingModel);
}
}
trainingModel.CleanUp();
trainingModel = null;
} /* CrossValidate */
} /* GradeAgainstGroundTruth */
private void GradeAgainstCrossValidationRun()
{
if (curSelModel == null)
{
curSelModel = new PicesTrainingConfigManaged(configFileName, runLog);
}
if (!curSelModel.Valid())
{
MessageBox.Show("Configuration File[" + configFileName + "] is not valid");
return;
}
bool changesMade = false;
PicesFeatureVectorList data = curSelModel.LoadFeatureDataFromTrainingLibraries(changesMade);
if (cancelProcessing)
{
data = null;
return;
}
if (data == null)
{
MessageBox.Show("No data was loaded from Configuration File[" + configFileName + "] is not valid");
return;
}
PicesFeatureVectorList examples = data.StratifyAmoungstClasses(numOfFolds);
int imageCount = examples.Count;
int numImagesPerFold = (imageCount + numOfFolds - 1) / numOfFolds;
int firstInGroup = 0;
int foldNum;
for (foldNum = 0; foldNum < numOfFolds; foldNum++)
{
int lastInGroup;
// If We are doing the last Fold Make sure that we are including all the examples
// that have not been tested.
if (foldNum == (numOfFolds - 1))
{
lastInGroup = imageCount;
}
else
{
lastInGroup = firstInGroup + numImagesPerFold - 1;
}
PicesFeatureVectorList trainingImages = new PicesFeatureVectorList();
PicesFeatureVectorList testImages = new PicesFeatureVectorList();
for (int x = 0; (x < imageCount) && (!cancelProcessing); x++)
{
PicesFeatureVector newImage = new PicesFeatureVector(examples[x]);
if ((x >= firstInGroup) && (x <= lastInGroup))
{
testImages.Add(newImage);
}
else
{
trainingImages.Add(newImage);
}
}
if (cancelProcessing)
{
break;
}
CrossValidate(testImages, trainingImages, foldNum);
firstInGroup = firstInGroup + numImagesPerFold;
}
if (!cancelProcessing)
{
if (trainingModel != null)
{
trainingModel = null;
}
msgQueue.AddMsg("\n" + "Building Final Classifier for User Query");
trainingModel = new TrainingModel2(runLog, curSelModel);
trainingModel.BuildTrainingModel(data);
}
if ((!cancelProcessing) && (trainingModel != null) && (!trainingModel.Valid))
{
MessageBox.Show("Error Loading Training Model; refer to dialog box");
cancelProcessing = true;
}
else if (!cancelProcessing)
{
confusionMatrix.SetTrainingModel(trainingModel);
}
// We no longer need the Training Model or Ground Truth loaded
BlockStart();
data = null;
BlockEnd();
} /* GradeAgainstCrossValidationRun */
} /* PerformPredictions*/
private void GradeAgainstGroundTruth()
{
if (!cancelProcessing)
{
if (trainingModel != null)
{
trainingModel = null;
}
trainingModel = new TrainingModel2(runLog, trainingModelName);
if (trainingModel.Valid)
{
if (trainingLevel < maxLevel)
{
trainingModel.LoadTrainingModelForGivenLevel(trainingLevel, msgQueue);
}
else
{
trainingModel.LoadExistingModelOtherwiseBuild(msgQueue);
}
}
}
if (!cancelProcessing)
{
groundTruth = new PicesFeatureVectorList();
groundTruth.LoadInSubDirectoryTree(GroundTruth.Text,
true, // true = useDirectoryNameForClassName,
runLog,
false // false = Don't rewiteRootFeatureFile
);
numGroundTruthImages = groundTruth.Count;
numGroundTruthImagesProcessed = 0;
}
if (cancelProcessing)
{
if (trainingModel != null)
{
BlockStart();
trainingModel = null;
BlockEnd();
}
}
else
{
if (!trainingModel.Valid)
{
MessageBox.Show("Error Loading Training Model; refer to dialog box");
cancelProcessing = true;
}
else if (!cancelProcessing)
{
confusionMatrix.SetTrainingModel(trainingModel);
PerformPredictions();
}
}
if (groundTruth != null)
{
BlockStart();
groundTruth = null;
BlockEnd();
}
} /* GradeAgainstGroundTruth */
} /* DialogTimer_Tick */
private void CreateTuningFeatureFiles()
{
tuningThreadRunning = true;
PicesDataBase.ThreadInit();
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Started");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Creating Destination Directory");
String destinationDirectory = OSservices.AddSlash(DestinationDirectory.Text) + OSservices.GetRootName(modelName);
OSservices.CreateDirectoryPath(destinationDirectory);
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Loading feature data for Model[" + modelName + "]");
config = new PicesTrainingConfigManaged(modelName, runLog);
if (!config.Valid())
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] is invalid.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
bool changesMade = false;
PicesFeatureVectorList data = config.LoadFeatureDataFromTrainingLibraries(changesMade);
if (data == null)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] Could not load Feature Data.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Process Canceled.");
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Feature Data Loaded.");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Checking that Depth Data Exists.");
MakeSureDepthFieldIsIncluded(data);
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Splitting into separate files.");
PicesFeatureVectorList trainData = new PicesFeatureVectorList();
PicesFeatureVectorList testData = new PicesFeatureVectorList();
PicesFeatureVectorList validationData = new PicesFeatureVectorList();
PicesClassList classes = data.ExtractListOfClasses();
if (classes == null)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Could not extract list of classes from data.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
foreach (PicesClass c in classes)
{
int trainCount = 0;
int testCount = 0;
int validationCount = 0;
PicesFeatureVectorList examplesForClass = data.ExtractExamplesForAGivenClass(c);
int n = examplesForClass.Count;
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Processing Class[" + c.Name + "] Count[" + n + "]");
trainCount = (int)((float)n * trainingDataPercentage / 100.0f + 0.5f);
if (trainingDataMaxImagesPerClass > 0)
{
if (trainCount > trainingDataMaxImagesPerClass)
{
trainCount = trainingDataMaxImagesPerClass;
}
}
if (validationDataPercentage == 0.0f)
{
testCount = n - trainCount;
}
else
{
testCount = (int)((float)n * testDataPercentage / 100.0f + 0.5f);
}
validationCount = n - (trainCount + testCount);
examplesForClass.RandomizeOrder();
int validationStart = trainCount + testCount;
int testStart = trainCount;
for (int idx = 0; idx < n; idx++)
{
PicesFeatureVector fv = examplesForClass[idx];
if (idx >= validationStart)
{
validationData.Add(fv);
}
else if (idx >= testStart)
{
testData.Add(fv);
}
else
{
trainData.Add(fv);
}
}
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Process Canceled.");
break;
}
}
if (!tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n" + DateTime.Now.ToShortTimeString() + " Stratifying Data");
PicesFeatureVectorList trainDataStratified = trainData.StratifyAmoungstClasses(numOfFolds);
PicesFeatureVectorList testDataStratified = testData.StratifyAmoungstClasses(numOfFolds);
PicesFeatureVectorList validationDataStratified = validationData.StratifyAmoungstClasses(numOfFolds);
String rootName = OSservices.AddSlash(destinationDirectory) + OSservices.GetRootName(modelName);
if ((trainDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
trainDataFileName = rootName + "_Train.data";
AddMsgToDialogMsgQueue("Saving Training Data[" + trainDataFileName + "]");
trainDataStratified.Save(trainDataFileName, "", runLog);
}
if ((testDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
testDataFileName = rootName + "_Test.data";
AddMsgToDialogMsgQueue("Saving Test Data[" + testDataFileName + "]");
testDataStratified.Save(testDataFileName, "", runLog);
}
if ((validationDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
validationDataFileName = rootName + "_Validation.data";
AddMsgToDialogMsgQueue("Saving Validation Data[" + validationDataFileName + "]");
validationDataStratified.Save(validationDataFileName, "", runLog);
}
}
AddMsgToDialogMsgQueue("\n\n");
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("Process *** CANCELED ***.");
}
else
{
AddMsgToDialogMsgQueue("Process Completed.");
}
PicesDataBase.ThreadEnd();
tuningThreadDone = true;
tuningThreadRunning = false;
} /* CreateTuningFeatureFiles */
} /* DialogTimer_Tick */
private void CreateFeatureDataFileThread()
{
savingThreadRunning = true;
PicesDataBase.ThreadInit();
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Started");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Loading feature data for Model[" + modelName + "]");
config = new PicesTrainingConfigManaged(modelName, runLog);
if (!config.Valid())
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] is invalid.");
savingThreadFailed = true;
savingThreadDone = true;
savingThreadRunning = false;
return;
}
bool changesMade = false;
PicesFeatureVectorList data = config.LoadFeatureDataFromTrainingLibraries(changesMade);
if (data == null)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] Count not load Feature Data.");
savingThreadFailed = true;
savingThreadDone = true;
savingThreadRunning = false;
return;
}
if (savingThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Process Canceled.");
savingThreadDone = true;
savingThreadRunning = false;
return;
}
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Feature Data Loaded.");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Checking that Depth Data Exists.");
MakeSureDepthFieldIsIncluded(data);
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Splitting into separate files.");
if (!savingThreadCancelReq)
{
data.Save(destinationFileName, FileFormat.Text, runLog);
}
AddMsgToDialogMsgQueue("\n\n");
if (savingThreadCancelReq)
{
AddMsgToDialogMsgQueue("Process *** CANCELED ***.");
}
else
{
AddMsgToDialogMsgQueue("Process Completed.");
}
PicesDataBase.ThreadEnd();
savingThreadDone = true;
savingThreadRunning = false;
} /* CreateFeatureDataFileThread */