} /* ExploreMatrixCell */ public void MakePrediction(PicesClass knownClass, PicesClass predictedClass, double predProb, PicesFeatureVector fv, TrainingModel2 trainModel ) { bool partOfOriginalClassifier = ClassInClassifierClasses(knownClass); if (!partOfOriginalClassifier) { String notPartClassName = "(V)" + knownClass.Name; knownClass = PicesClassList.GetUniqueClass(notPartClassName, ""); } int x = classes.LookUpIndex(knownClass); if (x < 0) { classes.Add(knownClass); } x = classes.LookUpIndex(predictedClass); if (x < 0) { classes.Add(predictedClass); } Prediction p = new Prediction(knownClass, predictedClass, predProb, fv, trainModel, partOfOriginalClassifier); predictions.Add(p); } /* featureVector */
} /* MakePredictions */ public static void MakePredictions(PicesFeatureVector featureVector, ref PicesPredictionList model1Predictions, ref PicesPredictionList model2Predictions, PicesRunLog runLog ) { model1Predictions = null; model2Predictions = null; if ((!ModelsAreAvailable()) || (featureVector == null)) { return; } if (model1 != null) { model1Predictions = model1.PredictProbabilities(featureVector); } if (model2 != null) { model2Predictions = model2.PredictProbabilities(featureVector); } return; } /* MakePredictions */
} /* MakePredictions */ public static void MakePredictions(PicesFeatureVector featureVector, ref PicesPrediction model1Prediction1, ref PicesPrediction model1Prediction2, ref PicesPrediction model2Prediction1, ref PicesPrediction model2Prediction2, PicesRunLog runLog ) { if (model1Prediction1 == null) { model1Prediction1 = new PicesPrediction(null, 0, 0.0f); } if (model1Prediction2 == null) { model1Prediction2 = new PicesPrediction(null, 0, 0.0f); } if (model2Prediction1 == null) { model2Prediction1 = new PicesPrediction(null, 0, 0.0f); } if (model2Prediction2 == null) { model2Prediction2 = new PicesPrediction(null, 0, 0.0f); } if (model1 != null) { model1.PredictClass(featureVector, model1Prediction1, model1Prediction2); } if (model2 != null) { model2.PredictClass(featureVector, model2Prediction1, model2Prediction2); } } /* MakePredictions */
public PredictionBreakDownDisplay(PicesDataBase _dbConn, PicesDataBaseImage _image, PicesRaster _raster, TrainingModel2 _trainingModel, PicesInstrumentData _instrumentData, /*!< Instrument data that goes with image, if null will read from database. */ PicesRunLog _runLog ) { if (_runLog == null) { runLog = new PicesRunLog(); } else { runLog = _runLog; } if (_image == null) { return; } fileName = _image.ImageFileName; instrumentData = _instrumentData; trainingModel = _trainingModel; PicesRaster raster = _raster; if (_dbConn != null) { if (raster == null) { raster = _dbConn.ImageFullSizeFind(fileName); } if (instrumentData == null) { instrumentData = _dbConn.InstrumentDataGetByScanLine(_image.SipperFileName, _image.TopLeftRow); } dbFeatureVector = _dbConn.FeatureDataRecLoad(fileName); } if (raster == null) { return; } //raster = raster.ConnectedComponent (); featureCalcImages = new PicesRasterList(); featureVector = new PicesFeatureVector(raster, fileName, featureCalcImages, runLog); if (instrumentData != null) { featureVector.AddInstrumentData(instrumentData); } InitializeComponent(); }
} /* LoadImageGrid */ private void UpdateDetailProbabilityDisplay() { if ((trainingModel == null) || (trainingModel.ModelType != TrainingModel2.ModelTypes.Dual)) { return; } PicesPredictionList class1Results = null; PicesPredictionList class2Results = null; String classifier1Desc = null; String classifier2Desc = null; PicesFeatureVector fvToUse = dbFeatureVector; if (fvToUse == null) { fvToUse = featureVector; } trainingModel.ProbabilitiesByClassDual(fvToUse, ref classifier1Desc, ref classifier2Desc, ref class1Results, ref class2Results ); this.Classifier1Name.Text = classifier1Desc; this.Classifier2Name.Text = classifier2Desc; if (class1Results != null) { Classifier1Probabilities.Rows.Clear(); foreach (PicesPrediction p in class1Results) { Object[] data = new Object[3]; data[0] = p.ClassName; data[1] = p.Probability; data[2] = p.Votes; Classifier1Probabilities.Rows.Add(data); } } if (class2Results != null) { Classifier2Probabilities.Rows.Clear(); foreach (PicesPrediction p in class2Results) { Object[] data = new Object[3]; data[0] = p.ClassName; data[1] = p.Probability; data[2] = p.Votes; Classifier2Probabilities.Rows.Add(data); } } } /* UpdateDetailProbabilityDisplay */
/// <summary> /// Using supplied 'Raster' data it will call each currently active model and return there predictions. /// </summary> /// <param name="raster">Raster of image to predict. .</param> /// <param name="instrumentData">Instrument data that was recorded with the 'raster' .</param> /// <param name="imageFileName">Name of the image file.</param> /// <param name="model1Predictions">The model1 predictions.</param> /// <param name="model2Predictions">The model2 predictions.</param> /// <param name="runLog">The run log.</param> public static void MakePredictions(PicesRaster raster, InstrumentData instrumentData, String imageFileName, ref PicesPredictionList model1Predictions, ref PicesPredictionList model2Predictions, PicesRunLog runLog ) { model1Predictions = null; model2Predictions = null; if (!ModelsAreAvailable()) { return; } ParsedImageFileName pifn = PicesFeatureVector.ParseImageFileName(imageFileName); String sipperFileName = pifn.sipperFileName; uint scanLineNum = pifn.scanLineNum; PicesFeatureVector fv = new PicesFeatureVector(raster, imageFileName, null, runLog); if (instrumentData != null) { fv.AddInstrumentData(instrumentData.Depth(), instrumentData.Salinity(), instrumentData.Oxygen(), instrumentData.Fluorescence() ); } else { PicesDataBase dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog); PicesInstrumentData pid = dbConn.InstrumentDataGetByScanLine(sipperFileName, scanLineNum); if (pid != null) { fv.AddInstrumentData(pid); } pid = null; dbConn.Close(); dbConn = null; } if (model1 != null) { model1Predictions = model1.PredictProbabilities(fv); } if (model2 != null) { model2Predictions = model2.PredictProbabilities(fv); } fv = null; return; } /* MakePredictions */
} /* CreateFeatureDataFileThread */ void MakeSureDepthFieldIsIncluded(PicesFeatureVectorList data) { PicesDataBase dbConn = null; PicesRunLog runLog = new PicesRunLog(dialogMsgQueue); data.SortByImageFileName(); String curSipperFileName = ""; float curSipperFileDepth = 0.0f; foreach (PicesFeatureVector fv in data) { if (fv.Depth == 0.0f) { if (dbConn == null) { dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog); } uint sacnLineNum = 0; uint scanCol = 0; String nextSipperFileName = ""; PicesFeatureVector.ParseImageFileName(fv.ExampleFileName, ref nextSipperFileName, ref sacnLineNum, ref scanCol); if (nextSipperFileName != curSipperFileName) { curSipperFileDepth = 0.0f; curSipperFileName = nextSipperFileName; PicesSipperFile sf = dbConn.SipperFileRecLoad(curSipperFileName); if (sf != null) { curSipperFileDepth = sf.Depth; } } PicesInstrumentData id = dbConn.InstrumentDataGetByScanLine(curSipperFileName, sacnLineNum); if ((id == null) || (id.Depth == 0.0)) { fv.Depth = curSipperFileDepth; } else { fv.Depth = id.Depth; } } } if (dbConn != null) { dbConn.Close(); dbConn = null; GC.Collect(); } } /* MakeSureDepthFieldIsIncluded */
private PicesPredictionList PredictClasses(PicesFeatureVector fv, PicesPrediction pred1, PicesPrediction pred2, String imageFileName, String imageFileRootName ) { PicesFeatureVector fvToUse = fv; PicesPredictionList probPredList = null; if (fvToUse.FeatureDataMissing) { fvToUse = null; // Feature data is missing. we need to compute from original image. PicesDataBaseImage pdbi = DbConn().ImageLoad(imageFileRootName); if (pdbi != null) { PicesRaster pr = DbConn().ImageFullSizeFind(imageFileRootName); if (pr == null) { ErrorMsgsAdd("Failed to Load FullSize image [" + imageFileRootName + "]"); } else { fvToUse = new PicesFeatureVector(pr, imageFileName, null, runLog); // Since we had to compute the FeatureDatya from the raster image we now need to // get the instrument data that matches it. PicesInstrumentData id = DbConn().InstrumentDataGetByScanLine(pdbi.SipperFileName, pdbi.TopLeftRow); if (id != null) { fvToUse.AddInstrumentData(id); } DbConn().FeatureDataInsertRow(pdbi.SipperFileName, fvToUse); pr.Dispose(); } pr = null; } } if (fvToUse != null) { classifier.PredictClass(fvToUse, pred1, pred2); probPredList = classifier.PredictProbabilities(fvToUse); } else { pred1.MLClass = PicesClassList.GetUnKnownClassStatic(); pred1.Probability = 1.0f; pred2.MLClass = PicesClassList.GetUnKnownClassStatic(); pred2.Probability = 0.0f; } return(probPredList); } /* PredictClasses */
public PredictionBreakDownDisplayDual(PicesDataBase _dbConn, String _fileName, TrainingModel2 _trainingModel, PicesInstrumentData _instrumentData, /**< Instrument data that goes with image, if null will read from dtabase. */ PicesRunLog _runLog ) { if (_runLog == null) { runLog = new PicesRunLog(); } else { runLog = _runLog; } fileName = _fileName; instrumentData = _instrumentData; trainingModel = _trainingModel; PicesDataBase dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog); PicesRaster raster = dbConn.ImageFullSizeFind(fileName); if (raster == null) { return; } if (_dbConn != null) { dbFeatureVector = _dbConn.FeatureDataRecLoad(_fileName); if (instrumentData == null) { PicesDataBaseImage image = dbConn.ImageLoad(fileName); if (image != null) { instrumentData = dbConn.InstrumentDataGetByScanLine(image.SipperFileName, image.TopLeftRow); } } } featureVector = new PicesFeatureVector(raster, fileName, null, runLog); if (instrumentData != null) { featureVector.AddInstrumentData(instrumentData); } dbConn.Close(); dbConn = null; GC.Collect(); InitializeComponent(); }
} /* StartThread */ private PicesFeatureVector GetNextFeatureVector() { PicesFeatureVector nextFv = null; lock (featureVectorQueue) { if (featureVectorQueue.Count > 0) { nextFv = featureVectorQueue.Dequeue(); } } return(nextFv); } /* GetNextFeatureVector*/
Prediction(PicesClass _knownClass, PicesClass _predClass, double _prob, PicesFeatureVector _fv, TrainingModel2 _trainModel, bool _partOfClassifier ) { knowClass = _knownClass; predClass = _predClass; prob = _prob; fv = _fv; trainModel = _trainModel; partOfClassifier = _partOfClassifier; }
} /* ProcessDirectory */ private void ProcessOneImage(String fullImageFileName) { String rootName = OSservices.GetRootName(fullImageFileName); PicesDataBaseImage i = dbConn.ImageLoad(rootName); if (i != null) { runStatsImagesInDataBase++; return; } String sipperFileName = null; uint scanLine = 0; uint scanCol = 0; PicesFeatureVector.ParseImageFileName(fullImageFileName, ref sipperFileName, ref scanLine, ref scanCol); if (String.IsNullOrEmpty(sipperFileName) || (scanLine < 1)) { // We are not going to be able to locate this image in the Database. runStatsImagesNotInDataBase++; reconcilingRunLog.Writeln(fullImageFileName + "\t" + "Name format will not let me derive needed info."); return; } String sqlStr = "call ImagesLocateClosestImage(" + "\"" + sipperFileName + "\"" + ", " + scanLine.ToString() + ", " + scanCol.ToString() + ")"; String[][] results = dbConn.QueryStatement(sqlStr, null); if ((results == null) || (results.Length < 1)) { runStatsImagesNotInDataBase++; reconcilingRunLog.Writeln(fullImageFileName + "\t" + "Could not find an appropriate image in the Database"); RemoveImage(fullImageFileName); return; } String correctImageFileName = results[0][1]; String newFullName = OSservices.AddSlash(OSservices.GetPathPartOfFile(fullImageFileName)) + correctImageFileName + "." + OSservices.GetFileExtension(fullImageFileName); OSservices.RenameFile(fullImageFileName, newFullName); runStatsImagesFixed++; } /* ProcessOneImage */
} /* GetNextFeatureVector*/ private void ThreadProcess() { threadRunning = true; PicesDataBase.ThreadInit(); if ((classifier == null) && (!String.IsNullOrEmpty(classifierName))) { PicesRunLog log = new PicesRunLog(); classifier = new TrainingModel2(log, classifierName); classifier.LoadExistingModelOtherwiseBuild(null); classifierWeOwnIt = true; } while (!cancelFlag) { PicesFeatureVector fv = GetNextFeatureVector(); if (fv != null) { ProcessOneFeatureVector(fv); } else { if (threadTerminate) { break; } Thread.Sleep(10); } } if (dbConn != null) { dbConn.Close(); dbConn = null; GC.Collect(); } PicesDataBase.ThreadEnd(); threadRunning = false; threadDone = true; } /* ThreadProcess */
} /* PredictClasses */ PicesClass AssignVallidatedClassAsPredictedClass(PicesFeatureVector fv) { PicesClass validatedClass = fv.MLClass; if ((validatedClass == null) || (classesInClassifier == null)) { return(validatedClass); } if (classesInClassifier.LookUpIndex(validatedClass) >= 0) { return(validatedClass); } // Lets see if the Validated Class is a decedent of one of the classes in the Classifier. PicesClass candidate = null; int numGenerations = 99999; foreach (PicesClass pc in classesInClassifier) { int zed = pc.NumGenerations(validatedClass); if (zed >= 0) { if (zed < numGenerations) { candidate = pc; numGenerations = zed; } } } if (candidate != null) { return(candidate); } return(validatedClass); } /* AssignVallidatedClassAsPredictedClass */
} /* MakePredictions */ public static void MakePredictions(PicesRaster raster, ref PicesPrediction model1Prediction1, ref PicesPrediction model1Prediction2, ref PicesPrediction model2Prediction1, ref PicesPrediction model2Prediction2, PicesRunLog runLog ) { String imageFileName = raster.FileName; ParsedImageFileName pifn = PicesFeatureVector.ParseImageFileName(imageFileName); String sipperFileName = pifn.sipperFileName; uint scanLineNum = pifn.scanLineNum; PicesFeatureVector fv = new PicesFeatureVector(raster, imageFileName, null, runLog); { PicesDataBase dbConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog); PicesInstrumentData pid = dbConn.InstrumentDataGetByScanLine(sipperFileName, scanLineNum); if (pid != null) { fv.AddInstrumentData(pid); } pid = null; dbConn.Close(); dbConn = null; } MakePredictions(fv, ref model1Prediction1, ref model1Prediction2, ref model2Prediction1, ref model2Prediction2, runLog ); fv = null; }
} /* GradeAgainstGroundTruth */ private void GradeAgainstCrossValidationRun() { if (curSelModel == null) { curSelModel = new PicesTrainingConfigManaged(configFileName, runLog); } if (!curSelModel.Valid()) { MessageBox.Show("Configuration File[" + configFileName + "] is not valid"); return; } bool changesMade = false; PicesFeatureVectorList data = curSelModel.LoadFeatureDataFromTrainingLibraries(changesMade); if (cancelProcessing) { data = null; return; } if (data == null) { MessageBox.Show("No data was loaded from Configuration File[" + configFileName + "] is not valid"); return; } PicesFeatureVectorList examples = data.StratifyAmoungstClasses(numOfFolds); int imageCount = examples.Count; int numImagesPerFold = (imageCount + numOfFolds - 1) / numOfFolds; int firstInGroup = 0; int foldNum; for (foldNum = 0; foldNum < numOfFolds; foldNum++) { int lastInGroup; // If We are doing the last Fold Make sure that we are including all the examples // that have not been tested. if (foldNum == (numOfFolds - 1)) { lastInGroup = imageCount; } else { lastInGroup = firstInGroup + numImagesPerFold - 1; } PicesFeatureVectorList trainingImages = new PicesFeatureVectorList(); PicesFeatureVectorList testImages = new PicesFeatureVectorList(); for (int x = 0; (x < imageCount) && (!cancelProcessing); x++) { PicesFeatureVector newImage = new PicesFeatureVector(examples[x]); if ((x >= firstInGroup) && (x <= lastInGroup)) { testImages.Add(newImage); } else { trainingImages.Add(newImage); } } if (cancelProcessing) { break; } CrossValidate(testImages, trainingImages, foldNum); firstInGroup = firstInGroup + numImagesPerFold; } if (!cancelProcessing) { if (trainingModel != null) { trainingModel = null; } msgQueue.AddMsg("\n" + "Building Final Classifier for User Query"); trainingModel = new TrainingModel2(runLog, curSelModel); trainingModel.BuildTrainingModel(data); } if ((!cancelProcessing) && (trainingModel != null) && (!trainingModel.Valid)) { MessageBox.Show("Error Loading Training Model; refer to dialog box"); cancelProcessing = true; } else if (!cancelProcessing) { confusionMatrix.SetTrainingModel(trainingModel); } // We no longer need the Training Model or Ground Truth loaded BlockStart(); data = null; BlockEnd(); } /* GradeAgainstCrossValidationRun */
} /* UpdatedataFields*/ private void MakePredictions() { if (!ActiveTrainingLibraries.ModelsAreAvailable()) { // Since there are NO training models loaded; there is no point doing the work for a prediction. return; } featureVector = dbConn.FeatureDataRecLoad(image); if (featureVector == null) { featureVector = new PicesFeatureVector(raster, image.ImageFileName, null, runLog); // Since we had to compute the FeatureDatya from the raster we now need to // get the instrument data that matches it. if (instrumentData != null) { featureVector.AddInstrumentData(instrumentData); } dbConn.FeatureDataInsertRow(image.SipperFileName, featureVector); } float esd = 0.0f; float eBv = 0.0f; float filledArea = image.PixelCount; float cropWidth = 3800.0f; // (3900.0f - 200.0f); float flowRate2 = 0.5f; if (featureVector != null) { filledArea = featureVector.FilledArea; } if (sipperFile != null) { scanRate = sipperFile.ScanRate; } esd = (float)(2.0 * Math.Sqrt(filledArea * (0.096 / cropWidth) * 1000.0 * (flowRate1 / sipperFile.ScanRate) * 1000.0 / 3.1415926)); eBv = (float)((4.0 / 3.0) * Math.PI * Math.Pow(Math.Sqrt(filledArea * (chamberWidth / cropWidth) * 1000 * (flowRate1 / scanRate) * 1000.0 / Math.PI), 3)); PicesPrediction model1Prediction1 = new PicesPrediction(null, 0, 0.0f); PicesPrediction model1Prediction2 = new PicesPrediction(null, 0, 0.0f); PicesPrediction model2Prediction1 = new PicesPrediction(null, 0, 0.0f); PicesPrediction model2Prediction2 = new PicesPrediction(null, 0, 0.0f); ActiveTrainingLibraries.MakePredictions(featureVector, ref model1Prediction1, ref model1Prediction2, ref model2Prediction1, ref model2Prediction2, runLog ); if (model1Prediction1 != null) { Lib1Pred1Class.Text = model1Prediction1.ClassName; Lib1Pred1Prob.Text = model1Prediction1.Probability.ToString("##0.00%"); Lib1Pred1Votes.Text = model1Prediction1.Votes.ToString("#,##0"); } if (model1Prediction2 != null) { Lib1Pred2Class.Text = model1Prediction2.ClassName; Lib1Pred2Prob.Text = model1Prediction2.Probability.ToString("##0.00%"); Lib1Pred2Votes.Text = model1Prediction2.Votes.ToString("#,##0"); } if (model2Prediction1 != null) { Lib2Pred1Class.Text = model2Prediction1.ClassName; Lib2Pred1Prob.Text = model2Prediction1.Probability.ToString("##0.00%"); Lib2Pred1Votes.Text = model2Prediction1.Votes.ToString("#,##0"); } if (model1Prediction2 != null) { Lib2Pred2Class.Text = model2Prediction2.ClassName; Lib2Pred2Prob.Text = model2Prediction2.Probability.ToString("##0.00%"); Lib2Pred2Votes.Text = model2Prediction2.Votes.ToString("#,##0"); } if (featureVector != null) { AreaMMSquare.Text = featureVector.AreaMMSquare.ToString("#,##0.000"); } ESD.Text = esd.ToString("#,##0.00"); EBv.Text = eBv.ToString("##0.0000"); return; } /* MakePredictions */
public FeatureDataDisplay(String imageFileName, PicesFeatureVector calcFV, PicesFeatureVector dbFV ) { InitializeComponent(); int numFeatures = 0; PicesFeatureVector descFV = null; if (calcFV != null) { numFeatures = calcFV.NumFeatures; descFV = calcFV; } if ((dbFV != null) && (dbFV.NumFeatures < numFeatures)) { numFeatures = dbFV.NumFeatures; descFV = dbFV; } if ((descFV == null) || (numFeatures < 1)) { return; } for (int x = 0; x < numFeatures; x++) { ListViewItem lvi = new ListViewItem(descFV.FeatureName(x)); double fvValueCalc = 0.0; double fvValueDb = 0.0; if (calcFV != null) { fvValueCalc = calcFV.FeatureValue(x); } if (dbFV != null) { fvValueDb = dbFV.FeatureValue(x); } double delta = fvValueCalc - fvValueDb; String formatStr = "f6"; if (Math.Max(Math.Abs(fvValueCalc), Math.Abs(fvValueDb)) < 1.0) { formatStr = "e4"; } lvi.SubItems.Add(fvValueCalc.ToString(formatStr)); lvi.SubItems.Add(fvValueDb.ToString(formatStr)); if (delta != 0) { lvi.SubItems.Add(delta.ToString(formatStr)); lvi.SubItems[lvi.SubItems.Count - 1].ForeColor = Color.Red; } FeatureDataList.Items.Add(lvi); } }
} /* ThreadProcess */ private void ProcessOneFeatureVector(PicesFeatureVector fv) { int x = midPoint.CompareTo(fv.CtdDateTime); PicesPredictionList probPredList = null; bool downCast = (x > 0); if (reClassify) { String imageFileName = fv.ExampleFileName; String imageFileRootName = OSservices.GetRootName(imageFileName); if ((fv.Validated) && (validatedTrumps)) { PicesClass validatedClass = AssignVallidatedClassAsPredictedClass(fv); probPredList = PredictClasses(fv, pred1, pred2, imageFileName, imageFileRootName); if (validatedClass != null) { if (pred1.MLClass != validatedClass) { pred2.MLClass = pred1.MLClass; } pred2.Probability = 0.0f; pred1.MLClass = validatedClass; pred1.Probability = 1.0f; } } else { probPredList = PredictClasses(fv, pred1, pred2, imageFileName, imageFileRootName); } String imageRootName = OSservices.GetRootName(fv.ExampleFileName); lock (predictionUpdateQueue) { if (pred1.MLClass.ClassId < 0) { pred1.MLClass = UpdateClassId(pred1.MLClass); } if (pred2.MLClass.ClassId < 0) { pred2.MLClass = UpdateClassId(pred2.MLClass); } predictionUpdateQueue.Enqueue(new PredictionResult(imageRootName, pred1.MLClass.ClassId, pred1.Probability, pred2.MLClass.ClassId, pred2.Probability, probPredList ) ); } } else { if ((fv.Validated) && (validatedTrumps)) { pred1.MLClass = fv.MLClass; pred1.Probability = 1.0f; } else { pred1.MLClass = fv.MLClass; pred1.Probability = fv.Probability; } } String className = pred1.ClassName; bool noiseClass = ((className.Length >= 5) && (className.Substring(0, 5).Equals("Noise", StringComparison.InvariantCultureIgnoreCase)) ); if (noiseClass) { } else { lock (classCounts) { classCounts.AddOne(pred1.MLClass); depthDistribution_1.Increment(pred1.MLClass, (int)fv.Depth); if (downCast) { depthDistribution_1Down.Increment(pred1.MLClass, (int)fv.Depth); sizeDistributionDown.Increment(pred1.MLClass, fv.AreaMMSquare); } else { depthDistribution_1Up.Increment(pred1.MLClass, (int)fv.Depth); sizeDistributionUp.Increment(pred1.MLClass, fv.AreaMMSquare); } depthDistribution_10.Increment(pred1.MLClass, (int)fv.Depth); numImagesClassified++; } } } /* ProcessOneFeatureVector */
} /* DialogTimer_Tick */ private void CreateTuningFeatureFiles() { tuningThreadRunning = true; PicesDataBase.ThreadInit(); AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Started"); AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Creating Destination Directory"); String destinationDirectory = OSservices.AddSlash(DestinationDirectory.Text) + OSservices.GetRootName(modelName); OSservices.CreateDirectoryPath(destinationDirectory); AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Loading feature data for Model[" + modelName + "]"); config = new PicesTrainingConfigManaged(modelName, runLog); if (!config.Valid()) { AddMsgToDialogMsgQueue("\n\n"); AddMsgToDialogMsgQueue("Model[" + modelName + "] is invalid."); tuningThreadFailed = true; tuningThreadDone = true; tuningThreadRunning = false; return; } bool changesMade = false; PicesFeatureVectorList data = config.LoadFeatureDataFromTrainingLibraries(changesMade); if (data == null) { AddMsgToDialogMsgQueue("\n\n"); AddMsgToDialogMsgQueue("Model[" + modelName + "] Could not load Feature Data."); tuningThreadFailed = true; tuningThreadDone = true; tuningThreadRunning = false; return; } if (tuningThreadCancelReq) { AddMsgToDialogMsgQueue("\n\n"); AddMsgToDialogMsgQueue("Process Canceled."); tuningThreadDone = true; tuningThreadRunning = false; return; } AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Feature Data Loaded."); AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Checking that Depth Data Exists."); MakeSureDepthFieldIsIncluded(data); AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Splitting into separate files."); PicesFeatureVectorList trainData = new PicesFeatureVectorList(); PicesFeatureVectorList testData = new PicesFeatureVectorList(); PicesFeatureVectorList validationData = new PicesFeatureVectorList(); PicesClassList classes = data.ExtractListOfClasses(); if (classes == null) { AddMsgToDialogMsgQueue("\n\n"); AddMsgToDialogMsgQueue("Could not extract list of classes from data."); tuningThreadFailed = true; tuningThreadDone = true; tuningThreadRunning = false; return; } foreach (PicesClass c in classes) { int trainCount = 0; int testCount = 0; int validationCount = 0; PicesFeatureVectorList examplesForClass = data.ExtractExamplesForAGivenClass(c); int n = examplesForClass.Count; AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Processing Class[" + c.Name + "] Count[" + n + "]"); trainCount = (int)((float)n * trainingDataPercentage / 100.0f + 0.5f); if (trainingDataMaxImagesPerClass > 0) { if (trainCount > trainingDataMaxImagesPerClass) { trainCount = trainingDataMaxImagesPerClass; } } if (validationDataPercentage == 0.0f) { testCount = n - trainCount; } else { testCount = (int)((float)n * testDataPercentage / 100.0f + 0.5f); } validationCount = n - (trainCount + testCount); examplesForClass.RandomizeOrder(); int validationStart = trainCount + testCount; int testStart = trainCount; for (int idx = 0; idx < n; idx++) { PicesFeatureVector fv = examplesForClass[idx]; if (idx >= validationStart) { validationData.Add(fv); } else if (idx >= testStart) { testData.Add(fv); } else { trainData.Add(fv); } } if (tuningThreadCancelReq) { AddMsgToDialogMsgQueue("\n\n"); AddMsgToDialogMsgQueue("Process Canceled."); break; } } if (!tuningThreadCancelReq) { AddMsgToDialogMsgQueue("\n\n" + DateTime.Now.ToShortTimeString() + " Stratifying Data"); PicesFeatureVectorList trainDataStratified = trainData.StratifyAmoungstClasses(numOfFolds); PicesFeatureVectorList testDataStratified = testData.StratifyAmoungstClasses(numOfFolds); PicesFeatureVectorList validationDataStratified = validationData.StratifyAmoungstClasses(numOfFolds); String rootName = OSservices.AddSlash(destinationDirectory) + OSservices.GetRootName(modelName); if ((trainDataStratified.Count > 0) && (!tuningThreadCancelReq)) { trainDataFileName = rootName + "_Train.data"; AddMsgToDialogMsgQueue("Saving Training Data[" + trainDataFileName + "]"); trainDataStratified.Save(trainDataFileName, "", runLog); } if ((testDataStratified.Count > 0) && (!tuningThreadCancelReq)) { testDataFileName = rootName + "_Test.data"; AddMsgToDialogMsgQueue("Saving Test Data[" + testDataFileName + "]"); testDataStratified.Save(testDataFileName, "", runLog); } if ((validationDataStratified.Count > 0) && (!tuningThreadCancelReq)) { validationDataFileName = rootName + "_Validation.data"; AddMsgToDialogMsgQueue("Saving Validation Data[" + validationDataFileName + "]"); validationDataStratified.Save(validationDataFileName, "", runLog); } } AddMsgToDialogMsgQueue("\n\n"); if (tuningThreadCancelReq) { AddMsgToDialogMsgQueue("Process *** CANCELED ***."); } else { AddMsgToDialogMsgQueue("Process Completed."); } PicesDataBase.ThreadEnd(); tuningThreadDone = true; tuningThreadRunning = false; } /* CreateTuningFeatureFiles */
private void ImportImage(PicesDataBase threadConn, String fileName ) { String rootName = OSservices.GetRootName(fileName); String picesRootName = sipperFileName + "_" + rootName; PicesDataBaseImage dbi = threadConn.ImageLoad(picesRootName); if (dbi != null) { return; } PicesRaster r = new PicesRaster(fileName); r = r.Padded(2); PicesFeatureVector fv = new PicesFeatureVector(r, picesRootName, null, runLog); fv.ExampleFileName = picesRootName; int imageId = 0; bool successful = false; uint centroidRow = (uint)(0.5f + fv.CentroidRow); uint centroidCol = (uint)(0.5f + fv.CentroidCol); if ((centroidRow < 0) || (centroidRow > r.Height)) { centroidRow = (uint)(r.Height / 2); } if ((centroidCol < 0) || (centroidCol > r.Width)) { centroidCol = (uint)(r.Width / 2); } threadConn.ImageInsert(r, picesRootName, sipperFileName, 0, 0, 0, (uint)r.Height, (uint)r.Width, (uint)fv.OrigSize, 3, extractionLogEntryId.LogEntryId, extractionLogEntryId.LogEntryId, centroidRow, centroidCol, unknownClass, 1.0f, null, 0.0f, null, 0.0f, // depth, 0.0f, // Image Size null, ref imageId, ref successful ); if (successful) { threadConn.FeatureDataInsertRow(sipperFileName, fv); } else { RunLogAddMsg("RootName[" + rootName + "] Failed: " + threadConn.LastErrorDesc() + "\n"); } } /* ImportImage */