} /* BackGroundThread */
private void SaveSampleImages(PicesFeatureVectorList featureData)
{
PicesClassList classes = featureData.ExtractListOfClasses();
if (classes == null)
{
return;
}
String sampleDir = OSservices.AddSlash(OSservices.GetPathPartOfFile(featureFileName)) +
OSservices.GetRootName(featureFileName) +
"_SampleImages";
sampleDir = OSservices.AddSlash(sampleDir);
OSservices.CreateDirectoryPath(sampleDir);
foreach (PicesClass pc in classes)
{
String classSampleDir = OSservices.AddSlash(sampleDir + pc.Name);
OSservices.CreateDirectoryPath(classSampleDir);
PicesFeatureVectorList examplesThisClass = featureData.ExtractExamplesForAGivenClass(pc);
examplesThisClass.RandomizeOrder();
int numWritten = 0;
foreach (PicesFeatureVector fv in examplesThisClass)
{
String imageRootName = OSservices.GetRootName(fv.ExampleFileName);
PicesRaster i = dbConn.ImageFullSizeFind(imageRootName);
if (i != null)
{
String imageFileName = classSampleDir + imageRootName + ".bmp";
i.Save(imageFileName);
++numWritten;
}
if (numWritten >= includeSampleImagesNumPerClass)
{
break;
}
}
}
} /* SaveSampleImages */
} /* ImagesInDirectoryAlready */
private void ValidateDestinationDirectory()
{
errorProvider1.SetError(DestinationDirectory, "");
if (String.IsNullOrEmpty(DestinationDirectory.Text))
{
errorsFound = true;
errorProvider1.SetError(DestinationDirectory, "You must provide a destination directory.");
return;
}
String parentDir = OSservices.GetParentDirectoryOfDirPath(DestinationDirectory.Text);
DirectoryInfo di = new DirectoryInfo(parentDir);
if (!di.Exists)
{
errorsFound = true;
errorProvider1.SetError(DestinationDirectory, "No such directory[" + parentDir + "].");
}
} /* ValidateDestinationDirectory */
} /* ValidateAll */
private void StartTheBackGroundProcedure()
{
DateTime n = DateTime.Now;
String logFileName = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + "ImportGroupAssignments_Log_" +
n.Year.ToString("0000") + "-" + n.Month.ToString("00") + "-" + n.Day.ToString("00") + "_" +
n.Hour.ToString("00") + "-" + n.Minute.ToString("00") + "-" + n.Hour.ToString("00") +
".txt";
backGroundLog = new PicesRunLog(logFileName);
cancelBackGround = false;
backGroundThread = new Thread(new ThreadStart(ImportAssignments));
backGroundThread.Start();
BackGroundTimer.Enabled = true;
SourceDirectory.Enabled = false;
BrowseSourceDir.Enabled = false;
GroupName.Enabled = false;
GroupDescription.Enabled = false;
} /* StartTheHarvestingProcedure */
} /* ProcessOneImage */
private void RemoveImage(String fullImageFileName)
{
String dir = OSservices.GetPathPartOfFile(fullImageFileName);
String rootNameWithExt = OSservices.GetRootNameWithExtension(fullImageFileName);
String subDir = "";
if (SourceDirectory.Text.Length < dir.Length)
{
subDir = dir.Substring(SourceDirectory.Text.Length);
if ((subDir.Length > 0) && (subDir[0] == '\\'))
{
subDir = subDir.Substring(1);
}
}
String destDir = removalDestRootDir;
if (subDir.Length > 0)
{
destDir = OSservices.AddSlash(destDir) + subDir;
OSservices.CreateDirectoryPath(destDir);
}
FileInfo fi = null;
try { fi = new FileInfo(fullImageFileName); } catch (Exception) { fi = null; }
if (fi != null)
{
String destFileName = OSservices.AddSlash(destDir) + rootNameWithExt;
try
{
fi.MoveTo(destFileName);
reconcilingRunLog.Writeln(rootNameWithExt + "\t" + "Removed");
}
catch (Exception e)
{
reconcilingRunLog.Writeln(rootNameWithExt + "\t" + "Error Removing[" + e.ToString() + "]");
MessageBox.Show("RemoveImage[" + fullImageFileName + "]" + "\n\n" + e.ToString());
}
}
} /* RemoveImage*/
public ImportImagesIntoDeployment(String _cruiseName,
String _stationName,
String _deploymentNum
)
{
msgQueue = new PicesMsgQueue("ImportImagesIntoDeployment");
runLog = new PicesRunLog(msgQueue);
cruiseName = _cruiseName;
stationName = _stationName;
deploymentNum = _deploymentNum;
mainWinConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog);
unknownClass = GetClassFromName(mainWinConn, "UnKnown");
configFileName = OSservices.AddSlash(PicesSipperVariables.ConfigurationDirectory()) + "ImportImagesIntoDeployment.cfg";
InitializeComponent();
//SourceDirectory.Text = PicesSipperVariables.HomeDir ();
}
public SaveImagesToDisk(PicesDataBase _dbConn,
String _cruise,
String _station,
String _deployment,
String _sipperFileName,
PicesDataBaseImageList _images
)
{
dbConn = _dbConn;
images = _images;
cruise = _cruise;
station = _station;
deployment = _deployment;
sipperFileName = _sipperFileName;
runLog = new PicesRunLog();
InitializeComponent();
DestinationDirectory.Text = OSservices.AddSlash(OSservices.AddSlash(PicesSipperVariables.HomeDir()) + "SavedImages");
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
// PicesTempDirectory ());
if (!String.IsNullOrEmpty(cruise))
{
DestinationDirectory.Text += cruise;
if (!String.IsNullOrEmpty(station))
{
DestinationDirectory.Text += "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
DestinationDirectory.Text += "-" + deployment;
if (!String.IsNullOrEmpty(sipperFileName))
{
DestinationDirectory.Text += "-" + sipperFileName;
}
}
}
}
ImageCount.Text = images.Count.ToString("###,##0");
}
private void StartReconciliationThread()
{
String rootName = OSservices.GetRootNameOfDirectory(SourceDirectory.Text);
DateTime d = DateTime.Now;
String logFileName = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + rootName + "_ReconcileImageNames_" +
d.ToString("yyyyMMdd-HHmmss") + ".txt";
// Any images that are not in the Database that are to be removed will be moved to the subdirectory structure below.
removalDestRootDir = "c:\\Temp\\PicesDatabaseImageReconciliationRemovedImages\\" + rootName;
OSservices.CreateDirectoryPath(removalDestRootDir);
reconcilingRunLog = new PicesRunLog(logFileName);
Start.Enabled = false;
SourceDirectory.ReadOnly = true;
SourceDirectoryBrowse.Enabled = false;
reconcilingThread = new Thread(new ThreadStart(ReconciliationThread));
reconcilingThread.Start();
Cancel.Enabled = true;
RunStatsTimer.Enabled = true;
} /* StartReconciliationThread */
} /* LoadNextImageFromDir */
private String LocateRootName(String rootName)
{
String subDir = null;
if (!String.IsNullOrEmpty(dir))
{
subDir = OSservices.LookForFile(rootName + ".bmp", dir);
}
if (String.IsNullOrEmpty(subDir))
{
if (!String.IsNullOrEmpty(dir2))
{
subDir = OSservices.LookForFile(rootName + ".bmp", dir2);
if (subDir == null)
{
return(null);
}
}
}
return(OSservices.AddSlash(subDir) + rootName + ".bmp");
} /* LocateRootName */
} /* ImportImage */
private String[] FilterListForImageFiles(String[] fileNames)
{
List <String> results = new List <string> ();
foreach (String s in fileNames)
{
String ext = OSservices.GetFileExtension(s);
if (ext.Equals("bmp", StringComparison.InvariantCultureIgnoreCase) ||
ext.Equals("jpg", StringComparison.InvariantCultureIgnoreCase)
)
{
results.Add(s);
}
}
if (results.Count < 1)
{
return(null);
}
else
{
return(results.ToArray());
}
} /* FilterListFirImageFiles */
public DisplayPicesImages(String _dir,
String _dir2,
ProbNamePair[] _nameList
)
{
InitializeComponent();
dir = OSservices.AddSlash(_dir);
dir2 = OSservices.AddSlash(_dir2);
nameList = _nameList;
Text = dir;
if (!String.IsNullOrEmpty(dir) && !String.IsNullOrEmpty(dir2))
{
Text = OSservices.GetRootNameOfDirectory(dir) + " vs " + OSservices.GetRootNameOfDirectory(dir2);
}
thumbNails = new List <FlowLayoutPanel> ();
DirectoryInfo di = null;
try { di = new DirectoryInfo(dir); }
catch (Exception) { di = null; }
if (di == null)
{
return;
}
try { files = di.GetFiles("*.*"); } catch (Exception) { files = null; }
files = ReduceToImageFiles(files);
if (files != null)
{
UpdateDisplayTimer.Enabled = true;
}
}
} /* ToCmdLine */
private void ImportValidatedClassAssignments()
{
importingRunning = true;
importingCompleted = false;
PicesDataBase.ThreadInit();
PicesRunLog runLog = new PicesRunLog();
PicesDataBase threadConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog);
extractionLogEntryId = threadConn.LogEntriesProcessStart("IV",
"ImportValidatedClass",
ExecutableDateTimeStamp(),
ToCmdLine(),
OSservices.GetRootName(sipperFileName)
);
totalImagesUpdated = 0;
ImportValidatedClassAssignmentsDir(threadConn, sourceDirectory);
if (cancelImporting)
{
RunLogAddMsg("\n" + "Importing Assignments has been Canceled." + "\n\n");
}
else
{
RunLogAddMsg("\n\n" + "Total Images[" + totalImagesUpdated.ToString("###,##0") + "\n\n");
}
threadConn.Close();
threadConn = null;
GC.Collect();
PicesDataBase.ThreadEnd();
importingRunning = false;
importingCompleted = true;
} /* ImportValidatedClassAssignments */
private void SaveActiveTrainingModelFeatureData_Load(object sender, EventArgs e)
{
modelNames = TrainingModel2.GetListOfTrainingModels();
if (modelNames != null)
{
foreach (String s in modelNames)
{
ModelName.Items.Add(s);
}
}
ModelName.Text = OSservices.GetRootName(initialModelName);
String[] fileFormats = PicesFeatureFileIO.RegisteredDriverNames(false, true);
if (fileFormats != null)
{
foreach (String ff in fileFormats)
{
FileFormat.Items.Add(ff);
}
}
FileFormat.Text = "Pices";
DestinationFileName.Text = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + ModelName.Text + ".data";
}
private void BackGroundThread()
{
PicesDataBase.ThreadInit();
backGroundThreadRunning = true;
imagesLoaded = 0;
if (group != null)
{
planktonData = RetrievePlanktonDataByGroup();
}
else
{
planktonData = RetrievePlanktonDataBySipperFiles();
}
if (!cancelRequested)
{
backGroundStatus = "Saving Data File";
String pathName = OSservices.GetPathPartOfFile(dataFileName);
if (!String.IsNullOrEmpty(pathName))
{
OSservices.CreateDirectoryPath(pathName);
}
if (IncludeSampleImages.Checked)
{
SaveSampleImages(planktonData);
}
SavePlanktonDataFile(planktonData);
}
PicesDataBase.ThreadEnd();
backgroundThreadCompleted = true;
backGroundThreadRunning = false;
} /* BackGroundThread */
private void SaveImagesByDepthImcrements()
{
images.Sort(PicesDataBaseImageList.SortOrderType.soDepth, false);
int idx = 0;
PicesDataBaseImage i = images[0];
while ((idx < images.Count) && (!cancelRequested))
{
int depthIndex = DepthIndex(i.Depth);
int curDepthIndex = depthIndex;
int fromDepth = depthIndex * increments;
int toDepth = (depthIndex + 1) * increments;
String depthSubDirName = fromDepth.ToString("000") + "-" + toDepth.ToString("000");
curDir = OSservices.AddSlash(DestinationDirectory.Text) + depthSubDirName;
try { Directory.CreateDirectory(curDir); } catch (Exception) {}
PicesDataBaseImageList imagesThisDepthIndex = new PicesDataBaseImageList();
while ((idx < images.Count) && (!cancelRequested) && (depthIndex == curDepthIndex))
{
imagesThisDepthIndex.Add(i);
idx++;
if (idx < images.Count)
{
i = images[idx];
depthIndex = DepthIndex(i.Depth);
}
}
SaveImagesInList(imagesThisDepthIndex, curDir);
}
} /* SaveImagesByDepthImcrements */
} /* LoadImageGrid */
private void UpdateDetailProbabilityDisplay(String fileName)
{
if (fileName == null)
{
return;
}
curSelImageFileName = fileName;
String fullFileName = null;
if ((fileName.Length > 1) && (fileName[1] == ':'))
{
fullFileName = fileName;
}
else
{
fullFileName = OSservices.AddSlash(rootDir) + fileName;
}
if (!File.Exists(fullFileName))
{
String fn = OSservices.GetRootName(fileName) + ".bmp";
fullFileName = OSservices.LookForFile(fn, rootDir);
}
Prediction prediction = GetPrediction(fileName);
// We need to call 'PredictProbabilities' twice. Once to get the intermediate images and the
// second time to perform prediction using the FeatureVector that was used in the original
// prediction. It has to be done this way because the version of 'PredictProbabilities' that
// derives the FeatureVector from an image does not always have access of instrumentation data.
PicesRasterList featureCalcImages = new PicesRasterList();
PicesPredictionList guesses2 = prediction.trainModel.PredictProbabilities(fullFileName, featureCalcImages);
// Perform Prediction using original FeatureVector.
PicesPredictionList guesses = prediction.trainModel.PredictProbabilities(prediction.fv);
ImageFileName.Text = fileName;
if (guesses == null)
{
return;
}
ClassificationProbabilities.Rows.Clear();
foreach (PicesPrediction p in guesses)
{
Object[] data = new Object[3];
data[0] = p.ClassName;
data[1] = p.Probability;
data[2] = p.Votes;
ClassificationProbabilities.Rows.Add(data);
}
FeatureCalcImages.Rows.Clear();
if ((featureCalcImages == null) || (featureCalcImages.Count < 1))
{
return;
}
for (int x = 0; x < featureCalcImages.Count; x++)
{
Object[] data = new Object[2];
data[0] = featureCalcImages[x].BuildBitmap();
data[1] = featureCalcImages[x].FileName;
FeatureCalcImages.Rows.Add(data);
}
} /* UpdateDetailProbabilityDisplay */
private void SaveFeatureData_Load(object sender, EventArgs e)
{
CruiseName.Text = cruise;
StationName.Text = station;
DeploymentNum.Text = deployment;
SipperFile.Text = sipperFileName;
if (group != null)
{
GroupName.Text = group.Name;
}
if (mlClass != null)
{
ClassName.Text = mlClass.Name;
}
ProbMin.Text = (100.0f * probMin).ToString("##0.0") + "%";
ProbMax.Text = (100.0f * probMax).ToString("##0.0") + "%";
SizeMin.Text = sizeMin.ToString("#,###,##0");
if (sizeMax <= 0)
{
SizeMax.Text = infinityStr;
}
else
{
SizeMax.Text = sizeMax.ToString("#,###,##0");
}
DepthMin.Text = depthMin.ToString("##,##0.00");
if (depthMax <= 0.0)
{
DepthMax.Text = infinityStr;
}
else
{
DepthMax.Text = depthMax.ToString("##,##0.00");
}
dataFileName = OSservices.AddSlash(PicesSipperVariables.PicesReportDir()) + "PlanktonDataFiles";
dataFileName = OSservices.AddSlash(dataFileName);
if (group != null)
{
dataFileName += group.Name;
}
else
{
if (!String.IsNullOrEmpty(cruise))
{
dataFileName += cruise;
if (!String.IsNullOrEmpty(station))
{
dataFileName += "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
dataFileName += "-" + deployment;
if (!String.IsNullOrEmpty(sipperFileName))
{
dataFileName += "-" + sipperFileName;
}
}
}
}
}
if (mlClass != null)
{
if (!String.IsNullOrEmpty(dataFileName))
{
dataFileName += "-";
}
dataFileName += mlClass.Name;
}
dataFileName += ".data";
FeatureFileName.Text = dataFileName;
IncludeSampleImagesNumPerClass.Text = includeSampleImagesNumPerClass.ToString();
StartLoadingStatusOnTotalImageCount();
} /* SaveImagesToDisk2_Load */
} /* StartTheBackGroundProcedure */
private void BackGroundProcess()
{
backGroundRunning = true;
PicesDataBase.ThreadInit();
OSservices.CreateDirectoryPath(reportFileDir);
DateTime n = DateTime.Now;
String logFileName = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + "InstrumentDataByDeployment_Log_" +
n.Year.ToString("0000") + "-" + n.Month.ToString("00") + "-" + n.Day.ToString("00") + "_" +
n.Hour.ToString("00") + "-" + n.Minute.ToString("00") + "-" + n.Hour.ToString("00") +
".txt";
backGroundLog = new PicesRunLog(logFileName);
PicesDataBase threadConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(backGroundLog);
RunLogAddMsg("Cruise [" + cruiseName + "]" + "\n");
RunLogAddMsg("Station [" + stationName + "]" + "\n");
RunLogAddMsg("Deployment [" + deploymentNum + "]" + "\n");
RunLogAddMsg("Start Date/Time [" + DateTime.Now.ToString("yyyy-MMM-dd HH:mm:ss") + "]" + "\n");
RunLogAddMsg("Active DataBase [" + threadConn.Server.Description + "]" + "\n");
PicesSipperDeploymentList deployments = null;
if (!String.IsNullOrEmpty(deploymentNum))
{
PicesSipperDeployment d = threadConn.SipperDeploymentLoad(cruiseName, stationName, deploymentNum);
if (d != null)
{
deployments = new PicesSipperDeploymentList();
deployments.Add(d);
}
}
else
{
deployments = threadConn.SipperDeploymentLoad(cruiseName, stationName);
}
if (deployments == null)
{
RunLogAddMsg("\n\nNo Deployments for Specified Criteria!\n\n");
return;
}
int reportNum = 0;
foreach (PicesSipperDeployment d in deployments)
{
if (cancelBackGround)
{
break;
}
reportNum++;
RunLogAddMsg("Generating Report " + reportNum.ToString() + " of " + deployments.Count.ToString());
SummarizeInstrumentDataForADeployment(threadConn, d.CruiseName, d.StationName, d.DeploymentNum);
}
threadConn.Close();
threadConn = null;
GC.Collect();
PicesDataBase.ThreadEnd();
backGroundRunning = false;
} /* BackGroundProcess */
} /* LoadImageDepthStats */
private void SummarizeInstrumentDataForADeployment(PicesDataBase threadConn,
String cruiseName,
String stationName,
String deploymentNum
)
{
RunLogAddMsg("\nStart Report " + cruiseName + "-" + stationName + "-" + deploymentNum);
PicesInstrumentDataList data = threadConn.InstrumentDataLoad(cruiseName, stationName, deploymentNum);
// We will assume that all entries are in order of date and time. We want
// to first decide where the break in UpCast and Down cast occur.
if (cancelBackGround)
{
return;
}
data = StripOutObviouslyBadrecs(data);
if ((data == null) || (data.Count < 1))
{
RunLogAddMsg("There is no Instrument Data for Cruise[" + cruiseName + "] Station[" + stationName + "] Deployment [" + deploymentNum + "]");
return;
}
RunLogAddMsg("Number Entries Loaded[" + data.Count.ToString() + "]");
data.Sort(DatePicesInstrumentComparer);
int idx;
float maxDepth = -9999.9f;
int maxDepthIdx = -9999;
for (idx = 0; idx < data.Count; idx++)
{
if (data[idx].Depth > maxDepth)
{
maxDepth = data[idx].Depth;
maxDepthIdx = idx;
}
}
DateTime refTime = new DateTime(1999, 1, 1);
DateTime startTime = new DateTime(1999, 1, 1);
bool startTimeSet = false;
InstrumentDataList downCast = new InstrumentDataList();
InstrumentDataList upCast = new InstrumentDataList();
for (idx = 0; idx < data.Count; idx++)
{
PicesInstrumentData pid = data[idx];
if (pid.CtdDate.CompareTo(refTime) < 0)
{
continue;
}
if (!startTimeSet)
{
startTime = pid.CtdDate;
startTimeSet = true;
}
if (idx <= maxDepthIdx)
{
downCast.Add(new InstrumentData(pid));
}
else
{
upCast.Add(new InstrumentData(pid));
}
}
DateTime endTime = data[data.Count - 1].CtdDate;
data = null;
String title = cruiseName + "-" + stationName + "-" + deploymentNum + " From [" + startTime.ToString("u") + "] to [" + endTime.ToString("u") + "]";
reportFileName = OSservices.AddSlash(reportFileDir) +
cruiseName + "-" +
stationName + "-" +
deploymentNum;
if (mlClass != null)
{
reportFileName += "-" + mlClass.Name;
}
reportFileName += "_InsrumentSumary.txt";
System.IO.StreamWriter o = null;
try
{
o = new System.IO.StreamWriter(reportFileName);
}
catch (Exception e)
{
RunLogAddMsg("Error opening file[" + reportFileName + "] for output." + "\n\n" + e.ToString());
return;
}
if (threadConn != null)
{
o.WriteLine("DataBase" + "\t" + threadConn.Server.DataBaseName);
}
o.WriteLine("Cruise" + "\t" + cruiseName);
o.WriteLine("Station" + "\t" + stationName);
o.WriteLine("Deployment" + "\t" + deploymentNum);
o.WriteLine("StartTime" + "\t" + startTime.ToString("u"));
o.WriteLine("EndTime" + "\t" + endTime.ToString("u"));
o.WriteLine();
float scanRateMin = 0.0f;
float scanRateMax = 0.0f;
float scanRate = 0.0f;
RunLogAddMsg("Getting Particle Counts");
List <ImagesDepthStats> downCastImageStats = null;
List <ImagesDepthStats> upCastImageStats = null;
LoadImageDepthStats(threadConn, cruiseName, stationName, deploymentNum, ref downCastImageStats, ref upCastImageStats);
RunLogAddMsg("Determining Scan Rate");
FindScanRates(threadConn, cruiseName, stationName, deploymentNum, ref scanRateMin, ref scanRateMax);
if (scanRateMin == scanRateMax)
{
scanRate = scanRateMin;
RunLogAddMsg("ScanRate [" + scanRateMin.ToString("###,##0.00") + "]");
}
else
{
RunLogAddMsg("Scan Rates Were not consistent, you should update Sipper Files.");
RunLogAddMsg(" scanRateMin [" + scanRateMin.ToString("###,##0.00") + "]" +
" scanRateMax [" + scanRateMax.ToString("###,##0.00") + "]"
);
o.WriteLine("Scan Rates Were not consistent, you should update Sipper Files.");
o.WriteLine(" scanRateMin [" + scanRateMin.ToString("###,##0.00") + "]" +
" scanRateMax [" + scanRateMax.ToString("###,##0.00") + "]"
);
o.WriteLine();
}
o.WriteLine("ScanRate" + "\t" + scanRateMin.ToString("###,##0.00"));
o.WriteLine();
o.WriteLine();
o.WriteLine("Depth" + "\t" +
"ScanLines" + "\t" +
"Duration" + "\t" +
"ImageCount" + "\t" +
"TotalPixelCount" + "\t" +
"FlowRate" + "\t" + "\t" +
"Temperature" + "\t" + "\t" +
"Oxygen" + "\t" + "\t" +
"OxygenSensor" + "\t" + "\t" +
"Salinity" + "\t" + "\t" +
"Density" + "\t" + "\t" +
"Fluoresce" + "\t" + "\t" +
"FluorensceSensor" + "\t" + "\t" +
"Transitivity" + "\t" + "\t" +
"TransmissivitySensor" + "\t" + "\t" +
"Turbidity" + "\t" + "\t" +
"TurbiditySensor" + "\t" + "\t" +
"RecordRate"
);
WriteDepthSummary(o, downCast, downCastImageStats, scanRate);
o.WriteLine();
if (!cancelBackGround)
{
WriteDepthSummary(o, upCast, upCastImageStats, scanRate);
}
if (cancelBackGround)
{
o.WriteLine();
o.WriteLine();
o.WriteLine();
o.WriteLine("Report Canceled by user request");
}
o.Close();
} /* SummarizeInstrumentDataForADeployment */
private void CreateTrainingModel()
{
String fileName = OSservices.AddSlash(PicesSipperVariables.TrainingModelsConfigurationDir()) + ModelName.Text + ".cfg";
config.Save(fileName);
} /* CreateTrainingModel */
private void MakePrediction(Prediction p)
{
String rootName = OSservices.GetRootName(p.FileName);
PicesClass knownClassPredLevel = p.knowClass.MLClassForGivenHierarchialLevel(predLevel);
PicesClass predClassPredLevel = p.predClass.MLClassForGivenHierarchialLevel(predLevel);
int knowClassIdx = classesPredLevel.LookUpIndex(knownClassPredLevel);
int predClassIdx = classesPredLevel.LookUpIndex(predClassPredLevel);
int exactMatch = 0;
float partialCredit = 0.0f;
if (knowClassIdx == predClassIdx)
{
partialCredit = 1.0f;
exactMatch = 1;
}
else
{
partialCredit = FractionThatIsAMatch2(knownClassPredLevel, predClassPredLevel);
}
PartialCreditIdx pcIdx = DerivePartialCreditIdx(partialCredit);
totalByCorectLevel[(int)pcIdx]++;
totalCount++;
totalCorrect += partialCredit;
totalCorrectNPC += exactMatch;
if (!p.partOfClassifier)
{
totalNotPartOfClassifier++;
}
else
{
totalPartOfClassifier++;
totalPartOfClassifierByCorectLevel[(int)pcIdx]++;
totalPartOfClassifierCorrect += partialCredit;
totalPartOfClassifierCorrectNPC += exactMatch;
if (p.knowClass.UnDefined)
{
totalNoise++;
totalNoiseCorrect += partialCredit;
totalNoiseCorrectNPC += exactMatch;
}
else
{
totalNonNoise++;
totalNonNoiseCorrect += partialCredit;
totalNonNoiseCorrectNPC += exactMatch;
}
}
if (knowClassIdx < 0)
{
MessageBox.Show("Invalid Class Specified[" + p.knowClass.Name + "]");
return;
}
if (predClassIdx < 0)
{
MessageBox.Show("Invalid Class Specified[" + p.predClass.Name + "]");
return;
}
knownClassCounts[knowClassIdx]++;
predClassCounts [predClassIdx]++;
predictionCounts[knowClassIdx][predClassIdx]++;
try { imagesByPred[knowClassIdx][predClassIdx].Add(rootName, p); }
catch (Exception e)
{
MessageBox.Show(this, "Exception occurred adding example[" + rootName + "]" + "\n" +
"Known Class[" + p.knowClass.Name + "] Pred Class[" + p.predClass.Name + "]" + "\n" +
"Exception [" + e.ToString() + "]" + "\n" +
e.StackTrace,
"Confusion Matrix",
MessageBoxButtons.OK
);
}
} /* MakePrediction */
public SipperConfigRec()
{
OSservices.CreateDirectory(PicesSipperVariables.ConfigurationDirectory());
sipperConfigurationFileName = PicesSipperVariables.ConfigrationFileName();
Load();
}
} /* SaveGivenImageList */
private void SaveImagesBySipperFiles()
{
bool includeStationAndDeploymentInDirName = false;
bool firstSipperFile = true;
PicesSipperFileList sipperFiles = null;
if (String.IsNullOrEmpty(sipperFileName))
{
sipperFiles = dbConn.SipperFileLoad(cruise, station, deployment);
}
else
{
PicesSipperFile sf = dbConn.SipperFileRecLoad(sipperFileName);
if (sf != null)
{
sipperFiles = new PicesSipperFileList();
sipperFiles.Add(sf);
}
}
if (sipperFiles != null)
{
String lastCruise = "";
String lastStation = "";
String lastDeployment = "";
String dirPath = "";
Dictionary <String, int> imagesPerClass = new Dictionary <string, int> ();
foreach (PicesSipperFile sf in sipperFiles)
{
bool startANewDir = false;
if (firstSipperFile)
{
startANewDir = true;
}
//else if (sf.CruiseName != lastCruise)
// startANewDir = true;
else if ((includeStationAndDeploymentInDirName) &&
((sf.StationName != lastStation) || (sf.DeploymentNum != lastDeployment))
)
{
startANewDir = true;
}
if (startANewDir)
{
lastCruise = sf.CruiseName;
lastStation = sf.StationName;
lastDeployment = sf.DeploymentNum;
dirPath = OSservices.AddSlash(DestinationDirectory.Text);
if (includeStationAndDeploymentInDirName)
{
if (!String.IsNullOrEmpty(lastCruise))
{
dirPath = OSservices.AddSlash(dirPath) + lastCruise;
}
if (!String.IsNullOrEmpty(lastStation))
{
dirPath = OSservices.AddSlash(dirPath) + lastStation;
}
if (!String.IsNullOrEmpty(lastDeployment))
{
dirPath = OSservices.AddSlash(dirPath) + lastDeployment;
}
}
OSservices.CreateDirectoryPath(dirPath);
imagesPerClass = new Dictionary <string, int> ();
}
SaveImagesForOneSipperFile(dirPath, sf, imagesPerClass);
firstSipperFile = false;
}
}
} /* SaveImagesBySipperFiles */
private void SaveOneBatchOfImages(String dirPath,
Dictionary <String, int> imagesPerClass,
PicesDataBaseImageList batch
)
{
if (classKeyToUse == 'V')
{
batch.Sort(PicesDataBaseImageList.SortOrderType.soValidatedClassName, false);
}
else
{
batch.Sort(PicesDataBaseImageList.SortOrderType.soClassName, false);
}
int idx = 0;
PicesDataBaseImage i = batch[0];
while ((idx < batch.Count) && (!cancelRequested))
{
String className = i.Class1Name;
if (classKeyToUse == 'V')
{
className = i.ValidatedClassName;
}
String curClassName = className;
String classDir = OSservices.AddSlash(dirPath) + className;
try { OSservices.CreateDirectoryPath(classDir); } catch (Exception) {}
int countThisClass = 0;
if (imagesPerClass.ContainsKey(curClassName))
{
countThisClass = imagesPerClass[curClassName];
}
else
{
imagesPerClass.Add(curClassName, 0);
}
int lastFoldNum = -1;
String foldDir = "";
while ((idx < batch.Count) && (!cancelRequested) && (className == curClassName))
{
int foldNum = countThisClass / imagsPerDir;
if (foldNum != lastFoldNum)
{
foldDir = OSservices.AddSlash(classDir) + curClassName + "_" + foldNum.ToString("000");
OSservices.CreateDirectoryPath(foldDir);
lastFoldNum = foldNum;
}
String fullName = OSservices.AddSlash(foldDir) + i.ImageFileName + ".bmp";
PicesRaster r = dbConn.ImageFullSizeFind(i.ImageFileName);
if (r == null)
{
imagesFailedToExtract++;
}
else
{
r.Save(fullName);
}
imagesSaved++;
// Get the next image to save.
idx++;
if (idx < batch.Count)
{
i = batch[idx];
className = i.Class1Name;
if (classKeyToUse == 'V')
{
className = i.ValidatedClassName;
}
}
countThisClass++;
}
imagesPerClass[curClassName] = countThisClass;
}
} /* SaveOneBatchOfImages */
private void ParseStatusLine(string line)
{
string startSectorStr = GetStatusLineField(line, "bs=");
if (startSectorStr == null)
{
return;
}
string currentSectorStr = GetStatusLineField(line, "bc=");
if (currentSectorStr == null)
{
return;
}
string endSectorStr = GetStatusLineField(line, "be=");
if (endSectorStr == null)
{
return;
}
string dateStr = GetStatusLineField(line, "da=");
if (dateStr == null)
{
return;
}
string timeStr = GetStatusLineField(line, "ti=");
if (timeStr == null)
{
return;
}
string recordStatusStr = GetStatusLineField(line, "rs=");
if (recordStatusStr == null)
{
return;
}
startSector = (uint)OSservices.HexStrToLong(startSectorStr);
currentSector = (uint)OSservices.HexStrToLong(currentSectorStr);
endSector = (uint)OSservices.HexStrToLong(endSectorStr);
recording = (recordStatusStr == "r");
statusDateTime = new SipperDateTimeStamp(dateStr, timeStr);
if ((currentFileControlBlock != null) && recording)
{
currentFileControlBlock.EndSector = currentSector;
}
if (updateCurrentFileControlBlockEndSector)
{
if (currentFileControlBlock != null)
{
currentFileControlBlock.EndSector = currentSector;
updateCurrentFileControlBlockEndSector = false;
}
}
} /* ParseStatusLine */
private static void SetConfigurationFileName()
{
configFileName = OSservices.AddSlash(PicesSipperVariables.ConfigurationDirectory()) + "ActiveTrainingLibraries.cfg";
} /* SetConfigurationFileName */
private void SaveImagesToDisk2_Load(object sender, EventArgs e)
{
CruiseName.Text = cruise;
StationName.Text = station;
DeploymentNum.Text = deployment;
SipperFile.Text = sipperFileName;
if (group != null)
{
GroupName.Text = group.Name;
}
ProbMin.Text = (100.0f * probMin).ToString("##0.0") + "%";
ProbMax.Text = (100.0f * probMax).ToString("##0.0") + "%";
SizeMin.Text = sizeMin.ToString("#,###,##0");
if (sizeMax <= 0)
{
SizeMax.Text = infinityStr;
}
else
{
SizeMax.Text = sizeMax.ToString("#,###,##0");
}
DepthMin.Text = depthMin.ToString("##,##0.00");
if (depthMax <= 0.0)
{
DepthMax.Text = infinityStr;
}
else
{
DepthMax.Text = depthMax.ToString("##,##0.00");
}
DestinationDirectory.Text = OSservices.AddSlash(OSservices.AddSlash(PicesSipperVariables.HomeDir()) + "SavedImages");
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
// PicesTempDirectory ());
if (group != null)
{
DestinationDirectory.Text += group.Name;
}
else
{
if (!String.IsNullOrEmpty(cruise))
{
DestinationDirectory.Text += cruise;
if (!String.IsNullOrEmpty(station))
{
DestinationDirectory.Text += "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
DestinationDirectory.Text += "-" + deployment;
if (!String.IsNullOrEmpty(sipperFileName))
{
DestinationDirectory.Text += "-" + sipperFileName;
}
}
}
}
}
StartLoadingStatusOnTotalImageCount();
} /* SaveImagesToDisk2_Load */
} /* WriteForGnuplotToStream */
private void SaveForGnuplot(Object sender, EventArgs e)
{
SaveFileDialog sfd = new SaveFileDialog();
sfd.Filter = "Comma delimited data to disk|*.txt";
sfd.Title = "Save Abundance info for gnuplot to disk.";
sfd.DefaultExt = "txt";
sfd.AddExtension = true;
sfd.OverwritePrompt = true;
String fn = cruise + "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
fn = fn + "-" + deployment;
}
fn = fn + "_AbundanceByDeployment";
if (classToPlot != null)
{
fn += ("_" + classToPlot.Name);
}
fn += ".txt";
sfd.FileName = fn;
//sfd.CheckFileExists = true;
DialogResult dr = sfd.ShowDialog(this);
if (dr == DialogResult.OK)
{
fn = sfd.FileName;
System.IO.StreamWriter sw = null;
try { sw = new System.IO.StreamWriter(fn, false, System.Text.Encoding.ASCII); }
catch (Exception) { sw = null; }
if (sw == null)
{
MessageBox.Show(this, "Could not open file[" + fn + "]", "Save for gnuplot", MessageBoxButtons.OK);
return;
}
WriteForGnuplotToStream(sw, bucketsDisplayed, true, false);
sw.Close();
sw = null;
String ext = OSservices.GetFileExtension(fn);
String ofn = OSservices.RemoveExtension(fn) + "_nVd." + ext;
try { sw = new System.IO.StreamWriter(ofn, false, System.Text.Encoding.ASCII); }
catch (Exception) { sw = null; }
if (sw == null)
{
MessageBox.Show(this, "Could not open file[" + ofn + "]", "Save for gnuplot", MessageBoxButtons.OK);
return;
}
WriteForGnuplotToStream(sw, bucketsDisplayed, true, true);
sw.Close();
sw = null;
MessageBox.Show(this, "Abundance Data for gnuplot written to \"" + fn + "\".", "", MessageBoxButtons.OK);
}
} /* SaveForGnuplot */
private void ImportImage(PicesDataBase threadConn,
String fileName
)
{
String rootName = OSservices.GetRootName(fileName);
String picesRootName = sipperFileName + "_" + rootName;
PicesDataBaseImage dbi = threadConn.ImageLoad(picesRootName);
if (dbi != null)
{
return;
}
PicesRaster r = new PicesRaster(fileName);
r = r.Padded(2);
PicesFeatureVector fv = new PicesFeatureVector(r, picesRootName, null, runLog);
fv.ExampleFileName = picesRootName;
int imageId = 0;
bool successful = false;
uint centroidRow = (uint)(0.5f + fv.CentroidRow);
uint centroidCol = (uint)(0.5f + fv.CentroidCol);
if ((centroidRow < 0) || (centroidRow > r.Height))
{
centroidRow = (uint)(r.Height / 2);
}
if ((centroidCol < 0) || (centroidCol > r.Width))
{
centroidCol = (uint)(r.Width / 2);
}
threadConn.ImageInsert(r,
picesRootName,
sipperFileName,
0, 0, 0,
(uint)r.Height, (uint)r.Width,
(uint)fv.OrigSize,
3,
extractionLogEntryId.LogEntryId, extractionLogEntryId.LogEntryId,
centroidRow, centroidCol,
unknownClass, 1.0f, null, 0.0f, null,
0.0f, // depth,
0.0f, // Image Size
null,
ref imageId,
ref successful
);
if (successful)
{
threadConn.FeatureDataInsertRow(sipperFileName, fv);
}
else
{
RunLogAddMsg("RootName[" + rootName + "] Failed: " + threadConn.LastErrorDesc() + "\n");
}
} /* ImportImage */
public ChartAbundancePredictedAndValidated(String _cruise,
String _station,
String _deployment,
PicesDataBaseImageGroup _group,
PicesClass _classToPlot,
int _sizeMin,
int _sizeMax,
float _probMin,
float _probMax,
float _depthMin,
float _depthMax,
PicesClassList _classes,
PicesClassList _activeClasses,
String _rootDir
)
{
cruise = _cruise;
station = _station;
deployment = _deployment;
group = _group;
if (group != null)
{
groupName = group.Name;
}
classToPlot = _classToPlot;
sizeMin = _sizeMin;
sizeMax = _sizeMax;
probMin = _probMin;
probMax = _probMax;
depthMin = _depthMin;
depthMax = _depthMax;
classes = _classes;
activeClasses = _activeClasses;
rootDir = _rootDir;
if ((sizeMin > 0) && (sizeMax <= 0))
{
sizeMax = int.MaxValue;
}
if ((probMin >= 0.0f) && (probMax <= 0.0f))
{
probMax = 1.0f;
}
if ((depthMin > 0.0f) && (depthMax <= 0.0f))
{
depthMax = float.MaxValue;
}
filterSize = (sizeMin > 0) || ((sizeMax > 0) && (sizeMax > sizeMin));
filterProb = ((probMin > 0.0f) || (probMax < 1.0f)) && (probMax > probMin);
filterDepth = ((depthMax > 0) && (depthMax > depthMin));
statusMsgs = new PicesMsgQueue("ChartAbundancePredictedAndValidated-StatusMsgs");
msgQueue = new PicesMsgQueue("ChartAbundancePredictedAndValidated-RunLog");
runLog = new PicesRunLog(msgQueue);
mainWinConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(runLog);
configFileName = OSservices.AddSlash(PicesSipperVariables.ConfigurationDirectory()) + "ChartAbundancePredictedAndValidated.cfg";
BuildCriteriaString();
InitializeComponent();
}
} /* ImportImage */
private void ImportValidatedClassAssignmentsDir(PicesDataBase threadConn,
String dirName
)
{
RunLogAddMsg("Dir[" + dirName + "]" + "\n");
String[] fileNames = null;
try { fileNames = Directory.GetFiles(dirName); }
catch (Exception e)
{
RunLogAddMsg("\n" + "Error retrieving file info for Directory[" + dirName + "]" + "\n");
RunLogAddMsg("Exception[" + e.ToString() + "]" + "\n\n");
fileNames = null;
}
if (fileNames != null)
{
String className = PicesClass.GetClassNameFromDirName(dirName);
PicesClass mlClass = null;
int numThisDir = 0;
int numFailedThisDir = 0;
foreach (String fn in fileNames)
{
String ext = OSservices.GetFileExtension(fn).ToLower();
if ((ext == "bmp") || (ext == "jpg"))
{
String rn = OSservices.GetRootName(fn);
if (mlClass == null)
{
mlClass = GetClassFromName(threadConn, className);
}
if (importImages)
{
numThisDir++;
ImportImage(threadConn, fn);
}
else
{
//rn = "TRAIN_" + rn;
threadConn.ImagesUpdateValidatedAndPredictClass(rn, mlClass, 1.0f);
if (threadConn.Successful())
{
numThisDir++;
}
else
{
numFailedThisDir++;
RunLogAddMsg("Dir[" + dirName + "] RootName[" + rn + "] Failed" + "\n");
RunLogAddMsg(threadConn.LastErrorDesc() + "\n");
}
}
if ((numThisDir % 100) == 0)
{
RunLogAddMsg("Dir[" + dirName + "] Files Updated[" + numThisDir.ToString("###,##0") + "]" + "\n");
}
}
}
RunLogAddMsg("Dir[" + dirName + "] Files Updated[" + numThisDir.ToString("###,##0") + "]" + "\n");
totalImagesUpdated += numThisDir;
}
if (!cancelImporting)
{
String[] directories = null;
try { directories = Directory.GetDirectories(dirName); }
catch (Exception) { directories = null; }
if (directories != null)
{
foreach (String subDir in directories)
{
if ((subDir == ".") || (subDir == ".."))
{
continue;
}
ImportValidatedClassAssignmentsDir(threadConn, subDir);
if (cancelImporting)
{
break;
}
}
}
}
} /* ImportValidatedClassAssignmentsDir */
