private void StartSavingImagesThread()
{
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
savingThread = new Thread(new ThreadStart(SaveImages));
savingThread.Start();
Thread.Sleep(10);
savingTimer.Enabled = true;
}
private void StartSavingImagesThread()
{
SaveProgress.Maximum = (int)imageCount;
SaveProgress.Value = 0;
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
savingThread = new Thread(new ThreadStart(SaveImages));
savingThread.Start();
Thread.Sleep(10);
savingTimer.Enabled = true;
}
private void BackGroundThread()
{
PicesDataBase.ThreadInit();
backGroundThreadRunning = true;
imagesLoaded = 0;
imagesFilteredOut = 0;
imagesMissingFeatureData = 0;
if (group != null)
{
featureData = SaveFeatureDataByGroup();
}
else
{
featureData = SaveFeatureDataBySipperFiles();
}
if (!cancelRequested)
{
backGroundStatus = "Saving Feature File";
String pathName = OSservices.GetPathPartOfFile(featureFileName);
if (!String.IsNullOrEmpty(pathName))
{
OSservices.CreateDirectoryPath(pathName);
}
if (IncludeSampleImages.Checked)
{
SaveSampleImages(featureData);
}
driver = new PicesFeatureFileIO(fileFormat);
if (driver != null)
{
if (NormalizeData.Checked)
{
driver.SaveFeatureFileNormalized(featureFileName, featureData, runLog);
}
else
{
driver.SaveFeatureFile(featureFileName, featureData, runLog);
}
}
}
PicesDataBase.ThreadEnd();
backgroundThreadCompleted = true;
backGroundThreadRunning = false;
} /* BackGroundThread */
//SMP751001034_02_00025491_3588
private void SaveSampleImages(PicesDataBaseImageList planktonData)
{
PicesClassList classes = planktonData.ExtractListOfClasses();
if (classes == null)
{
return;
}
String sampleDir = OSservices.AddSlash(OSservices.GetPathPartOfFile(dataFileName)) +
OSservices.GetRootName(dataFileName) +
"_SampleImages";
sampleDir = OSservices.AddSlash(sampleDir);
OSservices.CreateDirectoryPath(sampleDir);
foreach (PicesClass pc in classes)
{
String classSampleDir = OSservices.AddSlash(sampleDir + pc.Name);
OSservices.CreateDirectoryPath(classSampleDir);
PicesDataBaseImageList examplesThisClass = planktonData.ExtractExamplesForAGivenClass(pc);
examplesThisClass.RandomizeOrder();
int numWritten = 0;
foreach (PicesDataBaseImage i in examplesThisClass)
{
String imageRootName = OSservices.GetRootName(i.ImageFileName);
PicesRaster fullSizeImage = dbConn.ImageFullSizeFind(imageRootName);
if (fullSizeImage != null)
{
String imageFileName = classSampleDir + imageRootName + ".bmp";
fullSizeImage.Save(imageFileName);
++numWritten;
}
if (numWritten >= includeSampleImagesNumPerClass)
{
break;
}
}
}
} /* SaveSampleImages */
} /* ProcessOneImage */
private void RemoveImage(String fullImageFileName)
{
String dir = OSservices.GetPathPartOfFile(fullImageFileName);
String rootNameWithExt = OSservices.GetRootNameWithExtension(fullImageFileName);
String subDir = "";
if (SourceDirectory.Text.Length < dir.Length)
{
subDir = dir.Substring(SourceDirectory.Text.Length);
if ((subDir.Length > 0) && (subDir[0] == '\\'))
{
subDir = subDir.Substring(1);
}
}
String destDir = removalDestRootDir;
if (subDir.Length > 0)
{
destDir = OSservices.AddSlash(destDir) + subDir;
OSservices.CreateDirectoryPath(destDir);
}
FileInfo fi = null;
try { fi = new FileInfo(fullImageFileName); } catch (Exception) { fi = null; }
if (fi != null)
{
String destFileName = OSservices.AddSlash(destDir) + rootNameWithExt;
try
{
fi.MoveTo(destFileName);
reconcilingRunLog.Writeln(rootNameWithExt + "\t" + "Removed");
}
catch (Exception e)
{
reconcilingRunLog.Writeln(rootNameWithExt + "\t" + "Error Removing[" + e.ToString() + "]");
MessageBox.Show("RemoveImage[" + fullImageFileName + "]" + "\n\n" + e.ToString());
}
}
} /* RemoveImage*/
public SaveImagesToDisk(PicesDataBase _dbConn,
String _cruise,
String _station,
String _deployment,
String _sipperFileName,
PicesDataBaseImageList _images
)
{
dbConn = _dbConn;
images = _images;
cruise = _cruise;
station = _station;
deployment = _deployment;
sipperFileName = _sipperFileName;
runLog = new PicesRunLog();
InitializeComponent();
DestinationDirectory.Text = OSservices.AddSlash(OSservices.AddSlash(PicesSipperVariables.HomeDir()) + "SavedImages");
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
// PicesTempDirectory ());
if (!String.IsNullOrEmpty(cruise))
{
DestinationDirectory.Text += cruise;
if (!String.IsNullOrEmpty(station))
{
DestinationDirectory.Text += "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
DestinationDirectory.Text += "-" + deployment;
if (!String.IsNullOrEmpty(sipperFileName))
{
DestinationDirectory.Text += "-" + sipperFileName;
}
}
}
}
ImageCount.Text = images.Count.ToString("###,##0");
}
private void StartReconciliationThread()
{
String rootName = OSservices.GetRootNameOfDirectory(SourceDirectory.Text);
DateTime d = DateTime.Now;
String logFileName = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + rootName + "_ReconcileImageNames_" +
d.ToString("yyyyMMdd-HHmmss") + ".txt";
// Any images that are not in the Database that are to be removed will be moved to the subdirectory structure below.
removalDestRootDir = "c:\\Temp\\PicesDatabaseImageReconciliationRemovedImages\\" + rootName;
OSservices.CreateDirectoryPath(removalDestRootDir);
reconcilingRunLog = new PicesRunLog(logFileName);
Start.Enabled = false;
SourceDirectory.ReadOnly = true;
SourceDirectoryBrowse.Enabled = false;
reconcilingThread = new Thread(new ThreadStart(ReconciliationThread));
reconcilingThread.Start();
Cancel.Enabled = true;
RunStatsTimer.Enabled = true;
} /* StartReconciliationThread */
private void BackGroundThread()
{
PicesDataBase.ThreadInit();
backGroundThreadRunning = true;
imagesLoaded = 0;
if (group != null)
{
planktonData = RetrievePlanktonDataByGroup();
}
else
{
planktonData = RetrievePlanktonDataBySipperFiles();
}
if (!cancelRequested)
{
backGroundStatus = "Saving Data File";
String pathName = OSservices.GetPathPartOfFile(dataFileName);
if (!String.IsNullOrEmpty(pathName))
{
OSservices.CreateDirectoryPath(pathName);
}
if (IncludeSampleImages.Checked)
{
SaveSampleImages(planktonData);
}
SavePlanktonDataFile(planktonData);
}
PicesDataBase.ThreadEnd();
backgroundThreadCompleted = true;
backGroundThreadRunning = false;
} /* BackGroundThread */
private void SaveImagesToDisk2_Load(object sender, EventArgs e)
{
CruiseName.Text = cruise;
StationName.Text = station;
DeploymentNum.Text = deployment;
SipperFile.Text = sipperFileName;
if (group != null)
{
GroupName.Text = group.Name;
}
ProbMin.Text = (100.0f * probMin).ToString("##0.0") + "%";
ProbMax.Text = (100.0f * probMax).ToString("##0.0") + "%";
SizeMin.Text = sizeMin.ToString("#,###,##0");
if (sizeMax <= 0)
{
SizeMax.Text = infinityStr;
}
else
{
SizeMax.Text = sizeMax.ToString("#,###,##0");
}
DepthMin.Text = depthMin.ToString("##,##0.00");
if (depthMax <= 0.0)
{
DepthMax.Text = infinityStr;
}
else
{
DepthMax.Text = depthMax.ToString("##,##0.00");
}
DestinationDirectory.Text = OSservices.AddSlash(OSservices.AddSlash(PicesSipperVariables.HomeDir()) + "SavedImages");
OSservices.CreateDirectoryPath(DestinationDirectory.Text);
// PicesTempDirectory ());
if (group != null)
{
DestinationDirectory.Text += group.Name;
}
else
{
if (!String.IsNullOrEmpty(cruise))
{
DestinationDirectory.Text += cruise;
if (!String.IsNullOrEmpty(station))
{
DestinationDirectory.Text += "-" + station;
if (!String.IsNullOrEmpty(deployment))
{
DestinationDirectory.Text += "-" + deployment;
if (!String.IsNullOrEmpty(sipperFileName))
{
DestinationDirectory.Text += "-" + sipperFileName;
}
}
}
}
}
StartLoadingStatusOnTotalImageCount();
} /* SaveImagesToDisk2_Load */
private void SaveOneBatchOfImages(String dirPath,
Dictionary <String, int> imagesPerClass,
PicesDataBaseImageList batch
)
{
if (classKeyToUse == 'V')
{
batch.Sort(PicesDataBaseImageList.SortOrderType.soValidatedClassName, false);
}
else
{
batch.Sort(PicesDataBaseImageList.SortOrderType.soClassName, false);
}
int idx = 0;
PicesDataBaseImage i = batch[0];
while ((idx < batch.Count) && (!cancelRequested))
{
String className = i.Class1Name;
if (classKeyToUse == 'V')
{
className = i.ValidatedClassName;
}
String curClassName = className;
String classDir = OSservices.AddSlash(dirPath) + className;
try { OSservices.CreateDirectoryPath(classDir); } catch (Exception) {}
int countThisClass = 0;
if (imagesPerClass.ContainsKey(curClassName))
{
countThisClass = imagesPerClass[curClassName];
}
else
{
imagesPerClass.Add(curClassName, 0);
}
int lastFoldNum = -1;
String foldDir = "";
while ((idx < batch.Count) && (!cancelRequested) && (className == curClassName))
{
int foldNum = countThisClass / imagsPerDir;
if (foldNum != lastFoldNum)
{
foldDir = OSservices.AddSlash(classDir) + curClassName + "_" + foldNum.ToString("000");
OSservices.CreateDirectoryPath(foldDir);
lastFoldNum = foldNum;
}
String fullName = OSservices.AddSlash(foldDir) + i.ImageFileName + ".bmp";
PicesRaster r = dbConn.ImageFullSizeFind(i.ImageFileName);
if (r == null)
{
imagesFailedToExtract++;
}
else
{
r.Save(fullName);
}
imagesSaved++;
// Get the next image to save.
idx++;
if (idx < batch.Count)
{
i = batch[idx];
className = i.Class1Name;
if (classKeyToUse == 'V')
{
className = i.ValidatedClassName;
}
}
countThisClass++;
}
imagesPerClass[curClassName] = countThisClass;
}
} /* SaveOneBatchOfImages */
} /* SaveGivenImageList */
private void SaveImagesBySipperFiles()
{
bool includeStationAndDeploymentInDirName = false;
bool firstSipperFile = true;
PicesSipperFileList sipperFiles = null;
if (String.IsNullOrEmpty(sipperFileName))
{
sipperFiles = dbConn.SipperFileLoad(cruise, station, deployment);
}
else
{
PicesSipperFile sf = dbConn.SipperFileRecLoad(sipperFileName);
if (sf != null)
{
sipperFiles = new PicesSipperFileList();
sipperFiles.Add(sf);
}
}
if (sipperFiles != null)
{
String lastCruise = "";
String lastStation = "";
String lastDeployment = "";
String dirPath = "";
Dictionary <String, int> imagesPerClass = new Dictionary <string, int> ();
foreach (PicesSipperFile sf in sipperFiles)
{
bool startANewDir = false;
if (firstSipperFile)
{
startANewDir = true;
}
//else if (sf.CruiseName != lastCruise)
// startANewDir = true;
else if ((includeStationAndDeploymentInDirName) &&
((sf.StationName != lastStation) || (sf.DeploymentNum != lastDeployment))
)
{
startANewDir = true;
}
if (startANewDir)
{
lastCruise = sf.CruiseName;
lastStation = sf.StationName;
lastDeployment = sf.DeploymentNum;
dirPath = OSservices.AddSlash(DestinationDirectory.Text);
if (includeStationAndDeploymentInDirName)
{
if (!String.IsNullOrEmpty(lastCruise))
{
dirPath = OSservices.AddSlash(dirPath) + lastCruise;
}
if (!String.IsNullOrEmpty(lastStation))
{
dirPath = OSservices.AddSlash(dirPath) + lastStation;
}
if (!String.IsNullOrEmpty(lastDeployment))
{
dirPath = OSservices.AddSlash(dirPath) + lastDeployment;
}
}
OSservices.CreateDirectoryPath(dirPath);
imagesPerClass = new Dictionary <string, int> ();
}
SaveImagesForOneSipperFile(dirPath, sf, imagesPerClass);
firstSipperFile = false;
}
}
} /* SaveImagesBySipperFiles */
} /* StartTheBackGroundProcedure */
private void BackGroundProcess()
{
backGroundRunning = true;
PicesDataBase.ThreadInit();
OSservices.CreateDirectoryPath(reportFileDir);
DateTime n = DateTime.Now;
String logFileName = OSservices.AddSlash(PicesSipperVariables.TempDirectory()) + "InstrumentDataByDeployment_Log_" +
n.Year.ToString("0000") + "-" + n.Month.ToString("00") + "-" + n.Day.ToString("00") + "_" +
n.Hour.ToString("00") + "-" + n.Minute.ToString("00") + "-" + n.Hour.ToString("00") +
".txt";
backGroundLog = new PicesRunLog(logFileName);
PicesDataBase threadConn = PicesDataBase.GetGlobalDatabaseManagerNewInstance(backGroundLog);
RunLogAddMsg("Cruise [" + cruiseName + "]" + "\n");
RunLogAddMsg("Station [" + stationName + "]" + "\n");
RunLogAddMsg("Deployment [" + deploymentNum + "]" + "\n");
RunLogAddMsg("Start Date/Time [" + DateTime.Now.ToString("yyyy-MMM-dd HH:mm:ss") + "]" + "\n");
RunLogAddMsg("Active DataBase [" + threadConn.Server.Description + "]" + "\n");
PicesSipperDeploymentList deployments = null;
if (!String.IsNullOrEmpty(deploymentNum))
{
PicesSipperDeployment d = threadConn.SipperDeploymentLoad(cruiseName, stationName, deploymentNum);
if (d != null)
{
deployments = new PicesSipperDeploymentList();
deployments.Add(d);
}
}
else
{
deployments = threadConn.SipperDeploymentLoad(cruiseName, stationName);
}
if (deployments == null)
{
RunLogAddMsg("\n\nNo Deployments for Specified Criteria!\n\n");
return;
}
int reportNum = 0;
foreach (PicesSipperDeployment d in deployments)
{
if (cancelBackGround)
{
break;
}
reportNum++;
RunLogAddMsg("Generating Report " + reportNum.ToString() + " of " + deployments.Count.ToString());
SummarizeInstrumentDataForADeployment(threadConn, d.CruiseName, d.StationName, d.DeploymentNum);
}
threadConn.Close();
threadConn = null;
GC.Collect();
PicesDataBase.ThreadEnd();
backGroundRunning = false;
} /* BackGroundProcess */
} /* DialogTimer_Tick */
private void CreateTuningFeatureFiles()
{
tuningThreadRunning = true;
PicesDataBase.ThreadInit();
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Started");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Creating Destination Directory");
String destinationDirectory = OSservices.AddSlash(DestinationDirectory.Text) + OSservices.GetRootName(modelName);
OSservices.CreateDirectoryPath(destinationDirectory);
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Loading feature data for Model[" + modelName + "]");
config = new PicesTrainingConfigManaged(modelName, runLog);
if (!config.Valid())
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] is invalid.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
bool changesMade = false;
PicesFeatureVectorList data = config.LoadFeatureDataFromTrainingLibraries(changesMade);
if (data == null)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Model[" + modelName + "] Could not load Feature Data.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Process Canceled.");
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Feature Data Loaded.");
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Checking that Depth Data Exists.");
MakeSureDepthFieldIsIncluded(data);
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Splitting into separate files.");
PicesFeatureVectorList trainData = new PicesFeatureVectorList();
PicesFeatureVectorList testData = new PicesFeatureVectorList();
PicesFeatureVectorList validationData = new PicesFeatureVectorList();
PicesClassList classes = data.ExtractListOfClasses();
if (classes == null)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Could not extract list of classes from data.");
tuningThreadFailed = true;
tuningThreadDone = true;
tuningThreadRunning = false;
return;
}
foreach (PicesClass c in classes)
{
int trainCount = 0;
int testCount = 0;
int validationCount = 0;
PicesFeatureVectorList examplesForClass = data.ExtractExamplesForAGivenClass(c);
int n = examplesForClass.Count;
AddMsgToDialogMsgQueue(DateTime.Now.ToShortTimeString() + " Processing Class[" + c.Name + "] Count[" + n + "]");
trainCount = (int)((float)n * trainingDataPercentage / 100.0f + 0.5f);
if (trainingDataMaxImagesPerClass > 0)
{
if (trainCount > trainingDataMaxImagesPerClass)
{
trainCount = trainingDataMaxImagesPerClass;
}
}
if (validationDataPercentage == 0.0f)
{
testCount = n - trainCount;
}
else
{
testCount = (int)((float)n * testDataPercentage / 100.0f + 0.5f);
}
validationCount = n - (trainCount + testCount);
examplesForClass.RandomizeOrder();
int validationStart = trainCount + testCount;
int testStart = trainCount;
for (int idx = 0; idx < n; idx++)
{
PicesFeatureVector fv = examplesForClass[idx];
if (idx >= validationStart)
{
validationData.Add(fv);
}
else if (idx >= testStart)
{
testData.Add(fv);
}
else
{
trainData.Add(fv);
}
}
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n");
AddMsgToDialogMsgQueue("Process Canceled.");
break;
}
}
if (!tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("\n\n" + DateTime.Now.ToShortTimeString() + " Stratifying Data");
PicesFeatureVectorList trainDataStratified = trainData.StratifyAmoungstClasses(numOfFolds);
PicesFeatureVectorList testDataStratified = testData.StratifyAmoungstClasses(numOfFolds);
PicesFeatureVectorList validationDataStratified = validationData.StratifyAmoungstClasses(numOfFolds);
String rootName = OSservices.AddSlash(destinationDirectory) + OSservices.GetRootName(modelName);
if ((trainDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
trainDataFileName = rootName + "_Train.data";
AddMsgToDialogMsgQueue("Saving Training Data[" + trainDataFileName + "]");
trainDataStratified.Save(trainDataFileName, "", runLog);
}
if ((testDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
testDataFileName = rootName + "_Test.data";
AddMsgToDialogMsgQueue("Saving Test Data[" + testDataFileName + "]");
testDataStratified.Save(testDataFileName, "", runLog);
}
if ((validationDataStratified.Count > 0) && (!tuningThreadCancelReq))
{
validationDataFileName = rootName + "_Validation.data";
AddMsgToDialogMsgQueue("Saving Validation Data[" + validationDataFileName + "]");
validationDataStratified.Save(validationDataFileName, "", runLog);
}
}
AddMsgToDialogMsgQueue("\n\n");
if (tuningThreadCancelReq)
{
AddMsgToDialogMsgQueue("Process *** CANCELED ***.");
}
else
{
AddMsgToDialogMsgQueue("Process Completed.");
}
PicesDataBase.ThreadEnd();
tuningThreadDone = true;
tuningThreadRunning = false;
} /* CreateTuningFeatureFiles */
