private bool WriteDetectionResults(JsonData data, string outputFileName, SubsetJsonDetectorOutputOptions options)
{
// Write the detector ouput *data* to *output_filename*
if (!Path.IsPathRooted(outputFileName))
{
outputFileName = Path.GetTempPath() + outputFileName;
}
if (!options.OverwriteJsonFiles && File.Exists(outputFileName))
{
string msg = string.Format("File {0} exists", outputFileName);
SetStatusMessage(msg);
return(false);
}
outputFileName = outputFileName.Replace("/", "\\");
string directoryPath = outputFileName.Substring(0, outputFileName.LastIndexOf("\\"));
if (options.CopyJsonstoFolders && options.CopyJsonstoFoldersDirectoriesMustExist)
{
if (!Directory.Exists(directoryPath))
{
string msg = String.Format("Directory {0} does not exist", outputFileName);
SetStatusMessage(msg);
return(false);
}
}
else
{
Directory.CreateDirectory(directoryPath);
}
try
{
var ext = System.IO.Path.GetExtension(outputFileName);
if (ext == String.Empty)
{
string msg = "Please enter a valid output file name";
SetStatusMessage(msg);
return(false);
}
using (var fs = File.Create(outputFileName))
using (var sw = new StreamWriter(fs))
using (var jtw = new JsonTextWriter(sw)
{
Formatting = Formatting.Indented,
Indentation = 1,
IndentChar = ' '
})
{
(new JsonSerializer()).Serialize(jtw, data);
}
}
catch (Exception ex)
{
SetStatusMessage(ex.Message);
}
return(true);
} // WriteDetectionResults()
private Dictionary <string, List <Image> > FindUniqueFolders(SubsetJsonDetectorOutputOptions options, JsonData data)
{
Dictionary <string, List <Image> > folderstoImages = new Dictionary <string, List <Image> >();
foreach (var imagedata in data.images)
{
string filePath = imagedata.file.ToString();
string directoryName = string.Empty;
List <Image> imageList = new List <Image>();
if (options.SplitFolderMode.ToLower() == "bottom")
{
directoryName = Path.GetDirectoryName(filePath);
}
else if (options.SplitFolderMode.ToLower() == "nfrombottom")
{
directoryName = Path.GetDirectoryName(filePath);
for (int n = 0; n < options.nDirectoryParam; n++)
{
directoryName = Path.GetDirectoryName(directoryName);
}
}
else if (options.SplitFolderMode.ToLower() == "top")
{
directoryName = TopLevelFolder(filePath);
}
if (!folderstoImages.ContainsKey(directoryName))
{
imageList.Add(imagedata);
folderstoImages.Add(directoryName, imageList);
}
else
{
var currentImgArray = folderstoImages[directoryName];
currentImgArray.Add(imagedata);
folderstoImages[directoryName] = currentImgArray;
}
} // ...for each image
SetLabelProgressMsg(string.Format("Found {0} unique folders", folderstoImages.Count.ToString(),
folderstoImages.Count.ToString()), ProgressBarStyle.Marquee);
return(folderstoImages);
}
private bool WriteDetectionResults(JsonData data, string outputFileName, SubsetJsonDetectorOutputOptions options)
{
// Convert all paths to forward slashes unless they're absolute
if (options.UseForwardSlashesWhenPossible)
{
foreach (Image im in data.images)
{
if ((im.file.Contains("\\")) && (!(im.file.Contains("/"))) && (!Path.IsPathRooted(im.file)))
{
im.file = im.file.Replace('\\', '/');
}
}
}
// Write the detector ouput *data* to *output_filename*
if (!Path.IsPathRooted(outputFileName))
{
string msg = string.Format("Must specify an absolute output path");
SetStatusMessage(msg);
return(false);
}
if (!options.OverwriteJsonFiles && File.Exists(outputFileName))
{
string msg = string.Format("File {0} exists", outputFileName);
SetStatusMessage(msg);
return(false);
}
outputFileName = outputFileName.Replace("/", "\\");
string directoryPath = outputFileName.Substring(0, outputFileName.LastIndexOf("\\"));
if (options.CopyJsonstoFolders && options.CopyJsonstoFoldersDirectoriesMustExist)
{
if (!Directory.Exists(directoryPath))
{
string msg = String.Format("Directory {0} does not exist", outputFileName);
SetStatusMessage(msg);
return(false);
}
}
else
{
Directory.CreateDirectory(directoryPath);
}
try
{
String ext = System.IO.Path.GetExtension(outputFileName);
if (!(ext.Equals(".json")))
{
string msg = "Output file name must end in .json";
SetStatusMessage(msg);
return(false);
}
using (FileStream fs = File.Create(outputFileName))
using (StreamWriter sw = new StreamWriter(fs))
using (JsonTextWriter jtw = new JsonTextWriter(sw)
{
Formatting = Formatting.Indented,
Indentation = 1,
IndentChar = ' '
})
{
(new JsonSerializer()).Serialize(jtw, data);
}
}
catch (Exception ex)
{
SetStatusMessage(ex.Message);
}
return(true);
} // WriteDetectionResults()
private Dictionary <string, List <Image> > FindUniqueFolders(SubsetJsonDetectorOutputOptions options, JsonData data)
{
Dictionary <string, List <Image> > folderstoImages = new Dictionary <string, List <Image> >();
foreach (var imagedata in data.images)
{
string filePath = imagedata.file.ToString();
string directoryName = string.Empty;
List <Image> imageList = new List <Image>();
if (options.SplitFolderMode.ToLower() == "bottom")
{
directoryName = Path.GetDirectoryName(filePath);
}
else if (options.SplitFolderMode.ToLower() == "nfrombottom")
{
directoryName = Path.GetDirectoryName(filePath);
for (int n = 0; n < options.nDirectoryParam; n++)
{
if (directoryName.Length == 0)
{
string msg = string.Format("Error: cannot walk {0} folders from the bottom in path {1}",
options.nDirectoryParam, filePath);
SetStatusMessage(msg);
return(null);
}
directoryName = Path.GetDirectoryName(directoryName);
}
}
else if (options.SplitFolderMode.ToLower() == "nfromtop")
{
directoryName = Path.GetDirectoryName(filePath);
// Split string into folders, keeping delimiters
String delimiter = @"([\\/])";
String[] tokens = Regex.Split(directoryName, delimiter);
int nTokensToKeep = ((options.nDirectoryParam + 1) * 2) - 1;
if (nTokensToKeep > tokens.Length)
{
string msg = string.Format("Error: cannot walk {0} folders from the top in path {1}",
options.nDirectoryParam, filePath);
SetStatusMessage(msg);
return(null);
}
tokens = tokens.Take(nTokensToKeep).ToArray();
directoryName = String.Join("", tokens);
}
else if (options.SplitFolderMode.ToLower() == "top")
{
directoryName = TopLevelFolder(filePath);
}
if (!folderstoImages.ContainsKey(directoryName))
{
imageList.Add(imagedata);
folderstoImages.Add(directoryName, imageList);
}
else
{
var currentImgArray = folderstoImages[directoryName];
currentImgArray.Add(imagedata);
folderstoImages[directoryName] = currentImgArray;
}
} // ...for each image
SetLabelProgressMsg(string.Format("Found {0} unique folders", folderstoImages.Count.ToString(),
folderstoImages.Count.ToString()), ProgressBarStyle.Marquee);
return(folderstoImages);
}
public JsonData SubsetJsonDetectorOutputMain(string inputFileName, string outputFilename,
SubsetJsonDetectorOutputOptions options, JsonData data = null)
{
try
{
string progressMsg = string.Empty;
if (options == null)
{
options = new SubsetJsonDetectorOutputOptions();
}
if (options.SplitFolders)
{
if (File.Exists(outputFilename))
{
progressMsg = "When splitting by folders, output must be a valid directory name";
SetStatusMessage(progressMsg);
return(null);
}
}
if (data == null)
{
SetLabelProgressMsg("Reading Json file...", ProgressBarStyle.Marquee);
data = LoadJson(inputFileName);
SetLabelProgressMsg("Loaded Json data...", ProgressBarStyle.Marquee);
if (options.DebugMaxImages > 0)
{
data.images = data.images.Take(options.DebugMaxImages).ToList <Image>();
}
}
else
{
data = DeepClone(data);
}
if (!string.IsNullOrEmpty(options.Query))
{
data = SubsetJsonDetectorOutputbyQuery(data, options);
}
if (options.ConfidenceThreshold != -1)
{
data = SubsetJsonDetectorOutputbyConfidence(data, options);
}
if (!options.SplitFolders)
{
SetLabelProgressMsg("Writing to file...", ProgressBarStyle.Marquee);
bool result = WriteDetectionResults(data, outputFilename, options);
if (!result)
{
return(null);
}
SetStatusMessage("File written to " + outputFilename.Replace("/", "\\"));
return(data);
}
else
{
SetLabelProgressMsg("Finding unique folders...", ProgressBarStyle.Marquee);
Dictionary <string, List <Image> > foldersToImages = FindUniqueFolders(options, data);
if (foldersToImages == null)
{
return(null);
}
if (options.MakeFolderRelative)
{
foldersToImages = MakeFoldersRelative(foldersToImages);
}
var ext = System.IO.Path.GetExtension(outputFilename);
string directory = outputFilename;
if (ext != String.Empty)
{
directory = outputFilename.Replace("\\", "/");
if (directory.Contains("/"))
{
directory = outputFilename.Substring(0, outputFilename.LastIndexOf("/"));
}
else
{
directory = outputFilename;
}
}
Directory.CreateDirectory(directory);
var allImages = data.images;
SetLabelProgressMsg("Writing to file...", ProgressBarStyle.Marquee);
int count = 0;
int totalCount = foldersToImages.Count;
// For each folder...
foreach (var item in foldersToImages)
{
count++;
string directoryName = item.Key;
if (directoryName.Length == 0)
{
directoryName = "base";
}
JsonData dirData = new JsonData();
dirData.classification_categories = data.classification_categories;
dirData.detection_categories = data.detection_categories;
dirData.info = data.info;
dirData.images = foldersToImages[directoryName];
if (Path.IsPathRooted(directoryName))
{
string rootPath = Path.GetPathRoot(directoryName);
directoryName = directoryName.Replace(rootPath, "");
}
string jsonFileName = directoryName.Replace('/', '_').Replace('\\', '_') + ".json";
if (options.CopyJsonstoFolders)
{
jsonFileName = Path.Combine(outputFilename, directoryName, jsonFileName);
}
else
{
jsonFileName = Path.Combine(outputFilename, jsonFileName);
}
bool result = WriteDetectionResults(dirData, jsonFileName, options);
if (!result)
{
return(null);
}
SetStatusMessage(string.Format("Wrote {0} images to {1}",
dirData.images.Count().ToString(), jsonFileName), count, totalCount);
} // ...for each folder
data.images = allImages;
} // if we are/aren't splitting folders
return(data);
}
catch (Exception ex)
{
ShowError(ex);
return(null);
}
} // SubsetJsonDetectorOutputMain
public SubsetJsonDetectorOutput(BackgroundWorker progressReporter, SubsetJsonDetectorOutputOptions options)
{
this.progressReporter = progressReporter;
this.options = options;
}
} // SubsetJsonDetectorOutputbyQuery
private JsonData SubsetJsonDetectorOutputbyConfidence(JsonData data, SubsetJsonDetectorOutputOptions options)
{
if (options.ConfidenceThreshold == -1)
{
return(data);
}
var imagesIn = data.images;
List <Image> imagesOut = new List <Image>();
long count = 0;
int maxChanges = 0;
long totalCount = imagesIn.Count();
string progressMsg = string.Empty;
int[] progressPercentagesForDisplay = { 10, 20, 40, 80, 100 };
progressMsg = string.Format("Subsetting by confidence >= {0}", options.ConfidenceThreshold.ToString());
foreach (var item in imagesIn)
{
count++;
int percentage = SharedFunctions.GetProgressPercentage(count, totalCount);
if (progressPercentagesForDisplay.Contains(percentage))
{
SetLabelProgressMsg(progressMsg, count, totalCount, ProgressBarStyle.Blocks);
}
dynamic p;
dynamic pOrig = item.max_detection_conf;
List <Detection> detections = new List <Detection>();
// detections = [d for d in im['detections'] if d['conf'] >= options.confidence_threshold]
// Find all detections above threshold for this image
foreach (var d in item.detections)
{
dynamic conf = d.conf;
if (conf >= options.ConfidenceThreshold)
{
detections.Add(d);
}
}
// If there are no detections above threshold, set the max probability
// to -1, unless it already had a negative probability.
if (detections.Count == 0)
{
if (pOrig <= 0)
{
p = pOrig;
}
else
{
p = -1;
}
}
// Otherwise find the maximum confidence
else
{
p = -1;
foreach (var c in detections)
{
dynamic confidence = c.conf;
if (confidence > p)
{
p = confidence;
}
}
}
item.detections = detections.ToArray();
// Did this thresholding result in a max-confidence change?
if (Math.Abs(pOrig - p) > 0.00001)
{
// We should only be *lowering* max confidence values (i.e., making them negative)
if (!(pOrig <= 0 || p < pOrig))
{
string errmsg = string.Format("Confidence changed from {0} to {1}", pOrig, p);
// This will get handled by the UI
throw new Exception(errmsg);
}
maxChanges += 1;
}
item.max_detection_conf = p;
imagesOut.Add(item);
} // for each image
data.images = imagesOut;
SetStatusMessage(string.Format("Finished confidence search, found {0} matches (of {1}), {2} max conf changes", data.images.Count().ToString(),
imagesIn.Count().ToString(), maxChanges.ToString()));
return(data);
} // SubsetJsonDetectorOutputbyConfidence
} // SubsetJsonDetectorOutputMain
private JsonData SubsetJsonDetectorOutputbyQuery(JsonData data, SubsetJsonDetectorOutputOptions options)
{
// Subset to images whose filename matches options.query; replace all instances of
// options.query with options.replacement.
string progressMsg = string.Empty;
var imagesIn = data.images;
List <Image> imagesOut = new List <Image>();
long count = 0;
int percentage = 0;
long totalCount = imagesIn.Count();
int[] progressPercentagesForDisplay = { 10, 20, 40, 80, 90, 100 };
progressMsg = string.Format("Subsetting by query {0}, replacement {1} ...", options.Query, options.Replacement);
SetLabelProgressMsg(progressMsg, ProgressBarStyle.Marquee);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
foreach (var image in imagesIn)
{
count++;
string file = image.file.ToString();
if (!string.IsNullOrEmpty(options.Query) && !file.Contains(options.Query))
{
continue;
}
if (!string.IsNullOrEmpty(options.Replacement))
{
if (!string.IsNullOrEmpty(options.Query))
{
file = file.Replace(options.Query, options.Replacement);
}
else
{
file = options.Replacement + file;
}
}
percentage = SharedFunctions.GetProgressPercentage(count, totalCount);
if (progressPercentagesForDisplay.Contains(percentage))
{
progressMsg = string.Format("Subsetting by query {0}, replacement {1} processed {2} of {3}...",
options.Query, options.Replacement, count.ToString(), totalCount.ToString());
SetLabelProgressPercentageMsg(progressMsg, count, totalCount, ProgressBarStyle.Blocks);
}
image.file = file;
imagesOut.Add(image);
} // for each image
data.images = imagesOut;
SetStatusMessage(string.Format("Finished query search, found {0} matches (of {1})", data.images.Count(), imagesIn.Count()));
stopwatch.Stop();
Console.WriteLine(stopwatch.Elapsed.TotalSeconds.ToString());
return(data);
} // SubsetJsonDetectorOutputbyQuery
