private void extractTheme_Click(object sender, EventArgs e) { //Use the Palette Extractor to extract the theme PaletteExtractor extractor = new PaletteExtractor(dir,weightsDir,json); String[] files = Directory.GetFiles(dir, "*.png"); String outfile = dir+"\\palettes.tsv"; String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n"; File.WriteAllText(outfile, ""); File.AppendAllText(outfile, headers); int count = 0; foreach (String f in files) { count++; String basename = f.Replace(dir+"\\", ""); //The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists //i.e. if the image filename is A.png, the saliency map filename is A_Judd.png PaletteData data = extractor.HillClimbPalette(basename, "_Judd", true); //save to file String colorString = data.ToString(); File.AppendAllText(outfile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); } }
/** * Extract themes from images in the image directory */ private void extractTheme_Click(object sender, EventArgs e) { if (backgroundWorker.IsBusy) { statusBox.Text = "Please wait..."; return; } backgroundWorker = MakeBackgroundWorker(delegate { backgroundWorker.ReportProgress(0, "Extracting Themes from Image Dir"); //Use the Palette Extractor to extract the theme PaletteExtractor extractor = new PaletteExtractor(dir, weightsDir, json); String[] files = GetImageFiles(dir); String outfile = Path.Combine(dir, "optimized.tsv"); String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n"; File.WriteAllText(outfile, ""); File.AppendAllText(outfile, headers); int count = 0; foreach (String f in files) { count++; String basename = new FileInfo(f).Name; //The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists //i.e. if the image filename is A.png, the saliency map filename is A_Judd.png PaletteData data = extractor.HillClimbPalette(basename, "_Judd", debug); //save to file String colorString = data.ToString(); File.AppendAllText(outfile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); backgroundWorker.ReportProgress(100 * count / files.Count(),"Finished " + f); } }); RunWorker(); }
/** * Find the theme closest on average to a given set of themes (compare[key]) */ private PaletteData GetOracleTheme(String key, Dictionary<String, List<PaletteData>> compare) { //find the palette that has the lowest distance to all other palettes int trials = 20; PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json); PaletteData swatches = extractor.GetPaletteSwatches(key); List<CIELAB> shuffled = new List<CIELAB>(); foreach (CIELAB c in swatches.lab) shuffled.Add(c); Random random = new Random(); PaletteData best = new PaletteData(); double bestScore = Double.NegativeInfinity; Stopwatch watch = new Stopwatch(); //Generate all the random starts first List<PaletteData> starts = new List<PaletteData>(); PaletteData[] allOptions = new PaletteData[trials]; double[] allScores = new double[trials]; for (int t = 0; t < trials; t++) { //setup PaletteData option = new PaletteData(); //pick k random colors. First shuffle the colors and pick the top k for (int j = shuffled.Count() - 1; j >= 0; j--) { int idx = random.Next(j + 1); CIELAB temp = shuffled[j]; shuffled[j] = shuffled[idx]; shuffled[idx] = temp; } for (int i = 0; i < 5; i++) { option.lab.Add(shuffled[i]); option.colors.Add(Util.LABtoRGB(shuffled[i])); } starts.Add(option); allOptions[t] = new PaletteData(option); double optionScore = 1 - GetAvgDist(new List<PaletteData> { option }, compare[key]); allScores[t] = optionScore; } watch.Restart(); Parallel.For(0, trials, t => { //setup PaletteData option = new PaletteData(starts[t]); double optionScore = allScores[t]; //Now hill climb, for each swatch, consider replacing it with a better swatch //Pick the best replacement, and continue until we reach the top of a hill int changes = 1; int iters = 0; watch.Restart(); while (changes > 0) { changes = 0; for (int i = 0; i < option.lab.Count(); i++) { //find the best swatch replacement for this color double bestTempScore = optionScore; CIELAB bestRep = option.lab[i]; double[] scores = new double[swatches.lab.Count()]; for (int s = 0; s < swatches.lab.Count(); s++) { CIELAB r = swatches.lab[s]; PaletteData temp = new PaletteData(option); if (!temp.lab.Contains(r)) { temp.lab[i] = r; temp.colors[i] = swatches.colors[s]; double tempScore = 1 - GetAvgDist(new List<PaletteData> { temp }, compare[key]); scores[s] = tempScore; } else { scores[s] = Double.NegativeInfinity; } } //aggregate results for (int s = 0; s < scores.Count(); s++) { if (scores[s] > bestTempScore) { bestTempScore = scores[s]; bestRep = swatches.lab[s]; } } if (!option.lab[i].Equals(bestRep)) { option.lab[i] = bestRep; optionScore = bestTempScore; changes++; } } iters++; } if (optionScore > allScores[t]) { allOptions[t] = option; allScores[t] = optionScore; } }); //aggregate scores for (int i = 0; i < allScores.Count(); i++) { if (allScores[i] > bestScore) { bestScore = allScores[i]; best = allOptions[i]; } } //convert best lab to rgb best.colors = new List<Color>(); foreach (CIELAB l in best.lab) best.colors.Add(Util.LABtoRGB(l)); return best; }
/** * Extract k-means, c-means, model, and oracle themes for comparison */ private void extractThemesToCompare_Click(object sender, EventArgs e) { if (backgroundWorker.IsBusy) { statusBox.Text = "Please wait..."; return; } backgroundWorker.WorkerReportsProgress = true; backgroundWorker = MakeBackgroundWorker(delegate { backgroundWorker.ReportProgress(0, "Extracting themes to compare..."); PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json); Dictionary<String, Dictionary<String, List<PaletteData>>> referenceP = ItemsToDictionary(reference); String[] files = GetImageFiles(evalDir); String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n"; backgroundWorker.ReportProgress(0, "Extracting random themes..."); //extract random String randomFile = Path.Combine(evalDir, "random.tsv"); if (!File.Exists(randomFile) || File.ReadLines(randomFile).Count() != (40 * files.Count()) + 1) { int count = 0; File.WriteAllText(randomFile, ""); File.AppendAllText(randomFile, headers); foreach (String f in files) { String basename = new FileInfo(f).Name; PaletteData swatches = extractor.GetPaletteSwatches(basename); List<PaletteData> random = GenerateRandomPalettesNonBinned(40, swatches); foreach (PaletteData data in random) { count++; String colorString = data.ToString(); File.AppendAllText(randomFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); } } } else { Console.WriteLine("Random palettes file already exists"); backgroundWorker.ReportProgress(100, "Random themes file already exist...skipping"); Thread.Sleep(1000); } backgroundWorker.ReportProgress(0, "Extracting K-means themes..."); //extract kmeans String kmeansFile = Path.Combine(evalDir, "kmeans.tsv"); if (!File.Exists(kmeansFile) || File.ReadLines(kmeansFile).Count() != files.Count() + 1) { File.WriteAllText(kmeansFile, ""); File.AppendAllText(kmeansFile, headers); int count = 0; foreach (String f in files) { count++; String basename = new FileInfo(f).Name; PaletteData data = KMeansPalette(f); //save to file String colorString = data.ToString(); File.AppendAllText(kmeansFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); backgroundWorker.ReportProgress(100 * count / files.Count(), ""); } } else { Console.WriteLine("Kmeans palettes file already exists"); backgroundWorker.ReportProgress(100, "K-means themes already exist...skipping"); Thread.Sleep(1000); } //extract cmeans backgroundWorker.ReportProgress(0, "Extracting C-means themes..."); String cmeansFile = Path.Combine(evalDir, "cmeans.tsv"); if (!File.Exists(cmeansFile) || File.ReadLines(cmeansFile).Count() != files.Count() + 1) { File.WriteAllText(cmeansFile, ""); File.AppendAllText(cmeansFile, headers); int count = 0; foreach (String f in files) { count++; String basename = new FileInfo(f).Name; PaletteData data = CMeansPalette(f); //save to file String colorString = data.ToString(); File.AppendAllText(cmeansFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); backgroundWorker.ReportProgress(100 * count / files.Count(), ""); } } else { Console.WriteLine("Cmeans palettes file already exists"); backgroundWorker.ReportProgress(100, "C-means themes already exist...skipping"); Thread.Sleep(1000); } //extract model backgroundWorker.ReportProgress(0, "Extracting Optimized themes..."); String modelFile = Path.Combine(evalDir, "optimized.tsv"); if (!File.Exists(modelFile) || File.ReadLines(modelFile).Count() != files.Count() + 1) { File.WriteAllText(modelFile, ""); File.AppendAllText(modelFile, headers); int count = 0; foreach (String f in files) { count++; String basename = new FileInfo(f).Name; //The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists //i.e. if the image filename is A.png, the saliency map filename is A_Judd.png PaletteData data = extractor.HillClimbPalette(basename, "_Judd", debug); //save to file String colorString = data.ToString(); File.AppendAllText(modelFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); backgroundWorker.ReportProgress(100 * count / files.Count(), ""); } } else { Console.WriteLine("Optimized palettes file already exists"); backgroundWorker.ReportProgress(100, "Optimized themes already exist...skipping"); Thread.Sleep(1000); } //extract oracle foreach (String name in referenceP.Keys) { backgroundWorker.ReportProgress(0, "Extracting Oracle themes for..." + name); String oracleFile = Path.Combine(evalDir, "oracle-" + name + ".tsv"); if (!File.Exists(oracleFile) || File.ReadLines(oracleFile).Count() != referenceP[name].Keys.Count() + 1) { File.WriteAllText(oracleFile, ""); File.AppendAllText(oracleFile, headers); int count = 0; foreach (String f in files) { String basename = new FileInfo(f).Name; if (!referenceP[name].ContainsKey(basename)) continue; count++; PaletteData data = GetOracleTheme(basename, referenceP[name]); //save to file String colorString = data.ToString(); File.AppendAllText(oracleFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n"); backgroundWorker.ReportProgress(100 * count / files.Count(), ""); } } else { Console.WriteLine("Oracle palettes file already exists"); backgroundWorker.ReportProgress(100, "Oracle themes already exist...skipping"); Thread.Sleep(1000); } } }); RunWorker(); }
/** * Create diagrams of the themes showing source image, swatches, and aligned themes */ private void diagramThemes_Click(object sender, EventArgs e) { if (backgroundWorker.IsBusy) { statusBox.Text = "Please wait..."; return; } backgroundWorker = MakeBackgroundWorker(delegate { backgroundWorker.ReportProgress(0, "Diagramming themes..."); //create directory String diagramDir = Path.Combine(evalDir, "diagrams"); Directory.CreateDirectory(diagramDir); Dictionary<String, Dictionary<String, List<PaletteData>>> leftP = ItemsToDictionary(diagramLeft); Dictionary<String, Dictionary<String, List<PaletteData>>> rightP = ItemsToDictionary(diagramRight); int imageWidth = 200; int colorSize = 20; int padding = 5; int gutter = 50; int labelTop = 100; PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json); SortedSet<String> keys = new SortedSet<String>(); foreach (String key in leftP.Keys) keys.UnionWith(leftP[key].Keys); foreach (String key in rightP.Keys) keys.UnionWith(rightP[key].Keys); int progress = 0; foreach (String key in keys) { //check total number of themes to draw int totalThemes = 0; PaletteData swatches = extractor.GetPaletteSwatches(key); //Count total number of themes, and sort the themes by average distance foreach (String n in leftP.Keys) { if (!leftP[n].ContainsKey(key)) continue; totalThemes += leftP[n][key].Count(); leftP[n][key] = SortByDist(leftP[n][key]); } foreach (String n in rightP.Keys) { if (!rightP[n].ContainsKey(key)) continue; totalThemes += rightP[n][key].Count(); rightP[n][key] = SortByDist(rightP[n][key]); } int width = imageWidth + (leftP.Keys.Count() + rightP.Keys.Count() + 2) * gutter + colorSize * totalThemes; int height = labelTop + swatches.colors.Count()*colorSize; Bitmap result = new Bitmap(width, height); Graphics g = Graphics.FromImage(result); Bitmap image = new Bitmap(Path.Combine(evalDir, key)); //draw the source image g.DrawImage(image, 0, labelTop, imageWidth, imageWidth*image.Height / image.Width); int startX = imageWidth+gutter; //draw the themes to the left of the swatches foreach (EvalItem item in diagramLeft) { String name = item.Name; if (!leftP[name].ContainsKey(key)) continue; DrawThemes(g, name, leftP[name][key], swatches, startX, labelTop, colorSize, padding); startX += gutter + leftP[name][key].Count()*colorSize; } //draw the swatches for (int c = 0; c < swatches.colors.Count; c++) { int y = c * colorSize+labelTop; g.FillRectangle(new SolidBrush(swatches.colors[c]), startX, y, colorSize - padding, colorSize - padding); g.DrawRectangle(new Pen(Color.Black), startX, y, colorSize - padding, colorSize - padding); } startX += colorSize+gutter; //draw the themes to the right of the swatches foreach (EvalItem item in diagramRight) { String name = item.Name; if (!rightP[name].ContainsKey(key)) continue; DrawThemes(g, name, rightP[name][key], swatches, startX, labelTop, colorSize, padding); startX += gutter+ rightP[name][key].Count()*colorSize; } result.Save(Path.Combine(diagramDir, key)); progress++; backgroundWorker.ReportProgress(100*progress / keys.Count, "Diagrammed " + key); } }); RunWorker(); }
/** * Calculate average distance between themes from different sources, and average overlap over different thresholds */ private void compareThemes_Click(object sender, EventArgs e) { if (backgroundWorker.IsBusy) { statusBox.Text = "Please wait..."; return; } backgroundWorker.WorkerReportsProgress = true; backgroundWorker = MakeBackgroundWorker(delegate { backgroundWorker.ReportProgress(0, "Comparing themes..."); //load the reference palettes //load the palettes to compare against Dictionary<String, Dictionary<String, List<PaletteData>>> referenceP = ItemsToDictionary(reference); Dictionary<String, Dictionary<String, List<PaletteData>>> compareP = ItemsToDictionary(compare); Dictionary<String, List<PaletteData>> testKeys = LoadFilePalettes(refkeys); //compare each source in compare to each reference List<String> desc = new List<String>(); List<double> dists = new List<double>(); foreach (var r in reference) { foreach (var c in compare) { desc.Add(c.Name + "-" + r.Name); dists.Add(0); } } int progress = 0; foreach (String key in testKeys.Keys) { int idx = 0; foreach (var r in reference) { foreach (var c in compare) { //count number of keys in common between reference, compare, and test set if (referenceP[r.Name].ContainsKey(key) && compareP[c.Name].ContainsKey(key)) { double factor = referenceP[r.Name].Keys.Intersect(compareP[c.Name].Keys).Intersect(testKeys.Keys).Count(); dists[idx] += GetAvgDist(referenceP[r.Name][key], compareP[c.Name][key], c.Path == r.Path) / factor; } idx++; } } progress++; backgroundWorker.ReportProgress(50 * progress/testKeys.Keys.Count(), "Comparing themes...Calculating distance..."); } //Write out the average distances List<String> lines = new List<String>(); lines.Add(String.Join(",", desc.ToArray<String>())); lines.Add(String.Join(",", dists.Select(i => i.ToString()))); File.WriteAllLines(Path.Combine(evalDir, "avgDists.csv"), lines.ToArray<String>()); backgroundWorker.ReportProgress(50, "Comparing themes...Calculating overlap..."); //Output csv file for the overlap graph //clamp each palette to the closest color swatch //when computing overlap for the graph PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json); foreach (String key in testKeys.Keys) { PaletteData swatches = extractor.GetPaletteSwatches(key); foreach (var r in reference) { if (!referenceP[r.Name].ContainsKey(key)) continue; var palettes = referenceP[r.Name][key]; for (int i = 0; i < palettes.Count(); i++) palettes[i] = SnapColors(palettes[i], swatches); } foreach (var c in compare) { if (!compareP[c.Name].ContainsKey(key)) continue; var palettes = compareP[c.Name][key]; for (int i = 0; i < palettes.Count(); i++) palettes[i] = SnapColors(palettes[i], swatches); } } lines.Clear(); //compute the overlaps for different thresholds lines.Add("thresh,type,overlap"); for (int t = 1; t < 100; t += 2) { double[] overlaps = new double[reference.Count() * compare.Count()]; int idx = 0; foreach (var r in reference) { foreach (var c in compare) { foreach (String key in testKeys.Keys) if (compareP[c.Name].ContainsKey(key) && referenceP[r.Name].ContainsKey(key)) overlaps[idx] += GetAvgOverlap(compareP[c.Name][key], referenceP[r.Name][key], t, c.Path == r.Path); double factor = referenceP[r.Name].Keys.Intersect(compareP[c.Name].Keys).Intersect(testKeys.Keys).Count(); lines.Add(t + "," + desc[idx] + "," + overlaps[idx] / factor); idx++; } } backgroundWorker.ReportProgress(t/2+50, String.Format("Comparing themes...Calculating overlap for thresh {0}/{1}...",t,100)); } File.WriteAllLines(Path.Combine(evalDir, "overlaps.csv"), lines.ToArray<String>()); }); RunWorker(); }
private void CalculateFeatures() { backgroundWorker.ReportProgress(0, "First generating binned random palettes...checking for existing file"); PaletteExtractor extractor = new PaletteExtractor(trainInDir, weightsDir, json); Dictionary<String, List<PaletteData>> palettes = LoadFilePalettes(trainReference.Path); if (trainReference.Filter) { var keys = palettes.Keys.ToArray<String>(); foreach (String key in keys) palettes[key] = removeOutliers(palettes[key]); } SaveRandomPalettes(Path.Combine(trainOutDir, "random-binned.tsv"), palettes, extractor); Dictionary<String, List<PaletteData>> randomP = LoadFilePalettes(Path.Combine(trainOutDir, "random-binned.tsv")); ClearLog(Path.Combine(trainOutDir, "timelog.txt")); Features[] included = Enum.GetValues(typeof(Features)).Cast<Features>().ToArray<Features>(); SortedSet<Features> headers = new SortedSet<Features>(included); String saliencyPattern = "_Judd"; String imageNameFile = Path.Combine(trainOutDir, "imageNames.txt"); String outFile = Path.Combine(trainOutDir, "features.csv"); File.WriteAllText(outFile, ""); //clear the file File.WriteAllText(imageNameFile, ""); //write the headings in a separate file File.WriteAllText(Path.Combine(trainOutDir, "featurenames.txt"), String.Join("\n", headers.Select<Features, String>((a) => featureName.Name(a)))); String log = Path.Combine(trainOutDir,"log.txt"); Log(log, "Start"); backgroundWorker.ReportProgress(0, "Calculating features..."); int progress = 0; foreach (String key in palettes.Keys) { List<String> sources = new List<String>(); List<String> allFeatures = new List<String>(); List<double> scores = new List<double>(); List<double> tempScores = new List<double>(); double maxScore = Double.NegativeInfinity; double minScore = Double.PositiveInfinity; List<PaletteData> all = new List<PaletteData>(palettes[key]); foreach (PaletteData r in randomP[key]) all.Add(r); FeatureParams fp = extractor.SetupFeatureParams(headers, key, saliencyPattern, debug); foreach (PaletteData data in all) { //calculate the score double rep = 1 - GetAvgDist(new List<PaletteData>{data}, palettes[key]); tempScores.Add(rep); maxScore = Math.Max(rep, maxScore); minScore = Math.Min(rep, minScore); Dictionary<Features, double> features = extractor.CalculateFeatures(data, fp); List<String> featureString = new List<String>(); foreach (Features f in headers) featureString.Add(features[f].ToString()); allFeatures.Add(String.Join(",", featureString.ToArray<String>())); sources.Add(data.id.ToString()); } //normalize the scores foreach (double score in tempScores) scores.Add((score - minScore) / (maxScore - minScore)); List<String> outLines = new List<String>(); //write the scores thus far for (int i = 0; i < scores.Count(); i++) { outLines.Add(sources[i] + "," + scores[i] + "," + allFeatures[i]); } File.AppendAllLines(outFile, outLines.ToArray<String>()); Log(log, key + " - " + DateTime.Now.ToString() + "\n"); File.AppendAllText(imageNameFile, key + "\n"); progress++; backgroundWorker.ReportProgress(100*progress/palettes.Keys.Count(), String.Format("Calculated features for {0} ({1}/{2})",key,progress,palettes.Keys.Count())); } }
/** * We're doing rejection sampling here, so this is pretty slow for getting palettes that are scored very highly or very lowly. * */ private void SaveRandomPalettes(String filename, Dictionary<String, List<PaletteData>> palettes, PaletteExtractor extractor) { int numPalettes = 1000; if (File.Exists(filename) && File.ReadLines(filename).Count()==(palettes.Keys.Count()*numPalettes)+1) { Console.WriteLine("Binned random palettes already generated...skipping"); backgroundWorker.ReportProgress(100, "Random palettes already generated from before...skipping"); Thread.Sleep(1000); return; } int counter = -1; //for each key, generate random palettes and save them //pid, id, image, colors List<String> outLines = new List<String>(); outLines.Add("pid\tid\timage\tcolors\tnumColors\tlog"); int progress = 0; foreach (String key in palettes.Keys) { backgroundWorker.ReportProgress(-1, "Generating random palettes for " + key); List<PaletteData> rand = GenerateRandomPalettes(numPalettes, palettes[key], extractor.GetPaletteSwatches(key)); for (int i = 0; i < rand.Count(); i++) { List<String> colors = new List<String>(); foreach (Color c in rand[i].colors) { colors.Add(c.R + "," + c.G + "," + c.B); } outLines.Add(counter + "\t-1\t" + key + "\t" + String.Join(" ", colors.ToArray<String>()) + "\t5\t"); counter--; } progress++; backgroundWorker.ReportProgress(progress * 100 / palettes.Keys.Count(), "Generated Random Palettes for Image..."+progress+" of "+palettes.Keys.Count()); Log(Path.Combine(trainOutDir, "randlog.txt"), key + "-" + DateTime.Now.ToString()); } File.WriteAllLines(filename, outLines.ToArray<String>()); }