private void extractTheme_Click(object sender, EventArgs e)
{
//Use the Palette Extractor to extract the theme
PaletteExtractor extractor = new PaletteExtractor(dir,weightsDir,json);
String[] files = Directory.GetFiles(dir, "*.png");
String outfile = dir+"\\palettes.tsv";
String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n";
File.WriteAllText(outfile, "");
File.AppendAllText(outfile, headers);
int count = 0;
foreach (String f in files)
{
count++;
String basename = f.Replace(dir+"\\", "");
//The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists
//i.e. if the image filename is A.png, the saliency map filename is A_Judd.png
PaletteData data = extractor.HillClimbPalette(basename, "_Judd", true);
//save to file
String colorString = data.ToString();
File.AppendAllText(outfile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
}
}
/**
* Extract themes from images in the image directory
*/
private void extractTheme_Click(object sender, EventArgs e)
{
if (backgroundWorker.IsBusy)
{
statusBox.Text = "Please wait...";
return;
}
backgroundWorker = MakeBackgroundWorker(delegate
{
backgroundWorker.ReportProgress(0, "Extracting Themes from Image Dir");
//Use the Palette Extractor to extract the theme
PaletteExtractor extractor = new PaletteExtractor(dir, weightsDir, json);
String[] files = GetImageFiles(dir);
String outfile = Path.Combine(dir, "optimized.tsv");
String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n";
File.WriteAllText(outfile, "");
File.AppendAllText(outfile, headers);
int count = 0;
foreach (String f in files)
{
count++;
String basename = new FileInfo(f).Name;
//The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists
//i.e. if the image filename is A.png, the saliency map filename is A_Judd.png
PaletteData data = extractor.HillClimbPalette(basename, "_Judd", debug);
//save to file
String colorString = data.ToString();
File.AppendAllText(outfile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
backgroundWorker.ReportProgress(100 * count / files.Count(),"Finished " + f);
}
});
RunWorker();
}
/**
* Find the theme closest on average to a given set of themes (compare[key])
*/
private PaletteData GetOracleTheme(String key, Dictionary<String, List<PaletteData>> compare)
{
//find the palette that has the lowest distance to all other palettes
int trials = 20;
PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json);
PaletteData swatches = extractor.GetPaletteSwatches(key);
List<CIELAB> shuffled = new List<CIELAB>();
foreach (CIELAB c in swatches.lab)
shuffled.Add(c);
Random random = new Random();
PaletteData best = new PaletteData();
double bestScore = Double.NegativeInfinity;
Stopwatch watch = new Stopwatch();
//Generate all the random starts first
List<PaletteData> starts = new List<PaletteData>();
PaletteData[] allOptions = new PaletteData[trials];
double[] allScores = new double[trials];
for (int t = 0; t < trials; t++)
{
//setup
PaletteData option = new PaletteData();
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
CIELAB temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int i = 0; i < 5; i++)
{
option.lab.Add(shuffled[i]);
option.colors.Add(Util.LABtoRGB(shuffled[i]));
}
starts.Add(option);
allOptions[t] = new PaletteData(option);
double optionScore = 1 - GetAvgDist(new List<PaletteData> { option }, compare[key]);
allScores[t] = optionScore;
}
watch.Restart();
Parallel.For(0, trials, t =>
{
//setup
PaletteData option = new PaletteData(starts[t]);
double optionScore = allScores[t];
//Now hill climb, for each swatch, consider replacing it with a better swatch
//Pick the best replacement, and continue until we reach the top of a hill
int changes = 1;
int iters = 0;
watch.Restart();
while (changes > 0)
{
changes = 0;
for (int i = 0; i < option.lab.Count(); i++)
{
//find the best swatch replacement for this color
double bestTempScore = optionScore;
CIELAB bestRep = option.lab[i];
double[] scores = new double[swatches.lab.Count()];
for (int s = 0; s < swatches.lab.Count(); s++)
{
CIELAB r = swatches.lab[s];
PaletteData temp = new PaletteData(option);
if (!temp.lab.Contains(r))
{
temp.lab[i] = r;
temp.colors[i] = swatches.colors[s];
double tempScore = 1 - GetAvgDist(new List<PaletteData> { temp }, compare[key]);
scores[s] = tempScore;
}
else
{
scores[s] = Double.NegativeInfinity;
}
}
//aggregate results
for (int s = 0; s < scores.Count(); s++)
{
if (scores[s] > bestTempScore)
{
bestTempScore = scores[s];
bestRep = swatches.lab[s];
}
}
if (!option.lab[i].Equals(bestRep))
{
option.lab[i] = bestRep;
optionScore = bestTempScore;
changes++;
}
}
iters++;
}
if (optionScore > allScores[t])
{
allOptions[t] = option;
allScores[t] = optionScore;
}
});
//aggregate scores
for (int i = 0; i < allScores.Count(); i++)
{
if (allScores[i] > bestScore)
{
bestScore = allScores[i];
best = allOptions[i];
}
}
//convert best lab to rgb
best.colors = new List<Color>();
foreach (CIELAB l in best.lab)
best.colors.Add(Util.LABtoRGB(l));
return best;
}
/**
* Extract k-means, c-means, model, and oracle themes for comparison
*/
private void extractThemesToCompare_Click(object sender, EventArgs e)
{
if (backgroundWorker.IsBusy)
{
statusBox.Text = "Please wait...";
return;
}
backgroundWorker.WorkerReportsProgress = true;
backgroundWorker = MakeBackgroundWorker(delegate
{
backgroundWorker.ReportProgress(0, "Extracting themes to compare...");
PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json);
Dictionary<String, Dictionary<String, List<PaletteData>>> referenceP = ItemsToDictionary(reference);
String[] files = GetImageFiles(evalDir);
String headers = "pid\tid\timage\tcolors\tnumColors\tlog\n";
backgroundWorker.ReportProgress(0, "Extracting random themes...");
//extract random
String randomFile = Path.Combine(evalDir, "random.tsv");
if (!File.Exists(randomFile) || File.ReadLines(randomFile).Count() != (40 * files.Count()) + 1)
{
int count = 0;
File.WriteAllText(randomFile, "");
File.AppendAllText(randomFile, headers);
foreach (String f in files)
{
String basename = new FileInfo(f).Name;
PaletteData swatches = extractor.GetPaletteSwatches(basename);
List<PaletteData> random = GenerateRandomPalettesNonBinned(40, swatches);
foreach (PaletteData data in random)
{
count++;
String colorString = data.ToString();
File.AppendAllText(randomFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
}
}
}
else
{
Console.WriteLine("Random palettes file already exists");
backgroundWorker.ReportProgress(100, "Random themes file already exist...skipping");
Thread.Sleep(1000);
}
backgroundWorker.ReportProgress(0, "Extracting K-means themes...");
//extract kmeans
String kmeansFile = Path.Combine(evalDir, "kmeans.tsv");
if (!File.Exists(kmeansFile) || File.ReadLines(kmeansFile).Count() != files.Count() + 1)
{
File.WriteAllText(kmeansFile, "");
File.AppendAllText(kmeansFile, headers);
int count = 0;
foreach (String f in files)
{
count++;
String basename = new FileInfo(f).Name;
PaletteData data = KMeansPalette(f);
//save to file
String colorString = data.ToString();
File.AppendAllText(kmeansFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
backgroundWorker.ReportProgress(100 * count / files.Count(), "");
}
}
else
{
Console.WriteLine("Kmeans palettes file already exists");
backgroundWorker.ReportProgress(100, "K-means themes already exist...skipping");
Thread.Sleep(1000);
}
//extract cmeans
backgroundWorker.ReportProgress(0, "Extracting C-means themes...");
String cmeansFile = Path.Combine(evalDir, "cmeans.tsv");
if (!File.Exists(cmeansFile) || File.ReadLines(cmeansFile).Count() != files.Count() + 1)
{
File.WriteAllText(cmeansFile, "");
File.AppendAllText(cmeansFile, headers);
int count = 0;
foreach (String f in files)
{
count++;
String basename = new FileInfo(f).Name;
PaletteData data = CMeansPalette(f);
//save to file
String colorString = data.ToString();
File.AppendAllText(cmeansFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
backgroundWorker.ReportProgress(100 * count / files.Count(), "");
}
}
else
{
Console.WriteLine("Cmeans palettes file already exists");
backgroundWorker.ReportProgress(100, "C-means themes already exist...skipping");
Thread.Sleep(1000);
}
//extract model
backgroundWorker.ReportProgress(0, "Extracting Optimized themes...");
String modelFile = Path.Combine(evalDir, "optimized.tsv");
if (!File.Exists(modelFile) || File.ReadLines(modelFile).Count() != files.Count() + 1)
{
File.WriteAllText(modelFile, "");
File.AppendAllText(modelFile, headers);
int count = 0;
foreach (String f in files)
{
count++;
String basename = new FileInfo(f).Name;
//The saliency pattern "_Judd" is just an additional annotation after the image filename if it exists
//i.e. if the image filename is A.png, the saliency map filename is A_Judd.png
PaletteData data = extractor.HillClimbPalette(basename, "_Judd", debug);
//save to file
String colorString = data.ToString();
File.AppendAllText(modelFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
backgroundWorker.ReportProgress(100 * count / files.Count(), "");
}
}
else
{
Console.WriteLine("Optimized palettes file already exists");
backgroundWorker.ReportProgress(100, "Optimized themes already exist...skipping");
Thread.Sleep(1000);
}
//extract oracle
foreach (String name in referenceP.Keys)
{
backgroundWorker.ReportProgress(0, "Extracting Oracle themes for..." + name);
String oracleFile = Path.Combine(evalDir, "oracle-" + name + ".tsv");
if (!File.Exists(oracleFile) || File.ReadLines(oracleFile).Count() != referenceP[name].Keys.Count() + 1)
{
File.WriteAllText(oracleFile, "");
File.AppendAllText(oracleFile, headers);
int count = 0;
foreach (String f in files)
{
String basename = new FileInfo(f).Name;
if (!referenceP[name].ContainsKey(basename))
continue;
count++;
PaletteData data = GetOracleTheme(basename, referenceP[name]);
//save to file
String colorString = data.ToString();
File.AppendAllText(oracleFile, count + "\t-1\t" + basename + "\t" + colorString + "\t5\t\n");
backgroundWorker.ReportProgress(100 * count / files.Count(), "");
}
}
else
{
Console.WriteLine("Oracle palettes file already exists");
backgroundWorker.ReportProgress(100, "Oracle themes already exist...skipping");
Thread.Sleep(1000);
}
}
});
RunWorker();
}
/**
* Create diagrams of the themes showing source image, swatches, and aligned themes
*/
private void diagramThemes_Click(object sender, EventArgs e)
{
if (backgroundWorker.IsBusy)
{
statusBox.Text = "Please wait...";
return;
}
backgroundWorker = MakeBackgroundWorker(delegate
{
backgroundWorker.ReportProgress(0, "Diagramming themes...");
//create directory
String diagramDir = Path.Combine(evalDir, "diagrams");
Directory.CreateDirectory(diagramDir);
Dictionary<String, Dictionary<String, List<PaletteData>>> leftP = ItemsToDictionary(diagramLeft);
Dictionary<String, Dictionary<String, List<PaletteData>>> rightP = ItemsToDictionary(diagramRight);
int imageWidth = 200;
int colorSize = 20;
int padding = 5;
int gutter = 50;
int labelTop = 100;
PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json);
SortedSet<String> keys = new SortedSet<String>();
foreach (String key in leftP.Keys)
keys.UnionWith(leftP[key].Keys);
foreach (String key in rightP.Keys)
keys.UnionWith(rightP[key].Keys);
int progress = 0;
foreach (String key in keys)
{
//check total number of themes to draw
int totalThemes = 0;
PaletteData swatches = extractor.GetPaletteSwatches(key);
//Count total number of themes, and sort the themes by average distance
foreach (String n in leftP.Keys)
{
if (!leftP[n].ContainsKey(key))
continue;
totalThemes += leftP[n][key].Count();
leftP[n][key] = SortByDist(leftP[n][key]);
}
foreach (String n in rightP.Keys)
{
if (!rightP[n].ContainsKey(key))
continue;
totalThemes += rightP[n][key].Count();
rightP[n][key] = SortByDist(rightP[n][key]);
}
int width = imageWidth + (leftP.Keys.Count() + rightP.Keys.Count() + 2) * gutter + colorSize * totalThemes;
int height = labelTop + swatches.colors.Count()*colorSize;
Bitmap result = new Bitmap(width, height);
Graphics g = Graphics.FromImage(result);
Bitmap image = new Bitmap(Path.Combine(evalDir, key));
//draw the source image
g.DrawImage(image, 0, labelTop, imageWidth, imageWidth*image.Height / image.Width);
int startX = imageWidth+gutter;
//draw the themes to the left of the swatches
foreach (EvalItem item in diagramLeft)
{
String name = item.Name;
if (!leftP[name].ContainsKey(key))
continue;
DrawThemes(g, name, leftP[name][key], swatches, startX, labelTop, colorSize, padding);
startX += gutter + leftP[name][key].Count()*colorSize;
}
//draw the swatches
for (int c = 0; c < swatches.colors.Count; c++)
{
int y = c * colorSize+labelTop;
g.FillRectangle(new SolidBrush(swatches.colors[c]), startX, y, colorSize - padding, colorSize - padding);
g.DrawRectangle(new Pen(Color.Black), startX, y, colorSize - padding, colorSize - padding);
}
startX += colorSize+gutter;
//draw the themes to the right of the swatches
foreach (EvalItem item in diagramRight)
{
String name = item.Name;
if (!rightP[name].ContainsKey(key))
continue;
DrawThemes(g, name, rightP[name][key], swatches, startX, labelTop, colorSize, padding);
startX += gutter+ rightP[name][key].Count()*colorSize;
}
result.Save(Path.Combine(diagramDir, key));
progress++;
backgroundWorker.ReportProgress(100*progress / keys.Count, "Diagrammed " + key);
}
});
RunWorker();
}
/**
* Calculate average distance between themes from different sources, and average overlap over different thresholds
*/
private void compareThemes_Click(object sender, EventArgs e)
{
if (backgroundWorker.IsBusy)
{
statusBox.Text = "Please wait...";
return;
}
backgroundWorker.WorkerReportsProgress = true;
backgroundWorker = MakeBackgroundWorker(delegate
{
backgroundWorker.ReportProgress(0, "Comparing themes...");
//load the reference palettes
//load the palettes to compare against
Dictionary<String, Dictionary<String, List<PaletteData>>> referenceP = ItemsToDictionary(reference);
Dictionary<String, Dictionary<String, List<PaletteData>>> compareP = ItemsToDictionary(compare);
Dictionary<String, List<PaletteData>> testKeys = LoadFilePalettes(refkeys);
//compare each source in compare to each reference
List<String> desc = new List<String>();
List<double> dists = new List<double>();
foreach (var r in reference)
{
foreach (var c in compare)
{
desc.Add(c.Name + "-" + r.Name);
dists.Add(0);
}
}
int progress = 0;
foreach (String key in testKeys.Keys)
{
int idx = 0;
foreach (var r in reference)
{
foreach (var c in compare)
{
//count number of keys in common between reference, compare, and test set
if (referenceP[r.Name].ContainsKey(key) && compareP[c.Name].ContainsKey(key))
{
double factor = referenceP[r.Name].Keys.Intersect(compareP[c.Name].Keys).Intersect(testKeys.Keys).Count();
dists[idx] += GetAvgDist(referenceP[r.Name][key], compareP[c.Name][key], c.Path == r.Path) / factor;
}
idx++;
}
}
progress++;
backgroundWorker.ReportProgress(50 * progress/testKeys.Keys.Count(), "Comparing themes...Calculating distance...");
}
//Write out the average distances
List<String> lines = new List<String>();
lines.Add(String.Join(",", desc.ToArray<String>()));
lines.Add(String.Join(",", dists.Select(i => i.ToString())));
File.WriteAllLines(Path.Combine(evalDir, "avgDists.csv"), lines.ToArray<String>());
backgroundWorker.ReportProgress(50, "Comparing themes...Calculating overlap...");
//Output csv file for the overlap graph
//clamp each palette to the closest color swatch
//when computing overlap for the graph
PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json);
foreach (String key in testKeys.Keys)
{
PaletteData swatches = extractor.GetPaletteSwatches(key);
foreach (var r in reference)
{
if (!referenceP[r.Name].ContainsKey(key))
continue;
var palettes = referenceP[r.Name][key];
for (int i = 0; i < palettes.Count(); i++)
palettes[i] = SnapColors(palettes[i], swatches);
}
foreach (var c in compare)
{
if (!compareP[c.Name].ContainsKey(key))
continue;
var palettes = compareP[c.Name][key];
for (int i = 0; i < palettes.Count(); i++)
palettes[i] = SnapColors(palettes[i], swatches);
}
}
lines.Clear();
//compute the overlaps for different thresholds
lines.Add("thresh,type,overlap");
for (int t = 1; t < 100; t += 2)
{
double[] overlaps = new double[reference.Count() * compare.Count()];
int idx = 0;
foreach (var r in reference)
{
foreach (var c in compare)
{
foreach (String key in testKeys.Keys)
if (compareP[c.Name].ContainsKey(key) && referenceP[r.Name].ContainsKey(key))
overlaps[idx] += GetAvgOverlap(compareP[c.Name][key], referenceP[r.Name][key], t, c.Path == r.Path);
double factor = referenceP[r.Name].Keys.Intersect(compareP[c.Name].Keys).Intersect(testKeys.Keys).Count();
lines.Add(t + "," + desc[idx] + "," + overlaps[idx] / factor);
idx++;
}
}
backgroundWorker.ReportProgress(t/2+50, String.Format("Comparing themes...Calculating overlap for thresh {0}/{1}...",t,100));
}
File.WriteAllLines(Path.Combine(evalDir, "overlaps.csv"), lines.ToArray<String>());
});
RunWorker();
}
private void CalculateFeatures()
{
backgroundWorker.ReportProgress(0, "First generating binned random palettes...checking for existing file");
PaletteExtractor extractor = new PaletteExtractor(trainInDir, weightsDir, json);
Dictionary<String, List<PaletteData>> palettes = LoadFilePalettes(trainReference.Path);
if (trainReference.Filter)
{
var keys = palettes.Keys.ToArray<String>();
foreach (String key in keys)
palettes[key] = removeOutliers(palettes[key]);
}
SaveRandomPalettes(Path.Combine(trainOutDir, "random-binned.tsv"), palettes, extractor);
Dictionary<String, List<PaletteData>> randomP = LoadFilePalettes(Path.Combine(trainOutDir, "random-binned.tsv"));
ClearLog(Path.Combine(trainOutDir, "timelog.txt"));
Features[] included = Enum.GetValues(typeof(Features)).Cast<Features>().ToArray<Features>();
SortedSet<Features> headers = new SortedSet<Features>(included);
String saliencyPattern = "_Judd";
String imageNameFile = Path.Combine(trainOutDir, "imageNames.txt");
String outFile = Path.Combine(trainOutDir, "features.csv");
File.WriteAllText(outFile, ""); //clear the file
File.WriteAllText(imageNameFile, "");
//write the headings in a separate file
File.WriteAllText(Path.Combine(trainOutDir, "featurenames.txt"), String.Join("\n", headers.Select<Features, String>((a) => featureName.Name(a))));
String log = Path.Combine(trainOutDir,"log.txt");
Log(log, "Start");
backgroundWorker.ReportProgress(0, "Calculating features...");
int progress = 0;
foreach (String key in palettes.Keys)
{
List<String> sources = new List<String>();
List<String> allFeatures = new List<String>();
List<double> scores = new List<double>();
List<double> tempScores = new List<double>();
double maxScore = Double.NegativeInfinity;
double minScore = Double.PositiveInfinity;
List<PaletteData> all = new List<PaletteData>(palettes[key]);
foreach (PaletteData r in randomP[key])
all.Add(r);
FeatureParams fp = extractor.SetupFeatureParams(headers, key, saliencyPattern, debug);
foreach (PaletteData data in all)
{
//calculate the score
double rep = 1 - GetAvgDist(new List<PaletteData>{data}, palettes[key]);
tempScores.Add(rep);
maxScore = Math.Max(rep, maxScore);
minScore = Math.Min(rep, minScore);
Dictionary<Features, double> features = extractor.CalculateFeatures(data, fp);
List<String> featureString = new List<String>();
foreach (Features f in headers)
featureString.Add(features[f].ToString());
allFeatures.Add(String.Join(",", featureString.ToArray<String>()));
sources.Add(data.id.ToString());
}
//normalize the scores
foreach (double score in tempScores)
scores.Add((score - minScore) / (maxScore - minScore));
List<String> outLines = new List<String>();
//write the scores thus far
for (int i = 0; i < scores.Count(); i++)
{
outLines.Add(sources[i] + "," + scores[i] + "," + allFeatures[i]);
}
File.AppendAllLines(outFile, outLines.ToArray<String>());
Log(log, key + " - " + DateTime.Now.ToString() + "\n");
File.AppendAllText(imageNameFile, key + "\n");
progress++;
backgroundWorker.ReportProgress(100*progress/palettes.Keys.Count(), String.Format("Calculated features for {0} ({1}/{2})",key,progress,palettes.Keys.Count()));
}
}
/**
* We're doing rejection sampling here, so this is pretty slow for getting palettes that are scored very highly or very lowly.
* */
private void SaveRandomPalettes(String filename, Dictionary<String, List<PaletteData>> palettes, PaletteExtractor extractor)
{
int numPalettes = 1000;
if (File.Exists(filename) && File.ReadLines(filename).Count()==(palettes.Keys.Count()*numPalettes)+1)
{
Console.WriteLine("Binned random palettes already generated...skipping");
backgroundWorker.ReportProgress(100, "Random palettes already generated from before...skipping");
Thread.Sleep(1000);
return;
}
int counter = -1;
//for each key, generate random palettes and save them
//pid, id, image, colors
List<String> outLines = new List<String>();
outLines.Add("pid\tid\timage\tcolors\tnumColors\tlog");
int progress = 0;
foreach (String key in palettes.Keys)
{
backgroundWorker.ReportProgress(-1, "Generating random palettes for " + key);
List<PaletteData> rand = GenerateRandomPalettes(numPalettes, palettes[key], extractor.GetPaletteSwatches(key));
for (int i = 0; i < rand.Count(); i++)
{
List<String> colors = new List<String>();
foreach (Color c in rand[i].colors)
{
colors.Add(c.R + "," + c.G + "," + c.B);
}
outLines.Add(counter + "\t-1\t" + key + "\t" + String.Join(" ", colors.ToArray<String>()) + "\t5\t");
counter--;
}
progress++;
backgroundWorker.ReportProgress(progress * 100 / palettes.Keys.Count(), "Generated Random Palettes for Image..."+progress+" of "+palettes.Keys.Count());
Log(Path.Combine(trainOutDir, "randlog.txt"), key + "-" + DateTime.Now.ToString());
}
File.WriteAllLines(filename, outLines.ToArray<String>());
}