public ColorTemplate(Bitmap image, Segmentation seg, PaletteData palette)
{
slots = seg; //TODO: need to copy the segmentation?
originalPalette = new PaletteData(palette);
int width = image.Width;
int height = image.Height;
template = new CIELAB[width, height];
//initialize the template
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
template[i, j] = Util.RGBtoLAB(image.GetPixel(i, j));
}
}
segToColor = new int[seg.numSegments];
//map each segment to the closest palette color, based on its mean color
//TODO: if two adjacent segments have the same mean color, we could merge them...
for (int i = 0; i < seg.numSegments; i++)
{
CIELAB lab = seg.segToMeanColor[i];
int bestColor = -1;
double bestDist = Double.PositiveInfinity;
for (int j = 0; j < palette.lab.Count(); j++)
{
double dist = palette.lab[j].SqDist(lab);
if (dist < bestDist)
{
bestDist = dist;
bestColor = j;
}
}
segToColor[i] = bestColor;
}
//subtract the mean color from the template
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int coloridx = segToColor[slots.assignments[i, j]];
template[i, j] -= palette.lab[coloridx];
//template[i, j] /= palette.lab[coloridx];
}
}
}
CIELAB[,] template; //relative LAB difference (or change to RGB difference?)
#endregion Fields
#region Constructors
public ColorTemplate(Bitmap image, PaletteData palette)
{
//create the segmentation based on the palette
//TODO: There's a problem that sometimes a palette color won't appear in the image (perhaps due to color blending), and so the slot will have no pixels associated with it
int width = image.Width;
int height = image.Height;
slots = new Segmentation(palette.colors.Count(), width, height);
segToColor = new int[slots.numSegments];
template = new CIELAB[width, height];
CIELAB[,] recolored = ModeRecolor(Util.Map<Color, CIELAB>(Util.BitmapToArray(image), c => Util.RGBtoLAB(c)), palette);
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
template[i, j] = new CIELAB();
int bestIdx = -1;
CIELAB lab = recolored[i, j];
double bestDist = Double.PositiveInfinity;
for (int c = 0; c < palette.lab.Count(); c++)
{
double dist = palette.lab[c].SqDist(lab);
if (dist < bestDist)
{
bestDist = dist;
bestIdx = c;
if (dist < 0.00001)
break;
}
}
slots.assignments[i, j] = bestIdx;
slots.counts[bestIdx]++;
segToColor[bestIdx] = bestIdx;
}
}
originalPalette = new PaletteData(palette);
}
//render an image of the original pattern, plus columns of patterns generated by different models (trained on diff styles)
private void VisAllStyles()
{
Directory.CreateDirectory(outdir + "\\stylecolor\\");
Directory.CreateDirectory(outdir + "\\styles\\");
//read in the patterns and save out their layers
List<PatternItem> patterns = PatternIO.GetPatterns(Path.Combine(imagedir,"../styles"));
foreach (PatternItem p in patterns)
{
String basename = p.Name;
//Read the style description if available
String specs = Path.Combine(outdir, "styles", p.Directory, Util.ConvertFileName(basename, "", ".txt"));
if (!File.Exists(specs))
continue;
Bitmap image = new Bitmap(p.FullPath);
if (!palettes.ContainsKey(basename))
continue;
PaletteData palette = palettes[basename];
ColorTemplate template = new ColorTemplate(image, palette);
int[] slotToColor = new int[template.NumSlots()];
for (int i = 0; i < slotToColor.Count(); i++)
slotToColor[i] = i;
List<List<String>> lines = File.ReadAllLines(specs).Select(line => line.Split(',').ToList<String>()).ToList<List<String>>();
if (lines.Count() == 0)
continue;
Dictionary<String, int> styleToRowIdx = new Dictionary<String, int>();
String origStyle = "";
foreach (List<String> line in lines)
{
origStyle = line[0];
if (!styleToRowIdx.ContainsKey(line[1]))
styleToRowIdx.Add(line[1], 0);
}
//read the score and if it's the original or not
double score = 0;
int iwidth = 100;
int iheight = 100;
int padding = 15;
int headerSpace = 30;
int width = (styleToRowIdx.Keys.Count() + 1) * iwidth;
int height = (lines.Count() / styleToRowIdx.Keys.Count()) * iheight + headerSpace;
Font font = new Font("Arial", 10);
Bitmap vis = new Bitmap(width, height);
Graphics g = Graphics.FromImage(vis);
//write out the headings, highlight the original style heading
var headers = styleToRowIdx.Keys.ToList<String>();
int ncol = headers.Count()+1;
Color headerColor = Color.Black;
g.DrawString("original", font, new SolidBrush(headerColor), 0, 0);
for (int i=0; i<headers.Count(); i++)
{
if (headers[i] == origStyle)
g.DrawString(headers[i], font, new SolidBrush(headerColor), (i+1)*iwidth, 0);
else
g.DrawString(headers[i], font, new SolidBrush(headerColor), (i+1)*iwidth, 0);
}
//draw the original
Bitmap original = (renderFinal)? image: template.DebugQuantization();
g.DrawImage(original, 0, headerSpace, iwidth-padding, iheight-padding);
original.Dispose();
PaletteData data = new PaletteData();
Dictionary<int, int> groupToSlot = new Dictionary<int, int>();
int ngroups = 0;
for (int i = 0; i < slotToColor.Count(); i++)
{
slotToColor[i] = i;
data.colors.Add(new Color());
data.lab.Add(new CIELAB());
if (template.PixelsInSlot(i) > 0)
groupToSlot.Add(ngroups++, i);
}
foreach (List<String> line in lines)
{
//draw the colorized thumbnail in the right location
String style = line[1];
String[] colors = line[2].Split('^');
int colorIdx = 0;
foreach (String color in colors)
{
//rgb floats
int[] fields = color.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries).Select<String, int>(s => ((int)(Math.Round(double.Parse(s) * 255)))).ToArray<int>();
Color c = Color.FromArgb(fields[0], fields[1], fields[2]);
data.colors[groupToSlot[colorIdx]] = c;
data.lab[groupToSlot[colorIdx]] = Util.RGBtoLAB(c);
colorIdx++;
}
int x = (headers.IndexOf(style)+1)*iwidth;
int y = styleToRowIdx[style]*iheight+headerSpace;
styleToRowIdx[style]++;
Bitmap result = (renderFinal)?GetFinalRendering(Util.ConvertFileName(basename, "",""), data):template.SolidColor(data, slotToColor);
//sometimes we can't get the nice image, so render the quantized image in this case
if (result == null)
result = template.SolidColor(data, slotToColor);
g.DrawImage(result, x, y, iwidth-padding, iheight-padding);
result.Dispose();
}
PatternIO.SavePattern(vis, p, Path.Combine(outdir, "stylecolor"));
vis.Dispose();
}
}
private double PaletteDistanceHungarian(PaletteData a, PaletteData b, ref int[] matching, bool print, int thresh = 1000)
{
double score = 0;
int maxDist = thresh;
//Make a the smaller palette
if (a.colors.Count() > b.colors.Count())
{
//swap
PaletteData temp = a;
a = b;
b = temp;
}
//do the matching
List<int> foundMatching = HungarianAlgorithmColors(a.lab, b.lab, maxDist, false, print);
matching = foundMatching.ToArray<int>();
Debug.Assert(foundMatching.Count() == a.colors.Count());
//find the score
for (int i = 0; i < foundMatching.Count(); i++)
{
int idx = foundMatching[i];
if (idx < b.lab.Count())
{
double dist = Math.Sqrt(a.lab[i].SqDist(b.lab[idx]));
score += dist;
}
else
{
score += maxDist;
matching[i] = -1;
}
}
int nullDistances = (b.colors.Count() - a.colors.Count()) * maxDist;
//normalize between 0 and 1
return ((score + nullDistances) / b.lab.Count()) / maxDist;
}
public double PaletteImageScore(PaletteData palette, String key, String saliencyPattern, bool debug = false)
{
double score = 0;
//weights
Dictionary<Features, double> weights = new Dictionary<Features, double>();
weights = LoadWeights(weightsDir + "/weights-final-no.csv", weightsDir + "/featurenames-all.txt");
SortedSet<Features> included = new SortedSet<Features>(weights.Keys);
FeatureParams fparams = SetupFeatureParams(included, key, saliencyPattern, debug, 1);
score = ScorePalette(weights, CalculateFeatures(palette, fparams));
return score;
}
public PaletteData(PaletteData other)
{
colors = new List<Color>(other.colors);
lab = new List<CIELAB>(other.lab);
id = other.id;
}
private CIELAB[,] EdgeUnmixRecolor(CIELAB[,] image, PaletteData palette)
{
int width = image.GetLength(0);
int height = image.GetLength(1);
CIELAB[,] result = new CIELAB[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double[] scores = new double[palette.colors.Count()];
//check if the color is in the palette
CIELAB color = image[i, j];
if (palette.lab.Contains(color))
{
result[i, j] = color;
continue;
}
//otherwise, find the best color
//see if this is an edge
//if it is, assign the color to be one of the edge color candidates
HashSet<CIELAB> tbCandidates = new HashSet<CIELAB>();
HashSet<CIELAB> lrCandidates = new HashSet<CIELAB>();
List<HashSet<CIELAB>> unmixCandidates = new List<HashSet<CIELAB>> { tbCandidates, lrCandidates };
//check edge types
//horizontal and vertical
HashSet<CIELAB> oneSide = new HashSet<CIELAB>();
HashSet<CIELAB> otherSide = new HashSet<CIELAB>();
Point[] top = new Point[] { new Point(i - 1, j - 1), new Point(i, j - 1), new Point(i + 1, j - 1) };
Point[] bottom = new Point[] { new Point(i - 1, j + 1), new Point(i - 1, j), new Point(i - 1, j + 1) };
Point[] left = new Point[] { new Point(i - 1, j - 1), new Point(i - 1, j), new Point(i - 1, j + 1) };
Point[] right = new Point[] { new Point(i + 1, j - 1), new Point(i + 1, j), new Point(i + 1, j + 1) };
List<Point[]> oneCompare = new List<Point[]> { top, left };
List<Point[]> otherCompare = new List<Point[]> { bottom, right };
bool edge = false;
for (int c = 0; c < oneCompare.Count(); c++)
{
oneSide.Clear();
otherSide.Clear();
foreach (Point p in oneCompare[c])
{
if (Util.InBounds(p.X, p.Y, width, height))
{
CIELAB rc = ClosestColor(image[p.X, p.Y], palette.lab);
//check if in the set
if (oneSide.Contains(rc))
{
//yes, we found a majority
unmixCandidates[c].Add(image[p.X, p.Y]);
break;
}
else
{
oneSide.Add(rc);
}
}
}
foreach (Point p in otherCompare[c])
{
if (Util.InBounds(p.X, p.Y, width, height))
{
CIELAB rc = ClosestColor(image[p.X, p.Y], palette.lab);
//check if in the set
if (otherSide.Contains(rc))
{
//yes, we found a majority
unmixCandidates[c].Add(rc);
break;
}
else
{
otherSide.Add(rc);
}
}
}
//is it an edge?
if (unmixCandidates[c].Count() >= 2)
{
result[i, j] = ClosestColor(image[i, j], unmixCandidates[c].ToList<CIELAB>());
edge = true;
break;
}
}
//TODO:
//45 degrees
//45 degrees-flipped
if (!edge)
{
for (int dx = -1; dx <= 1; dx++)
{
for (int dy = -1; dy <= 1; dy++)
{
int x = i + dx;
int y = j + dy;
double weight = (dx == 0 && dy == 0) ? 4 : 1;
if (Util.InBounds(x, y, width, height))
{
int bestc = ClosestColorIndex(image[x, y], palette.lab);
scores[bestc] -= weight;
}
}
}
//pick the color with the min score
double minScore = Double.PositiveInfinity;
int bestIdx = 0;
for (int c = 0; c < palette.colors.Count(); c++)
{
if (scores[c] < minScore)
{
minScore = scores[c];
bestIdx = c;
}
}
result[i, j] = palette.lab[bestIdx];
}
}
}
return result;
}
/**
* Randomly pick image swatches, and return the theme
*/
private List<PaletteData> GenerateRandomPalettesNonBinned(int count, PaletteData swatches)
{
int k = 5;
List<PaletteData> data = new List<PaletteData>();
Random random = new Random();
List<int> shuffled = new List<int>();
for (int i = 0; i < swatches.lab.Count(); i++)
shuffled.Add(i);
for (int i = 0; i < count; i++)
{
PaletteData result = new PaletteData();
result.id = -1;
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
int temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int c = 0; c < k; c++)
{
result.colors.Add(swatches.colors[shuffled[c]]);
result.lab.Add(swatches.lab[shuffled[c]]);
}
data.Add(result);
}
return data;
}
/**
* Draw the themes aligned to the swatches, assume themes are already sorted in the right order
*/
private void DrawThemes(Graphics g, String name, List<PaletteData> themes, PaletteData swatches, int startX, int labelTop, int colorSize, int padding)
{
g.DrawString(name, new Font("Arial", 12.0f), new SolidBrush(Color.Black), startX+(colorSize*(themes.Count()-1)/2) - colorSize/2, labelTop/2);
int x = startX;
int y = labelTop;
for (int i = 0; i < themes.Count(); i++)
{
int[] matching = new int[themes[i].colors.Count];
PaletteDistanceHungarian(themes[i], swatches, ref matching, false);
for (int c = 0; c < themes[i].colors.Count; c++)
{
if (matching[c] < 0)
{
backgroundWorker.ReportProgress(-1, "Error: Some elements not matched");
continue;
}
y = labelTop + matching[c] * colorSize;
g.FillRectangle(new SolidBrush(themes[i].colors[c]), x, y, colorSize - padding, colorSize - padding);
g.DrawRectangle(new Pen(Color.Black), x, y, colorSize - padding, colorSize - padding);
}
x = startX + colorSize * (i+1);
}
}
private List<PaletteData> GenerateRandomPalettes(int count, List<PaletteData> palettes, PaletteData swatches)
{
PaletteScoreCache cache = new PaletteScoreCache(1000000);
int k = 5;
List<PaletteData> data = new List<PaletteData>();
Random random = new Random();
List<int> shuffled = new List<int>();
for (int i = 0; i < swatches.lab.Count(); i++)
shuffled.Add(i);
//find the max scoring human palette
double maxScore = Double.NegativeInfinity;
foreach (PaletteData p in palettes)
{
double score = 1 - GetAvgDist(new List<PaletteData> { p }, palettes);
maxScore = Math.Max(maxScore, score);
}
//generate random palettes to find the min score
double minScore = Double.PositiveInfinity;
for (int i = 0; i < 10000; i++)
{
PaletteData result = new PaletteData();
result.id = -1;
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
int temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int c = 0; c < k; c++)
{
result.colors.Add(swatches.colors[shuffled[c]]);
result.lab.Add(swatches.lab[shuffled[c]]);
}
double score = 1 - GetAvgDist(new List<PaletteData>{result}, palettes);
cache.SetScore(result.lab, score);
minScore = Math.Min(minScore, score);
maxScore = Math.Max(maxScore, score);
}
int numbins = 10;
int[] counts = new int[numbins];
int totalCount = 0;
int thresh = count / numbins;
int tries = 0;
//generate random palettes, given swatches
while (totalCount < count)
{
if (tries % 1000 == 0)
{
String log = Path.Combine(trainOutDir, "randlog.txt");
Log(log, tries + "-" + totalCount + " binned ");
Log(log, " bins --- " + counts[0] + "," + counts[1] + "," + counts[2] + "," + counts[3] + "," + counts[4] + "," + counts[5] + "," + counts[6] + "," + counts[7] + "," + counts[8] + "," + counts[9]);
backgroundWorker.ReportProgress(-1,"bin counts " + String.Join(",",counts));
}
tries++;
PaletteData result = new PaletteData();
result.id = -1;
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
int temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int c = 0; c < k; c++)
{
result.colors.Add(swatches.colors[shuffled[c]]);
result.lab.Add(swatches.lab[shuffled[c]]);
}
double score = 0;
if (cache.ContainsKey(result.lab))
score = cache.GetScore(result.lab);
else
{
score = 1 - GetAvgDist(new List<PaletteData> { result }, palettes);
cache.SetScore(result.lab, score);
}
score = (score - minScore) / (maxScore - minScore);
int bin = (int)Math.Min(Math.Max(0, Math.Floor(score * numbins)), numbins - 1);
if (counts[bin] >= thresh)
continue;
totalCount++;
counts[bin]++;
data.Add(result);
}
return data;
}
/**
* Run c-means clustering on the given image file, and return the theme
*/
private PaletteData CMeansPalette(String file, int k=5)
{
PaletteData result = new PaletteData();
List<CIELAB> colors = Util.BitmapTo1DArray(Util.GetImage(file,debug)).Select(c => Util.RGBtoLAB(c)).ToList<CIELAB>();
List<Cluster> final = null;
double bestScore = Double.PositiveInfinity;
int trials = 5;
for (int t = 0; t < trials; t++)
{
List<Cluster> seeds = Clustering.InitializePictureSeeds(colors, k);
double score = Clustering.CMeansPicture(colors, seeds);
if (score < bestScore)
{
final = seeds;
bestScore = score;
}
}
foreach (Cluster c in final)
{
result.colors.Add(Util.LABtoRGB(c.lab));
result.lab.Add(c.lab);
}
return result;
}
/**
* Return a theme with colors snapped to the given swatches
*/
private PaletteData SnapColors(PaletteData data, PaletteData swatches)
{
PaletteData result = new PaletteData();
List<CIELAB> matchedColors = new List<CIELAB>();
//match each color to closest swatch color
foreach (CIELAB kc in data.lab)
{
double bestDist = Double.PositiveInfinity;
CIELAB best = new CIELAB();
foreach (CIELAB s in swatches.lab)
{
double dist = s.SqDist(kc);
if (dist < bestDist)
{
bestDist = dist;
best = s;
}
}
matchedColors.Add(best);
}
result.lab = matchedColors;
foreach (CIELAB l in data.lab)
result.colors.Add(Util.LABtoRGB(l));
return result;
}
private double PaletteOverlap(PaletteData a, PaletteData b, int[] matching)
{
//return number of colors in common over the number of colors in the larger palette
int common = 0;
foreach (int idx in matching)
if (idx != -1)
common++;
return common / (double)(Math.Max(a.colors.Count(), b.colors.Count()));
}
private Dictionary<String, List<PaletteData>> LoadFilePalettes(String file)
{
//load art palettes
String[] lines = File.ReadAllLines(file);
Dictionary<String, List<PaletteData>> plist = new Dictionary<String, List<PaletteData>>();
for (int i = 1; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split('\t');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[0]);
data.workerNum = Int32.Parse(fields[1]);
String key = fields[2];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
foreach (String s in colors)
{
String[] comp = s.Split(',');
Color c = Color.FromArgb(Int32.Parse(comp[0]), Int32.Parse(comp[1]), Int32.Parse(comp[2]));
CIELAB l = Util.RGBtoLAB(c);
data.colors.Add(c);
data.lab.Add(l);
}
if (!plist.ContainsKey(key))
plist.Add(key, new List<PaletteData>());
plist[key].Add(data);
}
return plist;
}
/**
* Find the theme closest on average to a given set of themes (compare[key])
*/
private PaletteData GetOracleTheme(String key, Dictionary<String, List<PaletteData>> compare)
{
//find the palette that has the lowest distance to all other palettes
int trials = 20;
PaletteExtractor extractor = new PaletteExtractor(evalDir, weightsDir, json);
PaletteData swatches = extractor.GetPaletteSwatches(key);
List<CIELAB> shuffled = new List<CIELAB>();
foreach (CIELAB c in swatches.lab)
shuffled.Add(c);
Random random = new Random();
PaletteData best = new PaletteData();
double bestScore = Double.NegativeInfinity;
Stopwatch watch = new Stopwatch();
//Generate all the random starts first
List<PaletteData> starts = new List<PaletteData>();
PaletteData[] allOptions = new PaletteData[trials];
double[] allScores = new double[trials];
for (int t = 0; t < trials; t++)
{
//setup
PaletteData option = new PaletteData();
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
CIELAB temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int i = 0; i < 5; i++)
{
option.lab.Add(shuffled[i]);
option.colors.Add(Util.LABtoRGB(shuffled[i]));
}
starts.Add(option);
allOptions[t] = new PaletteData(option);
double optionScore = 1 - GetAvgDist(new List<PaletteData> { option }, compare[key]);
allScores[t] = optionScore;
}
watch.Restart();
Parallel.For(0, trials, t =>
{
//setup
PaletteData option = new PaletteData(starts[t]);
double optionScore = allScores[t];
//Now hill climb, for each swatch, consider replacing it with a better swatch
//Pick the best replacement, and continue until we reach the top of a hill
int changes = 1;
int iters = 0;
watch.Restart();
while (changes > 0)
{
changes = 0;
for (int i = 0; i < option.lab.Count(); i++)
{
//find the best swatch replacement for this color
double bestTempScore = optionScore;
CIELAB bestRep = option.lab[i];
double[] scores = new double[swatches.lab.Count()];
for (int s = 0; s < swatches.lab.Count(); s++)
{
CIELAB r = swatches.lab[s];
PaletteData temp = new PaletteData(option);
if (!temp.lab.Contains(r))
{
temp.lab[i] = r;
temp.colors[i] = swatches.colors[s];
double tempScore = 1 - GetAvgDist(new List<PaletteData> { temp }, compare[key]);
scores[s] = tempScore;
}
else
{
scores[s] = Double.NegativeInfinity;
}
}
//aggregate results
for (int s = 0; s < scores.Count(); s++)
{
if (scores[s] > bestTempScore)
{
bestTempScore = scores[s];
bestRep = swatches.lab[s];
}
}
if (!option.lab[i].Equals(bestRep))
{
option.lab[i] = bestRep;
optionScore = bestTempScore;
changes++;
}
}
iters++;
}
if (optionScore > allScores[t])
{
allOptions[t] = option;
allScores[t] = optionScore;
}
});
//aggregate scores
for (int i = 0; i < allScores.Count(); i++)
{
if (allScores[i] > bestScore)
{
bestScore = allScores[i];
best = allOptions[i];
}
}
//convert best lab to rgb
best.colors = new List<Color>();
foreach (CIELAB l in best.lab)
best.colors.Add(Util.LABtoRGB(l));
return best;
}
private void SavePaletteToImage(String dir, String key, String filename, PaletteData data)
{
int colorSize = 100;
int numColors = data.colors.Count();
int gridWidth = 10;
int padding = 20;
int imageSize = 500;
Bitmap image = new Bitmap(Image.FromFile(dir + "\\" + key));
int imageWidth = imageSize;
int imageHeight = imageSize;
if (image.Width > image.Height)
imageHeight = (int)Math.Round(imageSize / (double)image.Width * image.Height);
else
imageWidth = (int)Math.Round(imageSize / (double)image.Height * image.Width);
int width = Math.Max(colorSize * Math.Min(gridWidth, numColors), imageSize)+2*padding;
int height = imageHeight + 3*padding + colorSize * (int)(Math.Ceiling(numColors / (double)gridWidth));
Bitmap bitmap = new Bitmap(width, height);
Graphics g = Graphics.FromImage(bitmap);
//fill with black
g.FillRectangle(new SolidBrush(Color.Black), 0, 0, bitmap.Width, bitmap.Height);
//draw image
g.DrawImage(image, padding, padding, imageWidth, imageHeight);
//draw out the clusters
for (int i = 0; i < numColors; i++)
{
int row = (int)Math.Floor(i / (double)gridWidth);
int col = i - row * gridWidth;
Pen pen = new Pen(data.colors[i]);
g.FillRectangle(pen.Brush, col * colorSize+padding, imageHeight + 2*padding + row * colorSize, colorSize - padding, colorSize - padding);
double brightness = pen.Color.GetBrightness();
Brush brush = new SolidBrush(Color.White);
if (brightness > 0.5)
brush = new SolidBrush(Color.Black);
}
bitmap.Save(filename);
}
private Bitmap GetFinalRendering(String patternId, PaletteData palette)
{
String url = "http://www.colourlovers.com/patternPreview/";//41/CCCCCC/999999/666666/333333/000000.png
String[] colorHexes;
if (palette.colors.Count <= 5)
colorHexes = new String[] { "CCCCCC", "999999", "666666", "333333", "000000" };
else
colorHexes = new String[palette.colors.Count];
for (int i = 0; i < palette.colors.Count(); i++)
{
int slotIdx = i;
String hex = ColorTranslator.ToHtml(palette.colors[i]).Replace("#","");
colorHexes[slotIdx] = hex;
}
String finalUrl = url + patternToTemplate[patternId] + "/" + String.Join("/", colorHexes) + ".png";
return Util.BitmapFromWeb(finalUrl);
}
public Bitmap SolidColor(PaletteData palette, int[] slotToColor)
{
int width = Width();
int height = Height();
Bitmap result = new Bitmap(width, height);
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
CIELAB lab = palette.lab[slotToColor[slots.assignments[i, j]]];
result.SetPixel(i, j, Util.LABtoRGB(lab));
}
}
return result;
}
//Load PaletteData
private Dictionary<String, PaletteData> LoadFilePalettes(String file)
{
FileInfo finfo = new FileInfo(file);
Dictionary<String, PaletteData> plist = new Dictionary<String, PaletteData>();
String[] lines = File.ReadAllLines(file);
//extracted palettes file format
if (finfo.Extension == ".tsv")
{
for (int i = 1; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split('\t');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[0]);
data.workerNum = Int32.Parse(fields[1]);
String key = fields[2];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
foreach (String s in colors)
{
String[] comp = s.Split(',');
Color c = Color.FromArgb(Int32.Parse(comp[0]), Int32.Parse(comp[1]), Int32.Parse(comp[2]));
CIELAB l = Util.RGBtoLAB(c);
data.colors.Add(c);
data.lab.Add(l);
}
if (!plist.ContainsKey(key))
plist.Add(key, data);
else
throw new IOException("More than one palette per key");
}
}
else //pattern template file format, populate the pattern id to template id field
{
for (int i = 0; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split(',');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[1]);
data.workerName = fields[0];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
colors = colors.Distinct<String>().ToArray<String>();
String key = fields[1]+".png";
foreach (String s in colors)
{
Color c = ColorTranslator.FromHtml("#"+s);
data.colors.Add(c);
data.lab.Add(Util.RGBtoLAB(c));
}
//ignore missing palette data
if (!plist.ContainsKey(key) && colors.Count()>0)
plist.Add(key, data);
else if (plist.ContainsKey(key))
throw new IOException("More than one palette per key");
String templateId = fields[4];
if (!patternToTemplate.ContainsKey(data.id.ToString()))
patternToTemplate.Add(data.id.ToString(), templateId);
}
}
return plist;
}
private CIELAB[,] ModeRecolor(CIELAB[,] image, PaletteData palette)
{
int width = image.GetLength(0);
int height = image.GetLength(1);
CIELAB[,] result = new CIELAB[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double[] scores = new double[palette.colors.Count()];
//check if the color is in the palette
CIELAB color = image[i, j];
if (palette.lab.Contains(color))
{
result[i, j] = color;
continue;
}
//otherwise, find the best color by mode
for (int dx = -1; dx <= 1; dx++)
{
for (int dy = -1; dy <= 1; dy++)
{
int x = i + dx;
int y = j + dy;
double weight = (dx == 0 && dy == 0) ? 4 : 1;
if (Util.InBounds(x, y, width, height))
{
int bestc = ClosestColorIndex(image[x, y], palette.lab);
scores[bestc] -= weight;
}
}
}
//pick the color with the min score
double minScore = Double.PositiveInfinity;
int bestIdx = 0;
for (int c = 0; c < palette.colors.Count(); c++)
{
if (scores[c] < minScore)
{
minScore = scores[c];
bestIdx = c;
}
}
result[i, j] = palette.lab[bestIdx];
}
}
return result;
}
private void Recolor()
{
String suboutdir = Path.Combine(outdir, "recolored");
Directory.CreateDirectory(suboutdir);
//read in the patterns and save out their layers
List<PatternItem> patterns = PatternIO.GetPatterns(imagedir);
foreach (PatternItem p in patterns)
{
String basename = p.Name;
if (!palettes.ContainsKey(basename))
continue;
PaletteData palette = palettes[basename];
//Read the recoloring description if available
String specs = Path.Combine(outdir, "specs", p.Directory, Util.ConvertFileName(basename, "", ".txt"));
if (!File.Exists(specs))
continue;
Bitmap image = new Bitmap(p.FullPath);
//TODO: save and reload color templates functionality
ColorTemplate template = new ColorTemplate(image, palette);
PaletteData data = new PaletteData();
String[] lines = File.ReadAllLines(specs);
int[] slotToColor = new int[template.NumSlots()];
Dictionary<int, int> groupToSlot = new Dictionary<int, int>();
int ngroups = 0;
for (int i = 0; i < slotToColor.Count(); i++)
{
slotToColor[i] = i;
if (template.PixelsInSlot(i) > 0)
groupToSlot.Add(ngroups++, i);
}
//TODO: handle recoloring when # groups is less than number of original slots, because of quantization issues.
//Right now, this is rather ugly..
data.colors = new List<Color>();
data.lab = new List<CIELAB>();
for (int i = 0; i < slotToColor.Count(); i++)
{
data.colors.Add(new Color());
data.lab.Add(new CIELAB());
}
int groupid = 0;
foreach (String line in lines)
{
//rgb floats
int[] fields = line.Split(new string[]{" "},StringSplitOptions.RemoveEmptyEntries).Select<String, int>(s=>((int)(Math.Round(double.Parse(s)*255)))).ToArray<int>();
Color color = Color.FromArgb(fields[0], fields[1], fields[2]);
data.colors[groupToSlot[groupid]] = color;
data.lab[groupToSlot[groupid]] = Util.RGBtoLAB(color);
groupid++;
}
Bitmap orig = (renderFinal)? image : template.DebugQuantization();
PatternIO.SavePattern(orig, p, suboutdir, "_original");
orig.Dispose();
Bitmap result = (renderFinal)? GetFinalRendering(Util.ConvertFileName(basename, "",""), data): template.SolidColor(data, slotToColor);
//sometimes we can't get the nice image, so render the quantized image in this case. TODO: or maybe skip rendering this visualization at all
if (result == null)
result = template.SolidColor(data, slotToColor);
PatternIO.SavePattern(result, p, suboutdir, "_recolored");
result.Dispose();
}
}
//Load or create the candidate swatches
private PaletteData GetPaletteSwatches(string key, int maxIters=50)
{
PaletteData global = new PaletteData();
//load all the candidate colors (the json file)
String jsonFile = dir + "/swatches/" + Util.ConvertFileName(key,"",".json");
//check if it exists, if not, create the swatches
if (!File.Exists(jsonFile))
GenerateCandidates(key, maxIters);
String rawText = System.IO.File.ReadAllText(jsonFile);
//Get the rgb strings
String cleanedText = rawText.Replace("},{", "^");
cleanedText = cleanedText.Replace("[{", "");
cleanedText = cleanedText.Replace("}]", "");
String[] rgbStrings = cleanedText.Split('^');
foreach (String rgb in rgbStrings)
{
String[] fields = rgb.Split(',');
//assume that the first three fields are r,g,b
Color color = Color.FromArgb(Int32.Parse(fields[0].Split(':')[1]),
Int32.Parse(fields[1].Split(':')[1]),
Int32.Parse(fields[2].Split(':')[1]));
CIELAB lab = Util.RGBtoLAB(color);
global.lab.Add(lab);
global.colors.Add(color);
}
return global;
}
private void RenderTurkFile(String filename, String turkDir)
{
String[] lines = File.ReadAllLines(filename);
foreach (String line in lines)
{
String[] fields = line.Split('|');
String pid = fields[0];
String[] colors = fields[3].Split('^');
//render the template
Bitmap template = GetFinalRendering(pid, new PaletteData());
if (template != null)
{
template.Save(Path.Combine(turkDir, pid + "_t.png"));
//render the candidate pattern
String name = fields[1] + ".png";
PaletteData data = new PaletteData();
foreach (String color in colors)
{
int[] cfields = color.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries).Select<String, int>(s => ((int)(Math.Round(double.Parse(s) * 255)))).ToArray<int>();
data.colors.Add(Color.FromArgb(cfields[0], cfields[1], cfields[2]));
}
Bitmap cand = GetFinalRendering(pid, data);
cand.Save(Path.Combine(turkDir, name));
cand.Dispose();
template.Dispose();
}
}
}
public PaletteData HillClimbPalette(String key, String saliencyPattern, bool debug = false)
{
//Now initialize a palette and start hill climbing
int trials = 5;
PaletteData swatches = GetPaletteSwatches(key);
List<CIELAB> shuffled = new List<CIELAB>();
foreach (CIELAB c in swatches.lab)
shuffled.Add(c);
Random random = new Random();
//weights
Dictionary<Features, double> weights = new Dictionary<Features, double>();
weights = LoadWeights(weightsDir+"/weights-final-no.csv", weightsDir+"/featurenames-all.txt");
PaletteData best = new PaletteData();
double bestScore = Double.NegativeInfinity;
SortedSet<Features> included = new SortedSet<Features>(weights.Keys);
FeatureParams fparams = SetupFeatureParams(included, key, saliencyPattern, debug);
Stopwatch watch = new Stopwatch();
//Generate all the random starts first
List<PaletteData> starts = new List<PaletteData>();
PaletteData[] allOptions = new PaletteData[trials];
double[] allScores = new double[trials];
for (int t = 0; t < trials; t++)
{
//setup
PaletteData option = new PaletteData();
//pick k random colors. First shuffle the colors and pick the top k
for (int j = shuffled.Count() - 1; j >= 0; j--)
{
int idx = random.Next(j + 1);
CIELAB temp = shuffled[j];
shuffled[j] = shuffled[idx];
shuffled[idx] = temp;
}
for (int i = 0; i < 5; i++)
{
option.lab.Add(shuffled[i]);
option.colors.Add(Util.LABtoRGB(shuffled[i]));
}
starts.Add(option);
allOptions[t] = new PaletteData(option);
double optionScore = ScorePalette(weights, CalculateFeatures(option, fparams));
allScores[t] = optionScore;
}
watch.Restart();
for (int t = 0; t < trials; t++)
{
//setup
PaletteData option = new PaletteData(starts[t]);
double optionScore = allScores[t];
//Now hill climb, for each swatch, consider replacing it with a better swatch
//Pick the best replacement, and continue until we reach the top of a hill
int changes = 1;
int iters = 0;
watch.Restart();
while (changes > 0)
{
changes = 0;
for (int i = 0; i < option.lab.Count(); i++)
{
//find the best swatch replacement for this color
double bestTempScore = optionScore;
CIELAB bestRep = option.lab[i];
double[] scores = new double[swatches.lab.Count()];
for (int s = 0; s < swatches.lab.Count(); s++)
{
CIELAB r = swatches.lab[s];
PaletteData temp = new PaletteData(option);
if (!temp.lab.Contains(r))
{
temp.lab[i] = r;
temp.colors[i] = swatches.colors[s];
double tempScore = ScorePalette(weights, CalculateFeatures(temp, fparams));
scores[s] = tempScore;
}
else
{
scores[s] = Double.NegativeInfinity;
}
}
//aggregate results
for (int s = 0; s < scores.Count(); s++)
{
if (scores[s] > bestTempScore)
{
bestTempScore = scores[s];
bestRep = swatches.lab[s];
}
}
if (!option.lab[i].Equals(bestRep))
{
option.lab[i] = bestRep;
optionScore = bestTempScore;
changes++;
}
}
iters++;
}
if (optionScore > allScores[t])
{
allOptions[t] = option;
allScores[t] = optionScore;
}
Log(dir + "/out/convergelog.txt", "Trial " + t + " Key " + key + " BestScore: " + allScores[t] + " Steps: " + iters + " Time: " + watch.ElapsedMilliseconds, true);
}
//aggregate scores
for (int i = 0; i < allScores.Count(); i++)
{
if (allScores[i] > bestScore)
{
bestScore = allScores[i];
best = allOptions[i];
}
}
//convert best lab to rgb
best.colors = new List<Color>();
foreach (CIELAB l in best.lab)
best.colors.Add(Util.LABtoRGB(l));
return best;
}
private void Vis(bool figureQuality = false)
{
Directory.CreateDirectory(outdir + "\\viscolor\\");
Directory.CreateDirectory(outdir + "\\vis\\");
//read in the patterns and save out their layers
List<PatternItem> patterns = PatternIO.GetPatterns(imagedir);
foreach (PatternItem p in patterns)
{
String basename = p.Name;
//Read the vis permutation description if available
String specs = Path.Combine(outdir, "vis", p.Directory, Util.ConvertFileName(basename, "", ".txt"));
if (!File.Exists(specs))
continue;
Bitmap image = new Bitmap(p.FullPath);
if (!palettes.ContainsKey(basename))
continue;
PaletteData palette = palettes[basename];
ColorTemplate template = new ColorTemplate(image, palette);
int[] slotToColor = new int[template.NumSlots()];
for (int i = 0; i < slotToColor.Count(); i++)
slotToColor[i] = i;
Dictionary<int, int> groupToSlot = new Dictionary<int, int>();
PaletteData data = new PaletteData();
int ngroups = 0;
for (int i = 0; i < slotToColor.Count(); i++)
{
slotToColor[i] = i;
data.colors.Add(new Color());
data.lab.Add(new CIELAB());
if (template.PixelsInSlot(i) > 0)
groupToSlot.Add(ngroups++, i);
}
Bitmap vis = null;
Graphics g = null;
String[] lines = File.ReadAllLines(specs);
//read the score and if it's the original or not
double score = 0;
bool orig = false;
int nresult = 0;
int y = 0;
int x = 0;
int ncol = 8;
int iwidth = 200;
int iheight = 200;
int padding = (figureQuality) ? 8 : 15 ;
Font font = new Font("Arial", 8);
int colorIdx = 0;
foreach (String line in lines)
{
if (line.StartsWith("Count"))
{
int count = Int32.Parse(line.Split(' ').Last());
//initialize the result image
int nrow = count / ncol + 1;
vis = new Bitmap(ncol*iwidth, nrow*iheight);
g = Graphics.FromImage(vis);
} else if (line.StartsWith("Score"))
{
//add the result to the visualization
x = (nresult % ncol)*iwidth;
y = (nresult / ncol)*iheight;
if (colorIdx > 0)
{
Bitmap result = (renderFinal) ? GetFinalRendering(Util.ConvertFileName(basename, "",""), data): template.SolidColor(data, slotToColor);
//sometimes we can't get the nice image, so render the quantized image in this case
if (result == null)
result = template.SolidColor(data, slotToColor);
g.DrawImage(result, x, y, iwidth-padding, iheight-padding);
String label = String.Format("{0:0.00}", score);
Color color = Color.Black;
if (orig)
{
label += ", ***";
color = Color.Red;
}
if(!figureQuality) g.DrawString(label, font, new SolidBrush(color), x, y + iheight-padding);
result.Dispose();
colorIdx = 0;
//data.colors.Clear();
//data.lab.Clear();
nresult++;
}
score = Double.Parse(line.Split(' ')[1]);
orig = Boolean.Parse(line.Split(' ')[2]);
}
else
{
//rgb floats
int[] fields = line.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries).Select<String, int>(s => ((int)(Math.Round(double.Parse(s) * 255)))).ToArray<int>();
Color color = Color.FromArgb(fields[0], fields[1], fields[2]);
data.colors[groupToSlot[colorIdx]] = color;
data.lab[groupToSlot[colorIdx]] = Util.RGBtoLAB(color);
colorIdx++;
}
}
//save the last image
if (data.colors.Count() > 0)
{
x = (nresult % ncol) * iwidth;
y = (nresult / ncol) * iheight;
Bitmap result = (renderFinal)? GetFinalRendering(Util.ConvertFileName(basename, "",""), data): template.SolidColor(data, slotToColor);
//sometimes we can't get the nice image, so render the quantized image in this case
if (result == null)
result = template.SolidColor(data, slotToColor);
g.DrawImage(result, x, y, iwidth - padding, iheight - padding);
Color color = Color.Black;
String label = String.Format("{0:0.00}", score);
if (orig)
{
label += ", ***";
color = Color.Red;
}
if (!figureQuality) g.DrawString(label, font, new SolidBrush(color), x, y + iheight - padding);
result.Dispose();
//data.colors.Clear();
//data.lab.Clear();
colorIdx = 0;
nresult++;
}
PatternIO.SavePattern(vis, p, Path.Combine(outdir, "viscolor"));
vis.Dispose();
}
}
private Dictionary<Features, double> CalculateFeatures(PaletteData data, FeatureParams options)
{
String log = dir + "/out/timelog.txt";
Stopwatch watch = new Stopwatch();
//unpack options
SortedSet<Features> included = options.included;
CIELAB[,] imageLAB = options.imageLAB;
double[,] map = options.map;
double[] colorToSaliency = options.colorToSaliency;
int[] colorToCounts = options.colorToCounts;
double totalSaliency = options.totalSaliency;
ColorNames colorNames = options.colorNames;
double factor = options.factor;
double[] colorToCNSaliency = options.colorToCNSaliency;
double minE = options.minE;
double maxE = options.maxE;
Dictionary<CIELAB, int> swatchIndexOf = options.swatchIndexOf;
Dictionary<Tuple<int, int>, double> imageSwatchDist = options.imageSwatchDist;
double afactor = options.afactor;
double[] swatchToPurity = options.swatchToPurity;
Dictionary<int, double> pToScore = options.pToScore;
double maxN = options.maxN;
double avgN = options.avgN;
double avgE = options.avgE;
double[] segToSaliency = options.segmentToSaliency;
Segmentation seg = options.seg;
int[] swatchIdxToBin = options.swatchIdxToBin;
int[,] binAssignments = options.binAssignments;
double totalCapturable = options.totalCapturableSaliency;
double[,] pixelToDists = options.pixelToDists;
double[,] pixelToCNDists = options.pixelToNDists;
double[] segmentToSD = options.segmentToSD;
double totalSD = options.totalSD;
//calculate the palette features
Dictionary<Features, double> features = new Dictionary<Features, double>();
double sdmin2 = 1;
double sdmax2 = 0;
double sdavg2 = 0;
int width = options.imageLAB.GetLength(0);
int height = options.imageLAB.GetLength(1);
int n = data.lab.Count();
double[] elementToSaliency = new double[n];
int[] elementToCount = new int[n];
//Calculate the error of recoloring the image, each pixel weighted the same, and subtract from 1
double error = 0;
//Coverage weighted by saliency of pixel
double errorWeighted = 0;
//error of recoloring the image, distance is the names distance metric
double nerror = 0;
double nerrorWeighted = 0;
double errorSeg = 0;
double errorSegSal = 0;
double colorsPSeg = 0;
double colorsPSegW = 0;
double withinVariance = 0;
double betweenVariance = 0;
double sqerror = 0;
double sqerrorW = 0;
double sqerrorSeg = 0;
double sqerrorSegSal = 0;
double nsqerrorSeg = 0;
double nsqerrorSegSal = 0;
double nerrorSeg = 0;
double nerrorSegSal = 0;
double nsqerror = 0;
double nsqerrorW = 0;
double softerror = 0;
double softwerror = 0;
double softerrorseg = 0;
double softerrorwseg = 0;
double nsofterror = 0;
double nsoftwerror = 0;
double nsofterrorseg = 0;
double nsofterrorwseg = 0;
double nmeansegerror = 0;
double nmeansegwerror = 0;
double softerrorsegsd = 0;
double nsofterrorsegsd = 0;
double errorsegsd = 0;
double nerrorsegsd = 0;
double nsqerrorsegsd = 0;
double sqerrorsegsd = 0;
double meansegsd = 0;
double nmeansegsd = 0;
watch.Start();
int numPixels = width * height;
//compute the memberships
double[,] memberships = new double[numPixels, n];
double[,] nmemberships = new double[numPixels, n];
//LAB, and SAT coverage
double[] Lbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Abounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Bbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Satbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
for (int i = 0; i < n; i++)
{
CIELAB lab = data.lab[i];
Color c = data.colors[i];
double s = c.GetSaturation();
Lbounds[0] = Math.Min(Lbounds[0], lab.L);
Lbounds[1] = Math.Max(Lbounds[1], lab.L);
Abounds[0] = Math.Min(Abounds[0], lab.A);
Abounds[1] = Math.Max(Abounds[1], lab.A);
Bbounds[0] = Math.Min(Bbounds[0], lab.B);
Bbounds[1] = Math.Max(Bbounds[1], lab.B);
Satbounds[0] = Math.Min(Satbounds[0], s);
Satbounds[1] = Math.Max(Satbounds[1], s);
}
double Lspan = Lbounds[1] - Lbounds[0];
double Aspan = Abounds[1] - Abounds[0];
double Bspan = Bbounds[1] - Bbounds[0];
double Satspan = Satbounds[1] - Satbounds[0];
double Lcov = Math.Min(Lspan, options.Lspan) / Math.Max(Math.Max(Lspan, options.Lspan), 1);
double Acov = Math.Min(Aspan, options.Aspan) / Math.Max(Math.Max(Aspan, options.Aspan), 1);
double Bcov = Math.Min(Bspan, options.Bspan) / Math.Max(Math.Max(Bspan, options.Bspan), 1);
double Scov = Math.Min(Satspan, options.Satspan) / Math.Max(Math.Max(Satspan, options.Satspan), 1);
double meansegError = 0;
double meansegErrorWeighted = 0;
double[,] colorSegDist = new double[n, seg.numSegments];
double[] segToDist = new double[seg.numSegments];
for (int j = 0; j < seg.numSegments; j++)
{
//find the closest color for each segment
double bestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
colorSegDist[i, j] = Math.Sqrt(data.lab[i].SqDist(seg.segToMeanColor[j]));
segToDist[j] += colorSegDist[i, j];
bestDist = Math.Min(colorSegDist[i, j], bestDist);
}
double nbestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
double ndist = GetCNDist(colorNames.GetBin(data.lab[i]), colorNames.GetBin(seg.segToMeanColor[j]));
nbestDist = Math.Min(ndist, nbestDist);
}
meansegError += bestDist;
meansegErrorWeighted += bestDist * segToSaliency[j];
nmeansegerror += nbestDist;
nmeansegwerror += nbestDist * segToSaliency[j];
meansegsd += bestDist * segmentToSD[j];
nmeansegsd += nbestDist * segmentToSD[j];
}
meansegError /= seg.numSegments * factor;
meansegErrorWeighted /= totalSaliency * factor;
nmeansegerror /= seg.numSegments;
nmeansegwerror /= totalSaliency;
meansegsd /= totalSD * factor;
nmeansegsd /= totalSD;
//now calculate entropy per segment
//probability is 1-(dist/totaldist)
double[] segEntropy = new double[seg.numSegments];
for (int i = 0; i < seg.numSegments; i++)
{
for (int j = 0; j < n; j++)
{
double prob = 1 - colorSegDist[j, i] / segToDist[i];
segEntropy[i] += prob * Math.Log(prob);
}
}
//average and/or weight by saliency of segment
for (int i = 0; i < seg.numSegments; i++)
{
colorsPSeg += segEntropy[i];
colorsPSegW += segEntropy[i] * segToSaliency[i];
}
colorsPSeg /= seg.numSegments;
colorsPSegW /= totalSaliency;
double[,] nsubs = new double[width, height];
double[,] subs = new double[width, height];
if (included.Overlaps(options.NameCovFeatures) ||
included.Overlaps(options.CoverageFeatures) ||
included.Overlaps(options.SaliencyDensFeatures))
{
double[,] errors = new double[width, height];
int[,] assignment = new int[width, height]; //the pixel to the nearest palette color
double[,] nerrors = new double[width, height];
double[,] nerrorsWeighted = new double[width, height];
double[] clusterSqError = new double[n];
int[,] nassignment = new int[width, height];
Parallel.For(0, width, i =>
{
for (int j = 0; j < height; j++)
{
double bestDist = Double.PositiveInfinity;
int bestIdx = -1;
for (int s = 0; s < data.lab.Count(); s++)
{
double dist = data.lab[s].SqDist(options.imageLAB[i, j]);
if (dist < bestDist)
bestIdx = s;
bestDist = Math.Min(dist, bestDist);
}
assignment[i, j] = bestIdx;
double sqrtBestDist = Math.Sqrt(bestDist);
errors[i, j] = sqrtBestDist;
clusterSqError[bestIdx] += bestDist;
}
if (included.Overlaps(options.NameCovFeatures))
{
for (int j = 0; j < height; j++)
{
int bestIdx = -1;
double bestDist = Double.PositiveInfinity;
for (int s = 0; s < data.lab.Count(); s++)
{
int sIdx = swatchIndexOf[data.lab[s]];
double dist = GetCNDist(binAssignments[i, j], swatchIdxToBin[sIdx]);//imageSwatchDist[new Tuple<int, int>(binAssignments[i,j], swatchIdxToBin[sIdx])]; //imageSwatchDist[new Tuple<int, int>(colorNames.GetBin(imageLAB[i, j]), colorNames.GetBin(data.lab[s]))];
if (dist < bestDist)
bestIdx = s;
bestDist = Math.Min(dist, bestDist);
}
double val = map[i, j];
nerrors[i, j] = bestDist;
nerrorsWeighted[i, j] = bestDist * val;
nassignment[i, j] = bestIdx;
}
}
double epsilon = 0.0001;
for (int j = 0; j < height; j++)
{
//calculate the memberships
double[] dists = new double[n];
double[] ndists = new double[n];
double subtotal = 0;
double nsubtotal = 0;
int idx = j * width + i;
for (int k = 0; k < n; k++)
{
int sidx = swatchIndexOf[data.lab[k]];
dists[k] = Math.Max(pixelToDists[idx, sidx], epsilon);
ndists[k] = Math.Max(pixelToCNDists[idx, sidx], epsilon);
subtotal += (1.0 / dists[k]);
nsubtotal += (1.0 / ndists[k]);
}
for (int k = 0; k < n; k++)
{
memberships[idx, k] = 1.0 / (dists[k] * subtotal);
nmemberships[idx, k] = 1.0 / (ndists[k] * nsubtotal);
}
for (int k = 0; k < n; k++)
{
int sidx = swatchIndexOf[data.lab[k]];
double u = memberships[idx, k];
subs[i, j] += u * u * Math.Max(pixelToDists[idx, sidx], epsilon);
double nu = nmemberships[idx, k];
nsubs[i, j] += nu * nu * Math.Max(pixelToCNDists[idx, sidx], epsilon);
}
}
});
//find J
double[] segToJ = new double[seg.numSegments];
double[] nsegToJ = new double[seg.numSegments];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
double sub = subs[i, j];
double nsub = nsubs[i, j];
softerror += sub;
nsofterror += nsub;
softwerror += sub * map[i, j];
nsoftwerror += nsub * map[i, j];
int segIdx = seg.assignments[i, j];
segToJ[segIdx] += sub;
nsegToJ[segIdx] += nsub;
}
}
softerror /= numPixels * factor * factor;
nsofterror /= numPixels;
softwerror /= totalSaliency * factor * factor;
nsoftwerror /= totalSaliency;
for (int s = 0; s < seg.numSegments; s++)
{
softerrorseg += segToJ[s] / seg.counts[s];
softerrorwseg += segToSaliency[s] * segToJ[s] / seg.counts[s];
nsofterrorseg += nsegToJ[s] / seg.counts[s];
nsofterrorwseg += segToSaliency[s] * nsegToJ[s] / seg.counts[s];
softerrorsegsd += segmentToSD[s] * segToJ[s] / seg.counts[s];
nsofterrorsegsd += segmentToSD[s] * nsegToJ[s] / seg.counts[s];
}
softerrorseg /= seg.numSegments * factor * factor;
softerrorwseg /= totalSaliency * factor * factor;
nsofterrorseg /= seg.numSegments;
nsofterrorwseg /= totalSaliency;
softerrorsegsd /= totalSD * factor * factor;
nsofterrorsegsd /= totalSD;
//calculate within and between palette cluster variance
//between variance
CIELAB betweenMean = new CIELAB();
for (int j = 0; j < n; j++)
{
betweenMean = betweenMean + data.lab[j];
}
betweenMean = betweenMean / (double)n;
for (int j = 0; j < n; j++)
{
betweenVariance += betweenMean.SqDist(data.lab[j]);
}
betweenVariance /= (double)n * factor * factor;
double[] counts = new double[n];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
counts[assignment[i, j]]++;
}
}
for (int i = 0; i < n; i++)
{
withinVariance += clusterSqError[i] / Math.Max(1, counts[i]);
}
withinVariance /= n * (factor * factor);
double[] segmentToError = new double[segToSaliency.Count()];
double[] segToSqError = new double[segToSaliency.Count()];
Dictionary<int, SortedSet<int>> segToColors = new Dictionary<int, SortedSet<int>>();
//now aggregate
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double sqError = errors[i, j] * errors[i, j];
double val = map[i, j];
errorWeighted += val * errors[i, j];
sqerrorW += val * sqError;
error += errors[i, j];
sqerror += sqError;
int idx = assignment[i, j];
if (idx >= 0)
{
elementToSaliency[idx] += val;
elementToCount[idx]++;
}
int segIdx = seg.assignments[i, j];
segmentToError[segIdx] += errors[i, j] / seg.counts[segIdx];
segToSqError[segIdx] += sqError / seg.counts[segIdx];
}
}
for (int i = 0; i < seg.numSegments; i++)
{
errorSeg += segmentToError[i];
errorSegSal += segmentToError[i] * segToSaliency[i];
sqerrorSeg += segToSqError[i];
sqerrorSegSal += segToSqError[i] * segToSaliency[i]; //TODO: this is weighted by segment percent salient density
errorsegsd += segmentToError[i] * segmentToSD[i];
sqerrorsegsd += segToSqError[i] * segmentToSD[i];
}
errorSeg /= seg.numSegments * factor;
errorSegSal /= totalSaliency * factor;
sqerrorSeg /= seg.numSegments * factor * factor;
sqerrorSegSal /= totalSaliency * factor * factor;
errorWeighted /= totalSaliency * factor;
error /= width * height * factor;
sqerrorW /= totalSaliency * factor * factor;
sqerror /= width * height * factor * factor;
errorsegsd /= totalSD * factor;
sqerrorsegsd /= totalSD * factor * factor;
//now sum it all up
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
nerrorWeighted += nerrorsWeighted[i, j];
nerror += nerrors[i, j];
}
}
nerrorWeighted /= totalSaliency;
nerror /= width * height;
double[] nsegmentToError = new double[segToSaliency.Count()];
double[] nsegToSqError = new double[segToSaliency.Count()];
//now aggregate
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double sqError = nerrors[i, j] * nerrors[i, j];
double val = map[i, j];
nerrorWeighted += val * nerrors[i, j];
nsqerrorW += val * sqError;
nerror += nerrors[i, j];
nsqerror += sqError;
int idx = nassignment[i, j];
if (idx >= 0)
{
elementToSaliency[idx] += val;
elementToCount[idx]++;
}
int segIdx = seg.assignments[i, j];
nsegmentToError[segIdx] += nerrors[i, j] / seg.counts[segIdx];
nsegToSqError[segIdx] += sqError / seg.counts[segIdx];
}
}
for (int i = 0; i < seg.numSegments; i++)
{
nerrorSeg += nsegmentToError[i];
nerrorSegSal += nsegmentToError[i] * segToSaliency[i];
nsqerrorSeg += nsegToSqError[i];
nsqerrorSegSal += nsegToSqError[i] * segToSaliency[i]; //TODO: this is weighted by segment percent salient density
nerrorsegsd += nsegmentToError[i] * segmentToSD[i];
nsqerrorsegsd += nsegToSqError[i] * segmentToSD[i];
}
nerrorSeg /= seg.numSegments;
nerrorSegSal /= totalSaliency;
nsqerrorSeg /= seg.numSegments;
nsqerrorSegSal /= totalSaliency;
nerrorWeighted /= totalSaliency;
nerror /= width * height;
nsqerrorW /= totalSaliency;
nsqerror /= width * height;
nerrorsegsd /= totalSD;
nsqerrorsegsd /= totalSD;
}
Log(log, "Coverage and Name Coverage" + watch.ElapsedMilliseconds);
watch.Restart();
for (int i = 0; i < n; i++)
{
double dens = elementToSaliency[i] / Math.Max(1, elementToCount[i]);
sdmin2 = Math.Min(dens, sdmin2);
sdmax2 = Math.Max(dens, sdmax2);
sdavg2 += dens;
}
sdavg2 /= data.lab.Count();
Log(log, "SaliencyDensClust " + watch.ElapsedMilliseconds);
watch.Restart();
//total captured salience (salience sum/salience of the image)
double stotal = 0;
//average salience density of each swatch (and min/max)
double sdmin = 1;
double sdmax = 0;
double sdavg = 0;
foreach (CIELAB c in data.lab)
{
double val = colorToSaliency[swatchIndexOf[c]] / Math.Max(1, colorToCounts[swatchIndexOf[c]]);
sdmin = Math.Min(val, sdmin);
sdmax = Math.Max(val, sdavg);
sdavg += val;
stotal += colorToSaliency[swatchIndexOf[c]];
}
sdavg /= data.lab.Count();
stotal /= totalCapturable;
Log(log, "SaliencyDens " + watch.ElapsedMilliseconds);
watch.Restart();
double cnmin = 1;
double cnmax = 0;
double cnavg = 0;
for (int i = 0; i < n; i++)
{
for (int j = i + 1; j < n; j++)
{
double dist = 1 - colorNames.CosineDistance(data.lab[i], data.lab[j]);
cnavg += dist;
cnmin = Math.Min(cnmin, dist);
cnmax = Math.Max(cnmax, dist);
}
}
cnavg /= n * (n - 1) / 2;
Log(log, "NamesDiff " + watch.ElapsedMilliseconds);
watch.Restart();
//Calculate the color name diversity, avg color name distance to closest neighbor
double cndiv = 0;
for (int c = 0; c < n; c++)
{
double bestDist = 1;
for (int i = 0; i < n; i++)
{
if (c == i)
continue;
double curDist = 1 - colorNames.CosineDistance(data.lab[i], data.lab[c]);
if (curDist < bestDist)
bestDist = curDist;
}
cndiv += bestDist / n;
}
Log(log, "NamesClosestAvg " + watch.ElapsedMilliseconds);
watch.Restart();
//calculate color name salience (average, min, max)
double csmin = 1;
double csmax = 0;
double csavg = 0;
for (int i = 0; i < n; i++)
{
double salience = (colorToCNSaliency[swatchIndexOf[data.lab[i]]] - minE) / (maxE - minE);//colorNames.NormalizedSaliency(data.lab[i], minE, maxE);
csavg += salience;
csmin = Math.Min(csmin, salience);
csmax = Math.Max(csmax, salience);
}
csavg /= n;
Log(log, "NameSalience " + watch.ElapsedMilliseconds);
watch.Restart();
//Calculate the diversity, avg distance to closest neighbor
double div = 0;
double divr = 0;
for (int c = 0; c < n; c++)
{
double bestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
if (c == i)
continue;
double curDist = data.lab[c].SqDist(data.lab[i]);
if (curDist < bestDist)
bestDist = curDist;
}
div += Math.Sqrt(bestDist); // (n * factor);
}
double temp = div;
div = temp / (n * factor);
divr = temp / (n * afactor);
Log(log, "DiffClosestAvg " + watch.ElapsedMilliseconds);
watch.Restart();
//calculate average distance of one color to the rest of the colors
double dmin = 1;
double dmax = 0;
double davg = 0;
double dminr = 1;
double dmaxr = 0;
double davgr = 0;
for (int i = 0; i < n; i++)
{
for (int j = i + 1; j < n; j++)
{
double dist = Math.Sqrt(data.lab[i].SqDist(data.lab[j]));
davg += dist;
dmin = Math.Min(dmin, dist);
dmax = Math.Max(dmax, dist);
}
}
double dtemp = dmin;
dmin = dtemp / factor;
dminr = dtemp / afactor;
dtemp = dmax;
dmax = dtemp / factor;
dmaxr = dtemp / afactor;
davg /= n * (n - 1) / 2;
dtemp = davg;
davg = dtemp / factor;
davgr = dtemp / afactor;
Log(log, "LABDiff " + watch.ElapsedMilliseconds);
watch.Restart();
double puritymin = double.PositiveInfinity;
double puritymax = double.NegativeInfinity;
double purityavg = 0;
double purityminr = puritymin;
double puritymaxr = puritymax;
double purityavgr = 0;
foreach (CIELAB c in data.lab)
{
double val = swatchToPurity[swatchIndexOf[c]];
puritymin = Math.Min(puritymin, val);
puritymax = Math.Max(puritymax, val);
purityavg += val;
}
double ptemp = puritymin;
puritymin = ptemp / factor;
purityminr = ptemp / factor;
ptemp = puritymax;
puritymax = ptemp / factor;
puritymaxr = ptemp / afactor;
ptemp = purityavg / data.lab.Count();
purityavg = ptemp / factor;
purityavgr = ptemp / afactor;
//Record all the features
features.Add(Features.CovUnweighted, 1 - error);
features.Add(Features.CovWeightedSaliency, 1 - errorWeighted);
features.Add(Features.ErrorUnweighted, error);
features.Add(Features.ErrorWeightedSaliency, errorWeighted);
features.Add(Features.NCov, 1 - nerror);
features.Add(Features.NCovWeightedSaliency, 1 - nerrorWeighted);
features.Add(Features.NError, nerror);
features.Add(Features.NErrorWeightedSaliency, nerrorWeighted);
features.Add(Features.DAvg, davg);
features.Add(Features.DClosestAvg, div);
features.Add(Features.DMin, dmin);
features.Add(Features.DMax, dmax);
features.Add(Features.NClosestAvg, cndiv);
features.Add(Features.NDiffAvg, cnavg / maxN);
features.Add(Features.NDiffMin, cnmin / maxN);
features.Add(Features.NDiffMax, cnmax / maxN);
features.Add(Features.NSaliencyMin, csmin);
features.Add(Features.NSaliencyMax, csmax);
features.Add(Features.NSaliencyAvg, csavg);
features.Add(Features.SaliencyDensMin, sdmin);
features.Add(Features.SaliencyDensMax, sdmax);
features.Add(Features.SaliencyDensAvg, sdavg);
features.Add(Features.SaliencyTotal, stotal);
features.Add(Features.SaliencyDensClustMin, sdmin2);
features.Add(Features.SaliencyDensClustMax, sdmax2);
features.Add(Features.SaliencyDensClustAvg, sdavg2);
features.Add(Features.DMinR, dminr);
features.Add(Features.DMaxR, dmaxr);
features.Add(Features.DAvgR, davgr);
features.Add(Features.DClosestAvgR, divr);
features.Add(Features.PurityMin, puritymin);
features.Add(Features.PurityMax, puritymax);
features.Add(Features.PurityAvg, purityavg);
features.Add(Features.PurityMinR, purityminr);
features.Add(Features.PurityMaxR, puritymaxr);
features.Add(Features.PurityAvgR, purityavgr);
features.Add(Features.NDiffMinR, cnmin / avgN);
features.Add(Features.NDiffMaxR, cnmax / avgN);
features.Add(Features.NDiffAvgR, cnavg / avgN);
features.Add(Features.NSaliencyAvgR, csavg / avgE);
features.Add(Features.NSaliencyMaxR, csmax / avgE);
features.Add(Features.NSaliencyMinR, csmin / avgE);
features.Add(Features.NClosestAvgR, cndiv / avgE);
features.Add(Features.ErrorWeightedSegment, errorSegSal);
features.Add(Features.ErrorUnweightedSegment, errorSeg);
features.Add(Features.ColorsPerSeg, colorsPSeg);
features.Add(Features.ColorsPerSegWeighted, colorsPSegW);
features.Add(Features.MeanSegError, meansegError);
features.Add(Features.MeanSegErrorWeighted, meansegErrorWeighted);
features.Add(Features.LCov, Lcov);
features.Add(Features.ACov, Acov);
features.Add(Features.BCov, Bcov);
features.Add(Features.SatCov, Scov);
features.Add(Features.BetweenVar, betweenVariance);
features.Add(Features.WithinVar, withinVariance);
features.Add(Features.SqErrorUnweighted, sqerror);
features.Add(Features.SqErrorWeightedSaliency, sqerrorW);
features.Add(Features.SqErrorWSeg, sqerrorSegSal);
features.Add(Features.SqErrorSeg, sqerrorSeg);
features.Add(Features.NSqError, nsqerror);
features.Add(Features.NSqErrorW, nsqerrorW);
features.Add(Features.NSqErrorSeg, nsqerrorSeg);
features.Add(Features.NSqErrorWSeg, nsqerrorSegSal);
features.Add(Features.NErrorSeg, nerrorSeg);
features.Add(Features.NErrorWSeg, nerrorSegSal);
features.Add(Features.SoftError, softerror);
features.Add(Features.SoftWError, softwerror);
features.Add(Features.SoftErrorSeg, softerrorseg);
features.Add(Features.SoftErrorWSeg, softerrorwseg);
features.Add(Features.NSoftError, nsofterror);
features.Add(Features.NSoftWError, nsoftwerror);
features.Add(Features.NSoftErrorSeg, nsofterrorseg);
features.Add(Features.NSoftErrorWSeg, nsofterrorwseg);
features.Add(Features.NMeanSegError, nmeansegerror);
features.Add(Features.NMeanSegErrorW, nmeansegwerror);
features.Add(Features.MeanSegErrorDensity, meansegsd);
features.Add(Features.NMeanSegErrorDensity, nmeansegsd);
features.Add(Features.ErrorSegSD, errorsegsd);
features.Add(Features.NErrorSegSD, nerrorsegsd);
features.Add(Features.SqErrorSegSD, sqerrorsegsd);
features.Add(Features.NSqErrorSegSD, nsqerrorsegsd);
features.Add(Features.SoftErrorSegSD, softerrorsegsd);
features.Add(Features.NSoftErrorSegSD, nsofterrorsegsd);
return features;
}
private Dictionary<String, List<PaletteData>> LoadFilePalettes(String file)
{
//load art palettes
var lines = File.ReadLines(file);
Dictionary<String, List<PaletteData>> plist = new Dictionary<String, List<PaletteData>>();
int count = 0;
List<String> headers = new List<String>();
foreach (String line in lines)
{
if (count == 0)
{
count++;
headers = line.Replace("\"", "").Split('\t').ToList<String>();
continue;
}
String[] fields = line.Replace("\"", "").Split('\t');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[headers.IndexOf("pid")]);
data.workerNum = Int32.Parse(fields[headers.IndexOf("id")]);
String key = fields[headers.IndexOf("image")];
String[] colors = fields[headers.IndexOf("colors")].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
if (headers.IndexOf("log") >= 0)
data.log = fields[headers.IndexOf("log")];
foreach (String s in colors)
{
String[] comp = s.Split(',');
Color c = Color.FromArgb(Int32.Parse(comp[0]), Int32.Parse(comp[1]), Int32.Parse(comp[2]));
CIELAB l = Util.RGBtoLAB(c);
data.colors.Add(c);
data.lab.Add(l);
}
if (!plist.ContainsKey(key))
plist.Add(key, new List<PaletteData>());
plist[key].Add(data);
}
return plist;
}