private Dictionary <String, List <PaletteData> > LoadFilePalettes(String file)
{
//load art palettes
String[] lines = File.ReadAllLines(file);
Dictionary <String, List <PaletteData> > plist = new Dictionary <String, List <PaletteData> >();
for (int i = 1; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split('\t');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[0]);
data.workerNum = Int32.Parse(fields[1]);
String key = fields[2];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
foreach (String s in colors)
{
String[] comp = s.Split(',');
Color c = Color.FromArgb(Int32.Parse(comp[0]), Int32.Parse(comp[1]), Int32.Parse(comp[2]));
CIELAB l = Util.RGBtoLAB(c);
data.colors.Add(c);
data.lab.Add(l);
}
if (!plist.ContainsKey(key))
{
plist.Add(key, new List <PaletteData>());
}
plist[key].Add(data);
}
return(plist);
}
public static Color ToColor(CIELAB fab)
{
Color ret = fab.CovnertTo();
ret.a = fab.alpha;
return(ret);
}
public Cluster()
{
lab = new CIELAB();
sumlab = new CIELAB();
count = 0;
id = -1;
}
public void AddColor(CIELAB color, double weight=1)
{
sumlab.L += color.L*weight;
sumlab.A += color.A*weight;
sumlab.B += color.B*weight;
count += weight;
}
public double DistanceTo(CIELAB c)
{
double dL = L - c.L;
double da = a - c.a;
double db = b - c.b;
return(Math.Sqrt(0.5 * dL * dL + da * da + db * db));
}
public static CIELAB FromColor(Color color)
{
CIELAB ret = new CIELAB();
ret.CovnertFrom(color);
ret.alpha = color.a;
return(ret);
}
public static CIELAB ConvertToCIELAB(Color c)
{
if (cielabLookup.ContainsKey(c))
{
return(cielabLookup[c]);
}
double rf = ConvertChannel(c.R) * 100;
double gf = ConvertChannel(c.G) * 100;
double bf = ConvertChannel(c.B) * 100;
double var_X = rf * 0.4124 + gf * 0.3576 + bf * 0.1805;
double var_Y = rf * 0.2126 + gf * 0.7152 + bf * 0.0722;
double var_Z = rf * 0.0193 + gf * 0.1192 + bf * 0.9505;
var_X = var_X / 95.047; //Observer = 2°, Illuminant = D65
var_Y = var_Y / 100.000;
var_Z = var_Z / 108.883;
double toPow = 1.0 / 3.0;
if (var_X > 0.008856)
{
var_X = Math.Pow(var_X, toPow);
}
else
{
var_X = (7.787 * var_X) + (16 / 116);
}
if (var_Y > 0.008856)
{
var_Y = Math.Pow(var_Y, toPow);
}
else
{
var_Y = (7.787 * var_Y) + (16 / 116);
}
if (var_Z > 0.008856)
{
var_Z = Math.Pow(var_Z, toPow);
}
else
{
var_Z = (7.787 * var_Z) + (16 / 116);
}
CIELAB ans = new CIELAB();
ans.L = (116 * var_Y) - 16;
ans.a = 500 * (var_X - var_Y);
ans.b = 200 * (var_Y - var_Z);
cielabLookup[c] = ans;
return(ans);
}
CIELAB[,] template; //relative LAB difference (or change to RGB difference?)
#endregion Fields
#region Constructors
public ColorTemplate(Bitmap image, PaletteData palette)
{
//create the segmentation based on the palette
//TODO: There's a problem that sometimes a palette color won't appear in the image (perhaps due to color blending), and so the slot will have no pixels associated with it
int width = image.Width;
int height = image.Height;
slots = new Segmentation(palette.colors.Count(), width, height);
segToColor = new int[slots.numSegments];
template = new CIELAB[width, height];
CIELAB[,] recolored = ModeRecolor(Util.Map<Color, CIELAB>(Util.BitmapToArray(image), c => Util.RGBtoLAB(c)), palette);
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
template[i, j] = new CIELAB();
int bestIdx = -1;
CIELAB lab = recolored[i, j];
double bestDist = Double.PositiveInfinity;
for (int c = 0; c < palette.lab.Count(); c++)
{
double dist = palette.lab[c].SqDist(lab);
if (dist < bestDist)
{
bestDist = dist;
bestIdx = c;
if (dist < 0.00001)
break;
}
}
slots.assignments[i, j] = bestIdx;
slots.counts[bestIdx]++;
segToColor[bestIdx] = bestIdx;
}
}
originalPalette = new PaletteData(palette);
}
private bool ProcessImage(Color[,] image, CIELAB[,] imageLAB)
{
double thresh = 5;
UnionFind<CIELAB> uf = new UnionFind<CIELAB>((a,b)=>(a.SqDist(b)<=thresh));
int[,] assignments = uf.ConnectedComponents(imageLAB);//Util.Map<Color, CIELAB>(image, Util.RGBtoLAB));
int numC = -1;
for(int i=0; i<image.GetLength(0); i++)
for (int j=0; j<image.GetLength(1); j++)
numC = Math.Max(numC, assignments[i,j]+1);
if (numC >= 2)
RemoveBackground(image, imageLAB);
//if it is a black and white image (with num connected components >= 2), it's not a valid color image
return !(isBlackWhite(image, imageLAB) && numC >= 2);
}
private Dictionary<Features, double> CalculateFeatures(PaletteData data, FeatureParams options)
{
String log = dir + "/out/timelog.txt";
Stopwatch watch = new Stopwatch();
//unpack options
SortedSet<Features> included = options.included;
CIELAB[,] imageLAB = options.imageLAB;
double[,] map = options.map;
double[] colorToSaliency = options.colorToSaliency;
int[] colorToCounts = options.colorToCounts;
double totalSaliency = options.totalSaliency;
ColorNames colorNames = options.colorNames;
double factor = options.factor;
double[] colorToCNSaliency = options.colorToCNSaliency;
double minE = options.minE;
double maxE = options.maxE;
Dictionary<CIELAB, int> swatchIndexOf = options.swatchIndexOf;
Dictionary<Tuple<int, int>, double> imageSwatchDist = options.imageSwatchDist;
double afactor = options.afactor;
double[] swatchToPurity = options.swatchToPurity;
Dictionary<int, double> pToScore = options.pToScore;
double maxN = options.maxN;
double avgN = options.avgN;
double avgE = options.avgE;
double[] segToSaliency = options.segmentToSaliency;
Segmentation seg = options.seg;
int[] swatchIdxToBin = options.swatchIdxToBin;
int[,] binAssignments = options.binAssignments;
double totalCapturable = options.totalCapturableSaliency;
double[,] pixelToDists = options.pixelToDists;
double[,] pixelToCNDists = options.pixelToNDists;
double[] segmentToSD = options.segmentToSD;
double totalSD = options.totalSD;
//calculate the palette features
Dictionary<Features, double> features = new Dictionary<Features, double>();
double sdmin2 = 1;
double sdmax2 = 0;
double sdavg2 = 0;
int width = options.imageLAB.GetLength(0);
int height = options.imageLAB.GetLength(1);
int n = data.lab.Count();
double[] elementToSaliency = new double[n];
int[] elementToCount = new int[n];
//Calculate the error of recoloring the image, each pixel weighted the same, and subtract from 1
double error = 0;
//Coverage weighted by saliency of pixel
double errorWeighted = 0;
//error of recoloring the image, distance is the names distance metric
double nerror = 0;
double nerrorWeighted = 0;
double errorSeg = 0;
double errorSegSal = 0;
double colorsPSeg = 0;
double colorsPSegW = 0;
double withinVariance = 0;
double betweenVariance = 0;
double sqerror = 0;
double sqerrorW = 0;
double sqerrorSeg = 0;
double sqerrorSegSal = 0;
double nsqerrorSeg = 0;
double nsqerrorSegSal = 0;
double nerrorSeg = 0;
double nerrorSegSal = 0;
double nsqerror = 0;
double nsqerrorW = 0;
double softerror = 0;
double softwerror = 0;
double softerrorseg = 0;
double softerrorwseg = 0;
double nsofterror = 0;
double nsoftwerror = 0;
double nsofterrorseg = 0;
double nsofterrorwseg = 0;
double nmeansegerror = 0;
double nmeansegwerror = 0;
double softerrorsegsd = 0;
double nsofterrorsegsd = 0;
double errorsegsd = 0;
double nerrorsegsd = 0;
double nsqerrorsegsd = 0;
double sqerrorsegsd = 0;
double meansegsd = 0;
double nmeansegsd = 0;
watch.Start();
int numPixels = width * height;
//compute the memberships
double[,] memberships = new double[numPixels, n];
double[,] nmemberships = new double[numPixels, n];
//LAB, and SAT coverage
double[] Lbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Abounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Bbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Satbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
for (int i = 0; i < n; i++)
{
CIELAB lab = data.lab[i];
Color c = data.colors[i];
double s = c.GetSaturation();
Lbounds[0] = Math.Min(Lbounds[0], lab.L);
Lbounds[1] = Math.Max(Lbounds[1], lab.L);
Abounds[0] = Math.Min(Abounds[0], lab.A);
Abounds[1] = Math.Max(Abounds[1], lab.A);
Bbounds[0] = Math.Min(Bbounds[0], lab.B);
Bbounds[1] = Math.Max(Bbounds[1], lab.B);
Satbounds[0] = Math.Min(Satbounds[0], s);
Satbounds[1] = Math.Max(Satbounds[1], s);
}
double Lspan = Lbounds[1] - Lbounds[0];
double Aspan = Abounds[1] - Abounds[0];
double Bspan = Bbounds[1] - Bbounds[0];
double Satspan = Satbounds[1] - Satbounds[0];
double Lcov = Math.Min(Lspan, options.Lspan) / Math.Max(Math.Max(Lspan, options.Lspan), 1);
double Acov = Math.Min(Aspan, options.Aspan) / Math.Max(Math.Max(Aspan, options.Aspan), 1);
double Bcov = Math.Min(Bspan, options.Bspan) / Math.Max(Math.Max(Bspan, options.Bspan), 1);
double Scov = Math.Min(Satspan, options.Satspan) / Math.Max(Math.Max(Satspan, options.Satspan), 1);
double meansegError = 0;
double meansegErrorWeighted = 0;
double[,] colorSegDist = new double[n, seg.numSegments];
double[] segToDist = new double[seg.numSegments];
for (int j = 0; j < seg.numSegments; j++)
{
//find the closest color for each segment
double bestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
colorSegDist[i, j] = Math.Sqrt(data.lab[i].SqDist(seg.segToMeanColor[j]));
segToDist[j] += colorSegDist[i, j];
bestDist = Math.Min(colorSegDist[i, j], bestDist);
}
double nbestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
double ndist = GetCNDist(colorNames.GetBin(data.lab[i]), colorNames.GetBin(seg.segToMeanColor[j]));
nbestDist = Math.Min(ndist, nbestDist);
}
meansegError += bestDist;
meansegErrorWeighted += bestDist * segToSaliency[j];
nmeansegerror += nbestDist;
nmeansegwerror += nbestDist * segToSaliency[j];
meansegsd += bestDist * segmentToSD[j];
nmeansegsd += nbestDist * segmentToSD[j];
}
meansegError /= seg.numSegments * factor;
meansegErrorWeighted /= totalSaliency * factor;
nmeansegerror /= seg.numSegments;
nmeansegwerror /= totalSaliency;
meansegsd /= totalSD * factor;
nmeansegsd /= totalSD;
//now calculate entropy per segment
//probability is 1-(dist/totaldist)
double[] segEntropy = new double[seg.numSegments];
for (int i = 0; i < seg.numSegments; i++)
{
for (int j = 0; j < n; j++)
{
double prob = 1 - colorSegDist[j, i] / segToDist[i];
segEntropy[i] += prob * Math.Log(prob);
}
}
//average and/or weight by saliency of segment
for (int i = 0; i < seg.numSegments; i++)
{
colorsPSeg += segEntropy[i];
colorsPSegW += segEntropy[i] * segToSaliency[i];
}
colorsPSeg /= seg.numSegments;
colorsPSegW /= totalSaliency;
double[,] nsubs = new double[width, height];
double[,] subs = new double[width, height];
if (included.Overlaps(options.NameCovFeatures) ||
included.Overlaps(options.CoverageFeatures) ||
included.Overlaps(options.SaliencyDensFeatures))
{
double[,] errors = new double[width, height];
int[,] assignment = new int[width, height]; //the pixel to the nearest palette color
double[,] nerrors = new double[width, height];
double[,] nerrorsWeighted = new double[width, height];
double[] clusterSqError = new double[n];
int[,] nassignment = new int[width, height];
Parallel.For(0, width, i =>
{
for (int j = 0; j < height; j++)
{
double bestDist = Double.PositiveInfinity;
int bestIdx = -1;
for (int s = 0; s < data.lab.Count(); s++)
{
double dist = data.lab[s].SqDist(options.imageLAB[i, j]);
if (dist < bestDist)
bestIdx = s;
bestDist = Math.Min(dist, bestDist);
}
assignment[i, j] = bestIdx;
double sqrtBestDist = Math.Sqrt(bestDist);
errors[i, j] = sqrtBestDist;
clusterSqError[bestIdx] += bestDist;
}
if (included.Overlaps(options.NameCovFeatures))
{
for (int j = 0; j < height; j++)
{
int bestIdx = -1;
double bestDist = Double.PositiveInfinity;
for (int s = 0; s < data.lab.Count(); s++)
{
int sIdx = swatchIndexOf[data.lab[s]];
double dist = GetCNDist(binAssignments[i, j], swatchIdxToBin[sIdx]);//imageSwatchDist[new Tuple<int, int>(binAssignments[i,j], swatchIdxToBin[sIdx])]; //imageSwatchDist[new Tuple<int, int>(colorNames.GetBin(imageLAB[i, j]), colorNames.GetBin(data.lab[s]))];
if (dist < bestDist)
bestIdx = s;
bestDist = Math.Min(dist, bestDist);
}
double val = map[i, j];
nerrors[i, j] = bestDist;
nerrorsWeighted[i, j] = bestDist * val;
nassignment[i, j] = bestIdx;
}
}
double epsilon = 0.0001;
for (int j = 0; j < height; j++)
{
//calculate the memberships
double[] dists = new double[n];
double[] ndists = new double[n];
double subtotal = 0;
double nsubtotal = 0;
int idx = j * width + i;
for (int k = 0; k < n; k++)
{
int sidx = swatchIndexOf[data.lab[k]];
dists[k] = Math.Max(pixelToDists[idx, sidx], epsilon);
ndists[k] = Math.Max(pixelToCNDists[idx, sidx], epsilon);
subtotal += (1.0 / dists[k]);
nsubtotal += (1.0 / ndists[k]);
}
for (int k = 0; k < n; k++)
{
memberships[idx, k] = 1.0 / (dists[k] * subtotal);
nmemberships[idx, k] = 1.0 / (ndists[k] * nsubtotal);
}
for (int k = 0; k < n; k++)
{
int sidx = swatchIndexOf[data.lab[k]];
double u = memberships[idx, k];
subs[i, j] += u * u * Math.Max(pixelToDists[idx, sidx], epsilon);
double nu = nmemberships[idx, k];
nsubs[i, j] += nu * nu * Math.Max(pixelToCNDists[idx, sidx], epsilon);
}
}
});
//find J
double[] segToJ = new double[seg.numSegments];
double[] nsegToJ = new double[seg.numSegments];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
double sub = subs[i, j];
double nsub = nsubs[i, j];
softerror += sub;
nsofterror += nsub;
softwerror += sub * map[i, j];
nsoftwerror += nsub * map[i, j];
int segIdx = seg.assignments[i, j];
segToJ[segIdx] += sub;
nsegToJ[segIdx] += nsub;
}
}
softerror /= numPixels * factor * factor;
nsofterror /= numPixels;
softwerror /= totalSaliency * factor * factor;
nsoftwerror /= totalSaliency;
for (int s = 0; s < seg.numSegments; s++)
{
softerrorseg += segToJ[s] / seg.counts[s];
softerrorwseg += segToSaliency[s] * segToJ[s] / seg.counts[s];
nsofterrorseg += nsegToJ[s] / seg.counts[s];
nsofterrorwseg += segToSaliency[s] * nsegToJ[s] / seg.counts[s];
softerrorsegsd += segmentToSD[s] * segToJ[s] / seg.counts[s];
nsofterrorsegsd += segmentToSD[s] * nsegToJ[s] / seg.counts[s];
}
softerrorseg /= seg.numSegments * factor * factor;
softerrorwseg /= totalSaliency * factor * factor;
nsofterrorseg /= seg.numSegments;
nsofterrorwseg /= totalSaliency;
softerrorsegsd /= totalSD * factor * factor;
nsofterrorsegsd /= totalSD;
//calculate within and between palette cluster variance
//between variance
CIELAB betweenMean = new CIELAB();
for (int j = 0; j < n; j++)
{
betweenMean = betweenMean + data.lab[j];
}
betweenMean = betweenMean / (double)n;
for (int j = 0; j < n; j++)
{
betweenVariance += betweenMean.SqDist(data.lab[j]);
}
betweenVariance /= (double)n * factor * factor;
double[] counts = new double[n];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
counts[assignment[i, j]]++;
}
}
for (int i = 0; i < n; i++)
{
withinVariance += clusterSqError[i] / Math.Max(1, counts[i]);
}
withinVariance /= n * (factor * factor);
double[] segmentToError = new double[segToSaliency.Count()];
double[] segToSqError = new double[segToSaliency.Count()];
Dictionary<int, SortedSet<int>> segToColors = new Dictionary<int, SortedSet<int>>();
//now aggregate
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double sqError = errors[i, j] * errors[i, j];
double val = map[i, j];
errorWeighted += val * errors[i, j];
sqerrorW += val * sqError;
error += errors[i, j];
sqerror += sqError;
int idx = assignment[i, j];
if (idx >= 0)
{
elementToSaliency[idx] += val;
elementToCount[idx]++;
}
int segIdx = seg.assignments[i, j];
segmentToError[segIdx] += errors[i, j] / seg.counts[segIdx];
segToSqError[segIdx] += sqError / seg.counts[segIdx];
}
}
for (int i = 0; i < seg.numSegments; i++)
{
errorSeg += segmentToError[i];
errorSegSal += segmentToError[i] * segToSaliency[i];
sqerrorSeg += segToSqError[i];
sqerrorSegSal += segToSqError[i] * segToSaliency[i]; //TODO: this is weighted by segment percent salient density
errorsegsd += segmentToError[i] * segmentToSD[i];
sqerrorsegsd += segToSqError[i] * segmentToSD[i];
}
errorSeg /= seg.numSegments * factor;
errorSegSal /= totalSaliency * factor;
sqerrorSeg /= seg.numSegments * factor * factor;
sqerrorSegSal /= totalSaliency * factor * factor;
errorWeighted /= totalSaliency * factor;
error /= width * height * factor;
sqerrorW /= totalSaliency * factor * factor;
sqerror /= width * height * factor * factor;
errorsegsd /= totalSD * factor;
sqerrorsegsd /= totalSD * factor * factor;
//now sum it all up
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
nerrorWeighted += nerrorsWeighted[i, j];
nerror += nerrors[i, j];
}
}
nerrorWeighted /= totalSaliency;
nerror /= width * height;
double[] nsegmentToError = new double[segToSaliency.Count()];
double[] nsegToSqError = new double[segToSaliency.Count()];
//now aggregate
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double sqError = nerrors[i, j] * nerrors[i, j];
double val = map[i, j];
nerrorWeighted += val * nerrors[i, j];
nsqerrorW += val * sqError;
nerror += nerrors[i, j];
nsqerror += sqError;
int idx = nassignment[i, j];
if (idx >= 0)
{
elementToSaliency[idx] += val;
elementToCount[idx]++;
}
int segIdx = seg.assignments[i, j];
nsegmentToError[segIdx] += nerrors[i, j] / seg.counts[segIdx];
nsegToSqError[segIdx] += sqError / seg.counts[segIdx];
}
}
for (int i = 0; i < seg.numSegments; i++)
{
nerrorSeg += nsegmentToError[i];
nerrorSegSal += nsegmentToError[i] * segToSaliency[i];
nsqerrorSeg += nsegToSqError[i];
nsqerrorSegSal += nsegToSqError[i] * segToSaliency[i]; //TODO: this is weighted by segment percent salient density
nerrorsegsd += nsegmentToError[i] * segmentToSD[i];
nsqerrorsegsd += nsegToSqError[i] * segmentToSD[i];
}
nerrorSeg /= seg.numSegments;
nerrorSegSal /= totalSaliency;
nsqerrorSeg /= seg.numSegments;
nsqerrorSegSal /= totalSaliency;
nerrorWeighted /= totalSaliency;
nerror /= width * height;
nsqerrorW /= totalSaliency;
nsqerror /= width * height;
nerrorsegsd /= totalSD;
nsqerrorsegsd /= totalSD;
}
Log(log, "Coverage and Name Coverage" + watch.ElapsedMilliseconds);
watch.Restart();
for (int i = 0; i < n; i++)
{
double dens = elementToSaliency[i] / Math.Max(1, elementToCount[i]);
sdmin2 = Math.Min(dens, sdmin2);
sdmax2 = Math.Max(dens, sdmax2);
sdavg2 += dens;
}
sdavg2 /= data.lab.Count();
Log(log, "SaliencyDensClust " + watch.ElapsedMilliseconds);
watch.Restart();
//total captured salience (salience sum/salience of the image)
double stotal = 0;
//average salience density of each swatch (and min/max)
double sdmin = 1;
double sdmax = 0;
double sdavg = 0;
foreach (CIELAB c in data.lab)
{
double val = colorToSaliency[swatchIndexOf[c]] / Math.Max(1, colorToCounts[swatchIndexOf[c]]);
sdmin = Math.Min(val, sdmin);
sdmax = Math.Max(val, sdavg);
sdavg += val;
stotal += colorToSaliency[swatchIndexOf[c]];
}
sdavg /= data.lab.Count();
stotal /= totalCapturable;
Log(log, "SaliencyDens " + watch.ElapsedMilliseconds);
watch.Restart();
double cnmin = 1;
double cnmax = 0;
double cnavg = 0;
for (int i = 0; i < n; i++)
{
for (int j = i + 1; j < n; j++)
{
double dist = 1 - colorNames.CosineDistance(data.lab[i], data.lab[j]);
cnavg += dist;
cnmin = Math.Min(cnmin, dist);
cnmax = Math.Max(cnmax, dist);
}
}
cnavg /= n * (n - 1) / 2;
Log(log, "NamesDiff " + watch.ElapsedMilliseconds);
watch.Restart();
//Calculate the color name diversity, avg color name distance to closest neighbor
double cndiv = 0;
for (int c = 0; c < n; c++)
{
double bestDist = 1;
for (int i = 0; i < n; i++)
{
if (c == i)
continue;
double curDist = 1 - colorNames.CosineDistance(data.lab[i], data.lab[c]);
if (curDist < bestDist)
bestDist = curDist;
}
cndiv += bestDist / n;
}
Log(log, "NamesClosestAvg " + watch.ElapsedMilliseconds);
watch.Restart();
//calculate color name salience (average, min, max)
double csmin = 1;
double csmax = 0;
double csavg = 0;
for (int i = 0; i < n; i++)
{
double salience = (colorToCNSaliency[swatchIndexOf[data.lab[i]]] - minE) / (maxE - minE);//colorNames.NormalizedSaliency(data.lab[i], minE, maxE);
csavg += salience;
csmin = Math.Min(csmin, salience);
csmax = Math.Max(csmax, salience);
}
csavg /= n;
Log(log, "NameSalience " + watch.ElapsedMilliseconds);
watch.Restart();
//Calculate the diversity, avg distance to closest neighbor
double div = 0;
double divr = 0;
for (int c = 0; c < n; c++)
{
double bestDist = Double.PositiveInfinity;
for (int i = 0; i < n; i++)
{
if (c == i)
continue;
double curDist = data.lab[c].SqDist(data.lab[i]);
if (curDist < bestDist)
bestDist = curDist;
}
div += Math.Sqrt(bestDist); // (n * factor);
}
double temp = div;
div = temp / (n * factor);
divr = temp / (n * afactor);
Log(log, "DiffClosestAvg " + watch.ElapsedMilliseconds);
watch.Restart();
//calculate average distance of one color to the rest of the colors
double dmin = 1;
double dmax = 0;
double davg = 0;
double dminr = 1;
double dmaxr = 0;
double davgr = 0;
for (int i = 0; i < n; i++)
{
for (int j = i + 1; j < n; j++)
{
double dist = Math.Sqrt(data.lab[i].SqDist(data.lab[j]));
davg += dist;
dmin = Math.Min(dmin, dist);
dmax = Math.Max(dmax, dist);
}
}
double dtemp = dmin;
dmin = dtemp / factor;
dminr = dtemp / afactor;
dtemp = dmax;
dmax = dtemp / factor;
dmaxr = dtemp / afactor;
davg /= n * (n - 1) / 2;
dtemp = davg;
davg = dtemp / factor;
davgr = dtemp / afactor;
Log(log, "LABDiff " + watch.ElapsedMilliseconds);
watch.Restart();
double puritymin = double.PositiveInfinity;
double puritymax = double.NegativeInfinity;
double purityavg = 0;
double purityminr = puritymin;
double puritymaxr = puritymax;
double purityavgr = 0;
foreach (CIELAB c in data.lab)
{
double val = swatchToPurity[swatchIndexOf[c]];
puritymin = Math.Min(puritymin, val);
puritymax = Math.Max(puritymax, val);
purityavg += val;
}
double ptemp = puritymin;
puritymin = ptemp / factor;
purityminr = ptemp / factor;
ptemp = puritymax;
puritymax = ptemp / factor;
puritymaxr = ptemp / afactor;
ptemp = purityavg / data.lab.Count();
purityavg = ptemp / factor;
purityavgr = ptemp / afactor;
//Record all the features
features.Add(Features.CovUnweighted, 1 - error);
features.Add(Features.CovWeightedSaliency, 1 - errorWeighted);
features.Add(Features.ErrorUnweighted, error);
features.Add(Features.ErrorWeightedSaliency, errorWeighted);
features.Add(Features.NCov, 1 - nerror);
features.Add(Features.NCovWeightedSaliency, 1 - nerrorWeighted);
features.Add(Features.NError, nerror);
features.Add(Features.NErrorWeightedSaliency, nerrorWeighted);
features.Add(Features.DAvg, davg);
features.Add(Features.DClosestAvg, div);
features.Add(Features.DMin, dmin);
features.Add(Features.DMax, dmax);
features.Add(Features.NClosestAvg, cndiv);
features.Add(Features.NDiffAvg, cnavg / maxN);
features.Add(Features.NDiffMin, cnmin / maxN);
features.Add(Features.NDiffMax, cnmax / maxN);
features.Add(Features.NSaliencyMin, csmin);
features.Add(Features.NSaliencyMax, csmax);
features.Add(Features.NSaliencyAvg, csavg);
features.Add(Features.SaliencyDensMin, sdmin);
features.Add(Features.SaliencyDensMax, sdmax);
features.Add(Features.SaliencyDensAvg, sdavg);
features.Add(Features.SaliencyTotal, stotal);
features.Add(Features.SaliencyDensClustMin, sdmin2);
features.Add(Features.SaliencyDensClustMax, sdmax2);
features.Add(Features.SaliencyDensClustAvg, sdavg2);
features.Add(Features.DMinR, dminr);
features.Add(Features.DMaxR, dmaxr);
features.Add(Features.DAvgR, davgr);
features.Add(Features.DClosestAvgR, divr);
features.Add(Features.PurityMin, puritymin);
features.Add(Features.PurityMax, puritymax);
features.Add(Features.PurityAvg, purityavg);
features.Add(Features.PurityMinR, purityminr);
features.Add(Features.PurityMaxR, puritymaxr);
features.Add(Features.PurityAvgR, purityavgr);
features.Add(Features.NDiffMinR, cnmin / avgN);
features.Add(Features.NDiffMaxR, cnmax / avgN);
features.Add(Features.NDiffAvgR, cnavg / avgN);
features.Add(Features.NSaliencyAvgR, csavg / avgE);
features.Add(Features.NSaliencyMaxR, csmax / avgE);
features.Add(Features.NSaliencyMinR, csmin / avgE);
features.Add(Features.NClosestAvgR, cndiv / avgE);
features.Add(Features.ErrorWeightedSegment, errorSegSal);
features.Add(Features.ErrorUnweightedSegment, errorSeg);
features.Add(Features.ColorsPerSeg, colorsPSeg);
features.Add(Features.ColorsPerSegWeighted, colorsPSegW);
features.Add(Features.MeanSegError, meansegError);
features.Add(Features.MeanSegErrorWeighted, meansegErrorWeighted);
features.Add(Features.LCov, Lcov);
features.Add(Features.ACov, Acov);
features.Add(Features.BCov, Bcov);
features.Add(Features.SatCov, Scov);
features.Add(Features.BetweenVar, betweenVariance);
features.Add(Features.WithinVar, withinVariance);
features.Add(Features.SqErrorUnweighted, sqerror);
features.Add(Features.SqErrorWeightedSaliency, sqerrorW);
features.Add(Features.SqErrorWSeg, sqerrorSegSal);
features.Add(Features.SqErrorSeg, sqerrorSeg);
features.Add(Features.NSqError, nsqerror);
features.Add(Features.NSqErrorW, nsqerrorW);
features.Add(Features.NSqErrorSeg, nsqerrorSeg);
features.Add(Features.NSqErrorWSeg, nsqerrorSegSal);
features.Add(Features.NErrorSeg, nerrorSeg);
features.Add(Features.NErrorWSeg, nerrorSegSal);
features.Add(Features.SoftError, softerror);
features.Add(Features.SoftWError, softwerror);
features.Add(Features.SoftErrorSeg, softerrorseg);
features.Add(Features.SoftErrorWSeg, softerrorwseg);
features.Add(Features.NSoftError, nsofterror);
features.Add(Features.NSoftWError, nsoftwerror);
features.Add(Features.NSoftErrorSeg, nsofterrorseg);
features.Add(Features.NSoftErrorWSeg, nsofterrorwseg);
features.Add(Features.NMeanSegError, nmeansegerror);
features.Add(Features.NMeanSegErrorW, nmeansegwerror);
features.Add(Features.MeanSegErrorDensity, meansegsd);
features.Add(Features.NMeanSegErrorDensity, nmeansegsd);
features.Add(Features.ErrorSegSD, errorsegsd);
features.Add(Features.NErrorSegSD, nerrorsegsd);
features.Add(Features.SqErrorSegSD, sqerrorsegsd);
features.Add(Features.NSqErrorSegSD, nsqerrorsegsd);
features.Add(Features.SoftErrorSegSD, softerrorsegsd);
features.Add(Features.NSoftErrorSegSD, nsofterrorsegsd);
return features;
}
public static bool ColorsMatch(Color a, Color b, double distance)
{
CIELAB aLab = ConvertToCIELAB(a);
return(aLab.DistanceTo(b) < distance);
}
//Precompute some parameters
private FeatureParams SetupFeatureParams(SortedSet<Features> included, String key, String saliencyPattern, bool debug = false, int maxIters=50)
{
FeatureParams options = new FeatureParams();
//find the normalization factor for swatches
double factor = 0;
double afactor = 0;
PaletteData swatches = GetPaletteSwatches(key, maxIters);
int scount = swatches.lab.Count();
for (int i = 0; i < swatches.lab.Count(); i++)
{
for (int j = i + 1; j < swatches.lab.Count(); j++)
{
double dist = swatches.lab[i].SqDist(swatches.lab[j]);
factor = Math.Max(factor, dist);
afactor += Math.Sqrt(dist);
}
}
factor = Math.Sqrt(factor);
afactor /= ((scount - 1) * scount / 2);
//open the image
Bitmap orig = new Bitmap(Image.FromFile(dir + "/" + key));
Bitmap image = orig;
if (debug)
image = new Bitmap(orig, orig.Width / 4, orig.Height / 4);
CIELAB[,] imageLAB = new CIELAB[image.Width, image.Height];
//convert the image to LAB
for (int i = 0; i < image.Width; i++)
for (int j = 0; j < image.Height; j++)
imageLAB[i, j] = Util.RGBtoLAB(image.GetPixel(i, j));
double[,] pixelToDists = new double[image.Width * image.Height, scount];
double[,] pixelToNDists = new double[image.Width * image.Height, scount];
//calculate the assignment of a pixel to a swatch
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
int idx = j * image.Width + i;
for (int s = 0; s < swatches.lab.Count(); s++)
{
pixelToDists[idx, s] = imageLAB[i, j].SqDist(swatches.lab[s]);
int a = colorNames.GetBin(imageLAB[i, j]);
int b = colorNames.GetBin(swatches.lab[s]);
double ndist = GetCNDist(a, b);
pixelToNDists[idx, s] = ndist * ndist;
}
}
}
//calculate the unnormalized purity of each swatch
double[] purityAvg = new double[scount];
int purityCount = (int)Math.Round(image.Width * image.Height / 20.0);
//for each swatch, find the top 5% pixels closest to it (used by Donovan)
for (int s = 0; s < scount; s++)
{
CIELAB swatch = swatches.lab[s];
List<double> dists = new List<double>(image.Width * image.Height);
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
dists.Add(imageLAB[i, j].SqDist(swatch));
}
}
dists.Sort();
//now pop off the top
for (int i = 0; i < purityCount; i++)
{
purityAvg[s] += Math.Sqrt(dists[i]);
}
purityAvg[s] /= purityCount;
}
//read the saliency map, and calculate saliency for each swatch
String mapPath = Util.ConvertFileName(key, saliencyPattern);//"gbvs_" + key;
Bitmap map = new Bitmap(Image.FromFile(dir + "/saliency/" + mapPath), image.Width, image.Height);
double[,] mapData = new double[image.Width, image.Height];
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
mapData[i, j] = map.GetPixel(i, j).G / 255.0;
}
}
Dictionary<CIELAB, int> swatchIndexOf = new Dictionary<CIELAB, int>();
for (int i = 0; i < swatches.lab.Count(); i++)
{
swatchIndexOf.Add(swatches.lab[i], i);
}
//for each cluster, figure out it's total/average saliency
double[] colorToSaliency = new double[swatches.lab.Count()];
int[] colorToCounts = new int[swatches.lab.Count()];
double thresh = Double.PositiveInfinity;
double totalSaliency = 0;
for (int i = 0; i < map.Width; i++)
{
for (int j = 0; j < map.Height; j++)
{
//find the image color, and its closest swatch, within some max threshold
CIELAB c = imageLAB[i, j];
double bestDist = thresh;
int bestSwatch = -1;
for (int s = 0; s < swatches.lab.Count(); s++)
{
if (swatches.lab[s].SqDist(c) < bestDist)
{
bestDist = swatches.lab[s].SqDist(c);
bestSwatch = s;
}
}
double value = map.GetPixel(i, j).G / 255.0;
if (bestSwatch >= 0)
{
colorToSaliency[bestSwatch] += value;
colorToCounts[bestSwatch]++;
}
totalSaliency += value;
}
}
List<double> sorted = colorToSaliency.ToList<double>();
sorted.Sort();
sorted.Reverse();
double totalCapturable = 0;
for (int i = 0; i < 5; i++)
{
totalCapturable += sorted[i];
}
double minE = Double.PositiveInfinity;
double maxE = Double.NegativeInfinity;
double[] colorToCNSaliency = new double[swatches.lab.Count()];
double avgE = 0;
for (int i = 0; i < swatches.lab.Count(); i++)
{
CIELAB s = swatches.lab[i];
double sal = colorNames.Saliency(s);
minE = Math.Min(minE, sal);
maxE = Math.Max(maxE, sal);
avgE += sal;
colorToCNSaliency[i] = sal;
}
avgE /= swatches.lab.Count();
avgE = (avgE - minE) / (maxE - minE);
//find the avg and max color name distance between swatches
double maxN = Double.NegativeInfinity;
double avgN = 0;
for (int i = 0; i < scount; i++)
{
for (int j = i + 1; j < scount; j++)
{
double val = 1 - colorNames.CosineDistance(swatches.lab[i], swatches.lab[j]);
avgN += val;
maxN = Math.Max(maxN, val);
}
}
avgN /= ((scount - 1) * scount / 2);
//cache the distance between image colors and swatches
Dictionary<Tuple<int, int>, double> imageSwatchDist = new Dictionary<Tuple<int, int>, double>();
int[,] binAssignments = new int[image.Width, image.Height];
int[] swatchIdxToBin = new int[swatches.colors.Count()];
if (included.Overlaps(options.NameCovFeatures))
{
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
CIELAB c = imageLAB[i, j];
for (int s = 0; s < swatches.lab.Count(); s++)
{
int a = colorNames.GetBin(imageLAB[i, j]);
int b = colorNames.GetBin(swatches.lab[s]);
binAssignments[i, j] = a;
swatchIdxToBin[s] = b;
GetCNDist(a, b);
}
}
}
}
//segment to total saliency
Segmentation seg = LoadSegAssignments(key, image.Width, image.Height, imageLAB);
double[] segmentToSaliency = new double[seg.numSegments];
double[] segmentToSD = new double[seg.numSegments];
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
int id = seg.assignments[i, j];
segmentToSaliency[id] += mapData[i, j];
segmentToSD[id] += mapData[i, j];
}
}
double totalSD = 0;
for (int i = 0; i < seg.numSegments; i++)
{
segmentToSD[i] /= seg.counts[i];
totalSD += segmentToSD[i];
}
//find the l,a,b, sat bounds
double[] Lbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Abounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Bbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
double[] Satbounds = new double[] { double.PositiveInfinity, double.NegativeInfinity };
for (int i = 0; i < swatches.lab.Count(); i++)
{
CIELAB lab = swatches.lab[i];
Color rgb = swatches.colors[i];
Lbounds[0] = Math.Min(Lbounds[0], lab.L);
Lbounds[1] = Math.Max(Lbounds[1], lab.L);
Abounds[0] = Math.Min(Abounds[0], lab.A);
Abounds[1] = Math.Max(Abounds[1], lab.A);
Bbounds[0] = Math.Min(Bbounds[0], lab.B);
Bbounds[1] = Math.Max(Bbounds[1], lab.B);
double s = rgb.GetSaturation();
Satbounds[0] = Math.Min(Lbounds[0], s);
Satbounds[1] = Math.Max(Lbounds[1], s);
}
//pack up
options.included = included;
options.imageLAB = imageLAB;
options.map = mapData;
options.colorToSaliency = colorToSaliency;
options.colorToCounts = colorToCounts;
options.totalSaliency = totalSaliency;
options.colorNames = colorNames;
options.factor = factor;
options.colorToCNSaliency = colorToCNSaliency;
options.minE = minE;
options.maxE = maxE;
options.swatchIndexOf = swatchIndexOf;
options.imageSwatchDist = imageSwatchDist;
options.afactor = afactor;
options.swatchToPurity = purityAvg;
//options.pToScore = pToScore;
options.maxN = maxN;
options.avgN = avgN;
options.avgE = avgE;
options.segmentToSaliency = segmentToSaliency;
options.seg = seg;
options.Lspan = Lbounds[1] - Lbounds[0];
options.Aspan = Abounds[1] - Abounds[0];
options.Bspan = Bbounds[1] - Bbounds[0];
options.Satspan = Satbounds[1] - Satbounds[0];
options.swatchIdxToBin = swatchIdxToBin;
options.binAssignments = binAssignments;
options.totalCapturableSaliency = totalCapturable;
options.pixelToDists = pixelToDists;
options.pixelToNDists = pixelToNDists;
options.totalSD = totalSD;
options.segmentToSD = segmentToSD;
return options;
}
public void LoadFromFile(String imageFile, String segFile)
{
Bitmap image = new Bitmap(imageFile);
Bitmap map = new Bitmap(segFile);
int width = image.Width;
int height = image.Height;
Bitmap resized = Util.ResizeBitmapNearest(map, width, height);
assignments = new int[width, height];
List<Color> unique = new List<Color>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
Color c = resized.GetPixel(i, j);
if (!unique.Contains(c))
unique.Add(c);
int idx = unique.IndexOf(c);
assignments[i, j] = idx;
}
}
numSegments = unique.Count();
counts = new int[numSegments];
segToMeanColor = new CIELAB[numSegments];
for (int i = 0; i < numSegments; i++)
{
segToMeanColor[i] = new CIELAB();
}
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = assignments[i, j];
counts[idx]++;
segToMeanColor[idx] += Util.RGBtoLAB(image.GetPixel(i, j));
}
}
for (int i = 0; i < numSegments; i++)
{
segToMeanColor[i] /= counts[i];
}
}
private double[] GetProbabilities(String query, Func<CIELAB, CIELAB, double> distFunc, Kernel kernel, double whiteThresh=20)
{
CIELAB white = new CIELAB(100, 0, 0);
//load the histogram
double[, ,] hist = new double[Lbins, Abins, Bbins];
String histFile = Path.Combine(cacheDir, query)+".txt";
String[] hlines = File.ReadAllLines(histFile);
for (int i = 0; i < hlines.Count(); i++)
{
int L = i / (Abins * Bbins);
int plane = (i % (Abins * Bbins));
int A = plane / Bbins;
int B = plane % Bbins;
hist[L, A, B] = Double.Parse(hlines[i]);
}
int ncolors = paletteLAB.Count();
double[] freq = new double[20];
Parallel.For(0, ncolors, l =>
{
double count = 0;
for (int i = 0; i < Lbins; i++)
{
for (int j = 0; j < Abins; j++)
{
for (int k = 0; k < Bbins; k++)
{
double val = hist[i, j, k];
System.Diagnostics.Debug.Assert(!double.IsNaN(val));
CIELAB lab = new CIELAB(i * binSize, j * binSize - 100, k * binSize - 100);
if (white.SqDist(lab) < whiteThresh * whiteThresh)
continue;
if (val <= 0)
continue;
count += val;
freq[l] += val * kernel.Eval(distFunc(paletteLAB[l], lab));
}
}
}
if (count > 0)
{
freq[l] /= count;
}
});
//now renormalize
double totalFreq = 0;
for (int i=0; i<freq.Count(); i++)
totalFreq += freq[i];
//this should only happen if the histogram is empty or has no valid bins
if (totalFreq == 0)
totalFreq = 1;
for (int i=0; i<freq.Count(); i++)
freq[i] /= totalFreq;
return freq;
}
private int GetBinId(CIELAB color)
{
int Lbin = GetBin(color.L, minL, maxL, Lbins);
int Abin = GetBin(color.A, minA, maxA, Abins);
int Bbin = GetBin(color.B, minB, maxB, Bbins);
return (int)(Abins * Bbins * Lbin + Abins * Bbin + Abin);
}
public double CosineDistance(CIELAB a, CIELAB b)
{
int i = GetBin(a);
int j = GetBin(b);
int C = colors.Count();
int W = terms.Count();
double sa = 0, sb = 0, sc = 0;
int ta;
int tb;
for (var w = 0; w < W; w++)
{
ta = T[i * W + w];
tb = T[j * W + w];
sa += ta * ta;
sb += tb * tb;
sc += ta * tb;
}
return sc / Math.Max((Math.Sqrt(sa * sb)),1);
}
public double NormalizedSaliency(CIELAB a, double minE, double maxE)
{
//hardcoded to XKCD
//double minE = -4.5;
//double maxE = 0;
return (Saliency(a) - minE) / (maxE - minE);
}
public Bitmap Render()
{
//render the template (assuming possible values are between -200 and 200)
//must clamp to be valid RGB values
int width = template.GetLength(0);
int height = template.GetLength(1);
Bitmap result = new Bitmap(width, height);
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
CIELAB lab = template[i, j];
//add gray to the lab value
CIELAB gray = new CIELAB(53.5850, 0, 0);
//convert (first clamp to reasonable LAB values?)
Color rgb = Util.LABtoRGB(lab + gray);
//Color rgb = Util.LABtoRGB(lab * gray);
result.SetPixel(i, j, rgb);
}
}
return result;
}
public void Reset()
{
lab = MeanColor();
count = 0;
sumlab = new CIELAB(0, 0, 0);
}
private void RemoveBackground(Color[,] image, CIELAB[,] imageLAB)
{
//check perimeter to see if it's mostly black or white
//RGB to LAB
CIELAB black = new CIELAB(0, 0, 0);
CIELAB white = new CIELAB(100, 0, 0);
int width = image.GetLength(0);
int height = image.GetLength(1);
CIELAB[,] labs = imageLAB;//Util.Map<Color, CIELAB>(image, (c) => Util.RGBtoLAB(c));
int numBlack = 0;
int numWhite = 0;
int thresh = 3 * 3;
List<Point> perimeterIdx = new List<Point>();
double totalPerimeter = 4 * width + 4 * height;
double bgThresh = totalPerimeter*0.75;
for (int i = 0; i < width; i++)
{
//top
for (int j = 0; j < 2; j++)
{
if (black.SqDist(labs[i, j])<thresh)
numBlack++;
if (white.SqDist(labs[i, j]) < thresh)
numWhite++;
perimeterIdx.Add(new Point(i, j));
}
//bottom
for (int j = height - 2; j < height; j++)
{
perimeterIdx.Add(new Point(i, j));
if (black.SqDist(labs[i, j]) < thresh)
numBlack++;
if (white.SqDist(labs[i, j]) < thresh)
numWhite++;
}
}
for (int j = 0; j < height; j++)
{
//left
for (int i=0; i<2; i++)
{
perimeterIdx.Add(new Point(i,j));
if (black.SqDist(labs[i, j]) < thresh)
numBlack++;
if (white.SqDist(labs[i, j]) < thresh)
numWhite++;
}
//right
for (int i=width-2; i<width; i++)
{
perimeterIdx.Add(new Point(i, j));
if (black.SqDist(labs[i, j]) < thresh)
numBlack++;
if (white.SqDist(labs[i, j]) < thresh)
numWhite++;
}
}
if (numBlack >= bgThresh || numWhite >= bgThresh)
{
//connected components
UnionFind<CIELAB> uf = new UnionFind<CIELAB>((a,b) => a.SqDist(b)<thresh);
int[,] cc = uf.ConnectedComponents(labs);
SortedSet<int> ids = new SortedSet<int>();
//go around the perimeter to collect the right ids
foreach (Point p in perimeterIdx)
{
if (numWhite > numBlack)
{
if (labs[p.X, p.Y].SqDist(white) < thresh)
ids.Add(cc[p.X, p.Y]);
}
else
{
if (labs[p.X, p.Y].SqDist(black) < thresh)
ids.Add(cc[p.X, p.Y]);
}
}
//fill the bitmap with transparency
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
if (ids.Contains(cc[i, j]))
image[i, j] = Color.FromArgb(0, 0, 0, 0);
}
}
}
}
public PixelCluster(int i, CIELAB color)
{
id = i;
lab = color;
count = 1;
neighbors = new SortedSet<int>();
parentId = i;
children = new int[] { };
}
public PixelCluster()
{
id = -1;
lab = new CIELAB();
count = 1;
neighbors = new SortedSet<int>();
parentId = -1;
children = new int[] { };
}
private bool isBlackWhite(Color[,] image, CIELAB[,] imageLAB)
{
double nonGray = 0;
double thresh = 0.001;
CIELAB white = new CIELAB(100, 0, 0);
CIELAB black = new CIELAB(0,0,0);
int width = image.GetLength(0);
int height = image.GetLength(1);
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
Color color = image[i, j];
CIELAB lab = imageLAB[i, j];//Util.RGBtoLAB(color);
bool gray = color.GetSaturation() <= 0.2 || lab.SqDist(white) <= 5 || lab.SqDist(black) <= 5;
if (!gray)
nonGray++;
}
}
return nonGray/(width*height) < thresh;
}
public double DistanceTo(CIELAB c) {
double dL = L - c.L;
double da = a - c.a;
double db = b - c.b;
return Math.Sqrt(0.5 * dL * dL + da * da + db * db);
}
public static List<Cluster> InitializePictureSeeds(List<CIELAB> colors, int k)
{
//initialize k seeds, randomly choose colors in LAB space
//find extents
List<Cluster> seeds = new List<Cluster>();
Random random = new Random();
//sample colors in LAB bounding box
double Lmin = double.PositiveInfinity;
double Lmax = double.NegativeInfinity;
double Amin = double.PositiveInfinity;
double Amax = double.NegativeInfinity;
double Bmin = double.PositiveInfinity;
double Bmax = double.NegativeInfinity;
for (int i = 0; i < colors.Count(); i++)
{
CIELAB lab = colors[i];
Lmin = Math.Min(Lmin, lab.L);
Lmax = Math.Max(Lmax, lab.L);
Amin = Math.Min(Amin, lab.A);
Amax = Math.Max(Amax, lab.A);
Bmin = Math.Min(Bmin, lab.B);
Bmax = Math.Max(Bmax, lab.B);
}
//initialize the seeds (stratified) randomly
//within the bounding box
if (k <= 10)
{
for (int i = 0; i < k; i++)
{
double L = random.NextDouble() * (Lmax - Lmin) + Lmin;
double A = random.NextDouble() * (Amax - Amin) + Amin;
double B = random.NextDouble() * (Bmax - Bmin) + Bmin;
CIELAB seed = new CIELAB(L, A, B);
Cluster cluster = new Cluster();
cluster.id = i;
cluster.lab = seed;
seeds.Add(cluster);
}
}
else
{
//stratified
//find closest floor perfect square.
//TODO: need to generalize this better, doesn't work for non-perfect squares
int sideLength = 2;
int numSamples = (int)Math.Floor(k / (double)(sideLength*sideLength*sideLength));
int i = 0;
for (int l = 0; l < sideLength; l++)
{
double dLmax = (Lmax - Lmin) / sideLength * (l+1) + Lmin;
double dLmin = (Lmax - Lmin) / sideLength * l + Lmin;
for (int a = 0; a < sideLength; a++)
{
double dAmax = (Amax - Amin) / sideLength * (a + 1) + Amin;
double dAmin = (Amax - Amin) / sideLength * a + Amin;
for (int b = 0; b < sideLength; b++)
{
double dBmax = (Bmax - Bmin) / sideLength * (b + 1) + Bmin;
double dBmin = (Bmax - Bmin) / sideLength * b + Bmin;
int dSamples = numSamples;
if (b == sideLength - 1 && a == sideLength - 1 && l == sideLength - 1)
{
//figure out leftovers
dSamples = k - numSamples*(sideLength * sideLength * sideLength - 1);
}
for (int s = 0; s < dSamples; s++)
{
double L = random.NextDouble() * (dLmax - dLmin) + dLmin;
double A = random.NextDouble() * (dAmax - dAmin) + dAmin;
double B = random.NextDouble() * (dBmax - dBmin) + dBmin;
CIELAB seed = new CIELAB(L, A, B);
Cluster cluster = new Cluster();
cluster.id = i;
cluster.lab = seed;
seeds.Add(cluster);
i++;
}
}
}
}
}
return seeds;
}
public double GetCNDist(CIELAB a, CIELAB b)
{
int i = colorNames.GetBin(a);
int j = colorNames.GetBin(b);
if (CNcache[i, j] < 0)
{
double dist = 1 - colorNames.CosineDistance(i,j);
CNcache[i, j] = dist;
CNcache[j, i] = dist;
}
return CNcache[i, j];
}
private int ClosestColorIndex(CIELAB color, List<CIELAB> palette)
{
int bestc = 0;
Double bestDist = Double.PositiveInfinity;
for (int c = 0; c < palette.Count(); c++)
{
double dist = color.SqDist(palette[c]);
if (dist < bestDist)
{
bestDist = dist;
bestc = c;
}
}
return bestc;
}
//Load PaletteData
private Dictionary <String, PaletteData> LoadFilePalettes(String file)
{
FileInfo finfo = new FileInfo(file);
Dictionary <String, PaletteData> plist = new Dictionary <String, PaletteData>();
String[] lines = File.ReadAllLines(file);
//extracted palettes file format
if (finfo.Extension == ".tsv")
{
for (int i = 1; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split('\t');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[0]);
data.workerNum = Int32.Parse(fields[1]);
String key = fields[2];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
foreach (String s in colors)
{
String[] comp = s.Split(',');
Color c = Color.FromArgb(Int32.Parse(comp[0]), Int32.Parse(comp[1]), Int32.Parse(comp[2]));
CIELAB l = Util.RGBtoLAB(c);
data.colors.Add(c);
data.lab.Add(l);
}
if (!plist.ContainsKey(key))
{
plist.Add(key, data);
}
else
{
throw new IOException("More than one palette per key");
}
}
}
else //pattern template file format, populate the pattern id to template id field
{
for (int i = 0; i < lines.Count(); i++)
{
String line = lines[i];
String[] fields = line.Replace("\"", "").Split(',');
PaletteData data = new PaletteData();
data.id = Int32.Parse(fields[1]);
data.workerName = fields[0];
String[] colors = fields[3].Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
colors = colors.Distinct <String>().ToArray <String>();
String key = fields[1] + ".png";
foreach (String s in colors)
{
Color c = ColorTranslator.FromHtml("#" + s);
data.colors.Add(c);
data.lab.Add(Util.RGBtoLAB(c));
}
//ignore missing palette data
if (!plist.ContainsKey(key) && colors.Count() > 0)
{
plist.Add(key, data);
}
else if (plist.ContainsKey(key))
{
throw new IOException("More than one palette per key");
}
String templateId = fields[4];
if (!patternToTemplate.ContainsKey(data.id.ToString()))
{
patternToTemplate.Add(data.id.ToString(), templateId);
}
}
}
return(plist);
}
public static double CMeansPicture(List<CIELAB> colors, List<Cluster> seeds, int m=2)
{
//cluster colors given seeds
//return score
List<double> weights = new List<double>();
double[,] memberships = new double[colors.Count(), seeds.Count()]; //pixel to cluster
int numSeeds = seeds.Count();
int numColors = colors.Count();
int maxIters = 50;//100;
double epsilon = 0.0001;
double J = Double.PositiveInfinity;
int changes = 0;
for (int t = 0; t < maxIters; t++)
{
changes = 0;
for (int i = 0; i < colors.Count(); i++)
{
//calculate the memberships
double[] dists = new double[numSeeds];
double factor = 0;
for (int k = 0; k < numSeeds; k++)
{
dists[k] = Math.Max(epsilon, Math.Pow(Math.Sqrt(colors[i].SqDist(seeds[k].lab)), 2.0/(m-1)));
factor += (1.0 / dists[k]);
}
for (int k = 0; k < numSeeds; k++)
{
double oldval = memberships[i, k];
memberships[i, k] = 1.0 / (dists[k] * factor);
if (oldval != memberships[i, k])
changes++;
}
}
//update the centers
for (int k = 0; k < numSeeds; k++)
{
CIELAB center = new CIELAB();
double total = 0;
for (int i = 0; i < numColors; i++)
{
double u = Math.Pow(memberships[i, k], m);
center += colors[i]*u;
total += u;
}
center = center / total;
seeds[k].lab = center;
}
//find J
double thisJ = 0;
for (int i = 0; i < numColors; i++)
{
for (int k = 0; k < numSeeds; k++)
{
double u = memberships[i, k];
thisJ += Math.Pow(u, m)* Math.Max(epsilon, seeds[k].lab.SqDist(colors[i]));
}
}
if (thisJ >= J)
break;
J = thisJ;
if (changes == 0)
break;
}
return J;
}
private CIELAB[,] EdgeUnmixRecolor(CIELAB[,] image, PaletteData palette)
{
int width = image.GetLength(0);
int height = image.GetLength(1);
CIELAB[,] result = new CIELAB[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double[] scores = new double[palette.colors.Count()];
//check if the color is in the palette
CIELAB color = image[i, j];
if (palette.lab.Contains(color))
{
result[i, j] = color;
continue;
}
//otherwise, find the best color
//see if this is an edge
//if it is, assign the color to be one of the edge color candidates
HashSet<CIELAB> tbCandidates = new HashSet<CIELAB>();
HashSet<CIELAB> lrCandidates = new HashSet<CIELAB>();
List<HashSet<CIELAB>> unmixCandidates = new List<HashSet<CIELAB>> { tbCandidates, lrCandidates };
//check edge types
//horizontal and vertical
HashSet<CIELAB> oneSide = new HashSet<CIELAB>();
HashSet<CIELAB> otherSide = new HashSet<CIELAB>();
Point[] top = new Point[] { new Point(i - 1, j - 1), new Point(i, j - 1), new Point(i + 1, j - 1) };
Point[] bottom = new Point[] { new Point(i - 1, j + 1), new Point(i - 1, j), new Point(i - 1, j + 1) };
Point[] left = new Point[] { new Point(i - 1, j - 1), new Point(i - 1, j), new Point(i - 1, j + 1) };
Point[] right = new Point[] { new Point(i + 1, j - 1), new Point(i + 1, j), new Point(i + 1, j + 1) };
List<Point[]> oneCompare = new List<Point[]> { top, left };
List<Point[]> otherCompare = new List<Point[]> { bottom, right };
bool edge = false;
for (int c = 0; c < oneCompare.Count(); c++)
{
oneSide.Clear();
otherSide.Clear();
foreach (Point p in oneCompare[c])
{
if (Util.InBounds(p.X, p.Y, width, height))
{
CIELAB rc = ClosestColor(image[p.X, p.Y], palette.lab);
//check if in the set
if (oneSide.Contains(rc))
{
//yes, we found a majority
unmixCandidates[c].Add(image[p.X, p.Y]);
break;
}
else
{
oneSide.Add(rc);
}
}
}
foreach (Point p in otherCompare[c])
{
if (Util.InBounds(p.X, p.Y, width, height))
{
CIELAB rc = ClosestColor(image[p.X, p.Y], palette.lab);
//check if in the set
if (otherSide.Contains(rc))
{
//yes, we found a majority
unmixCandidates[c].Add(rc);
break;
}
else
{
otherSide.Add(rc);
}
}
}
//is it an edge?
if (unmixCandidates[c].Count() >= 2)
{
result[i, j] = ClosestColor(image[i, j], unmixCandidates[c].ToList<CIELAB>());
edge = true;
break;
}
}
//TODO:
//45 degrees
//45 degrees-flipped
if (!edge)
{
for (int dx = -1; dx <= 1; dx++)
{
for (int dy = -1; dy <= 1; dy++)
{
int x = i + dx;
int y = j + dy;
double weight = (dx == 0 && dy == 0) ? 4 : 1;
if (Util.InBounds(x, y, width, height))
{
int bestc = ClosestColorIndex(image[x, y], palette.lab);
scores[bestc] -= weight;
}
}
}
//pick the color with the min score
double minScore = Double.PositiveInfinity;
int bestIdx = 0;
for (int c = 0; c < palette.colors.Count(); c++)
{
if (scores[c] < minScore)
{
minScore = scores[c];
bestIdx = c;
}
}
result[i, j] = palette.lab[bestIdx];
}
}
}
return result;
}
public int GetBin(CIELAB x)
{
int L = (int)(5 * Math.Round(x.L/5));
int A = (int)(5 * Math.Round(x.A/5));
int B = (int)(5 * Math.Round(x.B/5));
String s = L+","+A+","+B;
if (map.ContainsKey(s))
{
return map[s];
}
else {
//look at nearby bins
CIELAB[] neighbors = new CIELAB[] {new CIELAB(L, A, B + 5), new CIELAB(L, A, B - 5), new CIELAB(L, A + 5, B), new CIELAB(L, A - 5, B), new CIELAB(L - 5, A, B), new CIELAB(L + 5, A, B)};
double bestDist = Double.PositiveInfinity;
String best = "";
for (int i = 0; i < neighbors.Count(); i++)
{
String key = neighbors[i].L + "," + neighbors[i].A + "," + neighbors[i].B;
double dist = neighbors[i].SqDist(x);
if (dist < bestDist && map.ContainsKey(key))
{
bestDist = dist;
best = key;
}
}
if (best != "")
{
//cache the best
return map[best];
}
else
//give up
return -1;
}
}
private CIELAB[,] ModeRecolor(CIELAB[,] image, PaletteData palette)
{
int width = image.GetLength(0);
int height = image.GetLength(1);
CIELAB[,] result = new CIELAB[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
double[] scores = new double[palette.colors.Count()];
//check if the color is in the palette
CIELAB color = image[i, j];
if (palette.lab.Contains(color))
{
result[i, j] = color;
continue;
}
//otherwise, find the best color by mode
for (int dx = -1; dx <= 1; dx++)
{
for (int dy = -1; dy <= 1; dy++)
{
int x = i + dx;
int y = j + dy;
double weight = (dx == 0 && dy == 0) ? 4 : 1;
if (Util.InBounds(x, y, width, height))
{
int bestc = ClosestColorIndex(image[x, y], palette.lab);
scores[bestc] -= weight;
}
}
}
//pick the color with the min score
double minScore = Double.PositiveInfinity;
int bestIdx = 0;
for (int c = 0; c < palette.colors.Count(); c++)
{
if (scores[c] < minScore)
{
minScore = scores[c];
bestIdx = c;
}
}
result[i, j] = palette.lab[bestIdx];
}
}
return result;
}
public double Saliency(CIELAB a)
{
double H = 0;
int W = terms.Count();
int i=GetBin(a);
for (int w = 0; w < W; w++)
{
double p = T[i * W + w];
p /= ccount[i];
if (p > 0)
H += p * Math.Log(p) / Math.Log(2);
}
return H;
}
//Load the segmentation file for a particular image
private Segmentation LoadSegAssignments(String key, int width, int height, CIELAB[,] imageLAB)
{
Bitmap map = new Bitmap(dir + "/segments/" + key);
Bitmap resized = Util.ResizeBitmapNearest(map, width, height);
Segmentation s = new Segmentation();
s.assignments = new int[width, height];
List<Color> unique = new List<Color>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
Color c = resized.GetPixel(i, j);
if (!unique.Contains(c))
unique.Add(c);
int idx = unique.IndexOf(c);
s.assignments[i, j] = idx;
}
}
s.numSegments = unique.Count();
s.counts = new int[s.numSegments];
s.segToMeanColor = new CIELAB[s.numSegments];
for (int i = 0; i < s.numSegments; i++)
{
s.segToMeanColor[i] = new CIELAB();
}
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = s.assignments[i, j];
s.counts[idx]++;
s.segToMeanColor[idx] += imageLAB[i, j];
}
}
for (int i = 0; i < s.numSegments; i++)
{
s.segToMeanColor[i] /= s.counts[i];
}
return s;
}
public static CIELAB ConvertToCIELAB(Color c) {
if (cielabLookup.ContainsKey(c)) {
return cielabLookup[c];
}
double rf = ConvertChannel(c.R) * 100;
double gf = ConvertChannel(c.G) * 100;
double bf = ConvertChannel(c.B) * 100;
double var_X = rf * 0.4124 + gf * 0.3576 + bf * 0.1805;
double var_Y = rf * 0.2126 + gf * 0.7152 + bf * 0.0722;
double var_Z = rf * 0.0193 + gf * 0.1192 + bf * 0.9505;
var_X = var_X / 95.047; //Observer = 2°, Illuminant = D65
var_Y = var_Y / 100.000;
var_Z = var_Z / 108.883;
double toPow = 1.0 / 3.0;
if (var_X > 0.008856) {
var_X = Math.Pow(var_X, toPow);
} else {
var_X = (7.787 * var_X) + (16 / 116);
}
if (var_Y > 0.008856) {
var_Y = Math.Pow(var_Y, toPow);
} else {
var_Y = (7.787 * var_Y) + (16 / 116);
}
if (var_Z > 0.008856) {
var_Z = Math.Pow(var_Z, toPow);
} else {
var_Z = (7.787 * var_Z) + (16 / 116);
}
CIELAB ans = new CIELAB();
ans.L = (116 * var_Y) - 16;
ans.a = 500 * (var_X - var_Y);
ans.b = 200 * (var_Y - var_Z);
cielabLookup[c] = ans;
return ans;
}
/**
* Return a theme with colors snapped to the given swatches
*/
private PaletteData SnapColors(PaletteData data, PaletteData swatches)
{
PaletteData result = new PaletteData();
List<CIELAB> matchedColors = new List<CIELAB>();
//match each color to closest swatch color
foreach (CIELAB kc in data.lab)
{
double bestDist = Double.PositiveInfinity;
CIELAB best = new CIELAB();
foreach (CIELAB s in swatches.lab)
{
double dist = s.SqDist(kc);
if (dist < bestDist)
{
bestDist = dist;
best = s;
}
}
matchedColors.Add(best);
}
result.lab = matchedColors;
foreach (CIELAB l in data.lab)
result.colors.Add(Util.LABtoRGB(l));
return result;
}
