/**
* Get images if needed
* */
private void SearchImages(Category category, int pages=3, bool ifNeeded = true)
{
int thresh = 10;
foreach (String member in category.members)
{
GoogleImageSearch engine = new GoogleImageSearch(apiKey, cxId);
String[] labels = { "", "_clipart" };
foreach (String label in labels)
{
String dirName = Path.Combine(imageDir, Encode(member) + label);
//images have been downloaded already
if (Directory.Exists(dirName) && Directory.GetFiles(dirName).Count() >= thresh)
{
Console.WriteLine("Already have images for " + member + label.Replace("_"," "));
continue;
}
List<String> urls = engine.Search(member+label.Replace("_"," "), pages);
Directory.CreateDirectory(dirName);
//save the images
for (int i = 0; i < urls.Count(); i++)
{
Bitmap image = Util.BitmapFromWeb(urls[i]);
if (image == null)
continue;
image.Save(Path.Combine(dirName, i + ".png"));
image.Dispose();
}
}
}
Console.WriteLine("Done getting images");
}
public ColorAssignment(Category c)
{
colors = new Dictionary<String, Color>();
category = c;
}
/**
* Compute histograms for the given category, both clipart and regular
**/
private void ComputeHistograms(Category category, bool loadIfPossible=true)
{
Stopwatch watch = new Stopwatch();
watch.Start();
Parallel.For(0, category.members.Count(), m =>
{
String member = category.members[m];
//Get the relevant images
String[] queryLabels = { "", "_clipart" };
foreach (String label in queryLabels)
{
String baseName = Encode(member) + label;
String outFile = Path.Combine(cacheDir, baseName + ".txt");
String memberDir = Path.Combine(imageDir, baseName);
if (File.Exists(outFile) && loadIfPossible)
{
Console.WriteLine("Reusing histogram for " + baseName);
continue;
}
double[, ,] histogram = new double[Lbins, Abins, Bbins];
String[] files = Directory.GetFiles(memberDir);
foreach (String f in files)
{
//TODO: maybe process the image here, or filter it out
try
{
Bitmap image = ResizeImage(f);
Color[,] imageRGB = Util.BitmapToArray(image);
CIELAB[,] imageLAB = Util.Map<Color, CIELAB>(imageRGB, Util.RGBtoLAB);
//Process the image
bool valid = ProcessImage(imageRGB, imageLAB);
if (!valid)
continue;
double pixelCount = 0;
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
if (imageRGB[i, j].A >= 1)
{
pixelCount++;
}
}
}
if (pixelCount > 0)
{
for (int i = 0; i < image.Width; i++)
{
for (int j = 0; j < image.Height; j++)
{
if (imageRGB[i, j].A <= 1)
continue;
CIELAB lab = imageLAB[i, j];//Util.RGBtoLAB(imageRGB[i,j]);
//update histogram
int L = (int)Math.Floor((lab.L / (double)binSize) + 0.5);
int A = (int)Math.Floor(((100 + lab.A) / (double)binSize) + 0.5);
int B = (int)Math.Floor(((100 + lab.B) / (double)binSize) + 0.5);
L = clamp(L, 0, Lbins - 1);
A = clamp(A, 0, Abins - 1);
B = clamp(B, 0, Bbins - 1);
histogram[L, A, B] += 1.0 / pixelCount;
}
}
}
//cleanup
image.Dispose();
}
catch (Exception)
{
Console.WriteLine("Could not process image " + f);
}
}
//save the histogram to a file
List<String> lines = new List<String>();
for (int i = 0; i < Lbins; i++)
{
for (int j = 0; j < Abins; j++)
{
for (int k = 0; k < Bbins; k++)
{
lines.Add(histogram[i, j, k].ToString());
}
}
}
File.WriteAllLines(outFile, lines.ToArray<String>());
}
});
watch.Stop();
Console.WriteLine("Done with histograms Time: " + watch.ElapsedMilliseconds / 1000.0);
}
private double[,] ComputeAffinities(Category category, double clipartPrior=0.7, double saturationThresh = 0.1, double sigma=0.2, double whiteThresh=20)
{
//get the color probabilities for regular and clipart queries
double[,] pcw_regular = new double[paletteHex.Count(), category.members.Count()];
double[,] pcw_clipart = new double[paletteHex.Count(), category.members.Count()];
double[,] pcw = new double[paletteHex.Count(), category.members.Count()];
Parallel.For(0, category.members.Count(), w =>
{
String query = Encode(category.members[w]);
double[] pc_regular = GetProbabilities(query, GetCNDist, new GaussianKernel(sigma), whiteThresh);
double[] pc_clipart = GetProbabilities(query+"_clipart", GetCNDist, new GaussianKernel(sigma), whiteThresh);
for (int c=0; c<paletteLAB.Count(); c++)
{
pcw_regular[c, w] = pc_regular[c];
pcw_clipart[c, w] = pc_clipart[c];
}
});
Console.WriteLine("Done computing probabilities");
//compute combined probability
for (int w=0; w<category.members.Count(); w++)
{
double rentropy = 0;
double centropy = 0;
for (int c=0; c<paletteHex.Count(); c++)
{
if (pcw_clipart[c,w] > 0)
centropy += pcw_clipart[c,w]*Math.Log(pcw_clipart[c,w]);
if (pcw_regular[c,w] > 0)
rentropy += pcw_regular[c,w]*Math.Log(pcw_regular[c,w]);
}
centropy *= -1;
rentropy *= -1;
//avoid divide by zero
centropy = Math.Max(centropy, epsilon);
rentropy = Math.Max(rentropy, epsilon);
double cw = clipartPrior/centropy;
double rw = (1-clipartPrior)/rentropy;
for (int c=0; c<paletteHex.Count(); c++)
{
CIELAB lab = paletteLAB[c];
double chroma = Math.Sqrt(lab.A*lab.A+lab.B*lab.B);
double saturation = chroma/Math.Max(Math.Sqrt(chroma*chroma+lab.L*lab.L), epsilon);
pcw[c, w] = Math.Max(saturation, saturationThresh)*(cw*pcw_clipart[c,w] + rw*pcw_regular[c,w]);
}
}
//renormalize
double[] memberSums = new double[category.members.Count()];
for (int c=0; c<paletteHex.Count(); c++)
for (int w=0; w<category.members.Count(); w++)
memberSums[w] += pcw[c,w];
for (int c = 0; c < paletteHex.Count(); c++)
for (int w = 0; w < category.members.Count(); w++)
pcw[c,w] /= memberSums[w];
//now compute affinities from the combined probabilities
double[,] affinities = new double[paletteHex.Count(), category.members.Count()];
double[,] pwc = new double[category.members.Count(), paletteHex.Count()];
//compute p(w|c)
//p(w|c) = p(c|w)*p(w)/p(c)
double colorZ = 0;
double[] colorSums = new double[paletteHex.Count()];
for (int c=0; c < paletteHex.Count(); c++)
{
for (int w=0; w<category.members.Count(); w++)
{
colorSums[c] += pcw[c,w];
colorZ += pcw[c,w];
}
}
for (int c=0; c<paletteHex.Count(); c++)
for (int w=0; w<category.members.Count(); w++)
pwc[w,c] = pcw[c,w]*(1.0/category.members.Count())/(Math.Max(colorSums[c],epsilon)/colorZ);
double minScore = Double.PositiveInfinity;
double maxScore = Double.NegativeInfinity;
//balance with entropy H(w|c)
for (int c = 0; c < paletteHex.Count(); c++)
{
double Hwc = 0;
for (int w = 0; w < category.members.Count(); w++)
{
if (pwc[w,c] > 0)
Hwc += pwc[w,c] * Math.Log(pwc[w,c]);
}
Hwc *= -1;
System.Diagnostics.Debug.Assert(!double.IsNaN(Hwc));
for (int w = 0; w < category.members.Count(); w++)
{
affinities[c, w] = pcw[c, w] / Math.Max(Hwc, epsilon);
minScore = Math.Min(minScore, affinities[c, w]);
maxScore = Math.Max(maxScore, affinities[c, w]);
}
}
System.Diagnostics.Debug.Assert(maxScore != minScore);
//scale the affinities between 0 and 1s (easier to visualize)
for (int c = 0; c < paletteHex.Count(); c++)
{
for (int w = 0; w < category.members.Count(); w++)
{
affinities[c, w] = (affinities[c, w] - minScore) / (maxScore - minScore);
if (Double.IsNaN(affinities[c,w]))
throw new FormatException("Affinity is NaN! Affinity " + category.members[w] + " " + paletteHex[c] );
}
}
Console.WriteLine("Done computing affinities");
return affinities;
}
public Bitmap RenderAffinities(Category category)
{
double[,] hist = ComputeAffinities(category);
int numMembers = category.members.Count();
int colorSize = 10;
int padding = 2;
int textWidth = 200;
int barSize = 100;
int paddingBottom = 10;
Bitmap result = new Bitmap(textWidth + paletteHex.Count() * colorSize, numMembers * barSize + padding+paddingBottom);
Graphics hg = Graphics.FromImage(result);
Brush black = new SolidBrush(Color.Black);
Brush gray = new SolidBrush(Color.Gray);
Font headers = new Font("Arial", 9);
hg.FillRectangle(new SolidBrush(Color.White), 0, 0, result.Width, result.Height);
//write out the concept titles
for (int w = 0; w < numMembers; w++)
{
hg.DrawString(category.members[w], headers, black, 0, w * barSize + barSize / 2);
}
//draw the bar charts
for (int w = 0; w < numMembers; w++)
{
for (int c = 0; c < paletteHex.Count(); c++)
{
int height = (int)Math.Round(hist[c, w] * (barSize - 5));
int offset = barSize - height;
hg.FillRectangle(new SolidBrush(paletteRGB[c]), c * colorSize + textWidth, w * barSize + offset, colorSize, height);
}
//draw a line
hg.DrawLine(new Pen(gray), textWidth, (w + 1) * barSize, 20 * colorSize + textWidth, (w + 1) * barSize);
}
return result;
}
//Reading Category Files
public List<Category> ReadCategories(String filename)
{
String[] lines = File.ReadAllLines(filename);
List<Category> categories = new List<Category>();
foreach (String line in lines)
{
String[] fields = line.Split(new string[] { "\",\"" }, StringSplitOptions.None);
Category c = new Category(fields[1].Replace("\"", ""), fields[0].Replace("\"", "").Split('|'));
categories.Add(c);
}
return categories;
}
public ColorAssignment AssignColors(Category category)
{
//check that number of category members is less than number of palette colors
if (category.members.Count() > paletteHex.Count())
throw new ArgumentException("Too many category members, or not enough palette colors!");
//Get the images, if needed
SearchImages(category, 3);
//Compute the histograms
ComputeHistograms(category);
//Compute affinities and assign colors
double[,] affinities = ComputeAffinities(category);
//Since the Hungarian Algorithm minimizes sum of affinities, let's invert the affinities
//Assume that the number of category members is less than the number of palette colors
double[,] matrix = new double[paletteHex.Count(), paletteHex.Count()];
for (int c=0; c<paletteHex.Count(); c++)
for (int w=0; w<category.members.Count(); w++)
matrix[w,c] = 1-affinities[c,w];
//create an assignment from colors to concept names
List<int> assignIds = HungarianAlgorithm.Solve(matrix);
ColorAssignment assignment = new ColorAssignment(category);
for (int w = 0; w < category.members.Count(); w++)
{
assignment.Set(category.members[w], paletteRGB[assignIds[w]]);
}
return assignment;
}