Пример #1
0
        /**
         * Get images if needed
         * */
        private void SearchImages(Category category, int pages=3, bool ifNeeded = true)
        {
            int thresh = 10;
            foreach (String member in category.members)
            {
                GoogleImageSearch engine = new GoogleImageSearch(apiKey, cxId);

                String[] labels = { "", "_clipart" };

                foreach (String label in labels)
                {

                    String dirName = Path.Combine(imageDir, Encode(member) + label);

                    //images have been downloaded already
                    if (Directory.Exists(dirName) && Directory.GetFiles(dirName).Count() >= thresh)
                    {
                        Console.WriteLine("Already have images for " + member + label.Replace("_"," "));
                        continue;
                    }

                    List<String> urls = engine.Search(member+label.Replace("_"," "), pages);

                    Directory.CreateDirectory(dirName);

                    //save the images
                    for (int i = 0; i < urls.Count(); i++)
                    {
                        Bitmap image = Util.BitmapFromWeb(urls[i]);

                        if (image == null)
                            continue;

                        image.Save(Path.Combine(dirName, i + ".png"));
                        image.Dispose();
                    }

                }
            }
            Console.WriteLine("Done getting images");
        }
Пример #2
0
 public ColorAssignment(Category c)
 {
     colors = new Dictionary<String, Color>();
     category = c;
 }
Пример #3
0
        /**
         * Compute histograms for the given category, both clipart and regular
         **/
        private void ComputeHistograms(Category category, bool loadIfPossible=true)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();

            Parallel.For(0, category.members.Count(), m =>
            {
                String member = category.members[m];

                //Get the relevant images
                String[] queryLabels = { "", "_clipart" };

                foreach (String label in queryLabels)
                {
                    String baseName = Encode(member) + label;
                    String outFile = Path.Combine(cacheDir, baseName + ".txt");
                    String memberDir = Path.Combine(imageDir, baseName);

                    if (File.Exists(outFile) && loadIfPossible)
                    {
                        Console.WriteLine("Reusing histogram for " + baseName);
                        continue;
                    }

                    double[, ,] histogram = new double[Lbins, Abins, Bbins];
                    String[] files = Directory.GetFiles(memberDir);

                    foreach (String f in files)
                    {
                        //TODO: maybe process the image here, or filter it out
                        try
                        {

                            Bitmap image = ResizeImage(f);

                            Color[,] imageRGB = Util.BitmapToArray(image);
                            CIELAB[,] imageLAB = Util.Map<Color, CIELAB>(imageRGB, Util.RGBtoLAB);

                            //Process the image
                            bool valid = ProcessImage(imageRGB, imageLAB);
                            if (!valid)
                                continue;

                            double pixelCount = 0;
                            for (int i = 0; i < image.Width; i++)
                            {
                                for (int j = 0; j < image.Height; j++)
                                {
                                    if (imageRGB[i, j].A >= 1)
                                    {
                                        pixelCount++;
                                    }
                                }
                            }

                            if (pixelCount > 0)
                            {
                                for (int i = 0; i < image.Width; i++)
                                {
                                    for (int j = 0; j < image.Height; j++)
                                    {
                                        if (imageRGB[i, j].A <= 1)
                                            continue;

                                        CIELAB lab = imageLAB[i, j];//Util.RGBtoLAB(imageRGB[i,j]);

                                        //update histogram
                                        int L = (int)Math.Floor((lab.L / (double)binSize) + 0.5);
                                        int A = (int)Math.Floor(((100 + lab.A) / (double)binSize) + 0.5);
                                        int B = (int)Math.Floor(((100 + lab.B) / (double)binSize) + 0.5);

                                        L = clamp(L, 0, Lbins - 1);
                                        A = clamp(A, 0, Abins - 1);
                                        B = clamp(B, 0, Bbins - 1);

                                        histogram[L, A, B] += 1.0 / pixelCount;
                                    }
                                }
                            }

                            //cleanup
                            image.Dispose();
                        }
                        catch (Exception)
                        {
                            Console.WriteLine("Could not process image " + f);
                        }

                    }

                    //save the histogram to a file
                    List<String> lines = new List<String>();

                    for (int i = 0; i < Lbins; i++)
                    {
                        for (int j = 0; j < Abins; j++)
                        {
                            for (int k = 0; k < Bbins; k++)
                            {
                                lines.Add(histogram[i, j, k].ToString());
                            }
                        }
                    }
                    File.WriteAllLines(outFile, lines.ToArray<String>());
                }
            });
            watch.Stop();
            Console.WriteLine("Done with histograms Time: " + watch.ElapsedMilliseconds / 1000.0);
        }
Пример #4
0
        private double[,] ComputeAffinities(Category category, double clipartPrior=0.7, double saturationThresh = 0.1, double sigma=0.2, double whiteThresh=20)
        {
            //get the color probabilities for regular and clipart queries
            double[,] pcw_regular = new double[paletteHex.Count(), category.members.Count()];
            double[,] pcw_clipart = new double[paletteHex.Count(), category.members.Count()];
            double[,] pcw = new double[paletteHex.Count(), category.members.Count()];

            Parallel.For(0, category.members.Count(), w =>
            {
                String query = Encode(category.members[w]);
                double[] pc_regular = GetProbabilities(query, GetCNDist, new GaussianKernel(sigma), whiteThresh);
                double[] pc_clipart = GetProbabilities(query+"_clipart", GetCNDist, new GaussianKernel(sigma), whiteThresh);

                for (int c=0; c<paletteLAB.Count(); c++)
                {
                    pcw_regular[c, w] = pc_regular[c];
                    pcw_clipart[c, w] = pc_clipart[c];
                }
            });

            Console.WriteLine("Done computing probabilities");

            //compute combined probability
            for (int w=0; w<category.members.Count(); w++)
            {

                double rentropy = 0;
                double centropy = 0;

                for (int c=0; c<paletteHex.Count(); c++)
                {
                    if (pcw_clipart[c,w] > 0)
                        centropy += pcw_clipart[c,w]*Math.Log(pcw_clipart[c,w]);
                    if (pcw_regular[c,w] > 0)
                        rentropy += pcw_regular[c,w]*Math.Log(pcw_regular[c,w]);
                }

                centropy *= -1;
                rentropy *= -1;

                //avoid divide by zero
                centropy = Math.Max(centropy, epsilon);
                rentropy = Math.Max(rentropy, epsilon);

                double cw = clipartPrior/centropy;
                double rw = (1-clipartPrior)/rentropy;

                for (int c=0; c<paletteHex.Count(); c++)
                {
                    CIELAB lab = paletteLAB[c];
                    double chroma = Math.Sqrt(lab.A*lab.A+lab.B*lab.B);
                    double saturation = chroma/Math.Max(Math.Sqrt(chroma*chroma+lab.L*lab.L), epsilon);

                    pcw[c, w] = Math.Max(saturation, saturationThresh)*(cw*pcw_clipart[c,w] + rw*pcw_regular[c,w]);
                }
            }

            //renormalize
            double[] memberSums = new double[category.members.Count()];
            for (int c=0; c<paletteHex.Count(); c++)
                for (int w=0; w<category.members.Count(); w++)
                    memberSums[w] += pcw[c,w];
            for (int c = 0; c < paletteHex.Count(); c++)
                for (int w = 0; w < category.members.Count(); w++)
                    pcw[c,w] /= memberSums[w];

            //now compute affinities from the combined probabilities
            double[,] affinities = new double[paletteHex.Count(), category.members.Count()];

            double[,] pwc = new double[category.members.Count(), paletteHex.Count()];

            //compute p(w|c)
            //p(w|c) = p(c|w)*p(w)/p(c)
            double colorZ = 0;
            double[] colorSums = new double[paletteHex.Count()];
            for (int c=0; c < paletteHex.Count(); c++)
            {
                for (int w=0; w<category.members.Count(); w++)
                {
                    colorSums[c] += pcw[c,w];
                    colorZ += pcw[c,w];
                }
            }

            for (int c=0; c<paletteHex.Count(); c++)
                for (int w=0; w<category.members.Count(); w++)
                    pwc[w,c] = pcw[c,w]*(1.0/category.members.Count())/(Math.Max(colorSums[c],epsilon)/colorZ);

            double minScore = Double.PositiveInfinity;
            double maxScore = Double.NegativeInfinity;

            //balance with entropy H(w|c)
            for (int c = 0; c < paletteHex.Count(); c++)
            {
                double Hwc = 0;
                for (int w = 0; w < category.members.Count(); w++)
                {
                    if (pwc[w,c] > 0)
                        Hwc += pwc[w,c] * Math.Log(pwc[w,c]);
                }
                Hwc *= -1;
                System.Diagnostics.Debug.Assert(!double.IsNaN(Hwc));

                for (int w = 0; w < category.members.Count(); w++)
                {
                    affinities[c, w] = pcw[c, w] / Math.Max(Hwc, epsilon);
                    minScore = Math.Min(minScore, affinities[c, w]);
                    maxScore = Math.Max(maxScore, affinities[c, w]);
                }

            }
            System.Diagnostics.Debug.Assert(maxScore != minScore);

            //scale the affinities between 0 and 1s (easier to visualize)
            for (int c = 0; c < paletteHex.Count(); c++)
            {
                for (int w = 0; w < category.members.Count(); w++)
                {
                    affinities[c, w] = (affinities[c, w] - minScore) / (maxScore - minScore);
                    if (Double.IsNaN(affinities[c,w]))
                        throw new FormatException("Affinity is NaN! Affinity " + category.members[w] + " " + paletteHex[c] );
                }
            }

            Console.WriteLine("Done computing affinities");
            return affinities;
        }
Пример #5
0
        public Bitmap RenderAffinities(Category category)
        {
            double[,] hist = ComputeAffinities(category);

               int numMembers = category.members.Count();
               int colorSize = 10;
               int padding = 2;
               int textWidth = 200;
               int barSize = 100;
               int paddingBottom = 10;

               Bitmap result = new Bitmap(textWidth + paletteHex.Count() * colorSize, numMembers * barSize + padding+paddingBottom);
               Graphics hg = Graphics.FromImage(result);

               Brush black = new SolidBrush(Color.Black);
               Brush gray = new SolidBrush(Color.Gray);
               Font headers = new Font("Arial", 9);

               hg.FillRectangle(new SolidBrush(Color.White), 0, 0, result.Width, result.Height);

               //write out the concept titles
               for (int w = 0; w < numMembers; w++)
               {
               hg.DrawString(category.members[w], headers, black, 0, w * barSize + barSize / 2);
               }

               //draw the bar charts
               for (int w = 0; w < numMembers; w++)
               {
               for (int c = 0; c < paletteHex.Count(); c++)
               {
                   int height = (int)Math.Round(hist[c, w] * (barSize - 5));
                   int offset = barSize - height;
                   hg.FillRectangle(new SolidBrush(paletteRGB[c]), c * colorSize + textWidth, w * barSize + offset, colorSize, height);
               }
               //draw a line
               hg.DrawLine(new Pen(gray), textWidth, (w + 1) * barSize, 20 * colorSize + textWidth, (w + 1) * barSize);
               }
               return result;
        }
Пример #6
0
 //Reading Category Files
 public List<Category> ReadCategories(String filename)
 {
     String[] lines = File.ReadAllLines(filename);
     List<Category> categories = new List<Category>();
     foreach (String line in lines)
     {
         String[] fields = line.Split(new string[] { "\",\"" }, StringSplitOptions.None);
         Category c = new Category(fields[1].Replace("\"", ""), fields[0].Replace("\"", "").Split('|'));
         categories.Add(c);
     }
     return categories;
 }
Пример #7
0
        public ColorAssignment AssignColors(Category category)
        {
            //check that number of category members is less than number of palette colors
            if (category.members.Count() > paletteHex.Count())
                throw new ArgumentException("Too many category members, or not enough palette colors!");

            //Get the images, if needed
            SearchImages(category, 3);

            //Compute the histograms
            ComputeHistograms(category);

            //Compute affinities and assign colors
            double[,] affinities = ComputeAffinities(category);

            //Since the Hungarian Algorithm minimizes sum of affinities, let's invert the affinities
            //Assume that the number of category members is less than the number of palette colors
            double[,] matrix = new double[paletteHex.Count(), paletteHex.Count()];
            for (int c=0; c<paletteHex.Count(); c++)
                for (int w=0; w<category.members.Count(); w++)
                    matrix[w,c] = 1-affinities[c,w];

            //create an assignment from colors to concept names
            List<int> assignIds = HungarianAlgorithm.Solve(matrix);

            ColorAssignment assignment = new ColorAssignment(category);
            for (int w = 0; w < category.members.Count(); w++)
            {
                assignment.Set(category.members[w], paletteRGB[assignIds[w]]);
            }

            return assignment;
        }