public DataSet getMicroData(HtmlDocument Document)
        {
            ClDataAcces objDataAccess = new ClDataAcces();
            string      Name = string.Empty;
            string      Ingredient = string.Empty;
            DataSet     recipeData = new DataSet();
            DataTable   recipe = new DataTable();
            DataTable   nutritionData = new DataTable();
            string      calories = string.Empty, fat = string.Empty, saturatefat = string.Empty,
                        fiber = string.Empty, carbohydrate = string.Empty, protein = string.Empty,
                        cholesterol = string.Empty, sugar = string.Empty, sodium = string.Empty;

            // RECIPE DATA
            recipe.Columns.Add("Name");
            recipe.Columns.Add("Ingrediente");

            // NUTRICION DATA

            nutritionData.Columns.Add("Calorias", typeof(decimal));
            nutritionData.Columns.Add("fat", typeof(decimal));
            nutritionData.Columns.Add("saturatefat", typeof(decimal));
            nutritionData.Columns.Add("fiber", typeof(decimal));
            nutritionData.Columns.Add("carbohydrate", typeof(decimal));
            nutritionData.Columns.Add("protein", typeof(decimal));
            nutritionData.Columns.Add("cholesterol", typeof(decimal));
            nutritionData.Columns.Add("sugar", typeof(decimal));
            nutritionData.Columns.Add("sodium", typeof(decimal));


            HtmlNode[] nodes    = Document.DocumentNode.SelectNodes(".//*[@itemtype='http://schema.org/Recipe']").ToArray();
            HtmlNode[] child    = null;
            HtmlNode   NameNode = Document.DocumentNode.SelectSingleNode("/html/body/div[2]/div/div[1]/div[3]/section[1]/section[2]/div/h3");

            Name = NameNode.InnerText;

            foreach (HtmlNode item in nodes)
            {
                child = item.SelectNodes(".//*[@itemprop='ingredients']").ToArray();
                Console.WriteLine("INGREDIENTES\r");
                Console.WriteLine("-----------------------------------------------------------------------");
                foreach (HtmlNode chldnode in child)
                {
                    for (int ind = 0; ind < chldnode.Attributes.Count; ind++)
                    {
                        Console.WriteLine(chldnode.Attributes[ind].Name + ": " +
                                          chldnode.Attributes[ind].Value.ToString() + " ---> " +
                                          chldnode.InnerText.Trim() + "\r");
                        Ingredient = chldnode.InnerText.Trim();
                        recipe.NewRow();
                        recipe.Rows.Add(Name, Ingredient);
                    }
                }
                nodes = Document.DocumentNode.SelectNodes(".//*[@itemtype='http://schema.org/NutritionInformation']").ToArray();

                Console.WriteLine("INFORMACION DE NUTRICION\r");
                Console.WriteLine("-----------------------------------------------------------------------");

                foreach (HtmlNode chldnode in nodes)
                {
                    child = chldnode.SelectNodes(".//*[@itemprop]").ToArray();
                    foreach (HtmlNode ntNodo in child)
                    {
                        for (int ind = 0; ind < ntNodo.Attributes.Count; ind++)
                        {
                            Console.WriteLine(ntNodo.Attributes[ind].Value + ": " +
                                              RegexTool.GetNumber(ntNodo.InnerText) + "\r");
                            switch (ntNodo.Attributes[ind].Value)
                            {
                            case "calories":
                                calories = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "fatContent":
                                fat = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "saturatedFatContent":
                                saturatefat = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "fiberContent":
                                fiber = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "carbohydrateContent":
                                carbohydrate = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "proteinContent":
                                protein = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "cholesterolContent":
                                cholesterol = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "sugarContent":
                                sugar = RegexTool.GetNumber(ntNodo.InnerText);
                                break;

                            case "sodiumContent":
                                sodium = RegexTool.GetNumber(ntNodo.InnerText);
                                break;
                            }
                        }
                    }
                }
                nutritionData.Rows.Add(Convert.ToDecimal(calories), Convert.ToDecimal(fat), Convert.ToDecimal(saturatefat),
                                       Convert.ToDecimal(fiber), Convert.ToDecimal(carbohydrate), Convert.ToDecimal(protein), Convert.ToDecimal(cholesterol)
                                       , Convert.ToDecimal(sugar), Convert.ToDecimal(sodium));
            }
            recipeData.Tables.Add(recipe);
            recipeData.Tables.Add(nutritionData);

            objDataAccess.insertRecipeData(recipeData, false);
            Console.WriteLine("Datos Registrados con exito");
            return(recipeData);
        }
Пример #2
0
        public DataSet GetMicroData(HtmlAgilityPack.HtmlDocument Document, int dataFormat)
        {
            string      dataError = string.Empty, pictureRoute = string.Empty, appRoute = string.Empty, filename = string.Empty;
            ClDataAcces objDataAccess = new ClDataAcces();
            string      Name          = string.Empty;
            string      Ingredient    = string.Empty;
            DataSet     recipeData    = new DataSet();
            DataTable   recipeTable   = new DataTable();
            DataTable   nutritionData = new DataTable();

            HtmlNode[]     nodes = null;
            HtmlNode[]     child = null;
            HtmlNode       NameNode = null;
            WebClient      cliente = new WebClient();
            IList <string> listIngredient = null;
            Uri            Url = null;
            string         calories = "0", fat = "0", saturatefat = "0",
                           fiber = "0", carbohydrate = "0", protein = "0",
                           cholesterol = "0", sugar = "0", sodium = "0", imagenUrl = string.Empty;

            // RECIPE DATA
            recipeTable.Columns.Add("Name", typeof(string));
            recipeTable.Columns.Add("Ingrediente", typeof(string));
            recipeTable.Columns.Add("recipeTipoPlatoData", typeof(string));
            recipeTable.Columns.Add("recipeCulturaData", typeof(string));
            recipeTable.Columns.Add("recipeNacionalidadData", typeof(string));
            recipeTable.Columns.Add("recipeMomentoData", typeof(string));
            recipeTable.Columns.Add("recipeTemporadaData", typeof(string));
            recipeTable.Columns.Add("Picture", typeof(string));
            recipeTable.Columns.Add("Origen", typeof(string));

            recipeTable.Columns.Add("esSopa", typeof(Boolean));
            recipeTable.Columns.Add("esPasta", typeof(Boolean));
            recipeTable.Columns.Add("esMarisco", typeof(Boolean));
            recipeTable.Columns.Add("esEnsalada", typeof(Boolean));
            recipeTable.Columns.Add("esBebida", typeof(Boolean));
            recipeTable.Columns.Add("esBajoColesterol", typeof(Boolean));
            recipeTable.Columns.Add("esBajoEnCalorias", typeof(Boolean));
            recipeTable.Columns.Add("esLibreGluten", typeof(Boolean));



            // NUTRICION DATA

            nutritionData.Columns.Add("Calorias", typeof(decimal));
            nutritionData.Columns.Add("fat", typeof(decimal));
            nutritionData.Columns.Add("saturatefat", typeof(decimal));
            nutritionData.Columns.Add("fiber", typeof(decimal));
            nutritionData.Columns.Add("carbohydrate", typeof(decimal));
            nutritionData.Columns.Add("protein", typeof(decimal));
            nutritionData.Columns.Add("cholesterol", typeof(decimal));
            nutritionData.Columns.Add("sugar", typeof(decimal));
            nutritionData.Columns.Add("sodium", typeof(decimal));

            appRoute = Application.StartupPath;

            switch (dataFormat)
            {
            case 0:

                nodes     = Document.DocumentNode.SelectNodes(".//*[@itemtype='http://schema.org/Recipe']").ToArray();
                NameNode  = Document.DocumentNode.SelectSingleNode(".//*[@class='recipeDetailHeader showOnTabletToDesktop']");
                imagenUrl = Document.DocumentNode.Descendants("img")
                            .Where(node => node.Attributes["class"] != null && node.Attributes["class"].Value == "recipeDetailSummaryImageMain")
                            .Select(node => node.Attributes["src"].Value)
                            .DefaultIfEmpty(string.Empty)
                            .FirstOrDefault()
                            .ToString();


                cliente.DownloadFileCompleted += new AsyncCompletedEventHandler(Cliente_DownloadFileCompleted);
                Url = new Uri(imagenUrl);

                filename     = System.IO.Path.GetFileName(Url.LocalPath);
                pictureRoute = appRoute.Remove(appRoute.Length - 10) + "\\Picture\\" + filename;
                cliente.DownloadFileAsync(Url, pictureRoute);
                Name = NameNode.InnerText;

                foreach (HtmlNode item in nodes)
                {
                    child = item.SelectNodes(".//*[@itemprop='ingredients']").ToArray();

                    foreach (HtmlNode chldnode in child)
                    {
                        for (int ind = 0; ind < chldnode.Attributes.Count; ind++)
                        {
                            Ingredient = chldnode.InnerText.Trim();
                            recipeTable.NewRow();
                            recipeTable.Rows.Add(Name, Ingredient, CboTipoPlato.Text, cboCultura.Text,
                                                 cboNacionalidad.Text, cboMomentoComida.Text, cboTemporada.Text, pictureRoute,
                                                 txtOrigenRegistro.Text, chkSopa.Checked,
                                                 chkPasta.Checked, chkPescadoMarisco.Checked, chkEnsalada.Checked,
                                                 chkBebida.Checked, chkBajoColesterol.Checked, chkBajoCalorias.Checked,
                                                 chkLibreGluten.Checked);
                        }
                    }
                    if (Document.DocumentNode.SelectNodes(".//*[@itemtype='http://schema.org/NutritionInformation']") != null)
                    {
                        nodes = Document.DocumentNode.SelectNodes(".//*[@itemtype='http://schema.org/NutritionInformation']").ToArray();

                        foreach (HtmlNode chldnode in nodes)
                        {
                            child = chldnode.SelectNodes(".//*[@itemprop]").ToArray();
                            foreach (HtmlNode ntNodo in child)
                            {
                                for (int ind = 0; ind < ntNodo.Attributes.Count; ind++)
                                {
                                    Console.WriteLine(ntNodo.Attributes[ind].Value + ": " +
                                                      RegexTool.GetNumber(ntNodo.InnerText) + "\r");
                                    if (ntNodo.InnerText != "")
                                    {
                                        switch (ntNodo.Attributes[ind].Value)
                                        {
                                        case "calories":
                                            calories = RegexTool.GetNumber(ntNodo.InnerText);

                                            break;

                                        case "fatContent":
                                            fat = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "saturatedFatContent":
                                            saturatefat = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "fiberContent":
                                            fiber = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "carbohydrateContent":
                                            carbohydrate = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "proteinContent":
                                            protein = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "cholesterolContent":
                                            cholesterol = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "sugarContent":
                                            sugar = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;

                                        case "sodiumContent":
                                            sodium = RegexTool.GetNumber(ntNodo.InnerText);
                                            break;
                                        }
                                    }
                                }
                            }
                        }
                        try
                        {
                            nutritionData.Rows.Add(Convert.ToDecimal(calories), Convert.ToDecimal(fat), Convert.ToDecimal(saturatefat),
                                                   Convert.ToDecimal(fiber), Convert.ToDecimal(carbohydrate), Convert.ToDecimal(protein), Convert.ToDecimal(cholesterol)
                                                   , Convert.ToDecimal(sugar), Convert.ToDecimal(sodium));
                        }
                        catch (Exception error)
                        {
                            dataError = error.Message;
                        }
                    }
                }
                if (nutritionData.Rows.Count > 0)
                {
                    recipeData.Tables.Add(recipeTable);
                    recipeData.Tables.Add(nutritionData);
                    dataError = objDataAccess.insertRecipeData(recipeData, chkClasificacion.Checked);
                    if (dataError != "")
                    {
                        errorCount++;
                        lbErrorCount.Text = errorCount.ToString() + "\r MicroData scraper ";
                        Ftool.WriteLogFile(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToShortTimeString() + " " + dataError);
                    }
                    else
                    {
                        contRegistroEncontrados++;
                        lbTotalRegistroEncontrados.Text = contRegistroEncontrados.ToString();
                    }
                }
                break;

            case 1:
                // RECUPERANDO EN VALOR JSON EN LA PAGINA

                string jsonValue = Document.DocumentNode.
                                   SelectSingleNode("//script[@type='application/ld+json']").InnerText;

                //SERIALIZANDO EL OBJETO JSON A UN OBJETO DINAMICO

                dynamic jsonData = JsonConvert.DeserializeObject(jsonValue);
                try
                {
                    jsonData.nutrition.Remove("@type");
                    Dictionary <string, string> values = jsonData.nutrition.ToObject <Dictionary <string, string> >();
                    listIngredient = jsonData.recipeIngredient.ToObject <IList <string> >();

                    // RECUPERANDO VALORES NUTRICIONALES

                    foreach (KeyValuePair <string, string> valor in values)
                    {
                        switch (valor.Key)
                        {
                        case "calories":
                            calories = RegexTool.GetNumber(valor.Value);
                            break;

                        case "fatContent":
                            fat = RegexTool.GetNumber(valor.Value);
                            break;

                        case "saturatedFatContent":
                            saturatefat = RegexTool.GetNumber(valor.Value);
                            break;

                        case "proteinContent":
                            protein = RegexTool.GetNumber(valor.Value);
                            break;

                        case "carbohydrateContent":
                            carbohydrate = RegexTool.GetNumber(valor.Value);
                            break;

                        case "sugarContent":
                            sugar = RegexTool.GetNumber(valor.Value);
                            break;

                        case "sodiumContent":
                            sodium = RegexTool.GetNumber(valor.Value);
                            break;

                        case "fiberContent":
                            fiber = RegexTool.GetNumber(valor.Value);
                            break;
                        }
                    }
                }
                catch (Exception e)
                {
                    dataError         = e.Message;
                    lbErrorCount.Text = errorCount.ToString() + "\r Json-Ld Scraper ";
                    Ftool.WriteLogFile(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToShortTimeString() + " " + dataError);
                }
                try
                {
                    nutritionData.Rows.Clear();
                    nutritionData.Rows.Add(Convert.ToDecimal(calories), Convert.ToDecimal(fat), Convert.ToDecimal(saturatefat),
                                           Convert.ToDecimal(fiber), Convert.ToDecimal(carbohydrate), Convert.ToDecimal(protein), Convert.ToDecimal(cholesterol)
                                           , Convert.ToDecimal(sugar), Convert.ToDecimal(sodium));


                    //RECUPERANDO INGREDIENTES

                    foreach (string ingrediente in listIngredient)
                    {
                        Ingredient = ingrediente;
                        Name       = jsonData.name;
                        recipeTable.NewRow();
                        recipeTable.Rows.Add(Name, Ingredient, CboTipoPlato.Text, cboCultura.Text,
                                             cboNacionalidad.Text, cboMomentoComida.Text, cboTemporada.Text, pictureRoute, txtOrigenRegistro.Text, chkSopa.Checked,
                                             chkPasta.Checked, chkPescadoMarisco.Checked, chkEnsalada.Checked, chkBebida.Checked, chkBajoColesterol.Checked,
                                             chkBajoCalorias.Checked, chkLibreGluten.Checked);
                    }
                }
                catch (Exception error)
                {
                    errorCount++;
                    dataError         = error.Message;
                    lbErrorCount.Text = errorCount.ToString() + "\r Json-Ld Scraper ";
                    Ftool.WriteLogFile(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToShortTimeString() + " " + dataError);
                }
                if (nutritionData.Rows.Count > 0)
                {
                    if (nutritionData.Rows[0]["Calorias"].ToString() != "0" || nutritionData.Rows[0]["fat"].ToString() != "0" ||
                        nutritionData.Rows[0]["saturatefat"].ToString() != "0" || nutritionData.Rows[0]["fiber"].ToString() != "0" ||
                        nutritionData.Rows[0]["protein"].ToString() != "0" || nutritionData.Rows[0]["carbohydrate"].ToString() != "0" ||
                        nutritionData.Rows[0]["cholesterol"].ToString() != "0" || nutritionData.Rows[0]["sugar"].ToString() != "0" ||
                        nutritionData.Rows[0]["sodium"].ToString() != "0"

                        )
                    {
                        //RECUPERANDO IMAGEN DEL LA RECETA
                        imagenUrl = jsonData.image;
                        cliente.DownloadFileCompleted += new AsyncCompletedEventHandler(Cliente_DownloadFileCompleted);
                        if (RegexTool.isUrl(imagenUrl))
                        {
                            Url          = new Uri(imagenUrl);
                            filename     = System.IO.Path.GetFileName(Url.LocalPath);
                            pictureRoute = appRoute.Remove(appRoute.Length - 10) + "\\Picture\\" + filename;
                            cliente.DownloadFileAsync(Url, pictureRoute);
                        }

                        recipeData.Tables.Add(recipeTable);
                        recipeData.Tables.Add(nutritionData);

                        dataError = objDataAccess.insertRecipeData(recipeData, chkClasificacion.Checked);
                        if (dataError != "")
                        {
                            errorCount++;
                            lbErrorCount.Text = errorCount.ToString();
                            Ftool.WriteLogFile(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToShortTimeString() + " " + dataError);
                        }
                        else
                        {
                            contRegistroEncontrados++;
                            lbTotalRegistroEncontrados.Text = contRegistroEncontrados.ToString();
                        }
                    }
                }
                break;
            }
            return(recipeData);
        }