public override void Convert() { using (var stream = new BinaryReader(File.OpenRead(Path), Encode)) { var size = stream.BaseStream.Length; ushort i = 0; var list = new List <RecipeJson>(); var txt = new StringBuilder(); while (stream.BaseStream.Position < size) { var temp = new RecipeJson { LoopId = i, ItemId = stream.ReadUInt32(), TargetItemId = stream.ReadUInt16(), TargetItemIdSup = stream.ReadUInt16(), Price = stream.ReadUInt32(), Quantity = stream.ReadByte(), MinLevel = stream.ReadByte(), Chance = stream.ReadUInt16(), SuperiorChance = stream.ReadUInt16(), DoubleChance = stream.ReadUInt16(), IngredientsItemId = new ushort[12], IngredientsQuantity = new uint[12] }; for (var j = 0; j < 12; j++) { temp.IngredientsItemId[j] = stream.ReadUInt16(); } for (var j = 0; j < 12; j++) { temp.IngredientsQuantity[j] = stream.ReadUInt32(); } i++; if (temp.ItemId == 0) { continue; } list.Add(temp); txt.AppendLine($"INSERT INTO `data_recipes` VALUES ({i}, {temp.ItemId}, {temp.TargetItemId}, {temp.TargetItemIdSup}, {temp.Price}, " + $"{temp.Quantity}, {temp.MinLevel}, {temp.Chance}, {temp.SuperiorChance}, {temp.DoubleChance});"); for (var j = 0; j < 12; j++) { if (temp.IngredientsItemId[j] > 0) { txt.AppendLine( $"INSERT INTO `data_recipe_ingredients` VALUES ({temp.ItemId}, {temp.IngredientsItemId[j]}, {temp.IngredientsQuantity[j]});"); } } } SqlData = txt.ToString(); JsonData = JsonConvert.SerializeObject(list, Formatting.Indented); } }
public static void ExecuteAllRecipesAndGenerateWebsite(string cookbookProjectFolder) { Console.WriteLine("Deleting old cookbook files..."); string outputFolder = Path.Combine(cookbookProjectFolder, "CookbookOutput"); if (Directory.Exists(outputFolder)) { Directory.Delete(outputFolder, recursive: true); } Directory.CreateDirectory(outputFolder); string jsonFilePath = Path.Combine(outputFolder, "recipes.json"); string imageFolderPath = Path.Combine(outputFolder, "images"); Directory.CreateDirectory(imageFolderPath); Console.WriteLine($"Generating Images..."); RecipeImages.Generate(imageFolderPath); Console.WriteLine($"Generating JSON..."); string json = RecipeJson.Generate(cookbookProjectFolder); File.WriteAllText(jsonFilePath, json); Console.WriteLine($"Reading JSON..."); RecipeSource[] recipes = RecipeJson.GetRecipes(jsonFilePath).Values.Select(x => x).ToArray(); Console.WriteLine($"Generating website..."); Website.Generate(outputFolder, recipes); Console.WriteLine($"SAVED IN: {outputFolder}"); }
private static void GenerateEverything(string outputImageFolder, string outputJsonFile, string cookbookFolder) { Stopwatch sw = Stopwatch.StartNew(); Console.WriteLine($"Generating images: {outputImageFolder}"); RecipeImages.Generate(outputImageFolder); Console.WriteLine($"Generating source: {outputJsonFile}"); RecipeJson.Generate(cookbookFolder, outputJsonFile); Console.WriteLine($"Finished in {sw.Elapsed.TotalSeconds:F3} seconds"); }
public void Test_Generate_Cookbook() { Console.WriteLine($"Generating cookbook in:\n{OUTPUT_FOLDER}"); // DELETE OLD COOKBOOK if (Directory.Exists(OUTPUT_FOLDER)) { Directory.Delete(OUTPUT_FOLDER, recursive: true); } Directory.CreateDirectory(OUTPUT_FOLDER); // GENERATE IMAGES Console.WriteLine($"Generating PNGs..."); Stopwatch sw = Stopwatch.StartNew(); IRecipe[] imageRecipes = RecipeImages.Generate(Path.Join(OUTPUT_FOLDER, "images")); Console.WriteLine($"Generated {imageRecipes.Length} PNGs in {sw.Elapsed.TotalSeconds:F4} sec"); // GENERATE JSON Console.Write($"Generating JSON..."); sw.Restart(); RecipeSource[] sourceRecipes = RecipeJson.Generate(COOKBOOK_PROJECT_FOLDER, JSON_FILE); Console.WriteLine($" {sw.Elapsed.TotalSeconds:F4} sec"); // READ JSON BACK Console.Write($"Validating JSON..."); sw.Restart(); List <string> readIDs = new(); using JsonDocument document = JsonDocument.Parse(File.ReadAllText(JSON_FILE)); string version = document.RootElement.GetProperty("version").GetString(); string generated = document.RootElement.GetProperty("generated").GetString(); foreach (JsonElement recipeElement in document.RootElement.GetProperty("recipes").EnumerateArray()) { string id = recipeElement.GetProperty("id").GetString(); string category = recipeElement.GetProperty("category").GetString(); string title = recipeElement.GetProperty("title").GetString(); string description = recipeElement.GetProperty("description").GetString(); string code = recipeElement.GetProperty("code").GetString(); readIDs.Add(id); } Console.WriteLine($" {sw.Elapsed.TotalSeconds:F4} sec"); // VALIDATE Assert.AreEqual(imageRecipes.Length, sourceRecipes.Length); Assert.AreEqual(sourceRecipes.Length, readIDs.Count); }
public void Test_Json_IsValid() { string jsonFilePath = Path.GetFullPath("cookbook-valid-test.json"); IRecipe[] recipes = Locate.GetRecipes(); string json = RecipeJson.Generate(COOKBOOK_PROJECT_FOLDER); File.WriteAllText(jsonFilePath, json); Dictionary <string, RecipeSource> readRecipes = RecipeJson.GetRecipes(new FileInfo(jsonFilePath)); Console.WriteLine($"Read {readRecipes.Count} recipes from JSON"); Assert.AreEqual(recipes.Length, readRecipes.Count); }
public static RecipeSource[] Generate(string cookbookProjectFolder, string outputFolder, bool regenerate = false) { string jsonFilePath = Path.Combine(outputFolder, "recipes.json"); string imageFolderPath = Path.Combine(outputFolder, "images"); if (regenerate) { if (Directory.Exists(imageFolderPath)) { Directory.Delete(imageFolderPath, recursive: true); } Directory.CreateDirectory(imageFolderPath); Console.WriteLine($"Generating Images: {imageFolderPath}"); RecipeImages.Generate(imageFolderPath); Console.WriteLine($"Generating JSON ..."); string json = RecipeJson.Generate(cookbookProjectFolder); File.WriteAllText(jsonFilePath, json); } Console.WriteLine($"Reading JSON ..."); return(RecipeJson.GetRecipes(new FileInfo(jsonFilePath)).Values.Select(x => x).ToArray()); }
// http://getschema.org/index.php?title=Recipe static void Main(string[] args) { var logger = LogManager.GetCurrentClassLogger(); List <RecipeMetaData> recipeMetaDataList = new List <RecipeMetaData> { //new RecipeMetaData {Name = "கேக்", PageCount = 17, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\cake"}, //new RecipeMetaData {Name = "ஐஸ்கிரீம்", PageCount = 3, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\icecream"}, //new RecipeMetaData {Name = "ஜாம்", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\jam"}, // new RecipeMetaData {Name = "பிஸ்கட்", PageCount = 6, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\biscuit"}, // new RecipeMetaData {Name = "பானம்", PageCount = 35, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\beverages"}, //new RecipeMetaData {Name = "பச்சடி", PageCount = 16, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\paachadi"}, // new RecipeMetaData {Name = "வற்றல்", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\vattal"}, // new RecipeMetaData {Name = "பொடி", PageCount = 14, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\podi"}, // new RecipeMetaData {Name = "கூட்டு", PageCount = 25, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\kootu"}, // new RecipeMetaData {Name = "சாலட்", PageCount = 12, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\salad"}, // new RecipeMetaData {Name = "ஊறுகாய்", PageCount = 12, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\pickla"}, // new RecipeMetaData {Name = "பொரியல்", PageCount = 37, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\porial"}, // new RecipeMetaData {Name = "வறுவல்", PageCount = 39, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\fry\veg"}, // new RecipeMetaData {Name = "சாதம்", PageCount = 58, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\rice"}, // new RecipeMetaData {Name = "கீரை", PageCount = 21, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\keerai"}, // new RecipeMetaData {Name = "அரேபியா", PageCount = 21, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Arabia"}, // new RecipeMetaData {Name = "இத்தாலி", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Italy"}, // new RecipeMetaData {Name = "இலங்கை", PageCount = 27, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Srilanka"}, // new RecipeMetaData {Name = "ப்ரான்ஸ்", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\france"}, // new RecipeMetaData {Name = "சீனா", PageCount = 5, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\china"}, // new RecipeMetaData {Name = "தாய்லாந்து", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\thailand"}, // new RecipeMetaData {Name = "மெக்சிகோ", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Mexico"}, // new RecipeMetaData {Name = "மற்றநாடுகள்", PageCount = 43, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\others"}, // new RecipeMetaData {Name = "துவையல்", PageCount = 16, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\thuyal"}, }; foreach (RecipeMetaData recipeMetaData in recipeMetaDataList) { Dictionary <string, string> recipeNameList = new Dictionary <string, string>(); List <string> manualLinks = new List <string>(); Dictionary <string, string> uniqueNames = new Dictionary <string, string>(); Dictionary <string, string> uniqueLinks = new Dictionary <string, string>(); Dictionary <string, RecipeJson> recipeJsonList = new Dictionary <string, RecipeJson>(); Dictionary <string, RecipeJsonEx> recipeJsonExList = new Dictionary <string, RecipeJsonEx>(); for (int i = 0; i <= recipeMetaData.PageCount; i++) { logger.Info("-----------------------------------------------------------------------------------------------"); String recipesPageUrl = String.Format(@"http://www.arusuvai.com/tamil/recipes/{0}/?page={1}", recipeMetaData.Name, i); logger.Info(recipesPageUrl); logger.Info("-----------------------------------------------------------------------------------------------"); HtmlWeb hw = new HtmlWeb(); HtmlDocument doc = hw.Load(recipesPageUrl); foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[starts-with(@href, '/tamil/node/')]")) { if (String.IsNullOrWhiteSpace(link.InnerText) == false) { WebClient myRecipesWebClient = new WebClient(); logger.Info(System.String.Format("{0} => http://www.arusuvai.com{1}", link.InnerText, link.Attributes.FirstOrDefault().Value)); byte[] myRecipesDataBuffer = myRecipesWebClient.DownloadData(System.String.Format("http://www.arusuvai.com{0}", link.Attributes.FirstOrDefault().Value)); HtmlAgilityPack.HtmlDocument recipesdoc = new HtmlAgilityPack.HtmlDocument(); recipesdoc.LoadHtml(Encoding.UTF8.GetString(myRecipesDataBuffer)); if (recipesdoc.DocumentNode.SelectNodes(@"/html/head/script[@type='application/ld+json']") != null) { String JsonString = recipesdoc.DocumentNode.SelectNodes(@"/html/head/script[@type='application/ld+json']").First().InnerText; JsonString = JsonString.Replace("@context", "context").Replace("@graph", "graph").Replace("@type", "type"); try { RecipeJson recipeJson = JsonConvert.DeserializeObject <RecipeJson>(JsonString); if (recipeJson != null) { String name = recipeJson.graph.FirstOrDefault().name.Replace(@"/", "-").Trim(); int index = 1; while (uniqueNames.ContainsKey(name) == true) { // Get Unique recipe name name = string.Format("{0} ({1})", name, index); index = index + 1; } recipeNameList.Add(name, name); recipeJson.graph.FirstOrDefault().name = name; recipeJsonList.Add(name, recipeJson); uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value); uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name); String filePath = System.String.Format(@"{0}\{1}.html", recipeMetaData.FolderName, name); try { System.IO.File.WriteAllText(filePath, recipeJson.RecipeString, Encoding.UTF8); } catch (System.Xml.XmlException e) { Console.WriteLine(e); } catch (Newtonsoft.Json.JsonSerializationException ex) { Console.WriteLine(ex.Message); } catch (Exception e) { Console.WriteLine(e); } } } catch (Newtonsoft.Json.JsonReaderException ex) { try { RecipeJsonEx recipeJsonEx = JsonConvert.DeserializeObject <RecipeJsonEx>(JsonString); if (recipeJsonEx != null) { String name = recipeJsonEx.graph.FirstOrDefault().name.Replace(@"/", "-") .Trim(); int index = 1; while (uniqueNames.ContainsKey(name) == true) { // Get Unique recipe name name = string.Format("{0} ({1})", name, index); index = index + 1; } recipeNameList.Add(name, name); recipeJsonEx.graph.FirstOrDefault().name = name; recipeJsonExList.Add(name, recipeJsonEx); uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value); uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name); String filePath1 = System.String.Format(@"{0}\{1}.html", recipeMetaData.FolderName, name); System.IO.File.WriteAllText(filePath1, recipeJsonEx.RecipeString, Encoding.UTF8); } } catch (System.Xml.XmlException e) { Console.WriteLine(e); } catch (Exception e) { Console.WriteLine(e); } } catch (Newtonsoft.Json.JsonSerializationException ex) { Console.WriteLine(ex.Message); } } else { String name = String.Empty; ; try { WebClient myRecipesWebClient1 = new WebClient(); byte[] myRecipesDataBuffer1 = myRecipesWebClient1.DownloadData(System.String.Format("http://www.arusuvai.com{0}", link.Attributes.FirstOrDefault().Value)); HtmlAgilityPack.HtmlDocument recipesdoc1 = new HtmlAgilityPack.HtmlDocument(); recipesdoc1.LoadHtml(Encoding.UTF8.GetString(myRecipesDataBuffer1)); String[] ingridientList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[3]/div/div").First().InnerText.Split(new char[] { '\n' }); String[] instructionList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[4]/div/div").First().InnerText.Split(new char[] { '\n' }); String[] noteList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[5]/div/div").First().InnerText.Split(new char[] { '\n' }); name = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/h1").First().InnerText.Trim().Replace(@"/", "-").Trim(); int index = 1; while (uniqueNames.ContainsKey(name) == true) { // Get Unique recipe name name = string.Format("{0} ({1})", name, index); index = index + 1; } uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value); uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name); StringBuilder retValue = new StringBuilder(); // Name StringBuilder sbname = new StringBuilder(string.Empty); sbname.AppendFormat("<p>{0}</p>", name); // Ingredients StringBuilder ingredients = new StringBuilder(string.Empty); foreach (string ingredient in ingridientList) { if (String.IsNullOrWhiteSpace(ingredient) == false) { ingredients.AppendFormat("<p>{0}</p>", ingredient.Trim()); } } // instructions StringBuilder instructions = new StringBuilder(string.Empty); foreach (string instruction in instructionList) { if (String.IsNullOrWhiteSpace(instruction) == false) { instructions.AppendFormat("<p>{0}</p>", instruction.Trim()); } } StringBuilder notes = new StringBuilder(string.Empty); foreach (string note in noteList) { if (String.IsNullOrWhiteSpace(note) == false) { notes.AppendFormat("<p>{0}</p>", note.Trim()); } } StringBuilder recipeBody = new StringBuilder(); recipeBody.Append("<div class=\"recipebody\">"); recipeBody.AppendFormat("<h3>தேவையான பொருட்கள்:</h3>{0}", ingredients.ToString()); recipeBody.AppendFormat("<h3>செய்முறை:</h3>{0}", instructions.ToString()); recipeBody.AppendFormat("<h3>குறிப்புகள்:</h3>{0}", String.IsNullOrWhiteSpace(notes.ToString()) == true ? "<p></p>" : notes.ToString()); recipeBody.Append("</div>"); retValue.AppendFormat(@"<html><body><h1>{0}</h1>{1}</body></html>", sbname.ToString(), recipeBody.ToString()); String filePath3 = System.String.Format(@"{0}\{1}.html", recipeMetaData.FolderName, name); System.IO.File.WriteAllText(filePath3, retValue.ToString(), Encoding.UTF8); } catch (Newtonsoft.Json.JsonSerializationException ex) { Console.WriteLine(ex.Message); } catch (Exception e) { manualLinks.Add(link.InnerText + " " + name); } } } } } String filePath2 = System.String.Format(@"{0}\{1}.txt", recipeMetaData.FolderName, "manual-links"); using (System.IO.StreamWriter file = new System.IO.StreamWriter(filePath2)) { // store manual links foreach (String link in manualLinks) { file.WriteLine(link); } } System.String test = ""; } }