public override void Convert()
{
using (var stream = new BinaryReader(File.OpenRead(Path), Encode))
{
var size = stream.BaseStream.Length;
ushort i = 0;
var list = new List <RecipeJson>();
var txt = new StringBuilder();
while (stream.BaseStream.Position < size)
{
var temp = new RecipeJson
{
LoopId = i,
ItemId = stream.ReadUInt32(),
TargetItemId = stream.ReadUInt16(),
TargetItemIdSup = stream.ReadUInt16(),
Price = stream.ReadUInt32(),
Quantity = stream.ReadByte(),
MinLevel = stream.ReadByte(),
Chance = stream.ReadUInt16(),
SuperiorChance = stream.ReadUInt16(),
DoubleChance = stream.ReadUInt16(),
IngredientsItemId = new ushort[12],
IngredientsQuantity = new uint[12]
};
for (var j = 0; j < 12; j++)
{
temp.IngredientsItemId[j] = stream.ReadUInt16();
}
for (var j = 0; j < 12; j++)
{
temp.IngredientsQuantity[j] = stream.ReadUInt32();
}
i++;
if (temp.ItemId == 0)
{
continue;
}
list.Add(temp);
txt.AppendLine($"INSERT INTO `data_recipes` VALUES ({i}, {temp.ItemId}, {temp.TargetItemId}, {temp.TargetItemIdSup}, {temp.Price}, " +
$"{temp.Quantity}, {temp.MinLevel}, {temp.Chance}, {temp.SuperiorChance}, {temp.DoubleChance});");
for (var j = 0; j < 12; j++)
{
if (temp.IngredientsItemId[j] > 0)
{
txt.AppendLine(
$"INSERT INTO `data_recipe_ingredients` VALUES ({temp.ItemId}, {temp.IngredientsItemId[j]}, {temp.IngredientsQuantity[j]});");
}
}
}
SqlData = txt.ToString();
JsonData = JsonConvert.SerializeObject(list, Formatting.Indented);
}
}
public static void ExecuteAllRecipesAndGenerateWebsite(string cookbookProjectFolder)
{
Console.WriteLine("Deleting old cookbook files...");
string outputFolder = Path.Combine(cookbookProjectFolder, "CookbookOutput");
if (Directory.Exists(outputFolder))
{
Directory.Delete(outputFolder, recursive: true);
}
Directory.CreateDirectory(outputFolder);
string jsonFilePath = Path.Combine(outputFolder, "recipes.json");
string imageFolderPath = Path.Combine(outputFolder, "images");
Directory.CreateDirectory(imageFolderPath);
Console.WriteLine($"Generating Images...");
RecipeImages.Generate(imageFolderPath);
Console.WriteLine($"Generating JSON...");
string json = RecipeJson.Generate(cookbookProjectFolder);
File.WriteAllText(jsonFilePath, json);
Console.WriteLine($"Reading JSON...");
RecipeSource[] recipes = RecipeJson.GetRecipes(jsonFilePath).Values.Select(x => x).ToArray();
Console.WriteLine($"Generating website...");
Website.Generate(outputFolder, recipes);
Console.WriteLine($"SAVED IN: {outputFolder}");
}
private static void GenerateEverything(string outputImageFolder, string outputJsonFile, string cookbookFolder)
{
Stopwatch sw = Stopwatch.StartNew();
Console.WriteLine($"Generating images: {outputImageFolder}");
RecipeImages.Generate(outputImageFolder);
Console.WriteLine($"Generating source: {outputJsonFile}");
RecipeJson.Generate(cookbookFolder, outputJsonFile);
Console.WriteLine($"Finished in {sw.Elapsed.TotalSeconds:F3} seconds");
}
public void Test_Generate_Cookbook()
{
Console.WriteLine($"Generating cookbook in:\n{OUTPUT_FOLDER}");
// DELETE OLD COOKBOOK
if (Directory.Exists(OUTPUT_FOLDER))
{
Directory.Delete(OUTPUT_FOLDER, recursive: true);
}
Directory.CreateDirectory(OUTPUT_FOLDER);
// GENERATE IMAGES
Console.WriteLine($"Generating PNGs...");
Stopwatch sw = Stopwatch.StartNew();
IRecipe[] imageRecipes = RecipeImages.Generate(Path.Join(OUTPUT_FOLDER, "images"));
Console.WriteLine($"Generated {imageRecipes.Length} PNGs in {sw.Elapsed.TotalSeconds:F4} sec");
// GENERATE JSON
Console.Write($"Generating JSON...");
sw.Restart();
RecipeSource[] sourceRecipes = RecipeJson.Generate(COOKBOOK_PROJECT_FOLDER, JSON_FILE);
Console.WriteLine($" {sw.Elapsed.TotalSeconds:F4} sec");
// READ JSON BACK
Console.Write($"Validating JSON...");
sw.Restart();
List <string> readIDs = new();
using JsonDocument document = JsonDocument.Parse(File.ReadAllText(JSON_FILE));
string version = document.RootElement.GetProperty("version").GetString();
string generated = document.RootElement.GetProperty("generated").GetString();
foreach (JsonElement recipeElement in document.RootElement.GetProperty("recipes").EnumerateArray())
{
string id = recipeElement.GetProperty("id").GetString();
string category = recipeElement.GetProperty("category").GetString();
string title = recipeElement.GetProperty("title").GetString();
string description = recipeElement.GetProperty("description").GetString();
string code = recipeElement.GetProperty("code").GetString();
readIDs.Add(id);
}
Console.WriteLine($" {sw.Elapsed.TotalSeconds:F4} sec");
// VALIDATE
Assert.AreEqual(imageRecipes.Length, sourceRecipes.Length);
Assert.AreEqual(sourceRecipes.Length, readIDs.Count);
}
public void Test_Json_IsValid()
{
string jsonFilePath = Path.GetFullPath("cookbook-valid-test.json");
IRecipe[] recipes = Locate.GetRecipes();
string json = RecipeJson.Generate(COOKBOOK_PROJECT_FOLDER);
File.WriteAllText(jsonFilePath, json);
Dictionary <string, RecipeSource> readRecipes = RecipeJson.GetRecipes(new FileInfo(jsonFilePath));
Console.WriteLine($"Read {readRecipes.Count} recipes from JSON");
Assert.AreEqual(recipes.Length, readRecipes.Count);
}
public static RecipeSource[] Generate(string cookbookProjectFolder, string outputFolder, bool regenerate = false)
{
string jsonFilePath = Path.Combine(outputFolder, "recipes.json");
string imageFolderPath = Path.Combine(outputFolder, "images");
if (regenerate)
{
if (Directory.Exists(imageFolderPath))
{
Directory.Delete(imageFolderPath, recursive: true);
}
Directory.CreateDirectory(imageFolderPath);
Console.WriteLine($"Generating Images: {imageFolderPath}");
RecipeImages.Generate(imageFolderPath);
Console.WriteLine($"Generating JSON ...");
string json = RecipeJson.Generate(cookbookProjectFolder);
File.WriteAllText(jsonFilePath, json);
}
Console.WriteLine($"Reading JSON ...");
return(RecipeJson.GetRecipes(new FileInfo(jsonFilePath)).Values.Select(x => x).ToArray());
}
// http://getschema.org/index.php?title=Recipe
static void Main(string[] args)
{
var logger = LogManager.GetCurrentClassLogger();
List <RecipeMetaData> recipeMetaDataList = new List <RecipeMetaData>
{
//new RecipeMetaData {Name = "கேக்", PageCount = 17, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\cake"},
//new RecipeMetaData {Name = "ஐஸ்கிரீம்", PageCount = 3, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\icecream"},
//new RecipeMetaData {Name = "ஜாம்", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\jam"},
// new RecipeMetaData {Name = "பிஸ்கட்", PageCount = 6, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\biscuit"},
// new RecipeMetaData {Name = "பானம்", PageCount = 35, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\beverages"},
//new RecipeMetaData {Name = "பச்சடி", PageCount = 16, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\paachadi"},
// new RecipeMetaData {Name = "வற்றல்", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\vattal"},
// new RecipeMetaData {Name = "பொடி", PageCount = 14, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\podi"},
// new RecipeMetaData {Name = "கூட்டு", PageCount = 25, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\kootu"},
// new RecipeMetaData {Name = "சாலட்", PageCount = 12, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\salad"},
// new RecipeMetaData {Name = "ஊறுகாய்", PageCount = 12, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\pickla"},
// new RecipeMetaData {Name = "பொரியல்", PageCount = 37, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\porial"},
// new RecipeMetaData {Name = "வறுவல்", PageCount = 39, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\fry\veg"},
// new RecipeMetaData {Name = "சாதம்", PageCount = 58, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\rice"},
// new RecipeMetaData {Name = "கீரை", PageCount = 21, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\keerai"},
// new RecipeMetaData {Name = "அரேபியா", PageCount = 21, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Arabia"},
// new RecipeMetaData {Name = "இத்தாலி", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Italy"},
// new RecipeMetaData {Name = "இலங்கை", PageCount = 27, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Srilanka"},
// new RecipeMetaData {Name = "ப்ரான்ஸ்", PageCount = 4, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\france"},
// new RecipeMetaData {Name = "சீனா", PageCount = 5, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\china"},
// new RecipeMetaData {Name = "தாய்லாந்து", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\thailand"},
// new RecipeMetaData {Name = "மெக்சிகோ", PageCount = 2, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\Mexico"},
// new RecipeMetaData {Name = "மற்றநாடுகள்", PageCount = 43, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\country\others"},
// new RecipeMetaData {Name = "துவையல்", PageCount = 16, FolderName = @"D:\2018\Source\Repos\Samayal365\RecipeHtmlParser\data\2019\thuyal"},
};
foreach (RecipeMetaData recipeMetaData in recipeMetaDataList)
{
Dictionary <string, string> recipeNameList = new Dictionary <string, string>();
List <string> manualLinks = new List <string>();
Dictionary <string, string> uniqueNames = new Dictionary <string, string>();
Dictionary <string, string> uniqueLinks = new Dictionary <string, string>();
Dictionary <string, RecipeJson> recipeJsonList = new Dictionary <string, RecipeJson>();
Dictionary <string, RecipeJsonEx> recipeJsonExList = new Dictionary <string, RecipeJsonEx>();
for (int i = 0; i <= recipeMetaData.PageCount; i++)
{
logger.Info("-----------------------------------------------------------------------------------------------");
String recipesPageUrl = String.Format(@"http://www.arusuvai.com/tamil/recipes/{0}/?page={1}", recipeMetaData.Name, i);
logger.Info(recipesPageUrl);
logger.Info("-----------------------------------------------------------------------------------------------");
HtmlWeb hw = new HtmlWeb();
HtmlDocument doc = hw.Load(recipesPageUrl);
foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[starts-with(@href, '/tamil/node/')]"))
{
if (String.IsNullOrWhiteSpace(link.InnerText) == false)
{
WebClient myRecipesWebClient = new WebClient();
logger.Info(System.String.Format("{0} => http://www.arusuvai.com{1}", link.InnerText, link.Attributes.FirstOrDefault().Value));
byte[] myRecipesDataBuffer = myRecipesWebClient.DownloadData(System.String.Format("http://www.arusuvai.com{0}", link.Attributes.FirstOrDefault().Value));
HtmlAgilityPack.HtmlDocument recipesdoc = new HtmlAgilityPack.HtmlDocument();
recipesdoc.LoadHtml(Encoding.UTF8.GetString(myRecipesDataBuffer));
if (recipesdoc.DocumentNode.SelectNodes(@"/html/head/script[@type='application/ld+json']") != null)
{
String JsonString = recipesdoc.DocumentNode.SelectNodes(@"/html/head/script[@type='application/ld+json']").First().InnerText;
JsonString = JsonString.Replace("@context", "context").Replace("@graph", "graph").Replace("@type", "type");
try
{
RecipeJson recipeJson = JsonConvert.DeserializeObject <RecipeJson>(JsonString);
if (recipeJson != null)
{
String name = recipeJson.graph.FirstOrDefault().name.Replace(@"/", "-").Trim();
int index = 1;
while (uniqueNames.ContainsKey(name) == true)
{
// Get Unique recipe name
name = string.Format("{0} ({1})", name, index);
index = index + 1;
}
recipeNameList.Add(name, name);
recipeJson.graph.FirstOrDefault().name = name;
recipeJsonList.Add(name, recipeJson);
uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value);
uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name);
String filePath = System.String.Format(@"{0}\{1}.html",
recipeMetaData.FolderName, name);
try
{
System.IO.File.WriteAllText(filePath, recipeJson.RecipeString,
Encoding.UTF8);
}
catch (System.Xml.XmlException e)
{
Console.WriteLine(e);
}
catch (Newtonsoft.Json.JsonSerializationException ex)
{
Console.WriteLine(ex.Message);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
catch (Newtonsoft.Json.JsonReaderException ex)
{
try
{
RecipeJsonEx recipeJsonEx =
JsonConvert.DeserializeObject <RecipeJsonEx>(JsonString);
if (recipeJsonEx != null)
{
String name = recipeJsonEx.graph.FirstOrDefault().name.Replace(@"/", "-")
.Trim();
int index = 1;
while (uniqueNames.ContainsKey(name) == true)
{
// Get Unique recipe name
name = string.Format("{0} ({1})", name, index);
index = index + 1;
}
recipeNameList.Add(name, name);
recipeJsonEx.graph.FirstOrDefault().name = name;
recipeJsonExList.Add(name, recipeJsonEx);
uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value);
uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name);
String filePath1 = System.String.Format(@"{0}\{1}.html",
recipeMetaData.FolderName, name);
System.IO.File.WriteAllText(filePath1, recipeJsonEx.RecipeString,
Encoding.UTF8);
}
}
catch (System.Xml.XmlException e)
{
Console.WriteLine(e);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
catch (Newtonsoft.Json.JsonSerializationException ex)
{
Console.WriteLine(ex.Message);
}
}
else
{
String name = String.Empty;
;
try
{
WebClient myRecipesWebClient1 = new WebClient();
byte[] myRecipesDataBuffer1 = myRecipesWebClient1.DownloadData(System.String.Format("http://www.arusuvai.com{0}", link.Attributes.FirstOrDefault().Value));
HtmlAgilityPack.HtmlDocument recipesdoc1 = new HtmlAgilityPack.HtmlDocument();
recipesdoc1.LoadHtml(Encoding.UTF8.GetString(myRecipesDataBuffer1));
String[] ingridientList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[3]/div/div").First().InnerText.Split(new char[] { '\n' });
String[] instructionList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[4]/div/div").First().InnerText.Split(new char[] { '\n' });
String[] noteList = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/div[1]/section/article/div[5]/div/div").First().InnerText.Split(new char[] { '\n' });
name = recipesdoc1.DocumentNode.SelectNodes(@"/html/body/div[3]/div/section/h1").First().InnerText.Trim().Replace(@"/", "-").Trim();
int index = 1;
while (uniqueNames.ContainsKey(name) == true)
{
// Get Unique recipe name
name = string.Format("{0} ({1})", name, index);
index = index + 1;
}
uniqueNames.Add(name, link.Attributes.FirstOrDefault().Value);
uniqueLinks.Add(link.Attributes.FirstOrDefault().Value, name);
StringBuilder retValue = new StringBuilder();
// Name
StringBuilder sbname = new StringBuilder(string.Empty);
sbname.AppendFormat("<p>{0}</p>", name);
// Ingredients
StringBuilder ingredients = new StringBuilder(string.Empty);
foreach (string ingredient in ingridientList)
{
if (String.IsNullOrWhiteSpace(ingredient) == false)
{
ingredients.AppendFormat("<p>{0}</p>", ingredient.Trim());
}
}
// instructions
StringBuilder instructions = new StringBuilder(string.Empty);
foreach (string instruction in instructionList)
{
if (String.IsNullOrWhiteSpace(instruction) == false)
{
instructions.AppendFormat("<p>{0}</p>", instruction.Trim());
}
}
StringBuilder notes = new StringBuilder(string.Empty);
foreach (string note in noteList)
{
if (String.IsNullOrWhiteSpace(note) == false)
{
notes.AppendFormat("<p>{0}</p>", note.Trim());
}
}
StringBuilder recipeBody = new StringBuilder();
recipeBody.Append("<div class=\"recipebody\">");
recipeBody.AppendFormat("<h3>தேவையான பொருட்கள்:</h3>{0}", ingredients.ToString());
recipeBody.AppendFormat("<h3>செய்முறை:</h3>{0}", instructions.ToString());
recipeBody.AppendFormat("<h3>குறிப்புகள்:</h3>{0}", String.IsNullOrWhiteSpace(notes.ToString()) == true ? "<p></p>" : notes.ToString());
recipeBody.Append("</div>");
retValue.AppendFormat(@"<html><body><h1>{0}</h1>{1}</body></html>", sbname.ToString(), recipeBody.ToString());
String filePath3 = System.String.Format(@"{0}\{1}.html", recipeMetaData.FolderName, name);
System.IO.File.WriteAllText(filePath3, retValue.ToString(), Encoding.UTF8);
}
catch (Newtonsoft.Json.JsonSerializationException ex)
{
Console.WriteLine(ex.Message);
}
catch (Exception e)
{
manualLinks.Add(link.InnerText + " " + name);
}
}
}
}
}
String filePath2 = System.String.Format(@"{0}\{1}.txt", recipeMetaData.FolderName, "manual-links");
using (System.IO.StreamWriter file = new System.IO.StreamWriter(filePath2))
{
// store manual links
foreach (String link in manualLinks)
{
file.WriteLine(link);
}
}
System.String test = "";
}
}
