private CatalogItem ParseJToken_Catalog(JsonObject thread, int pagenumber, string board) { CatalogItem ci = new CatalogItem(); //post number - no ci.ID = Convert.ToInt32(thread["no"]); // post time - now ci.Time = Common.ParseUTC_Stamp(Convert.ToInt32(thread["time"])); //name if (thread["name"] != null) { ci.Name = thread["name"].ToString(); } else { ci.Name = ""; } if (thread["com"] != null) { ci.Comment = thread["com"].ToString(); } else { ci.Comment = ""; } if (thread["trip"] != null) { ci.Trip = thread["trip"].ToString(); } else { ci.Trip = ""; } if (thread["id"] != null) { ci.PosterID = thread["id"].ToString(); } else { ci.PosterID = ""; } if (thread["filename"] != null) { PostFile pf = new PostFile(); pf.filename = thread["filename"].ToString(); pf.ext = thread["ext"].ToString().Replace(".", ""); pf.height = Convert.ToInt32(thread["h"]); pf.width = Convert.ToInt32(thread["w"]); pf.thumbW = Convert.ToInt32(thread["tn_w"]); pf.thumbH = Convert.ToInt32(thread["tn_h"]); pf.owner = ci; pf.thumbnail_tim = thread["tim"].ToString(); pf.board = board; pf.hash = thread["md5"].ToString(); pf.size = Convert.ToInt32(thread["fsize"]); ci.File = pf; } if (thread["last_replies"] != null) { JsonArray li = (JsonArray)thread["last_replies"]; List<GenericPost> repl = new List<GenericPost>(); foreach (JsonObject j in li) { repl.Add(ParseReply(j, board)); // HACK: parent must not be null. } ci.trails = repl.ToArray(); } if (thread["bumplimit"] != null) { ci.BumpLimit = Convert.ToInt32(thread["bumplimit"]); } else { ci.BumpLimit = 300; //most common one } if (thread["imagelimit"] != null) { ci.ImageLimit = Convert.ToInt32(thread["imagelimit"]); } else { ci.ImageLimit = 150; } ci.image_replies = Convert.ToInt32(thread["images"]); ci.text_replies = Convert.ToInt32(thread["replies"]); ci.page_number = pagenumber; return ci; /*{ "tim": 1385141348984, "time": 1385141348, "resto": 0, "bumplimit": 0, "imagelimit": 0, "omitted_posts": 1, "omitted_images": 0,*/ }
private PostFile ParseFile(JsonObject data, string board) { if (data["filename"] != null) { PostFile pf = new PostFile(); pf.filename = HttpUtility.HtmlDecode(data["filename"].ToString()); pf.ext = data["ext"].ToString().Substring(1); pf.height = Convert.ToInt32(data["h"]); pf.width = Convert.ToInt32(data["w"]); pf.thumbW = Convert.ToInt32(data["tn_w"]); pf.thumbH = Convert.ToInt32(data["tn_h"]); pf.thumbnail_tim = data["tim"].ToString(); pf.board = board; pf.hash = data["md5"].ToString(); pf.size = Convert.ToInt32(data["fsize"]); if (data["spoiler"] != null) { pf.IsSpoiler = Convert.ToInt32(data["spoiler"]) == 1; } return pf; } else { return null; } }
public static void dump_files(PostFile pf, bool thumbonly) { string md5 = base64tostring(pf.hash); //string file_path = Path.Combine(file_save_dir, md5 + "." + pf.ext); string reff = string.Format("http://boards.4chan.org/{0}/thread/{1}", pf.board, pf.owner); FileIndex.MarkPostAsync(md5, pf); #region thumb if (pf.ThumbLink != PostFile.NoFile) { if (is_file_banned(md5) && !Settings.SaveBannedFileThumbnail) goto skip; string thumb_path; if (!FileOperations.CheckThumbFileExist(md5, out thumb_path)) { string thumb_key = "thumb" + md5; if (!queued_files.ContainsKey(thumb_key)) { FileQueueStateInfo f = new FileQueueStateInfo(md5, pf); f.Type = FileQueueStateInfo.FileType.Thumbnail; queued_files.Add(thumb_key, f); f.ThreadBG = thumb_stp.QueueWorkItem(new Amib.Threading.Action(delegate { download_file(thumb_path, reff, f); })); } } } skip: #endregion if (!thumbonly) { if (!is_file_banned(md5)) { string file_name = md5 + "." + pf.ext; string file_path; bool file_no_exist = !FileOperations.CheckFullFileExist(file_name, out file_path); if (pf.ext == "gif") { //This file was a gif that were converted to webm //don't download it again if (File.Exists(file_path + ".webm")) { file_no_exist = false; } } if (file_no_exist) // GIF to webm check { string file_key = "file" + md5; if (!queued_files.ContainsKey(file_key)) { FileQueueStateInfo f = new FileQueueStateInfo(md5, pf); f.Type = FileQueueStateInfo.FileType.FullFile; var priority = get_file_priority(pf); f.Priority = priority; queued_files.Add(file_key, f); f.ThreadBG = file_stp.QueueWorkItem(new Amib.Threading.Action(delegate { download_file(file_path, reff, f); }), priority); } } } } }
private static Amib.Threading.WorkItemPriority get_file_priority(PostFile pf) { /* * max file size is 8388608 bytes. * new priority mapping: * 0 - 50k: level 14 [x] * 50k - 300k: level 13 [x] * 300k - 600k: level 12 [x] * 600k - 900k: level 11 [x] * 900k - 1200k: level 10 [x] * 1.2m - 1.5m: level 9 [x] * 1.5m - 1.8m: level 8 [x] * 1.8m - 2.0m: level 7 [x] * 2.0m - 2.3m: level 6 [x] * 2.3m - 2.6m: level 5 [x] * 2.6m - 3.0m: level 4 [x] * 3m - 4m: level 3 [x] * 4m - 6m: level 2 [x] * >6m: level 1 [x] */ const int kb = 1024; const int mb = 1048576; if (pf.size <= 50 * kb) { return Amib.Threading.WorkItemPriority.Level14; } else if (pf.size > 50 * kb && pf.size <= 300 * kb) { return Amib.Threading.WorkItemPriority.Level13; } else if (pf.size > 300 * kb && pf.size <= 600 * kb) { return Amib.Threading.WorkItemPriority.Level12; } else if (pf.size > 600 * kb && pf.size <= 900 * kb) { return Amib.Threading.WorkItemPriority.Level11; } else if (pf.size > 900 * kb && pf.size <= 1200 * kb) { return Amib.Threading.WorkItemPriority.Level10; } else if (pf.size > 1200 * kb && pf.size <= 1500 * kb) { return Amib.Threading.WorkItemPriority.Level9; } else if (pf.size > 1500 * kb && pf.size <= 1800 * kb) { return Amib.Threading.WorkItemPriority.Level8; } else if (pf.size > 1800 * kb && pf.size <= 2 * mb) { return Amib.Threading.WorkItemPriority.Level7; } else if (pf.size > 2 * mb && pf.size <= 2300 * kb) { return Amib.Threading.WorkItemPriority.Level6; } else if (pf.size > 2300 * kb && pf.size <= 2600 * kb) { return Amib.Threading.WorkItemPriority.Level5; } else if (pf.size > 2600 * kb && pf.size <= 3 * mb) { return Amib.Threading.WorkItemPriority.Level4; } else if (pf.size > 3 * mb && pf.size <= 4 * mb) { return Amib.Threading.WorkItemPriority.Level3; } else if (pf.size > 4 * mb && pf.size <= 6 * mb) { return Amib.Threading.WorkItemPriority.Level2; } else //if (pf.size > 6 * mb) { return Amib.Threading.WorkItemPriority.Level1; } //smaller than 50KB, Highest Priority //between 50KB and 400KB, High pr //between 400KB and 1MB, normal pr //larger than 1MB, low //larger than 3mb, lowest //if (pf.size < 50 * kb) //{ // return Amib.Threading.WorkItemPriority.Highest; //} //else if (pf.size > 50 * kb && pf.size < 400 * kb) //{ // return Amib.Threading.WorkItemPriority.AboveNormal; //} //else if (pf.size >= 400 * kb && pf.size <= 1 * mb) //{ // return Amib.Threading.WorkItemPriority.Normal; //} //else if (pf.size > 1 * mb && pf.size < 3 * mb) //{ // return Amib.Threading.WorkItemPriority.BelowNormal; //} //else if (pf.size >= 3 * mb) //{ // return Amib.Threading.WorkItemPriority.Lowest; //} //else //{ // return Amib.Threading.WorkItemPriority.Normal; //} }
private static PostFile parse_file(JsonObject data, FoolFuukaParserData ffp_data, GenericPost owner) { if (data["media"] != null) { JsonObject media = (JsonObject)data["media"]; if (media.Count == 0) { return null; } if (media["banned"].ToString() != "0") { return null; } if (media["media_status"].ToString() == "not-available") { return null; } PostFile pf = new PostFile(); pf.board = ffp_data.BOARD; pf.filename = media["media_filename_processed"].ToString(); pf.ext = pf.filename.Split('.').Last(); pf.filename = pf.filename.Split('.').First(); string thumb_link = media["thumb_link"].ToString(); string media_link = media["media_link"].ToString(); if (string.IsNullOrEmpty(media_link)) { return null; } pf.OverrideFileLinks(thumb_link, media_link); pf.hash = media["media_hash"].ToString(); pf.height = Convert.ToInt32(media["media_h"]); pf.width = Convert.ToInt32(media["media_w"]); if (media["spoiler"] != null) { pf.IsSpoiler = (media["spoiler"].ToString() != "0"); } pf.thumbH = Convert.ToInt32(media["preview_h"]); pf.thumbW = Convert.ToInt32(media["preview_w"]); pf.size = Convert.ToInt32(media["media_size"]); pf.thumbnail_tim = media["media"].ToString().Split('.').First(); pf.owner = owner; return pf; } else { return null; } }