private CatalogItem ParseJToken_Catalog(JsonObject thread, int pagenumber, string board)
{
CatalogItem ci = new CatalogItem();
//post number - no
ci.ID = Convert.ToInt32(thread["no"]);
// post time - now
ci.Time = Common.ParseUTC_Stamp(Convert.ToInt32(thread["time"]));
//name
if (thread["name"] != null)
{
ci.Name = thread["name"].ToString();
}
else
{
ci.Name = "";
}
if (thread["com"] != null)
{
ci.Comment = thread["com"].ToString();
}
else
{
ci.Comment = "";
}
if (thread["trip"] != null)
{
ci.Trip = thread["trip"].ToString();
}
else
{
ci.Trip = "";
}
if (thread["id"] != null)
{
ci.PosterID = thread["id"].ToString();
}
else
{
ci.PosterID = "";
}
if (thread["filename"] != null)
{
PostFile pf = new PostFile();
pf.filename = thread["filename"].ToString();
pf.ext = thread["ext"].ToString().Replace(".", "");
pf.height = Convert.ToInt32(thread["h"]);
pf.width = Convert.ToInt32(thread["w"]);
pf.thumbW = Convert.ToInt32(thread["tn_w"]);
pf.thumbH = Convert.ToInt32(thread["tn_h"]);
pf.owner = ci;
pf.thumbnail_tim = thread["tim"].ToString();
pf.board = board;
pf.hash = thread["md5"].ToString();
pf.size = Convert.ToInt32(thread["fsize"]);
ci.File = pf;
}
if (thread["last_replies"] != null)
{
JsonArray li = (JsonArray)thread["last_replies"];
List<GenericPost> repl = new List<GenericPost>();
foreach (JsonObject j in li)
{
repl.Add(ParseReply(j, board)); // HACK: parent must not be null.
}
ci.trails = repl.ToArray();
}
if (thread["bumplimit"] != null)
{
ci.BumpLimit = Convert.ToInt32(thread["bumplimit"]);
}
else
{
ci.BumpLimit = 300; //most common one
}
if (thread["imagelimit"] != null)
{
ci.ImageLimit = Convert.ToInt32(thread["imagelimit"]);
}
else
{
ci.ImageLimit = 150;
}
ci.image_replies = Convert.ToInt32(thread["images"]);
ci.text_replies = Convert.ToInt32(thread["replies"]);
ci.page_number = pagenumber;
return ci;
/*{
"tim": 1385141348984,
"time": 1385141348,
"resto": 0,
"bumplimit": 0,
"imagelimit": 0,
"omitted_posts": 1,
"omitted_images": 0,*/
}
private PostFile ParseFile(JsonObject data, string board)
{
if (data["filename"] != null)
{
PostFile pf = new PostFile();
pf.filename = HttpUtility.HtmlDecode(data["filename"].ToString());
pf.ext = data["ext"].ToString().Substring(1);
pf.height = Convert.ToInt32(data["h"]);
pf.width = Convert.ToInt32(data["w"]);
pf.thumbW = Convert.ToInt32(data["tn_w"]);
pf.thumbH = Convert.ToInt32(data["tn_h"]);
pf.thumbnail_tim = data["tim"].ToString();
pf.board = board;
pf.hash = data["md5"].ToString();
pf.size = Convert.ToInt32(data["fsize"]);
if (data["spoiler"] != null)
{
pf.IsSpoiler = Convert.ToInt32(data["spoiler"]) == 1;
}
return pf;
}
else
{
return null;
}
}
public static void dump_files(PostFile pf, bool thumbonly)
{
string md5 = base64tostring(pf.hash);
//string file_path = Path.Combine(file_save_dir, md5 + "." + pf.ext);
string reff = string.Format("http://boards.4chan.org/{0}/thread/{1}", pf.board, pf.owner);
FileIndex.MarkPostAsync(md5, pf);
#region thumb
if (pf.ThumbLink != PostFile.NoFile)
{
if (is_file_banned(md5) && !Settings.SaveBannedFileThumbnail)
goto skip;
string thumb_path;
if (!FileOperations.CheckThumbFileExist(md5, out thumb_path))
{
string thumb_key = "thumb" + md5;
if (!queued_files.ContainsKey(thumb_key))
{
FileQueueStateInfo f = new FileQueueStateInfo(md5, pf);
f.Type = FileQueueStateInfo.FileType.Thumbnail;
queued_files.Add(thumb_key, f);
f.ThreadBG = thumb_stp.QueueWorkItem(new Amib.Threading.Action(delegate
{
download_file(thumb_path, reff, f);
}));
}
}
}
skip:
#endregion
if (!thumbonly)
{
if (!is_file_banned(md5))
{
string file_name = md5 + "." + pf.ext;
string file_path;
bool file_no_exist = !FileOperations.CheckFullFileExist(file_name, out file_path);
if (pf.ext == "gif")
{
//This file was a gif that were converted to webm
//don't download it again
if (File.Exists(file_path + ".webm"))
{
file_no_exist = false;
}
}
if (file_no_exist) // GIF to webm check
{
string file_key = "file" + md5;
if (!queued_files.ContainsKey(file_key))
{
FileQueueStateInfo f = new FileQueueStateInfo(md5, pf);
f.Type = FileQueueStateInfo.FileType.FullFile;
var priority = get_file_priority(pf);
f.Priority = priority;
queued_files.Add(file_key, f);
f.ThreadBG = file_stp.QueueWorkItem(new Amib.Threading.Action(delegate
{
download_file(file_path, reff, f);
}), priority);
}
}
}
}
}
private static Amib.Threading.WorkItemPriority get_file_priority(PostFile pf)
{
/*
* max file size is 8388608 bytes.
* new priority mapping:
* 0 - 50k: level 14 [x]
* 50k - 300k: level 13 [x]
* 300k - 600k: level 12 [x]
* 600k - 900k: level 11 [x]
* 900k - 1200k: level 10 [x]
* 1.2m - 1.5m: level 9 [x]
* 1.5m - 1.8m: level 8 [x]
* 1.8m - 2.0m: level 7 [x]
* 2.0m - 2.3m: level 6 [x]
* 2.3m - 2.6m: level 5 [x]
* 2.6m - 3.0m: level 4 [x]
* 3m - 4m: level 3 [x]
* 4m - 6m: level 2 [x]
* >6m: level 1 [x]
*/
const int kb = 1024;
const int mb = 1048576;
if (pf.size <= 50 * kb)
{
return Amib.Threading.WorkItemPriority.Level14;
}
else if (pf.size > 50 * kb && pf.size <= 300 * kb)
{
return Amib.Threading.WorkItemPriority.Level13;
}
else if (pf.size > 300 * kb && pf.size <= 600 * kb)
{
return Amib.Threading.WorkItemPriority.Level12;
}
else if (pf.size > 600 * kb && pf.size <= 900 * kb)
{
return Amib.Threading.WorkItemPriority.Level11;
}
else if (pf.size > 900 * kb && pf.size <= 1200 * kb)
{
return Amib.Threading.WorkItemPriority.Level10;
}
else if (pf.size > 1200 * kb && pf.size <= 1500 * kb)
{
return Amib.Threading.WorkItemPriority.Level9;
}
else if (pf.size > 1500 * kb && pf.size <= 1800 * kb)
{
return Amib.Threading.WorkItemPriority.Level8;
}
else if (pf.size > 1800 * kb && pf.size <= 2 * mb)
{
return Amib.Threading.WorkItemPriority.Level7;
}
else if (pf.size > 2 * mb && pf.size <= 2300 * kb)
{
return Amib.Threading.WorkItemPriority.Level6;
}
else if (pf.size > 2300 * kb && pf.size <= 2600 * kb)
{
return Amib.Threading.WorkItemPriority.Level5;
}
else if (pf.size > 2600 * kb && pf.size <= 3 * mb)
{
return Amib.Threading.WorkItemPriority.Level4;
}
else if (pf.size > 3 * mb && pf.size <= 4 * mb)
{
return Amib.Threading.WorkItemPriority.Level3;
}
else if (pf.size > 4 * mb && pf.size <= 6 * mb)
{
return Amib.Threading.WorkItemPriority.Level2;
}
else //if (pf.size > 6 * mb)
{
return Amib.Threading.WorkItemPriority.Level1;
}
//smaller than 50KB, Highest Priority
//between 50KB and 400KB, High pr
//between 400KB and 1MB, normal pr
//larger than 1MB, low
//larger than 3mb, lowest
//if (pf.size < 50 * kb)
//{
// return Amib.Threading.WorkItemPriority.Highest;
//}
//else if (pf.size > 50 * kb && pf.size < 400 * kb)
//{
// return Amib.Threading.WorkItemPriority.AboveNormal;
//}
//else if (pf.size >= 400 * kb && pf.size <= 1 * mb)
//{
// return Amib.Threading.WorkItemPriority.Normal;
//}
//else if (pf.size > 1 * mb && pf.size < 3 * mb)
//{
// return Amib.Threading.WorkItemPriority.BelowNormal;
//}
//else if (pf.size >= 3 * mb)
//{
// return Amib.Threading.WorkItemPriority.Lowest;
//}
//else
//{
// return Amib.Threading.WorkItemPriority.Normal;
//}
}
private static PostFile parse_file(JsonObject data, FoolFuukaParserData ffp_data, GenericPost owner)
{
if (data["media"] != null)
{
JsonObject media = (JsonObject)data["media"];
if (media.Count == 0) { return null; }
if (media["banned"].ToString() != "0") { return null; }
if (media["media_status"].ToString() == "not-available") { return null; }
PostFile pf = new PostFile();
pf.board = ffp_data.BOARD;
pf.filename = media["media_filename_processed"].ToString();
pf.ext = pf.filename.Split('.').Last();
pf.filename = pf.filename.Split('.').First();
string thumb_link = media["thumb_link"].ToString();
string media_link = media["media_link"].ToString();
if (string.IsNullOrEmpty(media_link))
{
return null;
}
pf.OverrideFileLinks(thumb_link, media_link);
pf.hash = media["media_hash"].ToString();
pf.height = Convert.ToInt32(media["media_h"]);
pf.width = Convert.ToInt32(media["media_w"]);
if (media["spoiler"] != null) { pf.IsSpoiler = (media["spoiler"].ToString() != "0"); }
pf.thumbH = Convert.ToInt32(media["preview_h"]);
pf.thumbW = Convert.ToInt32(media["preview_w"]);
pf.size = Convert.ToInt32(media["media_size"]);
pf.thumbnail_tim = media["media"].ToString().Split('.').First();
pf.owner = owner;
return pf;
}
else { return null; }
}
