private static ThreadContainer parse_ffuuka_json(FoolFuukaParserData ffp_data)
{
ThreadContainer tc = null;
string data = fetch_api(ffp_data);
JsonObject response = JsonConvert.Import <JsonObject>(data);
JsonObject threadObject = (JsonObject)response[ffp_data.ThreadID.ToString()];
JsonObject opPost = (JsonObject)threadObject["op"];
tc = new ThreadContainer(parse_thread(opPost, ffp_data));
JsonObject postsObject = (JsonObject)threadObject["posts"];
foreach (string reply_id in postsObject.Names.Cast <string>())
{
JsonObject replyObject = (JsonObject)postsObject[reply_id];
GenericPost reply = parse_reply(replyObject, ffp_data);
tc.AddReply(reply);
continue;
}
return(tc);
}
private static GenericPost parse_reply(JsonObject data, FoolFuukaParserData ffp_data)
{
GenericPost gp = new GenericPost();
gp.Board = ffp_data.BOARD;
gp.ID = Convert.ToInt32(data["num"]);
if (data["comment_processed"] != null)
{
gp.Comment = data["comment_processed"].ToString();
}
if (data["email"] != null)
{
gp.Email = data["email"].ToString();
}
if (data["title"] != null)
{
gp.Subject = data["title"].ToString();
}
if (data["media"] != null)
{
gp.File = parse_file(data, ffp_data, gp);
}
if (data["capcode"] != null)
{
switch (data["capcode"].ToString())
{
case "N":
gp.Capcode = GenericPost.CapcodeEnum.None;
break;
default:
gp.Capcode = GenericPost.CapcodeEnum.None;
break;
}
}
if (data["name"] != null)
{
gp.Name = data["name"].ToString();
}
if (data["trip"] != null)
{
gp.Trip = data["trip"].ToString();
}
gp.Time = AniWrap.Common.ParseUTC_Stamp(Convert.ToInt32(data["timestamp"]));
return(gp);
}
private static CookieContainer get_cookies_for_board(FoolFuukaParserData ffp_data)
{
try
{
string url = ffp_data.GetBoardIndexUrl();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
wr.CookieContainer = new CookieContainer();
using (var w = wr.GetResponse())
{
return(wr.CookieContainer);
}
}
catch
{
return(new CookieContainer());
}
}
public static ThreadContainer Parse(FoolFuukaParserData data)
{
try
{
return(parse_ffuuka_json(data));
}
catch (Exception ex)
{
if (ex.Message == "404")
{
return(null);
//return parse_html(archive, board, threadID);
}
else
{
//TODO parse_html
return(null);
}
}
}
public static ThreadContainer Parse(FoolFuukaParserData data)
{
try
{
return parse_ffuuka_json(data);
}
catch (Exception ex)
{
if (ex.Message == "404")
{
return null;
//return parse_html(archive, board, threadID);
}
else
{
//TODO parse_html
return null;
}
}
}
private static string fetch_api(FoolFuukaParserData ffp_data)
{
string url = ffp_data.GetAPIUrl();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
wr.CookieContainer = get_cookies_for_board(ffp_data);
wr.AllowAutoRedirect = true;
wr.UserAgent = Program.get_random_user_agent();
//wr.Referer = string.Format("http://boards.4chan.org/{0}/thread/{1}", ffp_data.BOARD, ffp_data.ThreadID);
wr.Method = "GET";
byte[] data = null;
using (WebResponse wbr = wr.GetResponse())
{
using (Stream s = wbr.GetResponseStream())
{
using (MemoryStream memio = new MemoryStream())
{
s.CopyTo(memio);
data = memio.ToArray();
}
}
}
try
{
byte[] uncompressed = Decompress(data);
return(System.Text.Encoding.UTF8.GetString(uncompressed));
}
catch (Exception)
{
return(System.Text.Encoding.UTF8.GetString(data));
}
}
private static string fetch_api(FoolFuukaParserData ffp_data)
{
string url = ffp_data.GetAPIUrl();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
wr.CookieContainer = get_cookies_for_board(ffp_data);
wr.AllowAutoRedirect = true;
wr.UserAgent = Program.get_random_user_agent();
//wr.Referer = string.Format("http://boards.4chan.org/{0}/thread/{1}", ffp_data.BOARD, ffp_data.ThreadID);
wr.Method = "GET";
byte[] data = null;
using (WebResponse wbr = wr.GetResponse())
{
using (Stream s = wbr.GetResponseStream())
{
using (MemoryStream memio = new MemoryStream())
{
s.CopyTo(memio);
data = memio.ToArray();
}
}
}
try
{
byte[] uncompressed = Decompress(data);
return System.Text.Encoding.UTF8.GetString(uncompressed);
}
catch (Exception)
{
return System.Text.Encoding.UTF8.GetString(data);
}
}
public static AddThreadFromArchiveStatus AddThreadFromArchive(string board, int tid, bool outputConsoleMessages = false, ArchiveInfo info = null)
{
if (info == null)
{
var archives = ArchivesProvider.GetArchivesForBoard(board, false);
if (archives.Length == 0)
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot find an archive to use for board {0}", board);
}
return(AddThreadFromArchiveStatus.CannotFindAnArchive);
}
else
{
if (outputConsoleMessages)
{
Console.WriteLine("Found {0} archives for board {1}", archives.Length, board);
}
foreach (var archive_info in archives)
{
if (archive_info.Software == ArchiveInfo.ArchiverSoftware.FoolFuuka)
{
info = archive_info;
if (outputConsoleMessages)
{
Console.WriteLine("Selecting {0} (host: {1}) archive", info.Name, info.Domain);
}
break;
}
}
}
}
bool is_4chan_board = Program.ValidBoards.ContainsKey(board);
bool archive_found = info != null;
if (is_4chan_board && archive_found)
{
FoolFuukaParserData a = new FoolFuukaParserData(info, board, tid);
BoardWatcher bw = Program.GetBoardWatcher(board);
if (outputConsoleMessages)
{
Console.WriteLine("Adding thread {0} from board {1}...", a.ThreadID, a.BOARD);
}
ThreadContainer tc = FoolFuukaParser.Parse(a);
if (tc != null)
{
bw.AddStaticThread(tc, Settings.ThumbnailOnly);
if (outputConsoleMessages)
{
Console.WriteLine("Thread {0} from board {1} added.", a.ThreadID, a.BOARD);
}
return(AddThreadFromArchiveStatus.Success);
}
else
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot add this thread. Possible reasons:\n"
+ "- The thread ID is invalid\n"
+ "- The archive no longer archive this board.\n"
+ "- The archive has no JSON API support");
}
return(AddThreadFromArchiveStatus.Error);
}
}
else if (!is_4chan_board)
{
if (outputConsoleMessages)
{
Console.WriteLine("Unsupported board {0}", board);
}
return(AddThreadFromArchiveStatus.UnsupportedBoard);
}
else if (!archive_found)
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot find an archive with supported software");
}
return(AddThreadFromArchiveStatus.UnsupportedSoftware);
}
return(AddThreadFromArchiveStatus.Unknown);
}
public static AddThreadFromArchiveStatus AddThreadFromArchive(string board, int tid, bool outputConsoleMessages = false, ArchiveInfo info = null)
{
if (info == null)
{
var archives = ArchivesProvider.GetArchivesForBoard(board, false);
if (archives.Length == 0)
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot find an archive to use for board {0}", board);
}
return AddThreadFromArchiveStatus.CannotFindAnArchive;
}
else
{
if (outputConsoleMessages)
{
Console.WriteLine("Found {0} archives for board {1}", archives.Length, board);
}
foreach (var archive_info in archives)
{
if (archive_info.Software == ArchiveInfo.ArchiverSoftware.FoolFuuka)
{
info = archive_info;
if (outputConsoleMessages)
{
Console.WriteLine("Selecting {0} (host: {1}) archive", info.Name, info.Domain);
}
break;
}
}
}
}
bool is_4chan_board = Program.ValidBoards.ContainsKey(board);
bool archive_found = info != null;
if (is_4chan_board && archive_found)
{
FoolFuukaParserData a = new FoolFuukaParserData(info, board, tid);
BoardWatcher bw = Program.GetBoardWatcher(board);
if (outputConsoleMessages)
{
Console.WriteLine("Adding thread {0} from board {1}...", a.ThreadID, a.BOARD);
}
ThreadContainer tc = FoolFuukaParser.Parse(a);
if (tc != null)
{
bw.AddStaticThread(tc, Settings.ThumbnailOnly);
if (outputConsoleMessages)
{
Console.WriteLine("Thread {0} from board {1} added.", a.ThreadID, a.BOARD);
}
return AddThreadFromArchiveStatus.Success;
}
else
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot add this thread. Possible reasons:\n"
+ "- The thread ID is invalid\n"
+ "- The archive no longer archive this board.\n"
+ "- The archive has no JSON API support");
}
return AddThreadFromArchiveStatus.Error;
}
}
else if (!is_4chan_board)
{
if (outputConsoleMessages)
{
Console.WriteLine("Unsupported board {0}", board);
}
return AddThreadFromArchiveStatus.UnsupportedBoard;
}
else if (!archive_found)
{
if (outputConsoleMessages)
{
Console.WriteLine("Cannot find an archive with supported software");
}
return AddThreadFromArchiveStatus.UnsupportedSoftware;
}
return AddThreadFromArchiveStatus.Unknown;
}
private static PostFile parse_file(JsonObject data, FoolFuukaParserData ffp_data, GenericPost owner)
{
if (data["media"] != null)
{
JsonObject media = (JsonObject)data["media"];
if (media.Count == 0)
{
return(null);
}
if (media["banned"].ToString() != "0")
{
return(null);
}
if (media["media_status"].ToString() == "not-available")
{
return(null);
}
PostFile pf = new PostFile();
pf.board = ffp_data.BOARD;
pf.filename = media["media_filename_processed"].ToString();
pf.ext = pf.filename.Split('.').Last();
pf.filename = pf.filename.Split('.').First();
string thumb_link = media["thumb_link"].ToString();
string media_link = media["media_link"].ToString();
if (string.IsNullOrEmpty(media_link))
{
return(null);
}
pf.OverrideFileLinks(thumb_link, media_link);
pf.hash = media["media_hash"].ToString();
pf.height = Convert.ToInt32(media["media_h"]);
pf.width = Convert.ToInt32(media["media_w"]);
if (media["spoiler"] != null)
{
pf.IsSpoiler = (media["spoiler"].ToString() != "0");
}
pf.thumbH = Convert.ToInt32(media["preview_h"]);
pf.thumbW = Convert.ToInt32(media["preview_w"]);
pf.size = Convert.ToInt32(media["media_size"]);
pf.thumbnail_tim = media["media"].ToString().Split('.').First();
pf.owner = owner;
return(pf);
}
else
{
return(null);
}
}
private static Thread parse_thread(JsonObject data, FoolFuukaParserData ffp_data)
{
Thread t = new Thread();
t.OwnerThread = t;
t.Board = ffp_data.BOARD;
t.ID = Convert.ToInt32(data["thread_num"]);
if (data["comment_processed"] != null)
{
t.Comment = data["comment_processed"].ToString(); // raw html comment
}
if (data["email"] != null)
{
t.Email = data["email"].ToString();
}
if (data["media"] != null)
{
t.File = parse_file(data, ffp_data, t);
}
if (data["title"] != null)
{
t.Subject = data["title"].ToString();
}
if (data["capcode"] != null)
{
switch (data["capcode"].ToString())
{
case "N":
t.Capcode = GenericPost.CapcodeEnum.None;
break;
default:
t.Capcode = GenericPost.CapcodeEnum.None;
break;
}
}
if (data["sticky"] != null)
{
t.IsSticky = (data["sticky"].ToString() == "1");
}
if (data["name"] != null)
{
t.Name = data["name"].ToString();
}
if (data["trip"] != null)
{
t.Trip = data["trip"].ToString();
}
t.Time = AniWrap.Common.ParseUTC_Stamp(Convert.ToInt32(data["timestamp"]));
return(t);
}
private static CookieContainer get_cookies_for_board(FoolFuukaParserData ffp_data)
{
try
{
string url = ffp_data.GetBoardIndexUrl();
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
wr.CookieContainer = new CookieContainer();
using (var w = wr.GetResponse())
{
return wr.CookieContainer;
}
}
catch
{
return new CookieContainer();
}
}
private static Thread parse_thread(JsonObject data, FoolFuukaParserData ffp_data)
{
Thread t = new Thread();
t.OwnerThread = t;
t.Board = ffp_data.BOARD;
t.ID = Convert.ToInt32(data["thread_num"]);
if (data["comment_processed"] != null)
{
t.Comment = data["comment_processed"].ToString(); // raw html comment
}
if (data["email"] != null)
{
t.Email = data["email"].ToString();
}
if (data["media"] != null)
{
t.File = parse_file(data, ffp_data, t);
}
if (data["title"] != null)
{
t.Subject = data["title"].ToString();
}
if (data["capcode"] != null)
{
switch (data["capcode"].ToString())
{
case "N":
t.Capcode = GenericPost.CapcodeEnum.None;
break;
default:
t.Capcode = GenericPost.CapcodeEnum.None;
break;
}
}
if (data["sticky"] != null)
{
t.IsSticky = (data["sticky"].ToString() == "1");
}
if (data["name"] != null)
{
t.Name = data["name"].ToString();
}
if (data["trip"] != null)
{
t.Trip = data["trip"].ToString();
}
t.Time = AniWrap.Common.ParseUTC_Stamp(Convert.ToInt32(data["timestamp"]));
return t;
}
private static GenericPost parse_reply(JsonObject data, FoolFuukaParserData ffp_data)
{
GenericPost gp = new GenericPost();
gp.Board = ffp_data.BOARD;
gp.ID = Convert.ToInt32(data["num"]);
if (data["comment_processed"] != null)
{
gp.Comment = data["comment_processed"].ToString();
}
if (data["email"] != null)
{
gp.Email = data["email"].ToString();
}
if (data["title"] != null)
{
gp.Subject = data["title"].ToString();
}
if (data["media"] != null)
{
gp.File = parse_file(data, ffp_data, gp);
}
if (data["capcode"] != null)
{
switch (data["capcode"].ToString())
{
case "N":
gp.Capcode = GenericPost.CapcodeEnum.None;
break;
default:
gp.Capcode = GenericPost.CapcodeEnum.None;
break;
}
}
if (data["name"] != null)
{
gp.Name = data["name"].ToString();
}
if (data["trip"] != null)
{
gp.Trip = data["trip"].ToString();
}
gp.Time = AniWrap.Common.ParseUTC_Stamp(Convert.ToInt32(data["timestamp"]));
return gp;
}
private static PostFile parse_file(JsonObject data, FoolFuukaParserData ffp_data, GenericPost owner)
{
if (data["media"] != null)
{
JsonObject media = (JsonObject)data["media"];
if (media.Count == 0) { return null; }
if (media["banned"].ToString() != "0") { return null; }
if (media["media_status"].ToString() == "not-available") { return null; }
PostFile pf = new PostFile();
pf.board = ffp_data.BOARD;
pf.filename = media["media_filename_processed"].ToString();
pf.ext = pf.filename.Split('.').Last();
pf.filename = pf.filename.Split('.').First();
string thumb_link = media["thumb_link"].ToString();
string media_link = media["media_link"].ToString();
if (string.IsNullOrEmpty(media_link))
{
return null;
}
pf.OverrideFileLinks(thumb_link, media_link);
pf.hash = media["media_hash"].ToString();
pf.height = Convert.ToInt32(media["media_h"]);
pf.width = Convert.ToInt32(media["media_w"]);
if (media["spoiler"] != null) { pf.IsSpoiler = (media["spoiler"].ToString() != "0"); }
pf.thumbH = Convert.ToInt32(media["preview_h"]);
pf.thumbW = Convert.ToInt32(media["preview_w"]);
pf.size = Convert.ToInt32(media["media_size"]);
pf.thumbnail_tim = media["media"].ToString().Split('.').First();
pf.owner = owner;
return pf;
}
else { return null; }
}
private static ThreadContainer parse_ffuuka_json(FoolFuukaParserData ffp_data)
{
ThreadContainer tc = null;
string data = fetch_api(ffp_data);
JsonObject response = JsonConvert.Import<JsonObject>(data);
JsonObject threadObject = (JsonObject)response[ffp_data.ThreadID.ToString()];
JsonObject opPost = (JsonObject)threadObject["op"];
tc = new ThreadContainer(parse_thread(opPost, ffp_data));
JsonObject postsObject = (JsonObject)threadObject["posts"];
foreach (string reply_id in postsObject.Names.Cast<string>())
{
JsonObject replyObject = (JsonObject)postsObject[reply_id];
GenericPost reply = parse_reply(replyObject, ffp_data);
tc.AddReply(reply);
continue;
}
return tc;
}