public List <TCP> Parse_TCPPage(string URL, string imageRoot = @"H:\SiteParse\TheClassic")
{
List <TCP> Films = new List <TCP>();
string xpath = "/html/body/div[3]/div/div[2]/div[4]/ul";
string html = ahk.Download_HTML(URL);
List <string> resultLines = xml.Parse_HTML_XML(html, xpath); // extract sections of text from html xml
foreach (string line in resultLines) // pull segments of page (20 total)
{
TCP Film = new TCP();
List <string> ImageLinks = new List <string>();
//string FilmURL = ""; string CoverURL = ""; string Title = ""; List<string> ImageLinks = new List<string>();
//string VidID = "";
string seg = line.Replace("\">", "\">\n");
List <string> newLines = lst.Text_To_List(seg); bool TitleFound = false;
foreach (string newline in newLines)
{
//FILM URL
string videoURLCheck = "thumb-video-link\" href=\"/videos";
if (newline.Contains(videoURLCheck))
{
string videoURL = newline.Replace("<a class=\"thumb-video-link\" href=\"", "");
videoURL = videoURL.Replace("/\">", "");
Film.FilmURL = "https://theclassicporn.com" + videoURL; // + @"\";
//ahk.MsgBox("FilmURL: " + FilmURL);
}
// VIDEO ID
if (newline.Contains("covers.jpg"))
{
List <string> items = ahk.StringSplit_List(newline, "/");
foreach (string item in items)
{
if (!item.Contains("covers.jpg"))
{
Film.VideoID = item;
}
if (item.Contains("covers.jpg"))
{
break;
}
}
//ahk.MsgBox(vidID);
}
// VIDEO COVER URL
if (newline.Contains("covers.jpg"))
{
List <string> items = ahk.StringSplit_List(newline, "\"");
foreach (string item in items)
{
if (item.Contains("covers.jpg"))
{
Film.CoverURL = item; break;
}
}
//ahk.MsgBox(CoverURL);
}
// VIDEO TITLE
if (newline.Contains("class=\"link-blue link-no-border\""))
{
TitleFound = true; continue;
}
if (TitleFound)
{
Film.FilmName = newline.Replace("</a>", "");
//ahk.MsgBox(Title);
TitleFound = false;
}
if (newline.Contains(".jpg") && !newline.Contains("covers.jpg"))
{
string ImageLine = newline.Replace("\" alt=\"\">", "");
ImageLine = ImageLine.Replace("<img class=\"screen-thumb\" src=\"", "");
//ahk.MsgBox(ImageLine);
ImageLinks.Add(ImageLine);
}
}
string links = "";
if (imageRoot != "")
{
string saveDir = imageRoot + "\\" + Film.VideoID;
ahk.FileCreateDir(saveDir);
// download film images to dirs sorted by VidID
foreach (string image in ImageLinks)
{
string fileName = ahk.StringSplit(image, "/", 0, true);
ahk.Download_File(image, saveDir + "\\" + fileName);
if (links == "")
{
links = image;
}
else
{
links = links + "\n" + image;
}
}
// download film cover
if (Film.CoverURL != null)
{
string filename = ahk.StringSplit(Film.CoverURL, "/", 0, true);
ahk.Download_File(Film.CoverURL, saveDir + "\\" + filename);
}
}
else
{
// add image links to string list
foreach (string image in ImageLinks)
{
if (links == "")
{
links = image;
}
else
{
links = links + "\n" + image;
}
}
}
Film.ImageLinks = links;
Film.ImgLinks = ImageLinks;
Films.Add(Film);
bool added = TCP_UpdateInsert(Film);
}
return(Films);
}
public bool RGP(int PageNum = 1)
{
string url = "http://rgporn.com/index.php?newsid=" + PageNum;
//tel.Update(txtURL, url);
string html = web.DownloadHTML(url);
if (html == "")
{
return(false);
}
resultLines = xml.Parse_HTML_XML(html, "//body/div/div/div/div/table/tbody/tr/td/div/div/div/p"); // extract sections of text from html xml
// extract tags
string display = "TAGS: ";
List <string> tags = new List <string>();
string Tags = "";
foreach (string line in resultLines)
{
if (line.Contains("=tags"))
{
string tag = line.Replace("<a", "");
tag = tag.Replace("</a>", "");
tag = ahk.StringSplit(tag, ">", 1);
tag = tag.Trim();
tags.Add(tag);
display = display + "\n" + tag;
if (Tags != "")
{
Tags = Tags + "|" + tag;
}
if (Tags == "")
{
Tags = tag;
}
}
}
// extract title
List <string> titleLines = xml.Parse_HTML_XML(html, "//body/div/div/div/div/table/tbody/tr/td/div/div/div"); // extract sections of text from html xml
string title = lst.Return_List_Value(titleLines, 2);
title = title.Replace("<h1 class=\"shead\">", "");
title = title.Replace("</h1>", "");
string imagehtml = "";
foreach (string ima in titleLines)
{
if (ima.Contains(".jpg"))
{
imagehtml = imagehtml + "\n" + ima;
}
}
//ahk.MsgBox(title);
List <string> links = rg.Regex_RGLinks(html);
bool AllOnline = true;
int onlineLinkCount = 0;
string linkText = "";
foreach (string link in links)
{
RGInfo info = rg.RapidGatorCheckStatus(link);
if (!info.FileOnline)
{
AllOnline = false;
}
else
{
onlineLinkCount++;
}
string wlink = link + "|" + info.FileSize;
if (linkText != "")
{
linkText = linkText + "\n" + wlink;
}
if (linkText == "")
{
linkText = wlink;
}
}
string linksMsg = onlineLinkCount + "/" + links.Count + " Links Online";
// Extract / Display Images From Page
int downloaded = 0;
if (AllOnline)
{
string SaveDir = ahk.AppDir() + "\\Posts\\RGP\\" + ahk.AddLeadingZeros(PageNum, 5).ToString();
ahk.FileCreateDir(SaveDir);
ahk.Sleep(500);
ahk.FileAppend(Tags, SaveDir + "\\Tags.txt");
ahk.FileAppend(title, SaveDir + "\\Title.txt");
ahk.FileAppend(linkText, SaveDir + "\\Links.txt");
downloaded = DownloadImages(imagehtml, SaveDir, ahk.AddLeadingZeros(PageNum, 5).ToString());
//tel.Update(txtResults, "Title: " + title + "\n\nTags: " + Tags + "\n\n" + linksMsg);
}
//ahk.MsgBox("Found " + links.Count + " Links\nDownloaded " + downloaded.ToString() + " + Images");
return(AllOnline);
}
/// <summary>Creates a directory/folder. (Same As FileCreateDir)</summary>
/// <param name="DirName">Name of the directory to create, which is assumed to be in %A_WorkingDir% if an absolute path isn't specified.</param>
public static bool CreateDir(this string DirName)
{
_AHK ahk = new _AHK();
return(ahk.FileCreateDir(DirName));
}
// takes tiled screen shot to generate a single preview image - returns path to new preview image (blank if failed)
public string VidCap_Tiles(string videoFilePath, string saveDir = "", int TilesWide = 3, int TilesTall = 3, string Scale = "320:240", string SaveFormat = "jpg", string CaptureOpt = "1000")
{
if (!ahk.isVideo(videoFilePath))
{
sb(videoFilePath + " Not Recognized Video Format... Skipping");
return("");
}
VideoFile vidInfo = GetVideoInfo(videoFilePath);
// Extract FPS From video Info returned
//h264(Main)(avc1 / 0x31637661), yuv420p, 600x450, 750 kb / s, SAR 1:1 DAR 4:3, 29.97 fps, 29.97 tbr, 30k tbn, 59.94 tbc(default)
int seconds = vidInfo.Duration.TotalSeconds.ToInt(); //getDuration(videoFilePath);
int egs = TilesWide * TilesTall; // ex 3 x 3 = 9 segs - dfivide total time by 9 for even timing
int shotDistance = seconds / egs;
//TimeSpan t = TimeSpan.FromMilliseconds(shotDistance);
TimeSpan t = TimeSpan.FromSeconds(shotDistance);
int MinBetween = t.Minutes; // # of minutes between tile shots (in video)
int SecondsBetween = t.Seconds; // # of seconds between tile shots (in video)
int distanceBetweenFrames = (seconds * vidInfo.FPS).ToInt();
//string SaveFormat = "jpg";
//Scale = "0:0";
int FrameCount = 1;
//Scale = "320:240";
//int TileX = 2;
//int TileY = 3;
//string Scale = "320:240";
//string[] shots = new string[numShots];
// default save dir under neath original file
if (saveDir == "")
{
saveDir = ahk.FileDir(videoFilePath) + "\\Previews";
}
string outImage = saveDir + "\\" + ahk.FileNameNoExt(videoFilePath) + "." + SaveFormat;
if (File.Exists(outImage))
{
return("");
}
outImage = saveDir + "\\" + ahk.FileNameNoExt(videoFilePath) + "_%03d" + SaveFormat;
ahk.FileCreateDir(saveDir);
string args = "";
string opt = CaptureOpt;
//-i video.mkv -vf scale=500:-1 -t 10 -r 10 image.gif
// one every 1000 frames, works but didn't alway fill all the boxes
if (opt == "1000")
{
args = " -ss 00:" + MinBetween + ":" + SecondsBetween + " -i \"" + videoFilePath + "\" -frames: " + FrameCount + " -vf \"select=not(mod(n\\,1000)),scale=" + Scale + ",tile=" + TilesWide + "x" + TilesTall + "\" \"" + outImage + "\"";
}
// one every 3000 frames - works but SLOWER
if (opt == "3000")
{
args = " -ss 00:" + MinBetween + ":" + SecondsBetween + " -i \"" + videoFilePath + "\" -vf \"select=not(mod(n\\,3000)),scale=" + Scale + ",tile=" + TilesWide + "x" + TilesTall + "\" \"" + outImage + "\"";
}
//string args = " -ss 00:" + MinBetween + ":" + SecondsBetween + " -i \"" + videoFilePath + "\" -vf \"-r 0.0033,scale=" + Scale + ",tile=" + TilesWide + "x" + TilesTall + "\" \"" + outImage + "\"";
//string args = " -ss 00:" + MinBetween + ":" + SecondsBetween + " -i \"" + videoFilePath + "\" -vf \"select=not(mod(n\\,3500)),scale=" + Scale + ",tile=" + TilesWide + "x" + TilesTall + "\" \"" + outImage + "\"";
// one frame per video (thumbnails no rows)
if (opt == "1")
{
args = " -ss 00:" + MinBetween + ":" + SecondsBetween + " -i \"" + videoFilePath + "\" -vf thumbnail,scale=" + Scale + " -frames:v 1 \"" + outImage + "\"";
}
// export ALL action frames (about 1K images in 40min example video)
if (opt == "ActionFrames")
{
args = "-i \"" + videoFilePath + "\" -vf \"select = 'eq(pict_type,PICT_TYPE_I)'\" -vsync vfr \"" + saveDir + "\\" + ahk.FileNameNoExt(videoFilePath) + "_%04d." + SaveFormat + "\"";
}
// One screen shot at specific time stamp
if (opt == "TimeShot")
{
args = " -i \"" + videoFilePath + "\" -ss 00:02:05.000 -vframes 1 \"" + saveDir + "\\" + ahk.FileNameNoExt(videoFilePath) + "." + SaveFormat + "\"";
}
// mosasic option - works
if (opt == "Mosasic")
{
args = " -i \"" + videoFilePath + "\" -vf select='gt(scene\\,0.5)',scale=" + Scale + ",tile -frames:v 5 \"" + outImage + "\"";
}
//args = "-ss 3 -i \"" + videoFilePath + "\" -vf \"select = gt(scene\\, 0.5)\" -frames:v 5 -vsync vfr \"" + outImage + "\"";
//args = "-i \"" + videoFilePath + "\" -vf \"select = gt(scene\\, 0.4),scale = 640:360\" -frames:v 5 \"" + outImage + "\"";
//args = "-i \"" + videoFilePath + "\" -vf fps=1/60 \"" + outImage + "\""; // one image per minute
//int dur = getDuration(videoFilePath);
// 2,266 = 37 min video
string FPS = ""; // number of frames to capture
try
{
int totalFPS = vidInfo.Duration.TotalSeconds.ToInt() / 10; // divide total seconds by 10
FPS = "1/" + totalFPS.ToString();
}
catch { FPS = "1/60"; } // 1 frame per min
args = "-i \"" + videoFilePath + "\" -vf fps=" + FPS + " \"" + outImage + "\""; //
ahk.FileAppend(videoFilePath, ahk.AppDir() + "\\FFMpeg_Log.txt");
StartFFMpeg(args);
if (File.Exists(outImage))
{
return(outImage);
}
else
{
return("");
}
}
//_Parse.XML xml = new _Parse.XML();
//_AHK ahk = new _AHK();
//_Database.SQL sql = new _Database.SQL();
//_Lists lst = new _Lists();
//_Parse prs = new _Parse();
////_Images img = new _Images();
//_TelerikLib.RadProgress pro = new _TelerikLib.RadProgress();
//_TelerikLib tel = new _TelerikLib();
////_Apps.Chrome cr = new _Apps.Chrome();
//sharpAHK_Dev._Threads thr = new sharpAHK_Dev._Threads();
//_TelerikLib.RadTree tree = new _TelerikLib.RadTree();
//IAFD iafd = new IAFD();
//_Web.ADBSites.PBBForum pbb = new _Web.ADBSites.PBBForum();
//_Web.ADBSites.PRNWForum prnw = new _Web.ADBSites.PRNWForum();
#endregion
public void Download_Site(int startPage = 1, int LastPage = 230, bool SkipExisting = true, RadProgressBar Bar = null, RadProgressBar Bar2 = null, bool NewThread = true)
{
_AHK ahk = new _AHK();
_Lists lst = new _Lists();
_Sites.RapidGator rg = new _Sites.RapidGator();
_TelerikLib.RadProgress pro = new _TelerikLib.RadProgress();
if (NewThread)
{
Thread newThread = new Thread(() => Download_Site(startPage, LastPage, SkipExisting, Bar, Bar2, false)); // Function To Execute
newThread.IsBackground = true;
newThread.Start();
}
else
{
int pageNum = startPage;
do
{
int postNum = 0;
string html = ahk.Download_HTML("http://pornchil.com/page/" + pageNum + "/");
List <string> lines = lst.Text_To_List(html, true, true, false);
if (Bar != null)
{
pro.SetupProgressBar(Bar, 10);
} // # of posts/page
foreach (string line in lines)
{
if (line.Contains("<h1 class=\"entry-title\"><a href="))
{
PrnChill chill = new PrnChill();
//ahk.MsgBox(line);
string Line = line.Replace("<h1 class=\"entry-title\"><a href=\"", "");
chill.PostURL = ahk.StringSplit(Line, "\"", 0);
chill.PostName = ahk.StringSplit(Line, ">", 1);
chill.PostName = ahk.StringSplit(chill.PostName, "<", 0);
if (SkipExisting)
{
if (AlreadyParsed(chill.PostURL))
{
continue;
} // check to see if this post already has an entry, if so skip
}
if (Bar != null)
{
postNum++; pro.UpdateProgress(Bar, postNum + "/10");
} // # of posts remaining
string postHTML = ahk.Download_HTML(chill.PostURL);
List <string> links = rg.Regex_RGLinks(postHTML);
if (links.Count > 0)
{
if (Bar != null)
{
pro.ProgressText(Bar, postNum + "/10 | Verifying Links");
} // # of posts remaining
List <_Sites.RapidGator.RGInfo> checkedLinks = rg.RapidGator_BatchCheck(links, true, Bar2); // batch check list of rg links
string rglinks = ""; int goodLinkCount = 0;
foreach (_Sites.RapidGator.RGInfo link in checkedLinks)
{
if (rglinks == "")
{
rglinks = link.FileURL + "|" + link.FileSize; goodLinkCount++;
}
else
{
rglinks = rglinks + "\n" + link.FileURL + "|" + link.FileSize; goodLinkCount++;
}
}
chill.Links = rglinks; // online rg links with filepath|filesize return
chill.LinkCount = goodLinkCount;
chill.LinkCheckDate = DateTime.Now.ToString();
}
chill.InCollection = "false";
List <string> images = rg.JpgImageLinks(postHTML);
if (Bar != null)
{
pro.ProgressText(Bar, postNum + "/10 | Downloading Images (" + images.Count + ")");
} // # of posts remaining
if (images.Count > 0)
{
// create save dir
string saveDir = ahk.AppDir() + "\\PrnChillPosts";
ahk.FileCreateDir(saveDir);
saveDir = saveDir + "\\" + chill.PostName;
ahk.FileCreateDir(saveDir);
chill.ImageDir = saveDir;
int imgNum = 1;
foreach (string image in images)
{
ahk.Download_File(image, saveDir + "\\" + imgNum + ".jpg", true); imgNum++;
}
}
PrnChill_UpdateInsert(chill);
}
}
pageNum++;
} while (pageNum <= LastPage);
}
}
public void WriteMovieLink(string LinkURL, string MovieTitle = "MovieTitle (Year)", string IMDbID = "", bool AutoStart = true)
{
if (LinkURL == null)
{
return;
}
string FileLink = LinkURL;
if (FileLink.Contains("|"))
{
FileLink = ahk.StringSplit(FileLink, "|", 0);
}
string JDownloaderPath = @"C:\Users\jason\AppData\Local\JDownloader v2.0\JDownloader2.exe";
string DownloadRoot = "D:\\Media\\Movies";
MovieTitle = MovieTitle.Replace(":", "-");
MovieTitle = MovieTitle.Replace("?", "-");
ahk.FileCreateDir(DownloadRoot);
string FolderWatchPath = "D:\\WinFiles\\JDownloaderFolderWatch";
string fileName = prs.LinkFileName(FileLink).Replace(".html", "");
bool ExtractAfterDownload = true;
bool Enabled = true;
bool ForcedStart = true;
bool AutoConfirm = true;
//#this is a property file format. You can put all these entries in a jsonarray [{..},{...}] as well.
//#save this as myFile.crawljob
//#Start a new entry with anything but a comment (#...) or key=. An empty line would be ok as well
//#\ must be double encoded! \ -> \\
//->NEW ENTRY<-
// chunks = 6
//#a comment
//# extractPasswords=["Password1","Password2"]
// text = https://rapidgator.net/file/3aef7553f517698dad4d6d0c5d505e17/The.China.Hustle.2017.WEB.x264-MUSiCANA.mkv
// filename = The.China.Hustle.2017.WEB.x264 - MUSiCANA.mkv
// packageName = ShowName
// extractAfterDownload = TRUE
// downloadFolder = D:\\Media\\ThisShowName
//# priority=DEFAULT
//# forcedStart=UNSET
//# downloadPassword=null
//# use only if text contains one single link
//# overwritePackagizerEnabled=true
//# setBeforePackagizerEnabled=true
//# comment=null
//# deepAnalyseEnabled=true
//# addOfflineLink=true
// enabled = true
// autoStart = TRUE
// forcedStart = TRUE
// autoConfirm = TRUE
if (IMDbID != "") // Create IMDb .URL File in Download Dir
{
_Sites.OMDB omdb = new _Sites.OMDB();
omdb.OMDb_NewDirURL(IMDbID, DownloadRoot);
}
string lines = "# JDownloader FolderWatch Link";
lines = lines + "\n" + "->NEW ENTRY<-";
lines = lines + "\n" + "chunks=6";
lines = lines + "\n" + "text=" + FileLink;
// add missing file format?
//lines = lines + "\n" + "filename=" + fileName + ".mp4";
lines = lines + "\n" + "filename=" + fileName; // includes .rar
lines = lines + "\n" + "packageName=" + MovieTitle;
lines = lines + "\n" + "downloadFolder=" + DownloadRoot + "\\" + MovieTitle;
lines = lines + "\n" + "extractAfterDownload=" + ExtractAfterDownload.ToString();
lines = lines + "\n" + "enabled=" + Enabled.ToString();
lines = lines + "\n" + "autostart=" + AutoStart.ToString();
lines = lines + "\n" + "forcedStart=" + ForcedStart.ToString();
lines = lines + "\n" + "autoConfirm=" + AutoConfirm.ToString();
lines = lines + "\n" + "";
string saveName = fileName.Replace("|", "_");
saveName = ahk.Decode(fileName);
LinkCounter++;
string date = DateTime.Now.ToString("M_d_yyyy_mm_ss_fff");
//string date = DateTime.Now.Date.ToShortDateString();
string savePath = FolderWatchPath + "\\" + date + ".crawljob";
if (File.Exists(savePath))
{
ahk.Sleep(500);
savePath = DateTime.Now.ToString("M_d_yyyy_mm_ss_fff"); // try next current time
}
bool wroteLink = ahk.FileAppend(lines, savePath);
if (!wroteLink)
{
//ahk.Sleep(10)
//ahk.MsgBox("Writing Link File Failed, Retrying...");
//WriteMovieLink(LinkURL, MovieTitle, IMDbID, AutoStart);
}
}