private void RunThread()
{
ScrapperMin sm = ScrapperMin.InitWC();
sm.Log = new Action <string>(AddLog);
string[] result = sm.Multiple(Scripts, DataList);
}
public void AddLink(string str)
{
InitAll();
string[] ss = str.Split(new char[] { ' ' });
if (ss == null || ss.Length == 0 || ss.Length != 2)
{
return;
}
string uu = AppConfig.GetThreadLink();
Uri uri = null;
try
{
uri = new Uri(uu);
}
catch
{
AddLog("[Ignore] IsThreadUri:false, url:'" + uu + "'");
return;
}
string url = string.Format(uu, ss[0], ss[1]);
if (ALL.Contains(url))
{
return;
}
lock (LoginLock)
{
if (HasLogin == false && AppConfig.GetRequireLogin())
{
string scriptprefix = AppConfig.GetScriptPrefix();
if (File.Exists(Path.Combine(ScriptDir, scriptprefix + LoginScript)) == false)
{
return;
}
string[] strx = null;
try
{
strx = sm.Multiple(File.ReadAllText(Path.Combine(ScriptDir, scriptprefix + LoginScript)), new List <string> {
User, Pass
});
}
catch { }
if (strx != null && strx.Length == 1 && strx[0] != "SUCCESS LOGIN")
{
return;
}
HasLogin = true;
}
}
string page = string.Empty;
try
{
page = sm.WC.MethodPage(url, "GET", "", new Dictionary <string, string>());
}
catch { }
if (string.IsNullOrEmpty(page))
{
return;
}
string title = "";
string next = AppConfig.GotNext();
if (page.IndexOf(next) >= 0)
{
AddAllPage(url);
}
string[] tts = StringOps.TagMatch(page, "<title>", "</title>");
if (tts != null && tts.Length > 0)
{
title = tts[0];
}
if (SetText != null)
{
SetText("[" + ss[1].ToString() + "] " + title);
}
string regex = AppConfig.GetRegex();
var linkParser = new Regex(regex, RegexOptions.Compiled | RegexOptions.IgnoreCase);
var matchesCol = linkParser.Matches(page);
if (matchesCol == null || matchesCol.Count == 0)
{
AddLog("[NotFound] title:'" + title + "', url:'" + url + "'");
return;
}
List <string> domains = new List <string>();
string[] doms = AppConfig.GetDomains().Split(new char[] { '\n' });
string[] ignores = AppConfig.GetIgnoreDomains().Split(new char[] { '\n' });
if (ignores.Length > 0 && ignores[0] == "")
{
ignores = new string[0];
}
string[] bodies = AppConfig.GetIgnoreBodies().Split(new char[] { '\n' });
if (bodies.Length > 0 && bodies[0] == "")
{
bodies = new string[0];
}
string[] bodiesMatches = AppConfig.GetBodies().Split(new char[] { '\n' });
if (bodies.Length > 0 && bodies[0] == "")
{
bodies = new string[0];
}
if (doms.Length > 0 && doms[0] != "")
{
foreach (string dom in doms)
{
domains.Add(dom.Trim());
}
}
if (bodies.Length > 0 && bodies[0] != "")
{
foreach (string body in bodies)
{
if (page.ToLower().Contains(body.ToLower()))
{
AddLog("[Ignore] ContainsBody:'" + body + "', title:'" + title + "', url:'" + url + "'");
return;
}
}
}
if (bodiesMatches.Length > 0 && bodiesMatches[0] != "")
{
bool contains = false;
foreach (string body in bodiesMatches)
{
if (body.ToLower().StartsWith("http") && page.ToLower().Contains(body.ToLower()))
{
contains = true;
break;
}
else if (body.ToLower().StartsWith("http") == false)
{
if (page.ToLower().Contains(body.ToLower()))
{
contains = true;
break;
}
}
}
if (contains == false)
{
AddLog("[Ignore] NotContainsBody:true, title:'" + title + "', url:'" + url + "'");
return;
}
}
int count = 0;
foreach (Match m in matchesCol)
{
if (m == null || string.IsNullOrEmpty(m.Value))
{
continue;
}
string urlx = m.Value;
if (urlx.Contains("imgur"))
{
Console.WriteLine("a");
}
if (urlx.Contains("\""))
{
urlx = m.Value.Substring(0, m.Value.IndexOf("\""));
}
if (urlx.Contains("<"))
{
urlx = m.Value.Substring(0, m.Value.IndexOf("<"));
}
if (urlx.Contains(".") == false)
{
AddLog("[Ignore] ContainsDot:false, title:'" + title + "', url:'" + url + "', urlx:'" + urlx + "'");
continue;
}
if (urlx.Contains("..."))
{
AddLog("[Ignore] Contains3Dots:true, title:'" + title + "', url:'" + url + "', urlx:'" + urlx + "'");
continue;
}
try
{
Uri urlxx = new Uri(urlx);
}
catch
{
AddLog("[Ignore] IsUri:false, title:'" + title + "', url:'" + url + "', urlx:'" + urlx + "'");
continue;
}
bool ig = false;
foreach (string ig2 in ignores)
{
if (urlx.ToLower().Contains(ig2.ToLower()))
{
ig = true;
AddLog("[Ignore] ig2:'" + ig2 + "', title:'" + title + "', url:'" + url + "', urlx:'" + urlx + "'");
break;
}
}
if (ig)
{
continue;
}
if (domains.Count > 0)
{
foreach (string dom in domains)
{
if (urlx.ToLower().IndexOf(dom.ToLower()) < 0)
{
continue;
}
count++;
AddFile(title, url, urlx);
break;
}
}
else
{
count++;
AddFile(title, url, urlx);
}
}
if (count == 0)
{
AddLog("[NotFound] title:'" + title + "', url:'" + url + "'");
}
}