private static void ProcessWeb(string strWebAdd, int intIndex, Boolean boolTDin = false)
{
if (boolTDin)
{
intAllThreadNum++;
}
CollectionHelper.CollectionHelper ch = new CollectionHelper.CollectionHelper();
string strHtml = ch.gethtml(strWebAdd);
if (alKeyWords != null && alKeyWords.Count > 0)
{
foreach (string strKW in alKeyWords)
{
int intKeyNum = strHtml.IndexOf(strKW.ToString());
if (intKeyNum > 0)
{
alShowAdds.Add(strWebAdd);
alCheckAdds.Add(strWebAdd);
intAllValidNode++;
wl(strWebAdd + "||Deepth :" + intIndex.ToString() + " First Position : " + intKeyNum.ToString(), true);
ProcessDB(strWebAdd, strKW.ToString(), intIndex, intKeyNum, 0);
}
else
{
alCheckAdds.Add(strWebAdd);
//wl(strWebAdd + "||" + intKeyNum.ToString(), false, ConsoleColor.Yellow, ConsoleColor.Black);
}
}
}
if (boolTDin)
{
intAllThreadNum--;
intThreadNum--;
wl("Destroy Old Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
}
}
private static void MainProcess(string strOriAdd, int intIndex = 0, Boolean boolFirstRun = false, Boolean boolChildThreadFlag = false, int intDeepth = 3, int intThreadLimit = 5, Boolean boolTDin = false)
{
if (boolTDin)
{
intAllThreadNum++;
}
if (boolFirstRun)
{
//wl("Init..", false, ConsoleColor.Yellow, ConsoleColor.Black);
alAllWebAdds = new ArrayList();
alShowAdds = new ArrayList();
alCheckAdds = new ArrayList();
alAllWebAddsDeepth = new ArrayList();
alAllShowAdds = new ArrayList();
alAllWebAdds.Add(strOriAdd);
alAllWebAddsDeepth.Add(intIndex.ToString());
//ProcessWeb(strOriAdd);
boolFirstRun = false;
strOriWebAdd = strOriAdd;
}
else
{
try
{
if (alAllWebAdds.IndexOf(strOriAdd) >= 0)
{
intIndex = int.Parse(alAllWebAddsDeepth[alAllWebAdds.IndexOf(strOriAdd)].ToString());
}
}
catch
{ }
intIndex++;
}
//wl("Process Node:" + strOriAdd + "||Deepth is " + intIndex.ToString(), false, ConsoleColor.Yellow, ConsoleColor.Black);
CollectionHelper.CollectionHelper ch = new CollectionHelper.CollectionHelper();
string strBaseWeb = ch.getstr("://", "/", strOriAdd)[0].ToString();
int intErrNum = 0;
REBACK:
string strOri = ch.gethtml(strOriAdd, "utf-8", intErrNum);
if (!boolFirstRun)
{
try
{
if (intThreadNum < intThreadLimit)
{
List <Thread> tdPW_p = new List <Thread>();
if (tdPW != null)
{
tdPW_p = tdPW.ToList();
}
tdPW_p.Add(new Thread(() => ProcessWeb(strOriAdd, intDeepth, true)));
tdPW = tdPW_p.ToArray();
intThreadNum++;
wl("Create New Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
tdPW[tdPW.Count() - 1].Start();
}
else
{
ProcessWeb(strOriAdd, intDeepth);
}
}
catch (Exception ex)
{
wrr(ex.Message.ToString());
//if(intThreadNum > 0)
//{
// intThreadNum--;
// wl("Destroy Old Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
//}
ProcessWeb(strOriAdd, intDeepth);
}
}
if (strOri.Substring(0, 2) != "-1" && strOri.Substring(0, 2) != "-2")
{
ArrayList alNextWebAdds = new ArrayList();
int intAddNum = ch.GetHyperLinks(strOri).Count;
if (intAddNum > 0)
{
//wl("Child Node Process 1", false);
foreach (string strWebadd in ch.GetHyperLinks(strOri))
{
string strSeed = strWebadd;
if (!boolCrossWeb)
{
if (strSeed.Substring(strSeed.Length - 1, 1) != "/")
{
strSeed = strWebadd + "/";
}
string strCroseeWebSeed = ch.getstr("://", "/", strSeed)[0].ToString();
if (strBaseWeb == strCroseeWebSeed)
{
if (alAllWebAdds.IndexOf(strSeed.ToString()) < 0)
{
alNextWebAdds.Add(strSeed.ToString());
alAllWebAdds.Add(strSeed.ToString());
alAllWebAddsDeepth.Add(intIndex);
intAllAddress++;
}
}
}
else
{
if (alAllWebAdds.IndexOf(strSeed.ToString()) < 0)
{
alNextWebAdds.Add(strSeed.ToString());
alAllWebAdds.Add(strSeed.ToString());
alAllWebAddsDeepth.Add(intIndex);
intAllAddress++;
}
}
}
}
//wl("Child Node Process 2", false);
if (ch.getstr("\"/", "\"", strOri).Count > 0)
{
foreach (string strWebadd in ch.getstr("\"/", "\"", strOri))
{
if (alAllWebAdds.IndexOf(strOriWebAdd + strWebadd.ToString()) < 0)
{
alNextWebAdds.Add(strOriWebAdd + strWebadd.ToString());
alAllWebAdds.Add(strOriWebAdd + strWebadd.ToString());
alAllWebAddsDeepth.Add(intIndex);
intAllAddress++;
}
}
}
//wl("Current Node have " + alNextWebAdds.Count.ToString() + " Child Nodes, Deepth is " + (intIndex + 1).ToString(), false);
if (alNextWebAdds.Count > 0)
{
foreach (string strNextWebAdd in alNextWebAdds)
{
char charCheck = strNextWebAdd.Substring(strNextWebAdd.Length - 1, 1).ToCharArray()[0];
if ((char.IsLetterOrDigit(charCheck) || charCheck == '/') && (charCheck != ':'))
{
try
{
string strExName = System.IO.Path.GetExtension(strNextWebAdd);
if (!CheckStringList(strUnUseExName, strExName))
{
if (intIndex <= intDeepth)
{
wl_Thread("Add New Node:" + strNextWebAdd + "||Deepth is " + intIndex.ToString(), true, ConsoleColor.Yellow, ConsoleColor.Black);
alAllShowAdds.Add(strNextWebAdd);
//MainProcess(strNextWebAdd, intIndex, false);
if (intThreadNum >= intThreadLimit)
{
//tdChild.Join();
//, 0, true, false, intDeepth, intThreadLimit
MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit);
}
if (intThreadNum < intThreadLimit && intIndex <= intDeepth)
{
try
{
//tdPW = new Thread[intTDMax];
List <Thread> tdChild_p = new List <Thread>();
if (tdChild != null)
{
tdChild_p = tdChild.ToList();
}
tdChild_p.Add(new Thread(() => MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit, true)));
tdChild = tdChild_p.ToArray();
//tdChild[intTDMax - 1] = new Thread(() => MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit, true));
intThreadNum++;
wl("Create New Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
tdChild[tdChild.Count() - 1].Start();
//tdChild.Join();
//Thread.Sleep(500);
}
catch (Exception ex)
{
wrr(ex.Message.ToString());
//if (intThreadNum > 0)
//{
// intThreadNum--;
// wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
//}
//intThreadNum--;
//wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
}
}
}
else if (intIndex > intDeepth)
{
return;
}
}
}
catch (Exception ex)
{
//intThreadNum--;
//alCheckAdds.Add(strNextWebAdd);
//wl(strNextWebAdd + "::" + ex.Message.ToString(), false, ConsoleColor.Red, ConsoleColor.Black);
}
}
}
}
}
else if (strOri.Substring(0, 2) == "-2")
{
//wl("Current Node has no data", false);
intErrNum = int.Parse(strOri.Substring(strOri.Length - 1, 1));
goto REBACK;
}
if (boolChildThreadFlag)
{
intThreadNum--;
wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black);
}
if (boolTDin)
{
intAllThreadNum--;
}
}
