private static void ProcessWeb(string strWebAdd, int intIndex, Boolean boolTDin = false) { if (boolTDin) { intAllThreadNum++; } CollectionHelper.CollectionHelper ch = new CollectionHelper.CollectionHelper(); string strHtml = ch.gethtml(strWebAdd); if (alKeyWords != null && alKeyWords.Count > 0) { foreach (string strKW in alKeyWords) { int intKeyNum = strHtml.IndexOf(strKW.ToString()); if (intKeyNum > 0) { alShowAdds.Add(strWebAdd); alCheckAdds.Add(strWebAdd); intAllValidNode++; wl(strWebAdd + "||Deepth :" + intIndex.ToString() + " First Position : " + intKeyNum.ToString(), true); ProcessDB(strWebAdd, strKW.ToString(), intIndex, intKeyNum, 0); } else { alCheckAdds.Add(strWebAdd); //wl(strWebAdd + "||" + intKeyNum.ToString(), false, ConsoleColor.Yellow, ConsoleColor.Black); } } } if (boolTDin) { intAllThreadNum--; intThreadNum--; wl("Destroy Old Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); } }
private static void MainProcess(string strOriAdd, int intIndex = 0, Boolean boolFirstRun = false, Boolean boolChildThreadFlag = false, int intDeepth = 3, int intThreadLimit = 5, Boolean boolTDin = false) { if (boolTDin) { intAllThreadNum++; } if (boolFirstRun) { //wl("Init..", false, ConsoleColor.Yellow, ConsoleColor.Black); alAllWebAdds = new ArrayList(); alShowAdds = new ArrayList(); alCheckAdds = new ArrayList(); alAllWebAddsDeepth = new ArrayList(); alAllShowAdds = new ArrayList(); alAllWebAdds.Add(strOriAdd); alAllWebAddsDeepth.Add(intIndex.ToString()); //ProcessWeb(strOriAdd); boolFirstRun = false; strOriWebAdd = strOriAdd; } else { try { if (alAllWebAdds.IndexOf(strOriAdd) >= 0) { intIndex = int.Parse(alAllWebAddsDeepth[alAllWebAdds.IndexOf(strOriAdd)].ToString()); } } catch { } intIndex++; } //wl("Process Node:" + strOriAdd + "||Deepth is " + intIndex.ToString(), false, ConsoleColor.Yellow, ConsoleColor.Black); CollectionHelper.CollectionHelper ch = new CollectionHelper.CollectionHelper(); string strBaseWeb = ch.getstr("://", "/", strOriAdd)[0].ToString(); int intErrNum = 0; REBACK: string strOri = ch.gethtml(strOriAdd, "utf-8", intErrNum); if (!boolFirstRun) { try { if (intThreadNum < intThreadLimit) { List <Thread> tdPW_p = new List <Thread>(); if (tdPW != null) { tdPW_p = tdPW.ToList(); } tdPW_p.Add(new Thread(() => ProcessWeb(strOriAdd, intDeepth, true))); tdPW = tdPW_p.ToArray(); intThreadNum++; wl("Create New Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); tdPW[tdPW.Count() - 1].Start(); } else { ProcessWeb(strOriAdd, intDeepth); } } catch (Exception ex) { wrr(ex.Message.ToString()); //if(intThreadNum > 0) //{ // intThreadNum--; // wl("Destroy Old Thread for PW:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); //} ProcessWeb(strOriAdd, intDeepth); } } if (strOri.Substring(0, 2) != "-1" && strOri.Substring(0, 2) != "-2") { ArrayList alNextWebAdds = new ArrayList(); int intAddNum = ch.GetHyperLinks(strOri).Count; if (intAddNum > 0) { //wl("Child Node Process 1", false); foreach (string strWebadd in ch.GetHyperLinks(strOri)) { string strSeed = strWebadd; if (!boolCrossWeb) { if (strSeed.Substring(strSeed.Length - 1, 1) != "/") { strSeed = strWebadd + "/"; } string strCroseeWebSeed = ch.getstr("://", "/", strSeed)[0].ToString(); if (strBaseWeb == strCroseeWebSeed) { if (alAllWebAdds.IndexOf(strSeed.ToString()) < 0) { alNextWebAdds.Add(strSeed.ToString()); alAllWebAdds.Add(strSeed.ToString()); alAllWebAddsDeepth.Add(intIndex); intAllAddress++; } } } else { if (alAllWebAdds.IndexOf(strSeed.ToString()) < 0) { alNextWebAdds.Add(strSeed.ToString()); alAllWebAdds.Add(strSeed.ToString()); alAllWebAddsDeepth.Add(intIndex); intAllAddress++; } } } } //wl("Child Node Process 2", false); if (ch.getstr("\"/", "\"", strOri).Count > 0) { foreach (string strWebadd in ch.getstr("\"/", "\"", strOri)) { if (alAllWebAdds.IndexOf(strOriWebAdd + strWebadd.ToString()) < 0) { alNextWebAdds.Add(strOriWebAdd + strWebadd.ToString()); alAllWebAdds.Add(strOriWebAdd + strWebadd.ToString()); alAllWebAddsDeepth.Add(intIndex); intAllAddress++; } } } //wl("Current Node have " + alNextWebAdds.Count.ToString() + " Child Nodes, Deepth is " + (intIndex + 1).ToString(), false); if (alNextWebAdds.Count > 0) { foreach (string strNextWebAdd in alNextWebAdds) { char charCheck = strNextWebAdd.Substring(strNextWebAdd.Length - 1, 1).ToCharArray()[0]; if ((char.IsLetterOrDigit(charCheck) || charCheck == '/') && (charCheck != ':')) { try { string strExName = System.IO.Path.GetExtension(strNextWebAdd); if (!CheckStringList(strUnUseExName, strExName)) { if (intIndex <= intDeepth) { wl_Thread("Add New Node:" + strNextWebAdd + "||Deepth is " + intIndex.ToString(), true, ConsoleColor.Yellow, ConsoleColor.Black); alAllShowAdds.Add(strNextWebAdd); //MainProcess(strNextWebAdd, intIndex, false); if (intThreadNum >= intThreadLimit) { //tdChild.Join(); //, 0, true, false, intDeepth, intThreadLimit MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit); } if (intThreadNum < intThreadLimit && intIndex <= intDeepth) { try { //tdPW = new Thread[intTDMax]; List <Thread> tdChild_p = new List <Thread>(); if (tdChild != null) { tdChild_p = tdChild.ToList(); } tdChild_p.Add(new Thread(() => MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit, true))); tdChild = tdChild_p.ToArray(); //tdChild[intTDMax - 1] = new Thread(() => MainProcess(strNextWebAdd, intIndex, false, false, intDeepth, intThreadLimit, true)); intThreadNum++; wl("Create New Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); tdChild[tdChild.Count() - 1].Start(); //tdChild.Join(); //Thread.Sleep(500); } catch (Exception ex) { wrr(ex.Message.ToString()); //if (intThreadNum > 0) //{ // intThreadNum--; // wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); //} //intThreadNum--; //wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); } } } else if (intIndex > intDeepth) { return; } } } catch (Exception ex) { //intThreadNum--; //alCheckAdds.Add(strNextWebAdd); //wl(strNextWebAdd + "::" + ex.Message.ToString(), false, ConsoleColor.Red, ConsoleColor.Black); } } } } } else if (strOri.Substring(0, 2) == "-2") { //wl("Current Node has no data", false); intErrNum = int.Parse(strOri.Substring(strOri.Length - 1, 1)); goto REBACK; } if (boolChildThreadFlag) { intThreadNum--; wl("Destroy Old Thread for Main:" + intThreadNum, false, ConsoleColor.Cyan, ConsoleColor.Black); } if (boolTDin) { intAllThreadNum--; } }