public List <UrlList> GetMenuList(int i, string filter)
{
List <UrlList> list = new List <UrlList>();
var result = bll.Find().FindAll(n => n.Status == i);
if (!string.IsNullOrEmpty(filter))
{
result = result.FindAll(n => n.Url.Contains(filter) || n.Id.Contains(filter) || n.Name.Contains(filter) || n.Types.Contains(filter));
}
foreach (var item in result)
{
UrlList e = new UrlList();
e.Id = item.Id;
e.Name = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"名称……\" id=\"txtName_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlNameUpdateSave('{1}')\">Go</button></span></div>", item.Name, item.Id);
e.Url = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"网址……\" id=\"txtUrl_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlUpdateSave('{1}')\">Go</button></span></div>", item.Url, item.Id);
e.Source = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"网址描述……\" id=\"txtSource_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlSourceUpdateSave('{1}')\">Go</button></span></div>", item.Source, item.Id);
//e.Title = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"Title……\" id=\"txtTitle_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlTitleUpdateSave('{1}')\">Go</button></span></div>", item.Title, item.Id);
e.Types = string.Format("<a onclick=\"myModalModify('{0}')\">{1}</a>", item.Id, item.Types);
e.SortDesc = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"降序排列……\" id=\"txtSort_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"btnSortSave('{1}')\">Go</button></span></div>", item.SortDesc.ToString(), item.Id);
e.Status = string.Format("<button class=\"{2}\" id='btnStatus{0}' onclick=\"myStatusEdit('{0}');\">{1}</button>", item.Id, item.Status.ToEnumDiscription <EnumUrlStatus>(), GetStatusCss(item.Status));
e.Operation = string.Format("<a class=\"btn btn-info btn-circle\" href=\"{0}\" target=\"_blank\" title=\"{1}\" id=\"txtfwxtest_{2}\" onclick=\"BtnClearClass('{2}')\"> <i class=\"glyphicon glyphicon-ok\"></i></a>", item.Url, item.Title, item.Id);
list.Add(e);
}
return(list);
}
/// <summary>
/// 解析方法
/// </summary>
/// <returns></returns>
public string Analysis()
{
string result = Enums.成功;
UrlList.Clear();//清空解析的Url列表
for (int i = 0; i < Page; i++)
{
string html;
string res = reptile.GetHtml($"{Url}{(i > 0 ? $"&page={i + 1}" : string.Empty)}", HeadDic, out html);
if (res != Enums.成功)
{
result += $"\r\n获取第{i}页HTML失败\r\n原因:{res}";
continue;
}
res = GetUrlIDByZz(html);
if (res != Enums.成功)
{
result += $"\r\n获取第{i}页UrlID失败\r\n原因:{res}";
continue;
}
if (i < 1)
{
string countZz = "共([\\s\\S]*?)条数据";
DataCount = reptile.Analysis(countZz, "$1", html)[0];
int intCount = int.Parse(DataCount.Replace("+", string.Empty));
AllPage = intCount % 20 > 0 ? (intCount / 20 + 1).ToString() : (intCount / 20).ToString();
if (DataCount.IndexOf('+') > -1)
{
AllPage += "+";
}
}
}
LoadDataCount = 0;//清空已下载的数据条数
return(result);
}
private void LinearAddList(string text, int unityNumber)
{
for (int i = 1; i <= unityNumber; i++)
{
UrlList.Add(string.Format(text, i));
}
}
private void OpenURL_Shown(object sender, EventArgs e)
{
UrlList.Focus();
if (UrlList.Items.Count > 0)
{
UrlList.SelectedIndex = 0;
}
}
// Methods
/// <summary>
/// Add a site to the URL list and checks for the maximum size.
/// </summary>
/// <param name="url">Site to be added</param>
public void AddSiteToList(string url)
{
if (UrlList.Count >= ListSize)
{
UrlList.RemoveAt(0);
}
UrlList.Add(url);
}
private void UpdateCredentialsInPersistentStore()
{
_ps.UpdateCredentials(_netClient.Url, _netClient.TenancyName, _netClient.UserName, _netClient.Password, _netClient.AuthToken);
if (!UrlList.Contains(_netClient.Url))
{
UrlList.Add(_netClient.Url);
NotifyPropertyChanged(nameof(UrlList));
}
}
public void AddFeed(Feed feed)
{
if (feed != null)
{
if (!FeedList.Contains(feed))
{
UrlList.Add(feed.Link);
FeedList.Add(feed);
}
}
}
private void parseUrlAddress(string loc)
{
if (loc.Contains("xml"))
{
innerSiteMapUrlList.Add(loc);
}
else
{
UrlList.Add(loc);
}
}
internal void SslBlackList(FileInfo mfile)
{
string path = Common.GetPath();
//FileUrl = Common.GetURL();// Creates a dictionary of Filename and URL from which it got downloaded
using (var rd = new StreamReader(path + mfile.ToString()))
{
List <KeyValuePair <string, List <string> > > InfoList = new List <KeyValuePair <string, List <string> > >();
string fieldC = "category";
string fieldD = "description";
string fieldU = "url";
string fieldT = "date";
string IP = "";
string IpDescription = "";
string category = "";
string IpDate = "";
string URL = "";
string line1 = rd.ReadLine();
string line2 = rd.ReadLine();
string line3 = rd.ReadLine();
while (!rd.EndOfStream)
{
string line = rd.ReadLine();
string[] info = line.Split(',');
IpDate = line3.Split(':')[1];
if (info.Count() == 3)
{
IP = info[0];
IpDescription = info[2];
category = "C & C";
URL = "https://sslbl.abuse.ch";
DescriptionList.Add(IpDescription);
CategoryList.Add(category);
UrlList.Add(URL);
IpDateList.Add(IpDate);
InfoList.Add(new KeyValuePair <string, List <string> >(fieldC, CategoryList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldD, DescriptionList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldU, UrlList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldT, IpDateList));
if (!BlackListDB.ContainsKey(IP))
{
BlackListDB.Add(IP, InfoList);
totalip.Add(IP);
}
CategoryList = new List <string>();
DescriptionList = new List <string>();
UrlList = new List <string>();
InfoList = new List <KeyValuePair <string, List <string> > >();
IpDateList = new List <string>();
}
}
}
}
public Dictionary <string, List <KeyValuePair <string, List <string> > > > BadIpMain()
{
List <string> AllIP = new List <string>();
List <KeyValuePair <string, List <string> > > InfoList = new List <KeyValuePair <string, List <string> > >();
string fieldC = "category";
string fieldD = "description";
string fieldU = "url";
string fieldT = "date";
string IP = "";
string IpDescription = "";
string category = "";
string IpDate = "";
string URL = "";
Dictionary <string, string> CatDesc = new Dictionary <string, string>();
CatDesc = GetCatDesc();
foreach (string cat in CatDesc.Keys)
{
AllIP = Get("https://www.badips.com/get/list/" + cat + "/0");
foreach (string ip in AllIP)
{
IP = ip;
category = cat;
IpDescription = CatDesc[cat];
URL = "https://www.badips.com/get/list/" + cat + "/0";
IpDate = "No Date Provided";
DescriptionList.Add(IpDescription);
CategoryList.Add(category);
UrlList.Add(URL);
IpDateList.Add(IpDate);
InfoList.Add(new KeyValuePair <string, List <string> >(fieldC, CategoryList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldD, DescriptionList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldU, UrlList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldT, IpDateList));
if (IP != "")
{
if (!BlackListDB.ContainsKey(IP))
{
totalip.Add(IP);
BlackListDB.Add(IP, InfoList);
}
}
CategoryList = new List <string>();
DescriptionList = new List <string>();
UrlList = new List <string>();
InfoList = new List <KeyValuePair <string, List <string> > >();
IpDateList = new List <string>();
}
}
return(BlackListDB);
}
public override bool OpenUrl(UIApplication app, NSUrl url, NSDictionary options)
{
//ApplicationLogic.OpenFileUrl(url.AbsoluteString);
NSNotificationCenter.DefaultCenter.PostNotificationName("OpenUrl", url);
if (!UrlList.Contains(url.Path))
{
UrlList.Add(url.Path);
}
Debug.WriteLine("imported url = " + url.ToString());
return(true);
}
private void UrlList_DoubleClick(object sender, EventArgs e)
{
if (UrlList.SelectedItem == null)
{
return;
}
if (UrlList.IndexFromPoint(UrlList.PointToClient(Control.MousePosition)) == ListBox.NoMatches)
{
return;
}
if (UrlList.Items[UrlList.IndexFromPoint(UrlList.PointToClient(Control.MousePosition))] == null)
{
return;
}
OK_Button_Click(sender, e);
}
private void UrlList_DoubleClick(object sender, EventArgs e)
{
if (UrlList.SelectedItem == null)
{
return;
}
if (UrlList.IndexFromPoint(UrlList.PointToClient(MousePosition)) == ListBox.NoMatches)
{
return;
}
if (UrlList.Items[UrlList.IndexFromPoint(UrlList.PointToClient(MousePosition))] == null)
{
return;
}
SelectUrlOrCancelDialog();
}
public bool WriteLinks(string vsPath)
{
try
{
cFile oFile = new cFile(vsPath);
//oFile.FileName = msExportedLinkPath;
for (int iCnt = 0; iCnt < UrlList.Count(); iCnt++)
{
oFile.Write(UrlList[iCnt]);
}
return(true);
}
catch (Exception ex)
{
return(false);
throw ex;
}
}
private void UrlFinder(string Content)
{
string startString = "href=" + '\u0022';
string endString = '\u0022'.ToString();
Form1 form = new Form1();
int indexStart = 0, indexEnd = 0;
bool exit = false;
while (!exit)
{
indexStart = Content.IndexOf(startString);
Content = Content.Substring(indexStart + 7);
indexStart = 0;
indexEnd = Content.IndexOf(endString);
if (indexStart != -1 && indexEnd != -1)
{
string tempurl = null;
if ((indexEnd - 1) > indexEnd - indexStart)
{
tempurl = Content.Substring(indexStart - 1, indexEnd - indexStart);
}
if (UrlValidation(tempurl) == true && tempurl != null)
{
UrlList list = new UrlList();
urlList.Add(tempurl);
list.urllist = urlList;
Console.WriteLine(tempurl + "//////////////// \n");
}
Content = Content.Substring(indexEnd + endString.Length);
}
else
{
exit = true;
}
}
}
public void FetchAllUrls(int depth)
{
if (Source == null)
{
return;
}
using StreamReader sr = new StreamReader(Source);
MatchCollection matches = UrlPattern.Matches(sr.ReadToEnd());
foreach (Match match in matches)
{
string cleanUrl = CleanUrl(match.Value);
if (!string.IsNullOrEmpty(cleanUrl))
{
Url url = new Url(new Uri(cleanUrl), depth + 1, UrlStatus.Queue);
UrlList.Add(url);
_logger.LogInformation("Grabbed URL from page", url.uri.AbsoluteUri);
}
}
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
var urlList = new UrlList();
Configuration.GetSection("UrlList").Bind(urlList);
services.AddSingleton <UrlList>(urlList);
var appConfig = new AppConfig();
Configuration.GetSection("AppConfig").Bind(appConfig);
services.AddSingleton <AppConfig>(appConfig);
var consume = new Consume();
services.AddSingleton <Consume>(consume);
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", null);
});
}
/// <summary>
/// 通过正则获取html中的UrlId
/// </summary>
/// <param name="html">html</param>
/// <returns></returns>
public string GetUrlIDByZz(string html)
{
string result = 成功;
string zz = "<a title=\"([\\s\\S]*?)\" href=\"//h.bilibili.com/([\\s\\S]*?)\\?from=search\" target=\"_blank\" class=\"title\">";
string[] strings = reptile.Analysis(zz, "$2", html);//获取UrlId
string upZz = "<a href=\"([\\s\\S]*?)\" target=\"_blank\" class=\"up-name\">([\\s\\S]*?)</a>";
string[] ups = reptile.Analysis(upZz, "$2", html);//获取Up主
for (var i = 0; i < strings.Length; i++)
{
UrlList.Add(
new DownLoadData
{
UpName = ups[i]
.Replace("<em class=\"keyword\">", "")
.Replace("</em>", ""),
UrlId = strings[i],
State = "未下载"
});
}
return(result);
}
public void UrlButton_OnClick(object sender, RoutedEventArgs e)
{
if (TokenCache.CurrentToken.IsNullOrEmpty())
{
MessageBox.Show("请获取token后查询");
return;
}
var realurl = string.Format(RequestUrl.Contains("?") ? @"{0}&access_token={1}" : @"{0}?access_token={1}",
RequestUrl, TokenCache.CurrentToken);
if (RequestType.ToUpper() == "POST")
{
var responsejson = HttpHelper.HttpRequestPost(realurl, RequestBody);
ResponseBody = responsejson;
}
if (RequestType.ToUpper() == "GET")
{
var responsejson = HttpHelper.HttpRequestGet(realurl);
ResponseBody = responsejson;
}
UrlList.Add(RequestUrl);
}
public override bool TestLinks()
{
ErrorMessages = new List <ErrorMessage>();
var returnValue = true;
var i = 1;
using (var progress = new ConsoleProgressBar(UrlList.Count()))
{
Parallel.ForEach(UrlList, (item) =>
{
var retval = TestLink(item);
if (returnValue && !retval)
{
returnValue = retval;
}
progress.Report(i, item.Url);
Interlocked.Increment(ref i);
});
}
return(returnValue);
}
internal void blocklistDe(FileInfo mfile)
{
string path = Common.GetPath();
//FileUrl = Common.GetURL();// Creates a dictionary of Filename and URL from which it got downloaded
using (var rd = new StreamReader(path + mfile.ToString()))
{
List <KeyValuePair <string, List <string> > > InfoList = new List <KeyValuePair <string, List <string> > >();
string fieldC = "category";
string fieldD = "description";
string fieldU = "url";
string fieldT = "date";
string IP = "";
string IpDescription = "";
string category = "";
string IpDate = "";
string URL = "";
while (!rd.EndOfStream)
{
string line = rd.ReadLine();
IP = line.Trim();
IpDate = "Provider updates every 48hrs";
if (mfile.ToString() == "all.txt")
{
IpDescription = "All IP addresses that have attacked one of our customers/servers in the last 48 hours. ";
category = "attacks";
}
if (mfile.ToString() == "ssh.txt")
{
IpDescription = "All IP addresses which have been reported within the last 48 hours as having run attacks on the service";
category = "ssh attacks";
}
if (mfile.ToString() == "bots.txt")
{
IpDescription = "All IP addresses which have been reported within the last 48 hours as having run attacks attacks on the RFI-Attacks, REG-Bots, IRC-Bots or BadBots (BadBots = he has posted a Spam-Comment on a open Forum or Wiki). ";
category = "bots";
}
if (mfile.ToString() == "ircbot.txt")
{
IpDescription = "Irc bots found in last 48 hours. ";
category = "ircbot";
}
if (mfile.ToString() == "bruteforcelogin.txt")
{
IpDescription = "All IPs which attacks Joomlas, Wordpress and other Web-Logins with Brute-Force Logins. ";
category = "bruteforcelogin";
}
URL = "http://www.blocklist.de/en/export.html";
DescriptionList.Add(IpDescription);
CategoryList.Add(category);
UrlList.Add(URL);
IpDateList.Add(IpDate);
InfoList.Add(new KeyValuePair <string, List <string> >(fieldC, CategoryList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldD, DescriptionList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldU, UrlList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldT, IpDateList));
if (!BlackListDB.ContainsKey(IP))
{
totalip.Add(IP);
BlackListDB.Add(IP, InfoList);
}
CategoryList = new List <string>();
DescriptionList = new List <string>();
UrlList = new List <string>();
InfoList = new List <KeyValuePair <string, List <string> > >();
IpDateList = new List <string>();
}
}
}
/// <summary>
/// Обрабатывает выбранный элемент ссылок из страницы ria.ru.
/// </summary>
/// <param name="sender">Источник события.</param>
/// <param name="e">Аргументы события.</param>
private void UrlList_SelectedIndexChanged(object sender, EventArgs e)
{
UrlList.ClearSelected();
}
private List <UrlList> _geturlalldata(string parameter, string parameter2)
{
U_Url_ListBLL bll = new U_Url_ListBLL();
if (parameter == "undefined")
{
parameter = "1";
}
if (string.IsNullOrEmpty(parameter))
{
parameter2 = null;
}
List <UrlList> list = new List <UrlList>();
var i = 0;
int.TryParse(parameter, out i);
var result = bll.JoinTypeFind().FindAll(n => n.Status == i);
if (!string.IsNullOrEmpty(parameter2))
{
result = result.FindAll(n => n.Url.Contains(parameter2) || n.Id.Contains(parameter2) || n.Name.Contains(parameter2) || n.Types.Contains(parameter2));
}
foreach (var item in result)
{
UrlList e = new UrlList();
string statusCss = string.Empty;
if (item.Status == 0)
{
statusCss = "btn btn-rounded btn-danger btn-outline";
}
else if (item.Status == 1)
{
statusCss = "btn btn-rounded btn-success";
}
else if (item.Status == 2)
{
statusCss = "btn btn-rounded btn-warning";
}
else if (item.Status == 3)
{
statusCss = "btn btn-rounded btn-danger";
}
else if (item.Status == 4)
{
statusCss = "btn btn-rounded btn-primary";
}
else
{
statusCss = "btn btn-rounded ";
}
e.Id = item.Id;
e.Name = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"名称……\" id=\"txtName_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlNameUpdateSave('{1}')\">Go</button></span></div>", item.Name, item.Id);
e.Url = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"网址……\" id=\"txtUrl_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlUpdateSave('{1}')\">Go</button></span></div>", item.Url, item.Id);
e.Source = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"网址描述……\" id=\"txtSource_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlSourceUpdateSave('{1}')\">Go</button></span></div>", item.Source, item.Id);
//e.Title = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"Title……\" id=\"txtTitle_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"UrlTitleUpdateSave('{1}')\">Go</button></span></div>", item.Title, item.Id);
e.Types = string.Format("<a onclick=\"myModalModify('{0}')\">{1}</a>", item.Id, item.Types);
e.SortDesc = string.Format("<div class=\"input-group\"><input type=\"text\" value=\"{0}\" class=\"form-control\" placeholder=\"降序排列……\" id=\"txtSort_{1}\"><span class=\"input-group-btn\"><button class=\"btn btn-default\" type=\"button\" onclick=\"btnSortSave('{1}')\">Go</button></span></div>", item.SortDesc.ToString(), item.Id);
e.Status = string.Format("<button class=\"{2}\" id='btnStatus{0}' onclick=\"myStatusEdit('{0}');\">{1}</button>", item.Id, item.Status.ToEnumDiscription <EnumUrlStatus>(), statusCss);
e.Operation = string.Format("<a class=\"btn btn-info btn-circle\" href=\"{0}\" target=\"_blank\" title=\"{1}\" id=\"txtfwxtest_{2}\" onclick=\"BtnClearClass('{2}')\"> <i class=\"glyphicon glyphicon-ok\"></i></a>", item.Url, item.Title, item.Id);
list.Add(e);
}
return(list);
}
public IEnumerable <UrlCheckTask> GetTasksFromMemory()
{
return(UrlList.Distinct().Select(x => new UrlCheckTask {
Url = x
}).ToList());
}
public RestService(UrlList baseUrls)
{
this._baseUrls = baseUrls;
}
internal static DictionaryClient GetPooledClient(UrlList urls)
{
//list of all to detected connected, etc
UrlClientDictionaryState states = new UrlClientDictionaryState();
foreach (Uri url in urls)
{
states.Add(url, State.Unknown);
}
DictionaryClient result = null;
do
{
lock (lockObject)
{
//get all available and connected
List <DictionaryClient> available = new List <DictionaryClient>();
DictionaryClient dictClient;
foreach (Uri url in urls)
{
if (availableClients.TryGetValue(url, out dictClient))
{
if ((!errorClients.ContainsKey(url) ||
errorClients[url].AddSeconds(5) < DateTime.Now) &&
states[url] != State.Error)
{
available.Add(dictClient);
}
}
}
if (available.Count > 0)
{
if (available.Count > 1)
{
available.Sort(new DictionaryClientComparerByConnectTicks());
}
result = available[0];
}
if (result == null)
{
foreach (Uri url in urls)
{
if (states[url] != State.Error &&
!lockedClients.ContainsKey(url) &&
!availableClients.ContainsKey(url))
{
result = new DictionaryClient();
result.Url = url;
result.AutoConnect = false;
result.ClientName = "http://translate-net.appspot.com/";
break;
}
}
}
if (result != null)
{
lockedClients.Add(result.Url, result);
availableClients.Remove(result.Url);
states[result.Url] = State.Busy;
}
} //end lock
if (result == null)
{
int cnt = 0;
foreach (State state in states.Values)
{
if (state == State.Error)
{
cnt++;
}
else
{
break;
}
}
if (cnt < states.Count)
{
System.Threading.Thread.Sleep(200);
}
else
{
string errorMessage = "Can't connect to any dictd server : ";
foreach (Uri url in urls)
{
errorMessage += url + ", ";
}
errorMessage = errorMessage.Substring(0, errorMessage.Length - 2);
throw new TranslationException(errorMessage);
}
}
else
{ //try to connect
try
{
if (!result.Connected)
{
result.Connected = true;
}
}
catch
{
states[result.Url] = State.Error;
lock (lockObject)
{
errorClients[result.Url] = DateTime.Now;
availableClients.Add(result.Url, result);
lockedClients.Remove(result.Url);
}
result = null;
}
}
}while(result == null);
return(result);
}
internal void IPsetDB(FileInfo mfile)
{
string path = Common.GetPath();
FileUrl = Common.GetURL();// Creates a dictionary of Filename and URL from which it got downloaded
using (var rd = new StreamReader(path + mfile.ToString()))
{
List <KeyValuePair <string, List <string> > > InfoList = new List <KeyValuePair <string, List <string> > >();
string fieldC = "category";
string fieldD = "description";
string fieldU = "url";
string fieldT = "date";
string IP = "";
string IpDescription = "";
string category = "";
string IpDate = "";
string url = "";
List <string> tempDesc = new List <string>();
int i = 1;
string t = "";
string l = "";
int k = 0;
while (!rd.EndOfStream)
{
string x = rd.ReadLine();
if (i != k && k == 0)
{
if (i >= 6)
{
if (!x.Contains("Maintainer"))
{
tempDesc.Add(x);
}
else
{
k = i;
k++;
l = string.Join(" ", tempDesc);
l = l.Replace("#", " ");
IpDescription = l;
}
}
}
i++;
if (i == k)
{
k = 1;
}
category = x.Contains("Category") ? x.Split(':')[1].ToString().Trim() : category;
IpDate = x.Contains("Source File Date") ? x.Split(':')[1].ToString().Trim() : IpDate;
url = x.Contains("List source URL") ? x.Split(new char[] { ':' }, 2, StringSplitOptions.RemoveEmptyEntries)[1].ToString().Trim() : url;
var match = Regex.Match(x, @"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b");
if (match.ToString() != "")
{
IP = match.ToString().Trim();
DescriptionList.Add(IpDescription);
CategoryList.Add(category);
UrlList.Add(url);
IpDateList.Add(IpDate);
InfoList.Add(new KeyValuePair <string, List <string> >(fieldC, CategoryList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldD, DescriptionList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldU, UrlList));
InfoList.Add(new KeyValuePair <string, List <string> >(fieldT, IpDateList));
if (!BlackListDB.ContainsKey(IP))
{
totalip.Add(IP);
BlackListDB.Add(IP, InfoList);
}
else
{
InfoList = BlackListDB[IP];
foreach (KeyValuePair <string, List <string> > item in InfoList)
{
if (item.Key == "category")
{
if (!item.Value.Contains(category))
{
item.Value.Add(category);
}
}
if (item.Key == "url")
{
if (!item.Value.Contains(FileUrl[mfile.ToString()]))
{
item.Value.Add(FileUrl[mfile.ToString()]);
}
}
}
}
CategoryList = new List <string>();
DescriptionList = new List <string>();
UrlList = new List <string>();
InfoList = new List <KeyValuePair <string, List <string> > >();
IpDateList = new List <string>();
}
}
}
}
public ValuesController(UrlList urlList, AppConfig appConfig, Consume consume)
{
this.urlList = urlList;
this.appConfig = appConfig;
this.consume = consume;
}
public override void Run(Pattern[] patterns) //执行爬虫
{
Spider spider = new Spider(UrlList.ToArray(), DataFile);
spider.GetText(patterns, 5000);
}
