public override void CrawlData()
{
string baid = "-1";
for (int i = 0; i < MaxPageNumber; i++)
{
try
{
LinkParser linkParser;
do
{
ReqParametres reqParametres = new ReqParametres("https://ru.auctions.godaddy.com/trpSearchResults.aspx", HttpMethod.Post, $"t=5&action=search&hidAdvSearch=ddlAdvKeyword:3|txtKeyword:|ddlCharacters:0|txtCharacters:|txtMinTraffic:|txtMaxTraffic:|txtMinDomainAge:|txtMaxDomainAge:|txtMinPrice:|txtMaxPrice:|ddlCategories:0|chkAddBuyNow:false|chkAddFeatured:false|chkAddDash:true|chkAddDigit:true|chkAddWeb:false|chkAddAppr:false|chkAddInv:false|chkAddReseller:false|ddlPattern1:|ddlPattern2:|ddlPattern3:|ddlPattern4:|chkSaleOffer:false|chkSalePublic:false|chkSaleExpired:false|chkSaleCloseouts:false|chkSaleUsed:false|chkSaleBuyNow:false|chkSaleDC:false|chkAddOnSale:false|ddlAdvBids:0|txtBids:|txtAuctionID:|ddlDateOffset:|ddlSort:auctionendtimeD&rtr=7&baid={baid}&searchDir=1");
reqParametres.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.38 Safari/537.36");
reqParametres.RowRequest.ContentType = "application/x-www-form-urlencoded";
linkParser = new LinkParser(reqParametres.Request);
} while (!linkParser.Data.Contains("onclick=\"_s_baid"));
Sites.AddRange(linkParser.Data.ParsRegex("alt=\"\" />(.*?)<", 1).Select(x => new Site {
Link = x
}));
baid = linkParser.Data.ParsFromTo("onclick=\"_s_baid1=", ";");
if (string.IsNullOrEmpty(baid))
{
break;
}
}
catch (Exception e)
{
Thread.Sleep(5000);
Console.WriteLine(e.Message + "\n");
}
}
}
public static Sites Create(ICollection <Site> sites)
{
var result = new Sites();
result.AddRange(sites.Select(s => s.Point));
return(result);
}
/// <summary>
/// 初始化加载站点
/// </summary>
public void Initialize()
{
//设置加载站点配置解析
IniDataParser iniDataParser = new IniDataParser();
iniDataParser.Configuration.AllowCreateSectionsOnFly
= iniDataParser.Configuration.AllowDuplicateKeys
= iniDataParser.Configuration.AllowDuplicateSections
= iniDataParser.Configuration.SkipInvalidLines
= true;
iniParser = new FileIniDataParser(iniDataParser);
string[] dlls = { };
try { dlls = Directory.GetFiles(SitePacksPath, "SitePack*.dll", SearchOption.AllDirectories); } catch { }
#region 保证有基本站点包路径
if (dlls.Length < 1)
{
string basisdll = SitePacksPath + "SitePack.dll";
if (File.Exists(basisdll))
{
List <string> dlll = new List <string> {
basisdll
};
dlls = dlll.ToArray();
}
}
#endregion
foreach (string dll in dlls)
{
try
{
byte[] assemblyBuffer = File.ReadAllBytes(dll);
Type type = Assembly.Load(assemblyBuffer).GetType("SitePack.SiteProvider", true, false);
MethodInfo methodInfo = type.GetMethod("SiteList");
Sites.AddRange(methodInfo.Invoke(Activator.CreateInstance(type), new object[] { MainProxy }) as List <IMageSite>);
}
catch (Exception ex)
{
EchoErrLog("站点载入过程", ex);
}
}
}