protected override void MyInit(params string[] arguments)
{
AddStartUrl(string.Format("https://ccliquote.lflogistics.net/ajaxHandler.asp?act=pol"));
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("table", "]}", 7, 1));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=192.168.10.171 ;User ID=root;Password=sr@12345;Port=3306"));
AddPageProcessor(new FormProcessor());
}
protected override void MyInit(params string[] arguments)
{
var request = new Request("http://egi-freight.100jit.com/marketing-portal-rest/rest/efcl/queryFclList", new Dictionary <string, dynamic> {
{ "SpiderCompanyCode", "3" }, { "Currency", "CNY" }
});
request.Method = HttpMethod.Post;
var queryFreight = new QueryFreight
{
queryConditions = JsonConvert.SerializeObject(new QueryCondition
{
dischargeport = "AALBORG",
loadport = "NINGBO"
}),
pageSize = 30,
pageNum = 1,
sortColumn = "internetsellprice2",
sortBy = "asc",
biKey = "4842191518059353663",
isPage = false
};
request.PostBody = JsonConvert.SerializeObject(queryFreight);
AddStartRequest(request);
//Downloader = new CustomPostJsonHttpClientDownloader();
Downloader.AddAfterDownloadCompleteHandler(new CustomCutoutHandler("list", "firstPage", queryFreight, 6, 11));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=192.168.10.171 ;User ID=root;Password=sr@12345;Port=3306"));
AddEntityType <OceanFreightInfo>();
}
protected override void MyInit(params string[] arguments)
{
//船公司编号集合
// var shippingCompanyCodeArr = new string[] { "ANL","APL","ASL","BENLINE","BOHAI","CCL","CKS","CMA","CNC","CO-HEUNG","COSCO","CSC","CSCL","CUL","DJS","DYS","EAS","EMC","ESL",
// "HAM-SUD","HANSUNG","HASCO","HDS","HEUNG-A","HMM","HPL","IAL","JIAODONG","JJ","K-LINE","KMTC","MCC","MEL","MOL","MSC","MSK","MSL","NAMSUNG","NDS","NOS","NYK","NZL",
// "ONTO","OOCL","PAN CON","PIL","QMNS","RCL","RPS","RZF","SAF","SAMUDERA","SCI","SIMATECH","SINOKOR","SINOTRANS","SITC","SML","STAROCEAN","STX","SWIRE",
// "TCLC","TSLINE","UCL","WANHAI","WEIDONG","WFL","YANGZIJIANG","YML","ZIM" };
var shippingCompanyCodeArr = new string[] { "MSK", "COSCO" };
foreach (var item in shippingCompanyCodeArr)
{
//var startUrl = $"https://www.ezcarry.com/OceanFreight/OceanFreightSedAjax?load=&discharge=&country=&ShippingLineName=&shippingLineId=&companyCode=&seqStr=&oecanCarrierName=MSK%2C&loadingTerminal=&dischargeTerminal=&transhipPort=&etd=&eta=&isVip=&isPrise=&page=1&sortName=&sortType=&queryType=";
var startUrl = $"https://www.ezcarry.com/OceanFreight/OceanFreightSedAjax?country=&ShippingLineName=&shippingLineId=&companyCode=&oecanCarrierName={item}&etd=&eta=&page=1";
AddStartUrl(startUrl, new Dictionary <string, dynamic> {
{ "SpiderCompanyCode", "1" }
});
}
// var startUrl = $"https://www.ezcarry.com/OceanFreight/OceanFreightSedAjax?load=&discharge=&country=&ShippingLineName=&shippingLineId=&companyCode=&seqStr=&oecanCarrierName=MSK%2C&loadingTerminal=&dischargeTerminal=&transhipPort=&etd=&eta=&isVip=&isPrise=&page=1&sortName=&sortType=&queryType=";
// AddStartUrl(startUrl);
//var endStr= ","OceanFreightHistory":null,"PageListEntity":{"PageIndex":1,"PageSize":15,"RecordCount":2324,"PageCount":0},"OceanFreightInfo":null,"ServiceTypeAdjuestList":null,"SurchargeCurrencyList":null,"SurchargeList":null,"SurchargeType3List":null,"OceanScheduleList":null,"DiscussInteraction":null,"MaxPage":155},"Code":""}";
Downloader.AddAfterDownloadCompleteHandler(new CustomCutoutHandler("OceanFreightList", "OceanFreightHistory", 18, 21));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=192.168.10.171 ;User ID=root;Password=sr@12345;Port=3306"));
AddEntityType <OceanFreightInfo>();
}
protected override void MyInit(params string[] arguments)
{
var token = "b4b147bc522828731f1a016bfa72c073-1504522840550-0-364629a1e95e9f9450ab945ae3adeeb0-35ebd44cfa19c0450152121f332cc4fc-0-44cbce77ea242ed3b5ba50d4a78f31a1-0";
AddStartUrl(string.Format("http://www.51eumex.com/port/search_start_port.json?token={0}", token));
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("data", "message", 6, 9));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=192.168.10.171 ;User ID=root;Password=sr@12345;Port=3306"));
AddPageProcessor(new FormProcessor());
}
protected override void OnInit(params string[] arguments)
{
AddRequest($"http://api.search.sina.com.cn/?c=news&t=&q=赵丽颖&pf=2136012948&ps=2130770082&page=0&stime={DateTime.Now.AddYears(-7).AddDays(-1).ToString("yyyy-MM-dd")}&etime={DateTime.Now.AddDays(1).ToString("yyyy-MM-dd")}&sort=rel&highlight=1&num=10&ie=utf-8&callback=jQuery1720001955628746606708_1508996230766&_=1508996681484", new Dictionary <string, dynamic> {
{ "keyword", "赵丽颖" }
});
AddPipeline(new ConsoleEntityPipeline());
Downloader.AddAfterDownloadCompleteHandler(new ReplaceHandler());
AddEntityType <SinaNews>();
}
protected override void MyInit(params string[] arguments)
{
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("json(", ");", 5, 0));
AddPipeline(new ConsoleEntityPipeline());
AddStartUrlBuilder(new DbStartUrlsBuilder(Database.MySql, Env.DataConnectionString,
$"SELECT * FROM test.jd_sku", new[] { "sku" },
"http://chat1.jd.com/api/checkChat?my=list&pidList={0}&callback=json"));
AddEntityType <Item>();
}
protected override void MyInit(params string[] arguments)
{
AddStartUrl("http://chat1.jd.com/api/checkChat?my=list&pidList=3355984&callback=json");
AddStartUrl("http://chat1.jd.com/api/checkChat?my=list&pidList=3682523&callback=json");
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("json(", ");", 5, 2));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost ;User ID=root;Password=;Port=3306"));
AddEntityType <ProductUpdater>();
}
protected override void MyInit(params string[] arguments)
{
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("g_page_config = {", "g_srp_loadCss();", 16, 22));
Downloader.AddAfterDownloadCompleteHandler(new TargetUrlsHandler(new AutoIncrementTargetUrlsExtractor("&s=0", 44)));
Downloader.AddAfterDownloadCompleteHandler(new MyAfterDownloadHandler());
SkipTargetUrlsWhenResultIsEmpty = true;
AddStartRequest(new Request("https://s.taobao.com/search?q=妙可蓝多&imgfile=&js=1&stats_click=search_radio_all%3A1&ie=utf8&sort=sale-desc&s=0&tab=all", new Dictionary <string, dynamic> {
{ "bidwordstr", "妙可蓝多" }
}));
AddEntityType <TaobaoItem>(new MyDataHanlder());
AddPipeline(new ConsoleEntityPipeline());
}
protected override void MyInit(params string[] arguments)
{
Identity = Identity ?? Guid.NewGuid().ToString();
Downloader.AddAfterDownloadCompleteHandler(new SubContentHandler("json(", ");", 5, 0));
AddStartUrlBuilder(new DbStartUrlBuilder(Database.MySql,
Env.DataConnectionStringSettings.ConnectionString,
$"SELECT * FROM jd_sku_{DateTimeUtils.MondayOfCurrentWeek.ToString("yyyy_MM_dd")} WHERE ShopName is null or ShopId is null or ShopId = 0 order by sku", new[] { "sku" },
"http://chat1.jd.com/api/checkChat?my=list&pidList={0}&callback=json"));
AddPipeline(new MySqlEntityPipeline(Env.DataConnectionStringSettings.ConnectionString));
AddEntityType <ProductUpdater>();
}
protected override void MyInit(params string[] arguments)
{
Identity = Identity ?? Guid.NewGuid().ToString();
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("json(", ");", 5, 0));
AddStartUrlBuilder(new DbStartUrlsBuilder(Database.MySql,
"Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;",
$"SELECT * FROM test.jd_sku_{DateTimeUtil.Monday.ToString("yyyy_MM_dd")} WHERE ShopName is null or ShopId is null or ShopId = 0 order by sku", new[] { "sku" },
"http://chat1.jd.com/api/checkChat?my=list&pidList={0}&callback=json"));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Password=;Port=3306;SslMode=None;"));
AddEntityType <ProductUpdater>();
}
protected override void OnInit(params string[] arguments)
{
string line = "1";
AddRequest($"http://wapapp.dy4g.cn/bus/auto/test.php?t=busdb&busline={line}", new Dictionary <string, dynamic> {
{ "Keyword", $"{line}路线" }
});
Downloader.AddAfterDownloadCompleteHandler(new BusResultHandler());
AddEntityType <BusResultEntity>(new BusDataHanlder());
AddPipeline(new ConsoleEntityPipeline());
}
protected override void MyInit(params string[] arguments)
{
AddStartUrl("http://chat1.jd.com/api/checkChat?my=list&pidList=3355984&callback=json");
AddStartUrl("http://chat1.jd.com/api/checkChat?my=list&pidList=3682523&callback=json");
Downloader.AddAfterDownloadCompleteHandler(new SubContentHandler
{
StartPart = "json(",
EndPart = ");",
StartOffset = 5,
EndOffset = 2
});
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost ;User ID=root;Password=;Port=3306"));
AddEntityType(typeof(ProductUpdater));
}
protected override void OnInit(params string[] arguments)
{
AddHeaders("s.taobao.com", new Dictionary <string, object> {
{ "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" },
{ "Referer", "https://www.taobao.com/" },
{ "Cache-Control", "max-age=0" },
{ "Upgrade-Insecure-Requests", "1" }
});
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("g_page_config = {", "g_srp_loadCss();", 16, 22));
AddBeforeProcessor(new MyBeforeProcessorHandler());
SkipTargetRequestsWhenResultIsEmpty = true;
AddRequest(new Request("https://s.taobao.com/search?q=妙可蓝多&imgfile=&js=1&stats_click=search_radio_all%3A1&ie=utf8&sort=sale-desc&s=0&tab=all", new Dictionary <string, dynamic> {
{ "bidwordstr", "妙可蓝多" }
}));
AddEntityType <TaobaoItem>(new MyDataHanlder());
AddPipeline(new ConsoleEntityPipeline());
}
protected override void MyInit(params string[] arguments)
{
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("g_page_config = {", "g_srp_loadCss();", 16, 22));
Downloader.AddAfterDownloadCompleteHandler(new TargetUrlsHandler(new AutoIncrementTargetUrlsExtractor("&s=0", 44)));
Downloader.AddAfterDownloadCompleteHandler(new MyAfterDownloadHandler());
SkipTargetUrlsWhenResultIsEmpty = true;
if (!arguments.Contains("noprepare"))
{
//AddStartUrlBuilder(
// new DbStartUrlBuilder(Database.MySql, Env.DataConnectionStringSettings.ConnectionString,
// "SELECT * FROM taobao.result_keywords limit 10000", new[] { "bidwordstr", "tab" },
// "https://s.taobao.com/search?q={0}&imgfile=&js=1&stats_click=search_radio_all%3A1&ie=utf8&sort=sale-desc&s=0&tab={1}"));
AddStartRequest(new Request("https://s.taobao.com/search?q=妙可蓝多&imgfile=&js=1&stats_click=search_radio_all%3A1&ie=utf8&sort=sale-desc&s=0&tab=all", new Dictionary <string, dynamic> {
{ "bidwordstr", "妙可蓝多" }
}));
}
AddEntityType <TaobaoItem>(new MyDataHanlder());
}
protected override void MyInit(params string[] arguments)
{
//var request = new Request("http://www.51eumex.com/freight_search.json");
//request.Method = HttpMethod.Post;
//request.PostBody = "nowPage=1&pageSize=30&startPortCode=SHANGHAI&destPortCode=CHICAGOIL&sailTime=&orderName=&orderRule=&boatCompany=&shortPorts=0&cycle=0&sales_num=0&available_ratio=0&company_id=3006&token=26fb6d951556a2e9cc2db63750753fb2-1517918395224-3464-0ec4861c3b9f44ff1e0ee40cf480aceb-35ebd44cfa19c0450152121f332cc4fc-3006-5b699d3460713d71c59a411610097b44-0";
//AddStartRequest(request);
var startPortCode = "SHANGHAI";
var destPortCode = "CHICAGOIL";
AddStartUrl($"http://www.51eumex.com/freight_search.json?nowPage=1&pageSize=30&startPortCode={startPortCode}&destPortCode={destPortCode}&sailTime=&orderName=&orderRule=&boatCompany=&shortPorts=0&cycle=0&sales_num=0&available_ratio=0&company_id=3006&token=26fb6d951556a2e9cc2db63750753fb2-1517918395224-3464-0ec4861c3b9f44ff1e0ee40cf480aceb-35ebd44cfa19c0450152121f332cc4fc-3006-5b699d3460713d71c59a411610097b44-0"
, new Dictionary <string, dynamic> {
{ "SpiderCompanyCode", "4" }
});
Downloader.AddAfterDownloadCompleteHandler(new CutoutHandler("data", "count", 6, 7));
AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=192.168.10.171 ;User ID=root;Password=sr@12345;Port=3306"));
AddEntityType <OceanFreightInfo>();
}
