public ResponseBoolBase HttpRequestCfgAdd(HttpRequestCfg data)
{
ResponseBoolBase obj = new ResponseBoolBase();
try
{
datapro.EnterWriteLock();
HttpTaskDBContext db = new HttpTaskDBContext();
db.HttpRequestCfg.Add(data);
int i = db.SaveChanges();
if (i == 0)
{
obj.IsSuccess = false;
}
else
{
obj.IsSuccess = true;
}
}
catch (Exception ex)
{
obj.IsSuccess = false;
obj = ResponseBase.GetResponseByException <ResponseBoolBase>(ex);
logger.ErrorFormat("HttpRequestCfgAdd", ex);
}
finally
{
datapro.ExitWriteLock();
}
return(obj);
}
public ResponseBase HttpRequestCfgSave(HttpRequestCfgDataUI data)
{
ResponseBase obj = new ResponseBase();
try
{
datapro.EnterWriteLock();
using (HttpTaskDBContext db = new HttpTaskDBContext())
{
HttpRequestCfg d = new HttpRequestCfg()
{
UpdatedTime = data.UpdatedTime,
TaskStatus = data.TaskStatus,
SeqNo = data.SeqNo,
SearchKey = data.SearchKey,
Cookie = data.Cookie,
CreatedTime = data.CreatedTime,
CurrentDate = data.CurrentDate,
DateMax = data.DateMax,
CurrentPage = data.CurrentPage,
DateMin = data.DateMin,
DeletedTime = data.DeletedTime,
HasChildTask = data.HasChildTask,
Host = data.Host,
//HttpRequestChildCfgs =data.HttpRequestChildCfgs,
Id = data.Id,
info = data.info,
IsDelete = data.IsDelete,
Key = data.Key,
Level = data.Level,
PageMax = data.PageMax,
PageMin = data.PageMin,
ProcessName = data.ProcessName,
Quartz = data.Quartz,
SchedulerType = data.SchedulerType,
SearchKeys = data.SearchKeys,
TaskTimeOut = data.TaskTimeOut,
WebName = data.WebName
};
db.HttpRequestCfg.Add(d);
db.SaveChanges();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
obj = ResponseBase.GetResponseByException <ResponseBase>(ex);
}
finally
{
datapro.ExitWriteLock();
}
return(obj);
}
private static HttpResultCfg NewMethod1(HttpRequestCfg item, HttpResult result)
{
HttpResultCfg re = new HttpResultCfg();
re.Binary = result.ResultStream.ToArray();
re.CreateTime = DateTime.Now;
re.FullText = result.Html;
re.HttpStatusCode = (int)result.StatusCode;
re.Key = item.Key;
re.TaskId = item.Id;
return(re);
}
protected override void Seed(HttpTaskDbContext.HttpTaskDBContext context)
{
var co = new List <Cookie>();
co.Add(new Cookie()
{
Name = "PHPSESSID", Value = "qs4f6ttm7g5vl2rqiqh7aia7m1", Domain = "www.chinesesfpm.com"
});
Guid guid = Guid.NewGuid();
HttpRequestCfg obj = new HttpRequestCfg()
{
CreatedTime = DateTime.Now, HasChildTask = false, Host = "www.chinesesfpm.com", PageMax = 20, WebName = "chinesesfpm", TaskStatus = HttpTaskModel.TaskStatus.Created, PageMin = 1, Key = guid, SchedulerType = HttpTaskManage.SchedulerType.Page
};
//obj.PageMax = 20;
//obj.PageMin = 1;
obj.HttpRequestChildCfgs.Add(new HttpRequestChildCfg()
{
Key = guid,
Accept = "application/json, text/javascript, */*; q=0.01",
Accept_Encoding = "gzip, deflate",
Accept_Language = "zh-CN,zh;q=0.9",
ContentType = "application/x-www-form-urlencoded; charset=UTF-8",
Host = "www.chinesesfpm.com",
Method = HttpMethod.POST,
//UserAgent = "",
UACPU = "",
Encoding = "utf-8",
Url = "http://www.chinesesfpm.com/index/index/getAjaxSearch.html",
ResponseType = FilenameExtension.Text,
PostData = "court_sheng=&court_city=&court_arer=&province=&city=&district=&min_price=&max_price=&do_paimai=1&do_s_type=&biaodi_type=&do_isajax=1&page={page}&do_label=0",
PostDataType = PostDataType.String,
x_requested_with = "XMLHttpRequest",
//PHPSESSID=qs4f6ttm7g5vl2rqiqh7aia7m1
Cookie = co.ToJson()
});
var re = context.HttpRequestCfg.Add(obj);
context.SaveChanges();
// This method will be called after migrating to the latest version.
// You can use the DbSet<T>.AddOrUpdate() helper extension method
// to avoid creating duplicate seed data.
}
public HttpResultCfg HttpSpiderTaskReq(HttpRequestCfg item, System.Net.CookieContainer Cookies = null)
{
_LoginServer.Url = item.Url;
_LoginServer.Method = item.Method.ToString();
_LoginServer.Accept = item.Accept;
_LoginServer.Origin = item.Origin;
_LoginServer.Referer = item.Referer;
_LoginServer.ContentType = item.ContentType;
_LoginServer.PostData = item.PostData;
_LoginServer.AcceptEncoding = item.Accept_Encoding;
_LoginServer.AcceptLanguage = item.Accept_Language;
_LoginServer.CerPath = item.CrePath;
_LoginServer.Allowautoredirect = item.Allowautoredirect;
_LoginServer.Host = item.Host;
_LoginServer.PostDataType = (HttpHelper.PostDataType)Enum.ToObject(typeof(HttpHelper.PostDataType), (int)(item.PostDataType));
_LoginServer.KeepAlive = item.KeepAlive;
_LoginServer.x_requested_with = item.x_requested_with;
_LoginServer.UserAgent = item.UserAgent;
_LoginServer.UACPU = item.UACPU;
_LoginServer.PostdataByte = item.PostdataByte;
_LoginServer.Cookies = new System.Net.CookieContainer();
foreach (var coo in item.CookieCollection)
{
_LoginServer.Cookies.Add(new System.Net.Cookie(coo.Name, coo.Value)
{
Domain = coo.Domain
});
}
foreach (var head in item.Headers)
{
_LoginServer.HeaderCollection.Add(head.Key, head.Value);
}
if (Cookies != null)
{
_LoginServer.Cookies = Cookies;
}
var result = _LoginServer.GetHttpResult();
HttpResultCfg re = NewMethod1(item, result);
return(re);
}
static void Main(string[] args)
{
ChannelFactory <IRequestContract> channelFactory = new ChannelFactory <IRequestContract>("Request");
var client = channelFactory.CreateChannel();
var co = new List <Cookie>();
co.Add(new Cookie()
{
Name = "PHPSESSID", Value = "qs4f6ttm7g5vl2rqiqh7aia7m1", Domain = "www.chinesesfpm.com"
});
Guid guid = Guid.NewGuid();
HttpRequestCfg obj = new HttpRequestCfg()
{
CreatedTime = DateTime.Now, HasChildTask = false, Host = "www.chinesesfpm.com", PageMax = 20, WebName = "chinesesfpm", TaskStatus = HttpTaskModel.TaskStatus.Created, PageMin = 1, Key = guid, SchedulerType = HttpTaskManage.SchedulerType.Page
};
//obj.PageMax = 20;
//obj.PageMin = 1;
obj.HttpRequestChildCfgs.Add(new HttpRequestChildCfg()
{
Key = guid,
Accept = "application/json, text/javascript, */*; q=0.01",
Accept_Encoding = "gzip, deflate",
Accept_Language = "zh-CN,zh;q=0.9",
ContentType = "application/x-www-form-urlencoded; charset=UTF-8",
Host = "www.chinesesfpm.com",
Method = HttpMethod.POST,
//UserAgent = "",
UACPU = "",
Encoding = "utf-8",
Url = "http://www.chinesesfpm.com/index/index/getAjaxSearch.html",
ResponseType = FilenameExtension.Text,
PostData = "court_sheng=&court_city=&court_arer=&province=&city=&district=&min_price=&max_price=&do_paimai=1&do_s_type=&biaodi_type=&do_isajax=1&page={page}&do_label=0",
PostDataType = PostDataType.String,
x_requested_with = "XMLHttpRequest",
//PHPSESSID=qs4f6ttm7g5vl2rqiqh7aia7m1
Cookie = co.ToJson()
});
var re = client.HttpRequestCfgAdd(obj);
}
protected override void Seed(HttpTaskDataBase.HttpTaskDbContext context)
{
// This method will be called after migrating to the latest version.
// You can use the DbSet<T>.AddOrUpdate() helper extension method
// to avoid creating duplicate seed data. E.g.
//
// context.People.AddOrUpdate(
// p => p.FullName,
// new Person { FullName = "Andrew Peters" },
// new Person { FullName = "Brice Lambson" },
// new Person { FullName = "Rowan Miller" }
// );
//
var e = new HttpRequestCfg()
{
Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", CreateTime = DateTime.Now, Url = "http://m.ctrip.com/html5/hotel/sitemap-qingdao7", Host = "m.ctrip.com", Cookie = "", Encoding = "utf-8", Method = HttpMethod.GET, Referer = "", Origin = "", UACPU = "", UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0", x_requested_with = "", WebName = "", ResponseType = FilenameExtension.Text, Upgrade_Insecure_Requests = "1", Accept_Encoding = "gzip, deflate, sdch", Accept_Language = "zh-CN,zh;q=0.8", Cache_Control = "max-age=0", Connection = "keep-live"
};
context.HttpRequestCfg.AddOrUpdate(e);
context.SaveChanges();
}
private bool saveresult(HttpResult _Result, HttpTaskModel.HttpRequestChildCfg item, HttpRequestCfg data, out string info)
{
info = "";
try
{
HttpResultCfgDataUI re = new HttpResultCfgDataUI()
{
Binary = _Result.ResultByte, CreatedTime = DateTime.Now, FullText = _Result.Html, HttpStatusCode = _Result.StatusCode, Key = item.Key, RequstChildId = item.Id, ResponseType = item.ResponseType, SeqNo = item.SeqNo, TaskStatus = HttpTaskModel.TaskStatus.Complete, Page = data.CurrentPage, Date = data.CurrentDate, SearchKey = data.SearchKey, WebName = data.WebName, info = data.info
};
var r = _RequestContract.HttpResultCfgAdd(re);
return(r.IsSuccess);
}
catch (Exception ex)
{
info = ex.Message;
logger.ErrorFormat("提交请求结果失败", ex);
return(false);
}
}