protected Page SaveFile(Request request, HttpResponseMessage response, ISpider spider)
{
var intervalPath = request.Url.LocalPath.Replace("//", "/").Replace("/", Infrastructure.Environment.PathSeperator);
string filePath = $"{DownloadFolder}{Infrastructure.Environment.PathSeperator}{spider.Identity}{intervalPath}";
if (!File.Exists(filePath))
{
try
{
string folder = Path.GetDirectoryName(filePath);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
File.WriteAllBytes(filePath, response.Content.ReadAsByteArrayAsync().Result);
}
catch (Exception e)
{
spider.Log("保存文件失败。", LogLevel.Warn, e);
}
}
spider.Log($"下载文件: {request.Url} 成功.", LogLevel.Info);
return(new Page(request, ContentType.File, null)
{
IsSkip = true
});
}
public override void InitPipeline(ISpider spider)
{
if (!IsEnabled)
{
return;
}
if (string.IsNullOrEmpty(ConnectString))
{
if (UpdateConnectString == null)
{
throw new SpiderException("Can't find ConnectString or IUpdateConnectString.");
}
else
{
for (int i = 0; i < 5; ++i)
{
try
{
ConnectString = UpdateConnectString.GetNew();
break;
}
catch (Exception e)
{
spider.Log("Update ConnectString failed.", LogLevel.Error, e);
Thread.Sleep(1000);
}
}
if (string.IsNullOrEmpty(ConnectString))
{
throw new SpiderException("Can't updadate ConnectString via IUpdateConnectString.");
}
}
}
base.InitPipeline(spider);
if (Mode == PipelineMode.Update)
{
return;
}
NetworkCenter.Current.Execute("db-init", () =>
{
using (DbConnection conn = CreateConnection())
{
var command = conn.CreateCommand();
command.CommandText = GetCreateSchemaSql();
command.CommandType = CommandType.Text;
command.ExecuteNonQuery();
command.CommandText = GetCreateTableSql();
command.CommandType = CommandType.Text;
command.ExecuteNonQuery();
conn.Close();
}
});
}
public virtual void Inject(ISpider spider, bool stopSpider = true)
{
if (stopSpider)
{
spider.Pause(() =>
{
spider.Site.Cookies = GetCookies(spider.Site);
spider.Log("注入 Cookies 成功。", LogLevel.Info);
spider.Contiune();
});
}
else
{
spider.Site.Cookies = GetCookies(spider.Site);
spider.Log("注入 Cookies 成功。", LogLevel.Info);
}
}
private void RegisterControl(ISpider spider)
{
if (RedisConnection != null)
{
try
{
RedisConnection.Subscriber.Subscribe($"{spider.Identity}", (c, m) =>
{
switch (m)
{
case "PAUSE":
{
spider.Pause();
break;
}
case "CONTINUE":
{
spider.Contiune();
break;
}
case "RUNASYNC":
{
spider.RunAsync();
break;
}
case "EXIT":
{
spider.Exit();
break;
}
}
});
}
catch (Exception e)
{
spider.Log("Register contol failed.", LogLevel.Error, e);
}
}
}
protected override Page DowloadContent(Request request, ISpider spider)
{
Site site = spider.Site;
HttpResponseMessage response = null;
var proxy = site.GetHttpProxy();
request.PutExtra(Request.Proxy, proxy);
try
{
var httpMessage = GenerateHttpRequestMessage(request, site);
response = NetworkCenter.Current.Execute("http", message =>
{
HttpClient httpClient = _httpClientPool.GetHttpClient(proxy);
var requestTask = httpClient.SendAsync(message);
requestTask.Wait(site.Timeout);
if (requestTask.Status == TaskStatus.RanToCompletion)
{
return(requestTask.Result);
}
else
{
return(new HttpResponseMessage(HttpStatusCode.RequestTimeout));
}
}, httpMessage);
response.EnsureSuccessStatusCode();
if (!site.AcceptStatCode.Contains(response.StatusCode))
{
throw new DownloadException($"下载 {request.Url} 失败. Code {response.StatusCode}");
}
var httpStatusCode = response.StatusCode;
request.PutExtra(Request.StatusCode, httpStatusCode);
Page page;
if (response.Content.Headers.ContentType != null && !MediaTypes.Contains(response.Content.Headers.ContentType.MediaType))
{
if (!site.DownloadFiles)
{
spider.Log($"Miss request: {request.Url} because media type is not text.", LogLevel.Debug);
return(new Page(request, site.ContentType, null)
{
IsSkip = true
});
}
else
{
page = SaveFile(request, response, spider);
}
}
else
{
page = HandleResponse(request, response, httpStatusCode, site);
}
if (string.IsNullOrEmpty(page.Content))
{
spider.Log($"下载 {request.Url} 内容为空。", LogLevel.Warn);
}
// need update
page.TargetUrl = request.Url.ToString();
//page.SetRawText(File.ReadAllText(@"C:\Users\Lewis\Desktop\taobao.html"));
// 这里只要是遇上登录的, 则在拨号成功之后, 全部抛异常在Spider中加入Scheduler调度
// 因此如果使用多线程遇上多个Warning Custom Validate Failed不需要紧张, 可以考虑用自定义Exception分开
// 结束后要置空, 这个值存到Redis会导致无限循环跑单个任务
request.PutExtra(Request.CycleTriedTimes, null);
//#if !NET_CORE
// httpWebRequest.ServicePoint.ConnectionLimit = int.MaxValue;
//#endif
return(page);
//正常结果在上面已经Return了, 到此处必然是下载失败的值.
//throw new SpiderExceptoin("Download failed.");
}
catch (DownloadException)
{
throw;
}
catch (HttpRequestException he)
{
throw new DownloadException(he.Message);
}
catch (Exception e)
{
Page page = new Page(request, site.ContentType, null)
{
Exception = e
};
return(page);
}
finally
{
// 先Close Response, 避免前面语句异常导致没有关闭.
try
{
//ensure the connection is released back to pool
//check:
//EntityUtils.consume(httpResponse.getEntity());
response?.Dispose();
}
catch (Exception e)
{
spider.Log("Close response fail.", LogLevel.Warn, e);
}
}
}
public override void InitPipeline(ISpider spider)
{
if (string.IsNullOrEmpty(ConnectString))
{
if (UpdateConnectString == null)
{
throw new SpiderException("Can't find ConnectString or IUpdateConnectString.");
}
else
{
for (int i = 0; i < 5; ++i)
{
try
{
ConnectString = UpdateConnectString.GetNew();
break;
}
catch (Exception e)
{
spider.Log("Update ConnectString failed.", LogLevel.Error, e);
Thread.Sleep(1000);
}
}
if (string.IsNullOrEmpty(ConnectString))
{
throw new SpiderException("Can't updadate ConnectString via IUpdateConnectString.");
}
}
}
base.InitPipeline(spider);
foreach (var metadata in DbMetadatas.Values)
{
if (!metadata.IsInsertModel)
{
continue;
}
NetworkCenter.Current.Execute("db-init", () =>
{
using (DbConnection conn = CreateConnection())
{
var command = conn.CreateCommand();
command.CommandText = GetIfSchemaExistsSql(metadata, conn.ServerVersion);
if (Convert.ToInt16(command.ExecuteScalar()) == 0)
{
command.CommandText = GetCreateSchemaSql(metadata, conn.ServerVersion);
command.CommandType = CommandType.Text;
command.ExecuteNonQuery();
}
command.CommandText = GetCreateTableSql(metadata);
command.CommandType = CommandType.Text;
command.ExecuteNonQuery();
}
});
}
}
protected override Page DowloadContent(Request request, ISpider spider)
{
Site site = spider.Site;
try
{
lock (this)
{
if (_webDriver == null)
{
_webDriver = WebDriverUtil.Open(_browser, _option);
}
if (!_isLogined && SignIn != null)
{
_isLogined = SignIn.Handle(_webDriver as RemoteWebDriver);
if (!_isLogined)
{
throw new SpiderException("Login failed. Please check your login codes.");
}
}
}
//中文乱码URL
Uri uri = request.Url;
string query = string.IsNullOrEmpty(uri.Query) ? "" : $"?{HttpUtility.UrlPathEncode(uri.Query.Substring(1, uri.Query.Length - 1))}";
string realUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}{uri.AbsolutePath}{query}";
var domainUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}";
var options = _webDriver.Manage();
if (options.Cookies.AllCookies.Count == 0 && spider.Site.Cookies.PairPart.Count > 0)
{
_webDriver.Url = domainUrl;
options.Cookies.DeleteAllCookies();
foreach (var c in spider.Site.Cookies.PairPart)
{
options.Cookies.AddCookie(new Cookie(c.Key, c.Value));
}
}
if (UrlHandler != null)
{
realUrl = UrlHandler(realUrl);
}
NetworkCenter.Current.Execute("wd-d", () =>
{
_webDriver.Navigate().GoToUrl(realUrl);
NavigateCompeleted?.Handle((RemoteWebDriver)_webDriver);
});
Thread.Sleep(_webDriverWaitTime);
Page page = new Page(request, spider.Site.ContentType, site.RemoveOutboundLinks ? site.Domains : null)
{
Content = _webDriver.PageSource,
TargetUrl = _webDriver.Url,
Title = _webDriver.Title
};
// 结束后要置空, 这个值存到Redis会导置无限循环跑单个任务
request.PutExtra(Request.CycleTriedTimes, null);
return(page);
}
catch (DownloadException de)
{
Page page = new Page(request, site.ContentType, null)
{
Exception = de
};
if (site.CycleRetryTimes > 0)
{
page = Spider.AddToCycleRetry(request, site);
}
spider.Log($"下载 {request.Url} 失败: {de.Message}", Core.Infrastructure.LogLevel.Warn);
return(page);
}
catch (HttpRequestException he)
{
Page page = new Page(request, site.ContentType, null)
{
Exception = he
};
if (site.CycleRetryTimes > 0)
{
page = Spider.AddToCycleRetry(request, site);
}
spider.Log($"下载 {request.Url} 失败: {he.Message}", Core.Infrastructure.LogLevel.Warn);
return(page);
}
catch (Exception e)
{
Page page = new Page(request, site.ContentType, null)
{
Exception = e
};
return(page);
}
}