/// <summary>
/// アクセスのペースを調整するために許可時間のタイムスタンプを設定する
/// </summary>
/// <param name="cReq"></param>
private static void WaitSiteAccess(CrawlerRequest cReq)
{
var key = cReq.Request.RequestUri.Host;
if (string.IsNullOrEmpty(key)) return;
SiteAccessBlock site;
lock (DicSiteAccessBlock) {
if (!DicSiteAccessBlock.TryGetValue(key, out site)) {
site = new SiteAccessBlock();
DicSiteAccessBlock.Add(key, site);
}
}
lock (site) {
var wait = site.PassTime - DateTimeOffset.Now;
if (wait > TimeSpan.Zero) Thread.Sleep(wait);
site.PassTime = DateTimeOffset.Now + PassSpan;
}
}
private static IObservable<CrawlerRequest> WaitSiteAccess(this IObservable<CrawlerRequest> rxReq)
{
return rxReq
.Select(cReq =>
{
//待機時間を決定する
var key = cReq.Request.RequestUri.Host;
SiteAccessBlock site;
lock (DicSiteAccessBlock) {
if (!DicSiteAccessBlock.TryGetValue(key, out site)) {
site = new SiteAccessBlock();
DicSiteAccessBlock.Add(key, site);
}
}
lock (site) {
var passTime = site.PassTime;
site.PassTime += PassSpan;
return new Timestamped<CrawlerRequest>(cReq, passTime);
}
})
.Do(pass=>{
string.Format(
"[wait=>{0:HH:mm:ss.fff}] uri={1}"
, pass.Timestamp
, pass.Value.Request.RequestUri.AbsoluteUri)
.TraceInfo();
})
.DelayTimestamped()
;
}
