private async Task collectChat(string board, string oneOfTheServer, DateTimeOffset startTime, DateTimeOffset endTime, CancellationToken cancellationToken)
{
if (!this.pastThreadListerCache.TryGetValue(board, out var threadLister))
{
threadLister = new Nichan.PastThreadLister(board, oneOfTheServer, this.backTime);
await threadLister.Initialize(cancellationToken);
this.pastThreadListerCache.Add(board, threadLister);
}
IEnumerable <Nichan.Thread> threads = await threadLister.GetBetween(startTime, endTime, cancellationToken).ConfigureAwait(false);
IEnumerable <string> currentThreadUrls;
lock (this.threadList)
currentThreadUrls = this.threadList.Select(x => x.Uri.ToString()).ToArray();
IEnumerable <Nichan.Thread> newThreads = threads.Where(x => !currentThreadUrls.Contains(x.Uri.ToString()));
foreach (var newThread in newThreads)
{
var(server, board_, threadId) = getServerBoardThreadFromThreadUrl(newThread.Uri.ToString());
Nichan.Thread thread = await getThread(server, board_, threadId, cancellationToken);
thread.Uri = newThread.Uri; // キャッシュにヒットするようにthreadListerの返したUriで記憶する
lock (this.threadList)
this.threadList.Add(thread);
}
}
private async Task ResCollectLoop(CancellationToken cancellationToken)
{
while (true)
{
if (this.lastGetChannel == null)
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
continue;
}
string[] threadUrls = (await this.threadSelector.Get(
this.lastGetChannel, this.lastGetTime, cancellationToken
).ConfigureAwait(false)).ToArray();
this.currentThreadUrls = threadUrls;
IEnumerable <string> existingThreadUrls;
lock (this.threadList)
existingThreadUrls = this.threadList.Select(x => x.Uri.ToString()).ToArray();
IEnumerable <string> newThreadUrls = threadUrls.Where(x => !existingThreadUrls.Contains(x));
foreach (var newThreadUrl in newThreadUrls)
{
var(server, board_, threadId) = GetServerBoardThreadFromThreadUrl(newThreadUrl);
Nichan.Thread thread = await GetThread(server, board_, threadId, cancellationToken).ConfigureAwait(false);
thread.Uri = new Uri(newThreadUrl); // キャッシュにヒットするようにthreadSelectorの返したUriで記憶する
lock (this.threadList)
this.threadList.Add(thread);
}
await Task.Delay(this.threadSelectionUpdateInterval, cancellationToken);
}
}
private static async Task <Nichan.Thread> getThread(string server, string board, string thread, CancellationToken cancellationToken)
{
Nichan.Thread ret;
// まず2ch.scのdatから取得する
string datUrl = $"http://{server}.2ch.sc/{board}/dat/{thread}.dat";
string datResponse = null;
System.Diagnostics.Debug.WriteLine($"[PastNichanChatCollectService] HTTP Get {datUrl}");
try
{
datResponse = await httpClient.GetStringAsync(datUrl, cancellationToken).ConfigureAwait(false);
}
catch (HttpRequestException)
{
}
if (datResponse != null)
{
ret = new Nichan.Thread();
var datParser = new Nichan.DatParser();
datParser.Feed(datResponse);
ret.Uri = new Uri(datUrl);
ret.Title = datParser.ThreadTitle;
while (true)
{
Nichan.Res res = datParser.PopRes();
if (res == null)
{
break;
}
ret.Res.Add(res);
}
ret.ResCount = ret.Res.Count;
return(ret);
}
// 2ch.scがダメだった場合、5ch.netのスクレイピング
string gochanUrl = $"http://{server}.5ch.net/test/read.cgi/{board}/{thread}/";
string response;
System.Diagnostics.Debug.WriteLine($"[PastNichanChatCollectService] HTTP Get {gochanUrl}");
try
{
response = await httpClient.GetStringAsync(gochanUrl, cancellationToken).ConfigureAwait(false);
}
catch (HttpRequestException e)
{
if (e.StatusCode == null)
{
throw new Nichan.NetworkException(gochanUrl, null, e);
}
else
{
throw new Nichan.HttpErrorResponseException((int)e.StatusCode.Value, null, gochanUrl, null, e);
}
}
using var textReader = new StringReader(response);
ret = Nichan.ThreadParser.ParseFromStream(textReader);
ret.Uri = new Uri(gochanUrl);
return(ret);
}
