private async Task collectChat(string board, string oneOfTheServer, DateTimeOffset startTime, DateTimeOffset endTime, CancellationToken cancellationToken) { if (!this.pastThreadListerCache.TryGetValue(board, out var threadLister)) { threadLister = new Nichan.PastThreadLister(board, oneOfTheServer, this.backTime); await threadLister.Initialize(cancellationToken); this.pastThreadListerCache.Add(board, threadLister); } IEnumerable <Nichan.Thread> threads = await threadLister.GetBetween(startTime, endTime, cancellationToken).ConfigureAwait(false); IEnumerable <string> currentThreadUrls; lock (this.threadList) currentThreadUrls = this.threadList.Select(x => x.Uri.ToString()).ToArray(); IEnumerable <Nichan.Thread> newThreads = threads.Where(x => !currentThreadUrls.Contains(x.Uri.ToString())); foreach (var newThread in newThreads) { var(server, board_, threadId) = getServerBoardThreadFromThreadUrl(newThread.Uri.ToString()); Nichan.Thread thread = await getThread(server, board_, threadId, cancellationToken); thread.Uri = newThread.Uri; // キャッシュにヒットするようにthreadListerの返したUriで記憶する lock (this.threadList) this.threadList.Add(thread); } }
private async Task ResCollectLoop(CancellationToken cancellationToken) { while (true) { if (this.lastGetChannel == null) { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); continue; } string[] threadUrls = (await this.threadSelector.Get( this.lastGetChannel, this.lastGetTime, cancellationToken ).ConfigureAwait(false)).ToArray(); this.currentThreadUrls = threadUrls; IEnumerable <string> existingThreadUrls; lock (this.threadList) existingThreadUrls = this.threadList.Select(x => x.Uri.ToString()).ToArray(); IEnumerable <string> newThreadUrls = threadUrls.Where(x => !existingThreadUrls.Contains(x)); foreach (var newThreadUrl in newThreadUrls) { var(server, board_, threadId) = GetServerBoardThreadFromThreadUrl(newThreadUrl); Nichan.Thread thread = await GetThread(server, board_, threadId, cancellationToken).ConfigureAwait(false); thread.Uri = new Uri(newThreadUrl); // キャッシュにヒットするようにthreadSelectorの返したUriで記憶する lock (this.threadList) this.threadList.Add(thread); } await Task.Delay(this.threadSelectionUpdateInterval, cancellationToken); } }
private static async Task <Nichan.Thread> getThread(string server, string board, string thread, CancellationToken cancellationToken) { Nichan.Thread ret; // まず2ch.scのdatから取得する string datUrl = $"http://{server}.2ch.sc/{board}/dat/{thread}.dat"; string datResponse = null; System.Diagnostics.Debug.WriteLine($"[PastNichanChatCollectService] HTTP Get {datUrl}"); try { datResponse = await httpClient.GetStringAsync(datUrl, cancellationToken).ConfigureAwait(false); } catch (HttpRequestException) { } if (datResponse != null) { ret = new Nichan.Thread(); var datParser = new Nichan.DatParser(); datParser.Feed(datResponse); ret.Uri = new Uri(datUrl); ret.Title = datParser.ThreadTitle; while (true) { Nichan.Res res = datParser.PopRes(); if (res == null) { break; } ret.Res.Add(res); } ret.ResCount = ret.Res.Count; return(ret); } // 2ch.scがダメだった場合、5ch.netのスクレイピング string gochanUrl = $"http://{server}.5ch.net/test/read.cgi/{board}/{thread}/"; string response; System.Diagnostics.Debug.WriteLine($"[PastNichanChatCollectService] HTTP Get {gochanUrl}"); try { response = await httpClient.GetStringAsync(gochanUrl, cancellationToken).ConfigureAwait(false); } catch (HttpRequestException e) { if (e.StatusCode == null) { throw new Nichan.NetworkException(gochanUrl, null, e); } else { throw new Nichan.HttpErrorResponseException((int)e.StatusCode.Value, null, gochanUrl, null, e); } } using var textReader = new StringReader(response); ret = Nichan.ThreadParser.ParseFromStream(textReader); ret.Uri = new Uri(gochanUrl); return(ret); }