public async Task <IEnumerable <ScanResult> > ProcessCategoriesAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token) { if (_settings.EhamNet.CategorySearch.MaxPosts <= 0) { return(null); } _thisScan = new ScanInfo { Ids = new List <int>() }; #if DEBUG && CLEARHIST File.Delete(_settings.EhamNet.CategorySearch.ResultFile); #endif if (File.Exists(_settings.EhamNet.CategorySearch.ResultFile)) { _lastCategoryScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.EhamNet.CategorySearch.ResultFile)); _lastCategoryScan.OtherIds = new List <int>(_lastKeywordScan.Ids); } var res = new List <ScanResult>(); foreach (var category in _settings.EhamNet.CategorySearch.Categories.Split(',')) { if (token.IsCancellationRequested) { break; } _newPosts = new List <Post>(); res.Add(await ProcessCategory(httpClient, category, cookies, token)); } return(res); }
public async Task <ScanResult> ProcessKeywordsAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token) { _thisScan = new ScanInfo { Ids = new List <int>(), OtherIds = new List <int>() }; _newPosts = new List <Post>(); #if DEBUG && CLEARHIST File.Delete(_settings.EhamNet.KeywordSearch.ResultFile); #endif if (_settings.EhamNet.KeywordSearch.MaxPosts <= 0) { return(null); } var postNum = 0; if (File.Exists(_settings.EhamNet.KeywordSearch.ResultFile)) { _lastKeywordScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.EhamNet.KeywordSearch.ResultFile)); _lastKeywordScan.OtherIds = new List <int>(); } var sessionCookie = await GetSessionCookie(httpClient, cookies); while (postNum < _settings.EhamNet.KeywordSearch.MaxPosts) { _logger.LogDebug($"Fetching \"{_settings.EhamNet.KeywordSearch.Keywords}\" page {postNum / PAGE_SIZE + 1} of maximum {_settings.EhamNet.KeywordSearch.MaxPosts / PAGE_SIZE} from eHam.net"); var uri = new Uri($"https://www.eham.net/classifieds/?view=detail&page={postNum / PAGE_SIZE + 1}"); var message = new HttpRequestMessage(HttpMethod.Get, uri); message.Headers.Add("Cache-Control", "no-cache"); message.Headers.Add("Cookie", $"{sessionCookie.Name}={sessionCookie.Value}"); var res = await httpClient.SendAsync(message, token); //var res = await httpClient.GetAsync(uri, token); if (token.IsCancellationRequested) { break; } var msg = await res.Content.ReadAsStringAsync(); postNum += PAGE_SIZE; if (!await ScanResults(msg, ScanType.Keyword, httpClient)) { break; } } _newPosts.ForEach(x => _thisScan.Ids.Add(x.Id)); #if !DEBUG || SAVEHIST if (_newPosts.Count > 0) { File.WriteAllText(_settings.EhamNet.KeywordSearch.ResultFile, JsonConvert.SerializeObject(_thisScan)); } #endif return(BuildResults(_lastKeywordScan)); }
private ScanResult BuildResults(ScanInfo lastScan) { if (_newPosts.Count == 0) { _logger.LogDebug("No new posts found"); return(null); } _logger.LogDebug($"{_newPosts.Count} new posts"); string imgFmt; if (!string.IsNullOrEmpty(_settings.ResourceUrl) && !string.IsNullOrEmpty(_settings.EhamNet.Cache?.ImageFolder)) { imgFmt = $"{_settings.ResourceUrl}/{_settings.EhamNet.Cache.ImageFolder}/{{0}}.jpg"; } else { imgFmt = "https://www.eham.net/data/classifieds/images/{0}.t.jpg"; } var sb = new StringBuilder(); foreach (var post in _newPosts) { sb.AppendLine("<table>"); sb.AppendLine(" <tr>"); sb.AppendLine(" <td rowspan='3' class='thumb'>"); if (post.HasImage) { sb.AppendFormat($" <a href='https://www.eham.net/classifieds/detail/{{0}}' target='_blank'><img src='{imgFmt}'></a>{Environment.NewLine}", post.Id); } sb.AppendLine(" </td>"); sb.AppendLine(" <td class='title'>"); sb.Append($" <a class='link' href='https://www.eham.net/classifieds/detail/{post.Id}' target='_blank'>{post.Title}</a>"); if (post.Price != null) { sb.Append($" ${post.Price}"); } var category = _categories.FirstOrDefault(x => x.Key == post.Category); if (category.Key != null) { sb.AppendLine($"<a class='cat' href='https://www.eham.net/classifieds/results/{category.Value}' target='_blank'>{post.Category}</a>"); } sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine(" <tr class='content'>"); sb.AppendLine(" <td>"); sb.AppendLine($" {post.Description}"); sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine(" <tr>"); sb.AppendLine(" <td class='info'>"); sb.Append(" Submitted "); if (post.CallSign != null) { sb.Append($"by <a class='call' href='https://www.qrz.com/lookup?tquery={post.CallSign}&mode=callsign' target='_blank'>{post.CallSign}</a> "); } sb.AppendLine($"on {post.SubmittedOn:d}"); sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine("</table>\n"); } return(new ScanResult { Title = _settings.EhamNet.Title, Items = _newPosts.Count, LastScan = lastScan.Date, Html = sb.ToString() }); }
private ScanResult BuildResults(ScanInfo lastScan) { if (_newPosts.Count == 0) { _logger.LogDebug("No new posts found"); return(null); } _logger.LogDebug($"{_newPosts.Count} new posts"); string imgFmt; if (!string.IsNullOrEmpty(_settings.ResourceUrl) && !string.IsNullOrEmpty(_settings.QthCom.Cache?.ImageFolder)) { imgFmt = $"{_settings.ResourceUrl}/{_settings.QthCom.Cache.ImageFolder}/{{0}}.jpg"; } else { imgFmt = "https://swap.qth.com/segamida/thumb_{0}.jpg"; } var sb = new StringBuilder(); foreach (var post in _newPosts) { sb.AppendLine("<table>"); sb.AppendLine(" <tr>"); sb.AppendLine(" <td rowspan='3' class='thumb'>"); if (post.HasImage) { sb.AppendFormat($" <a href='https://swap.qth.com/view_ad.php?counter={{0}}' target='_blank'><img src='{imgFmt}'></a>{Environment.NewLine}", post.Id); } sb.AppendLine(" </td>"); sb.AppendLine(" <td class='title'>"); sb.Append($" <a class='link' href='https://swap.qth.com/view_ad.php?counter={post.Id}' target='_blank'>{post.Title}</a>"); if (post.Price != null) { sb.Append($" ${post.Price}"); } if (post.Category != null) { sb.AppendLine($"<a class='cat' href='https://swap.qth.com/c_{post.Category}.php' target='_blank'>{post.Category}</a>"); } sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine(" <tr class='content'>"); sb.AppendLine(" <td>"); sb.AppendLine($" {post.Description}"); sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine(" <tr>"); sb.AppendLine(" <td class='info'>"); sb.Append(" Submitted "); if (post.CallSign != null) { sb.Append($"by <a class='call' href='https://www.qrz.com/lookup?tquery={post.CallSign}&mode=callsign' target='_blank'>{post.CallSign}</a> "); } sb.Append($"on {post.SubmittedOn:d}"); if (post.ModifiedOn.HasValue) { sb.Append($" Modified: <span class='modified'>{post.ModifiedOn:d}</span>"); } sb.AppendLine(""); sb.AppendLine(" </td>"); sb.AppendLine(" </tr>"); sb.AppendLine("</table>\n"); } return(new ScanResult { Title = _settings.QthCom.Title, Items = _newPosts.Count, LastScan = lastScan.Date, Html = sb.ToString() }); }
public async Task <ScanResult> ProcessKeywordsAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token) { if (_settings.QthCom.KeywordSearch.MaxPosts <= 0) { return(null); } _thisScan = new ScanInfo { Ids = new List <int>(), OtherIds = new List <int>() }; _newPosts = new List <Post>(); #if DEBUG && CLEARHIST File.Delete(_settings.QthCom.KeywordSearch.ResultFile); #endif int startIndex = 0, postNum = 0; var uri = new Uri("https://swap.qth.com/advsearchresults.php"); if (File.Exists(_settings.QthCom.KeywordSearch.ResultFile)) { _lastKeywordScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.QthCom.KeywordSearch.ResultFile)); _lastKeywordScan.OtherIds = new List <int>(); } while (postNum < _settings.QthCom.KeywordSearch.MaxPosts) { var formData = new List <KeyValuePair <string, string> > { new KeyValuePair <string, string>("anywords", string.Join(' ', _settings.QthCom.KeywordSearch.Keywords.Split(','))) }; if (startIndex > 0) { formData.AddRange(new[] { new KeyValuePair <string, string>("startnum", startIndex.ToString()), new KeyValuePair <string, string>("submit", "Next 10 Ads") }); } var content = new FormUrlEncodedContent(formData); _logger.LogDebug($"Fetching \"{_settings.QthCom.KeywordSearch.Keywords}\" page {postNum/PAGE_SIZE + 1} of maximum {_settings.QthCom.KeywordSearch.MaxPosts/PAGE_SIZE} from qth.com"); var res = await httpClient.PostAsync(uri, content, token); if (token.IsCancellationRequested) { break; } var msg = await res.Content.ReadAsStringAsync(); startIndex += 10; postNum += PAGE_SIZE; if (!await ScanResults(msg, ScanType.Keyword, httpClient)) { break; } } _newPosts.ForEach(x => _thisScan.Ids.Add(x.Id)); #if !DEBUG || SAVEHIST if (_newPosts.Count > 0) { File.WriteAllText(_settings.QthCom.KeywordSearch.ResultFile, JsonConvert.SerializeObject(_thisScan)); } #endif _lastKeywordScan.Ids = new List <int>(_thisScan.Ids); return(BuildResults(_lastKeywordScan)); }