public async Task <IEnumerable <ScanResult> > ProcessCategoriesAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token)
{
if (_settings.EhamNet.CategorySearch.MaxPosts <= 0)
{
return(null);
}
_thisScan = new ScanInfo {
Ids = new List <int>()
};
#if DEBUG && CLEARHIST
File.Delete(_settings.EhamNet.CategorySearch.ResultFile);
#endif
if (File.Exists(_settings.EhamNet.CategorySearch.ResultFile))
{
_lastCategoryScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.EhamNet.CategorySearch.ResultFile));
_lastCategoryScan.OtherIds = new List <int>(_lastKeywordScan.Ids);
}
var res = new List <ScanResult>();
foreach (var category in _settings.EhamNet.CategorySearch.Categories.Split(','))
{
if (token.IsCancellationRequested)
{
break;
}
_newPosts = new List <Post>();
res.Add(await ProcessCategory(httpClient, category, cookies, token));
}
return(res);
}
public async Task <ScanResult> ProcessKeywordsAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token)
{
_thisScan = new ScanInfo {
Ids = new List <int>(), OtherIds = new List <int>()
};
_newPosts = new List <Post>();
#if DEBUG && CLEARHIST
File.Delete(_settings.EhamNet.KeywordSearch.ResultFile);
#endif
if (_settings.EhamNet.KeywordSearch.MaxPosts <= 0)
{
return(null);
}
var postNum = 0;
if (File.Exists(_settings.EhamNet.KeywordSearch.ResultFile))
{
_lastKeywordScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.EhamNet.KeywordSearch.ResultFile));
_lastKeywordScan.OtherIds = new List <int>();
}
var sessionCookie = await GetSessionCookie(httpClient, cookies);
while (postNum < _settings.EhamNet.KeywordSearch.MaxPosts)
{
_logger.LogDebug($"Fetching \"{_settings.EhamNet.KeywordSearch.Keywords}\" page {postNum / PAGE_SIZE + 1} of maximum {_settings.EhamNet.KeywordSearch.MaxPosts / PAGE_SIZE} from eHam.net");
var uri = new Uri($"https://www.eham.net/classifieds/?view=detail&page={postNum / PAGE_SIZE + 1}");
var message = new HttpRequestMessage(HttpMethod.Get, uri);
message.Headers.Add("Cache-Control", "no-cache");
message.Headers.Add("Cookie", $"{sessionCookie.Name}={sessionCookie.Value}");
var res = await httpClient.SendAsync(message, token);
//var res = await httpClient.GetAsync(uri, token);
if (token.IsCancellationRequested)
{
break;
}
var msg = await res.Content.ReadAsStringAsync();
postNum += PAGE_SIZE;
if (!await ScanResults(msg, ScanType.Keyword, httpClient))
{
break;
}
}
_newPosts.ForEach(x => _thisScan.Ids.Add(x.Id));
#if !DEBUG || SAVEHIST
if (_newPosts.Count > 0)
{
File.WriteAllText(_settings.EhamNet.KeywordSearch.ResultFile, JsonConvert.SerializeObject(_thisScan));
}
#endif
return(BuildResults(_lastKeywordScan));
}
private ScanResult BuildResults(ScanInfo lastScan)
{
if (_newPosts.Count == 0)
{
_logger.LogDebug("No new posts found");
return(null);
}
_logger.LogDebug($"{_newPosts.Count} new posts");
string imgFmt;
if (!string.IsNullOrEmpty(_settings.ResourceUrl) && !string.IsNullOrEmpty(_settings.EhamNet.Cache?.ImageFolder))
{
imgFmt = $"{_settings.ResourceUrl}/{_settings.EhamNet.Cache.ImageFolder}/{{0}}.jpg";
}
else
{
imgFmt = "https://www.eham.net/data/classifieds/images/{0}.t.jpg";
}
var sb = new StringBuilder();
foreach (var post in _newPosts)
{
sb.AppendLine("<table>");
sb.AppendLine(" <tr>");
sb.AppendLine(" <td rowspan='3' class='thumb'>");
if (post.HasImage)
{
sb.AppendFormat($" <a href='https://www.eham.net/classifieds/detail/{{0}}' target='_blank'><img src='{imgFmt}'></a>{Environment.NewLine}", post.Id);
}
sb.AppendLine(" </td>");
sb.AppendLine(" <td class='title'>");
sb.Append($" <a class='link' href='https://www.eham.net/classifieds/detail/{post.Id}' target='_blank'>{post.Title}</a>");
if (post.Price != null)
{
sb.Append($" ${post.Price}");
}
var category = _categories.FirstOrDefault(x => x.Key == post.Category);
if (category.Key != null)
{
sb.AppendLine($"<a class='cat' href='https://www.eham.net/classifieds/results/{category.Value}' target='_blank'>{post.Category}</a>");
}
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine(" <tr class='content'>");
sb.AppendLine(" <td>");
sb.AppendLine($" {post.Description}");
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine(" <tr>");
sb.AppendLine(" <td class='info'>");
sb.Append(" Submitted ");
if (post.CallSign != null)
{
sb.Append($"by <a class='call' href='https://www.qrz.com/lookup?tquery={post.CallSign}&mode=callsign' target='_blank'>{post.CallSign}</a> ");
}
sb.AppendLine($"on {post.SubmittedOn:d}");
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine("</table>\n");
}
return(new ScanResult {
Title = _settings.EhamNet.Title, Items = _newPosts.Count, LastScan = lastScan.Date, Html = sb.ToString()
});
}
private ScanResult BuildResults(ScanInfo lastScan)
{
if (_newPosts.Count == 0)
{
_logger.LogDebug("No new posts found");
return(null);
}
_logger.LogDebug($"{_newPosts.Count} new posts");
string imgFmt;
if (!string.IsNullOrEmpty(_settings.ResourceUrl) && !string.IsNullOrEmpty(_settings.QthCom.Cache?.ImageFolder))
{
imgFmt = $"{_settings.ResourceUrl}/{_settings.QthCom.Cache.ImageFolder}/{{0}}.jpg";
}
else
{
imgFmt = "https://swap.qth.com/segamida/thumb_{0}.jpg";
}
var sb = new StringBuilder();
foreach (var post in _newPosts)
{
sb.AppendLine("<table>");
sb.AppendLine(" <tr>");
sb.AppendLine(" <td rowspan='3' class='thumb'>");
if (post.HasImage)
{
sb.AppendFormat($" <a href='https://swap.qth.com/view_ad.php?counter={{0}}' target='_blank'><img src='{imgFmt}'></a>{Environment.NewLine}", post.Id);
}
sb.AppendLine(" </td>");
sb.AppendLine(" <td class='title'>");
sb.Append($" <a class='link' href='https://swap.qth.com/view_ad.php?counter={post.Id}' target='_blank'>{post.Title}</a>");
if (post.Price != null)
{
sb.Append($" ${post.Price}");
}
if (post.Category != null)
{
sb.AppendLine($"<a class='cat' href='https://swap.qth.com/c_{post.Category}.php' target='_blank'>{post.Category}</a>");
}
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine(" <tr class='content'>");
sb.AppendLine(" <td>");
sb.AppendLine($" {post.Description}");
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine(" <tr>");
sb.AppendLine(" <td class='info'>");
sb.Append(" Submitted ");
if (post.CallSign != null)
{
sb.Append($"by <a class='call' href='https://www.qrz.com/lookup?tquery={post.CallSign}&mode=callsign' target='_blank'>{post.CallSign}</a> ");
}
sb.Append($"on {post.SubmittedOn:d}");
if (post.ModifiedOn.HasValue)
{
sb.Append($" Modified: <span class='modified'>{post.ModifiedOn:d}</span>");
}
sb.AppendLine("");
sb.AppendLine(" </td>");
sb.AppendLine(" </tr>");
sb.AppendLine("</table>\n");
}
return(new ScanResult {
Title = _settings.QthCom.Title, Items = _newPosts.Count, LastScan = lastScan.Date, Html = sb.ToString()
});
}
public async Task <ScanResult> ProcessKeywordsAsync(HttpClient httpClient, CookieContainer cookies, CancellationToken token)
{
if (_settings.QthCom.KeywordSearch.MaxPosts <= 0)
{
return(null);
}
_thisScan = new ScanInfo {
Ids = new List <int>(), OtherIds = new List <int>()
};
_newPosts = new List <Post>();
#if DEBUG && CLEARHIST
File.Delete(_settings.QthCom.KeywordSearch.ResultFile);
#endif
int startIndex = 0, postNum = 0;
var uri = new Uri("https://swap.qth.com/advsearchresults.php");
if (File.Exists(_settings.QthCom.KeywordSearch.ResultFile))
{
_lastKeywordScan = JsonConvert.DeserializeObject <ScanInfo>(File.ReadAllText(_settings.QthCom.KeywordSearch.ResultFile));
_lastKeywordScan.OtherIds = new List <int>();
}
while (postNum < _settings.QthCom.KeywordSearch.MaxPosts)
{
var formData = new List <KeyValuePair <string, string> >
{
new KeyValuePair <string, string>("anywords", string.Join(' ', _settings.QthCom.KeywordSearch.Keywords.Split(',')))
};
if (startIndex > 0)
{
formData.AddRange(new[]
{
new KeyValuePair <string, string>("startnum", startIndex.ToString()),
new KeyValuePair <string, string>("submit", "Next 10 Ads")
});
}
var content = new FormUrlEncodedContent(formData);
_logger.LogDebug($"Fetching \"{_settings.QthCom.KeywordSearch.Keywords}\" page {postNum/PAGE_SIZE + 1} of maximum {_settings.QthCom.KeywordSearch.MaxPosts/PAGE_SIZE} from qth.com");
var res = await httpClient.PostAsync(uri, content, token);
if (token.IsCancellationRequested)
{
break;
}
var msg = await res.Content.ReadAsStringAsync();
startIndex += 10;
postNum += PAGE_SIZE;
if (!await ScanResults(msg, ScanType.Keyword, httpClient))
{
break;
}
}
_newPosts.ForEach(x => _thisScan.Ids.Add(x.Id));
#if !DEBUG || SAVEHIST
if (_newPosts.Count > 0)
{
File.WriteAllText(_settings.QthCom.KeywordSearch.ResultFile, JsonConvert.SerializeObject(_thisScan));
}
#endif
_lastKeywordScan.Ids = new List <int>(_thisScan.Ids);
return(BuildResults(_lastKeywordScan));
}
