public bool AddDataToCahce(string domain, WebCrawlerNode crawlerNode)
{
try
{
var cacheData = this.GetMainObjFromCache();
if (cacheData == null)
{
cacheData = new WebCrawlerMemContainer();
}
//check if domain exists on cache
if (cacheData.WebCrawlerDict.ContainsKey(domain))
{
var crawlerData = cacheData.WebCrawlerDict[domain];
//handle when uri non exists on domain
if (!crawlerData.CrawlerData.Exists(a => a.Url == crawlerNode.Url))
{
crawlerData.LastUpdate = DateTime.Now;
crawlerData.CrawlerData.Add(new WebCrawlerNode()
{
Url = crawlerNode.Url,
Title = crawlerNode.Title
});
}
}
else
{
cacheData.WebCrawlerDict.Add(domain, new WebCrawlerData()
{
Domain = domain,
CrawlerData = { new WebCrawlerNode()
{
Title = crawlerNode.Title, Url = crawlerNode.Url
} },
LastUpdate = DateTime.Now
});
}
var cacheEntryOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromMinutes(30));
// Save data in cache.
this.cacheObj.Set("crawlerCacheData", cacheData, cacheEntryOptions);
return(true);
}
catch (Exception ex)
{
Log.Error($"Error on AddDataToCahce() => {ex}");
return(false);
}
}
public async Task <WebCrawlerNode> GetRestCallAsyncResponse(string uriRequest)
{
try
{
WebCrawlerNode result = new WebCrawlerNode();
using (var client = new HttpClient())
{
var jsonRequest = new { url = uriRequest };
var stringContent = new StringContent(JsonSerializer.Serialize(jsonRequest), Encoding.UTF8, "application/json");
HttpRequestMessage request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = this.baseUri,
Content = stringContent
};
HttpResponseMessage response = await client.SendAsync(request);
using (HttpContent content = response.Content)
{
var jsonString = await response.Content.ReadAsStringAsync();
result = JsonSerializer.Deserialize <WebCrawlerNode>(jsonString);
}
}
return(result);
}
catch (HttpRequestException ex) // Non success
{
Log.Error($"error at GetJsonHttpClient {ex}");
}
catch (NotSupportedException ex) // When content type is not valid
{
Log.Error($"error at GetJsonHttpClient {ex}");
}
catch (JsonException ex) // Invalid JSON
{
Log.Error($"error at GetJsonHttpClient {ex}");
}
return(null);
}
