private void Log(LogData logData)
{
try
{
if (!WebSpyder.DataMatchExists(logData))
{
WebSpyder.CleanAndSaveLogData(logData);
}
else
{
WebSpyder.Urls.TryTake(out logData.Url);
}
}
finally
{
Thread.Sleep(LogLoopDelay);
}
}
static void Main(string[] args)
{
var spyder = new WebSpyder("https://news.google.com");
spyder.CollectData();
// Good endpoints to find data
// https://www.cnn.com
// https://www.amazon.com
// https://www.foxnews.com
// https://www.wikipedia.org
// https://news.google.com
// https://www.npr.org
// https://abcnews.go.com
// https://news.yahooo.com
// https://www.msn.com
// https://www.usnews.com
// https://www.usatoday.com
// https://docs.microsoft.com/en-us/dotnet/csharp/
Console.ReadLine();
}
