示例#1
0
 private void Log(LogData logData)
 {
     try
     {
         if (!WebSpyder.DataMatchExists(logData))
         {
             WebSpyder.CleanAndSaveLogData(logData);
         }
         else
         {
             WebSpyder.Urls.TryTake(out logData.Url);
         }
     }
     finally
     {
         Thread.Sleep(LogLoopDelay);
     }
 }
示例#2
0
        static void Main(string[] args)
        {
            var spyder = new WebSpyder("https://news.google.com");

            spyder.CollectData();

            // Good endpoints to find data
            // https://www.cnn.com
            // https://www.amazon.com
            // https://www.foxnews.com
            // https://www.wikipedia.org
            // https://news.google.com
            // https://www.npr.org
            // https://abcnews.go.com
            // https://news.yahooo.com
            // https://www.msn.com
            // https://www.usnews.com
            // https://www.usatoday.com
            // https://docs.microsoft.com/en-us/dotnet/csharp/

            Console.ReadLine();
        }