private static void getSites(TweetDataEntities db, params string[] query) { HashSet<string> seenUrls = new HashSet<string>(); var twitter = new TwitterSearch(); int count = 0; foreach (var tweet in twitter.Search(100, 10, query)) { Debug.Print("Tweet number: " + (++count).ToString()); if (db.Tweets.Any(i => i.TweetID == tweet.TweetID)) { continue; } Tweet t = new Tweet() { Text = tweet.Text, TweetID = tweet.TweetID }; Debug.Print("Tweet: " + tweet.Text); Regex linkParser = new Regex(@"\b(?:http://|www\.)\S+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); foreach (Match m in linkParser.Matches(tweet.Text)) { string fullUrl = ""; try { fullUrl = m.Value.ExpandUrl(); } catch { continue; } if (db.Websites.Any(i => i.Url == fullUrl)) { continue; } Debug.Print("Website: " + fullUrl); var page = new PageScraper(fullUrl); var website = new Website() { Url = page.Url, Title = page.Title() }; db.Websites.AddObject(website); foreach (var m2 in page.Media()) { if (db.Media.Any(i => i.Url == m2.Link)) { continue; } Medium media = new Medium() { Type = m2.Type, Url = m2.Link, SourceSite = website.Url }; if (m2.Type == "image") { var request = WebRequest.Create(m2.Link); using (var response = request.GetResponse()) using (var stream = response.GetResponseStream()) using (var b = Bitmap.FromStream(stream)) { int area = b.Width * b.Height; media.ImageArea = area; } } db.Media.AddObject(media); Debug.Print("Media element: " + m2.Link); } t.LinkSite = website.Url; } db.Tweets.AddObject(t); db.SaveChanges(); } }
static void Main(string[] args) { var twitter = new TwitterSearch(); foreach (var tweet in twitter.Search(100, "data","visualization")) { Console.WriteLine(tweet); Regex linkParser = new Regex(@"\b(?:http://|www\.)\S+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); foreach (Match m in linkParser.Matches(tweet.Text)) { var fullUrl = m.Value.ExpandUrl(); var scraper = new PageScraper(fullUrl); foreach (var img in scraper.GetImages()) { Debug.Print(img); } } } }
public IEnumerable<PageScraper> GetSites(params string[] query) { HashSet<string> seenUrls = new HashSet<string>(); var twitter = new TwitterSearch(); foreach (var tweet in twitter.Search(10, 1, query)) { Regex linkParser = new Regex(@"\b(?:http://|www\.)\S+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); foreach (Match m in linkParser.Matches(tweet.Text)) { var fullUrl = m.Value.ExpandUrl(); Debug.Print(fullUrl); if (seenUrls.Contains(fullUrl)) { continue; } yield return new PageScraper(fullUrl); } } }