Beispiel #1
0
        private static void getSites(TweetDataEntities db, params string[] query)
        {
            HashSet<string> seenUrls = new HashSet<string>();
            var twitter = new TwitterSearch();
            int count = 0;
            foreach (var tweet in twitter.Search(100, 10, query)) {
                Debug.Print("Tweet number: " + (++count).ToString());
                if (db.Tweets.Any(i => i.TweetID == tweet.TweetID)) {
                    continue;
                }
                Tweet t = new Tweet() { Text = tweet.Text, TweetID = tweet.TweetID };
                Debug.Print("Tweet: " + tweet.Text);
                Regex linkParser = new Regex(@"\b(?:http://|www\.)\S+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase);
                foreach (Match m in linkParser.Matches(tweet.Text)) {
                    string fullUrl = "";
                    try {
                        fullUrl = m.Value.ExpandUrl();
                    } catch {
                        continue;
                    }
                    if (db.Websites.Any(i => i.Url == fullUrl)) {
                        continue;
                    }
                    Debug.Print("Website: " + fullUrl);
                    var page = new PageScraper(fullUrl);
                    var website = new Website() { Url = page.Url, Title = page.Title() };
                    db.Websites.AddObject(website);
                    foreach (var m2 in page.Media()) {
                        if (db.Media.Any(i => i.Url == m2.Link)) {
                            continue;
                        }
                        Medium media = new Medium() { Type = m2.Type, Url = m2.Link, SourceSite = website.Url };
                        if (m2.Type == "image") {
                            var request = WebRequest.Create(m2.Link);

                            using (var response = request.GetResponse())
                            using (var stream = response.GetResponseStream())
                            using (var b = Bitmap.FromStream(stream)) {
                                int area = b.Width * b.Height;
                                media.ImageArea = area;
                            }
                        }
                        db.Media.AddObject(media);
                        Debug.Print("Media element: " + m2.Link);
                    }
                    t.LinkSite = website.Url;

                }
                db.Tweets.AddObject(t);
                db.SaveChanges();
            }
        }
 /// <summary>
 /// Create a new Website object.
 /// </summary>
 /// <param name="url">Initial value of the Url property.</param>
 public static Website CreateWebsite(global::System.String url)
 {
     Website website = new Website();
     website.Url = url;
     return website;
 }
 /// <summary>
 /// Deprecated Method for adding a new object to the Websites EntitySet. Consider using the .Add method of the associated ObjectSet&lt;T&gt; property instead.
 /// </summary>
 public void AddToWebsites(Website website)
 {
     base.AddObject("Websites", website);
 }