private static void addTweet(VisualizationEntities db, DateTime date, string handle, string tweet, string url) { if (counter++ < Properties.Settings.Default.TweetIndex - 1) { return; } Debug.Print("Tweet counter: " + counter); Debug.Print("Tweet: " + tweet); TweetData tweetData = new TweetData(tweet); string tweetId = url.Split('/').Last(); Tweet newTweet = new DB2.Tweet() { TweetText = tweet, Date = date, Username = handle.Split(' ').First(), TweetID = tweetId }; if (!db.Tweets.Select(i => i.TweetID).Contains(tweetId)) { db.Tweets.AddObject(newTweet); db.SaveChanges(); Properties.Settings.Default.TweetIndex = counter; Properties.Settings.Default.Save(); } else { return; } foreach (var hashtag in tweetData.Hashtags()) { Hashtag extantTag; if (!db.Hashtags.Select(i => i.Tag).Contains(hashtag)) { Debug.Print("new tag: " + hashtag); extantTag = new Hashtag() { Tag = hashtag }; db.Hashtags.AddObject(extantTag); db.SaveChanges(); } else { extantTag = db.Hashtags.Where(i => i.Tag == hashtag).Single(); } db.TweetHashtags.AddObject(new TweetHashtag() { TweetID = newTweet.ID, HashtagID = extantTag.ID }); db.SaveChanges(); } foreach (string matchedUrl in tweetData.Urls()) { string uniqueUrl = ""; try { uniqueUrl = matchedUrl.ExpandUrl().UnuiqeUrl(); } catch (Exception ex) { continue; } ///Find matched websites: var knownSite = db.Websites.Where(i => i.Url == uniqueUrl); if (knownSite.Count() > 0) { /// var tweetWebsite = new TweetWebsite() { TweetID = newTweet.ID, WebsiteID = knownSite.First().ID }; db.TweetWebsites.AddObject(tweetWebsite); db.SaveChanges(); continue; } ///This is a brand new website if (uniqueUrl.Count() > 600 || db.Websites.Any(i => i.Url == uniqueUrl)) { continue; } Debug.Print("Website: " + uniqueUrl); string websiteUrl; try { var page = new PageScraper(uniqueUrl); string longUrlTitle = ""; string description = ""; try { longUrlTitle = page.Url.GetTitle(); } catch { Debug.Print("Failed to get title"); } try { description = page.Url.GetDescription(); } catch { Debug.Print("Failed to get description"); } var website = new DB2.Website() { Url = page.Url, Title = string.Concat((page.Title() ?? "").Take(300)), TweetID = newTweet.ID, LongUrlTitle = string.Concat(longUrlTitle.Take(100)), Description = string.Concat(description.Take(500)) }; db.Websites.AddObject(website); db.SaveChanges(); var tweetWebsite = new TweetWebsite() { TweetID = newTweet.ID, WebsiteID = website.ID }; db.TweetWebsites.AddObject(tweetWebsite); db.SaveChanges(); foreach (var m2 in page.Media()) { if (m2.Link.Count() > 600 || db.Media.Any(i => i.Url == m2.Link)) { continue; } try { DB2.Medium media = new DB2.Medium() { Type = m2.Type, Url = m2.Link, SourceSiteID = website.ID }; if (m2.Type == "image") { var request = WebRequest.Create(m2.Link); using (var response = request.GetResponse()) using (var stream = response.GetResponseStream()) using (var b = System.Drawing.Bitmap.FromStream(stream)) { media.Width = b.Width; media.Height = b.Height; } } db.Media.AddObject(media); db.SaveChanges(); Debug.Print("Media element: " + m2.Link); } catch { break; } } websiteUrl = website.Url; } catch { Debug.Print("Failed to scrape page: " + uniqueUrl); } } db.SaveChanges(); }
/// <summary> /// Deprecated Method for adding a new object to the TweetWebsites EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. /// </summary> public void AddToTweetWebsites(TweetWebsite tweetWebsite) { base.AddObject("TweetWebsites", tweetWebsite); }
private static void associateTweetsAndSites() { var db = GetDataContext(); foreach (var website in db.Websites.ToList()) { TweetWebsite tweetWebsite = new TweetWebsite() { TweetID = website.TweetID, WebsiteID = website.ID }; db.TweetWebsites.AddObject(tweetWebsite); db.SaveChanges(); } }
/// <summary> /// Create a new TweetWebsite object. /// </summary> /// <param name="id">Initial value of the ID property.</param> /// <param name="tweetID">Initial value of the TweetID property.</param> /// <param name="websiteID">Initial value of the WebsiteID property.</param> public static TweetWebsite CreateTweetWebsite(global::System.Int32 id, global::System.Int32 tweetID, global::System.Int32 websiteID) { TweetWebsite tweetWebsite = new TweetWebsite(); tweetWebsite.ID = id; tweetWebsite.TweetID = tweetID; tweetWebsite.WebsiteID = websiteID; return tweetWebsite; }