public static void ScrappingData() { DataTable rssData = new DataTable(); BLLFeed objBLLFeed = new BLLFeed(); rssData = objBLLFeed.GetFeedUrls(2); if (rssData.Rows.Count > 0) { foreach (DataRow dataRow in rssData.Rows) { ModelFeed modelFeed = new ModelFeed(); modelFeed.FeedURL = dataRow["FeedURL"].ToString(); modelFeed.FeedTitlePath = dataRow["FeedTitlePath"].ToString(); modelFeed.FeedLastUpdateDateTimePath = dataRow["FeedLastUpdateDateTimePath"].ToString(); modelFeed.FeedCoverImagePath = dataRow["FeedCoverImagePath"].ToString(); modelFeed.FeedItemsPath = dataRow["FeedItemsPath"].ToString(); modelFeed.FeedShortDescPath = dataRow["FeedShortDescPath"].ToString(); modelFeed.FeedDetailPageURLPath = dataRow["FeedDetailPageURLPath"].ToString(); modelFeed.FeedPubDatePath = dataRow["FeedPubDatePath"].ToString(); modelFeed.FeedImagePath = dataRow["FeedImagePath"].ToString(); modelFeed.FeedDetailPageImagePath = dataRow["FeedDetailPageImagePath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedPostDetailsPath"].ToString(); modelFeed.FeedCategoryPath = dataRow["FeedCategoryPath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedSubCategoryPath"].ToString(); List<ModelFeedsData> feedsDetailList = StartScrapper(modelFeed); } } }
public List<ModelFeed> GetFeedDataList(int page = 0) { List<ModelFeed> ModelFeedList = new List<ModelFeed>(); DataTable dt = new DataTable(); SqlParameter[] param = new SqlParameter[1]; param[0] = new SqlParameter("@FeedId", page); dt = DALCommon.GetDataUsingDataTable("[sp_Admin_GetFeedList]", param); if (dt.Rows.Count > 0) { foreach (DataRow item in dt.Rows) { ModelFeed objModelFeed = new ModelFeed(); objModelFeed.FeedId = Convert.ToDouble(item["FeedId"]); objModelFeed.FeedChannelName = Convert.ToString(item["FeedChannelName"]); objModelFeed.FeedCategory = Convert.ToString(item["FeedCategory"]); objModelFeed.FeedTitlePath = Convert.ToString(item["FeedTitlePath"]); objModelFeed.FeedImagePath = Convert.ToString(item["FeedImagePath"]); objModelFeed.FeedAddedDate = Convert.ToDateTime(item["FeedAddedDate"]); ModelFeedList.Add(objModelFeed); } } return ModelFeedList; }
public ActionResult AddRssLink(ModelFeed objModelFeed) { if (ModelState.IsValid) { try { if (Session[DALVariables.AdminUserNo] == null) { return RedirectToAction("Login", "Admin"); } bllFeed = new BLLFeed(); decimal result = 0; result = bllFeed.InsertFeedData(objModelFeed); if (result > 0) { return View(); } ModelState.AddModelError("", "No User Found!,Re-check login details"); return View(); } catch (Exception ex) { DALUtility.ErrorLog(ex.Message, "AdminController, Login"); } } else { ModelState.AddModelError("", "Check error of form; Please correct to continue!"); } return View(); }
public decimal InsertFeedData(ModelFeed objModelFeed) { SqlParameter[] param = new SqlParameter[15]; param[0] = new SqlParameter("@FeedChannelName", objModelFeed.FeedChannelName); param[1] = new SqlParameter("@FeedCategory", objModelFeed.FeedCategory); param[2] = new SqlParameter("@FeedURL", objModelFeed.FeedURL); param[3] = new SqlParameter("@FeedTitlePath", objModelFeed.FeedTitlePath); param[4] = new SqlParameter("@FeedCoverImagePath", objModelFeed.FeedCoverImagePath); param[5] = new SqlParameter("@FeedShortDescPath", objModelFeed.FeedShortDescPath); param[6] = new SqlParameter("@FeedDetailPageURLPath", objModelFeed.FeedDetailPageURLPath); param[7] = new SqlParameter("@FeedPubDatePath", objModelFeed.FeedPubDatePath); param[8] = new SqlParameter("@FeedImagePath", objModelFeed.FeedImagePath); param[9] = new SqlParameter("@FeedDetailPageImagePath", objModelFeed.FeedDetailPageImagePath); param[10] = new SqlParameter("@FeedPostDetailsPath", objModelFeed.FeedPostDetailsPath); param[11] = new SqlParameter("@FeedCategoryPath", objModelFeed.FeedCategoryPath); param[12] = new SqlParameter("@FeedSubCategoryPath", objModelFeed.FeedSubCategoryPath); param[13] = new SqlParameter("@FeedAddedBy", Convert.ToDecimal("1")); //param[14] = new SqlParameter("@FeedStatus", objModelFeed.FeedStatus); param[14] = new SqlParameter("@FeedStatus", 1); return DALCommon.ExecuteNonQueryReturnIdentity("sp_Merchant_InsertFeedData", param); }
private static List<ModelFeedsData> StartScrapper(ModelFeed feedsSetting) { List<ModelFeedsData> FeedList = new List<ModelFeedsData>(); ModelFeedsData FeedData = new ModelFeedsData(); try { CommunicationResponse resp = CommunicationResponse.GetPage(feedsSetting.FeedURL); if (resp.IsValid) { //XmlDocument RSSXml = new XmlDocument(); //RSSXml.Load(url); //XmlDocument xd = resp.ToXml(); XmlDocument xd = new XmlDocument(); xd.Load(feedsSetting.FeedURL); if (xd != null) { //XmlNodeList nodes1 = RSSXml.SelectNodes("rss/channel/item"); //XmlNode rssSubNode = nodes1[0].SelectSingleNode("/span/feed/title"); //FeedData.MainTitle = rssSubNode != null ? rssSubNode.InnerText : ""; //XmlDocument xdInner = new XmlDocument(); //xdInner.LoadXml(xd.InnerXml); //XmlNodeList rssSubNodeXml = xdInner.SelectNodes("//channel/item"); //XmlNode rssSubNode = xdInner.SelectSingleNode("//channel/title"); //if (rssSubNode != null) //{ // FeedData.MainTitle = rssSubNode.InnerText; //} //else //{ // FeedData.MainTitle = String.Empty; //} ModelFeedDetail modelfeedDetail = new ModelFeedDetail(); XmlElement xmlDoc = xd.DocumentElement; XmlNodeList xmlNodeLists = xd.SelectNodes(feedsSetting.FeedItemsPath); if (xmlNodeLists.Count > 0 && xmlNodeLists != null) { foreach (XmlNode item in xmlNodeLists) { if (!String.IsNullOrEmpty(feedsSetting.FeedTitlePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedTitlePath) != null) { //getting item Title from RSS Feed Page modelfeedDetail.PostTitle = item.SelectSingleNode(feedsSetting.FeedTitlePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedShortDescPath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedShortDescPath) != null) { //getting short description from RSS Feed Page modelfeedDetail.ShortDescription = item.SelectSingleNode(feedsSetting.FeedShortDescPath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedPubDatePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedPubDatePath) != null) { //getting item publish date from RSS Feed Page modelfeedDetail.PublishDate = item.SelectSingleNode(feedsSetting.FeedPubDatePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedImagePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedImagePath) != null) { //getting item image from RSS Feed Page modelfeedDetail.RssImage = item.SelectSingleNode(feedsSetting.FeedImagePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedDetailPageURLPath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath) != null) { //getting Detail Page URL from Feed modelfeedDetail.DetailPageURL = item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath).InnerText; } //If Detail page URL not empty Then if (!String.IsNullOrEmpty(modelfeedDetail.DetailPageURL)) { //web Request on Detial page URL //XmlDocument xdDetailPage = new XmlDocument(); //xdDetailPage.Load(modelfeedDetail.DetailPageURL); CommunicationResponse GetdetailPageResponse = CommunicationResponse.GetPage(modelfeedDetail.DetailPageURL); XmlDocument xdDetailPage = GetdetailPageResponse.ToXml(); if (xdDetailPage != null) { //Detail Page Will screape here } } } } } /* XmlElement xmlDoc = xd.DocumentElement; foreach (XmlElement node in xd.DocumentElement) { if (node.Name == "title") { string feedTile = node.InnerText; } if (node.Name == "updated") { string feedcategory = node.InnerText; } XmlNodeList xmlNodeList = node.SelectNodes("/entry"); if (xmlNodeList.Count > 0) { } //if (!String.IsNullOrEmpty(node.Attributes[0].Value)) //{ // string feedTile = node["title"].Value; //} //if (!String.IsNullOrEmpty(node["updated"].Value)) //{ //string feedcategory = node["updated"].Value; //} } string xmlNode = xmlDoc.ParentNode.InnerText; XmlNodeList xmlNodeLists = xd.SelectNodes("/feed/entry"); //XmlNode titleNode = xmlNodeList.Count.ToString(); //string title = xmlNodeLists.Count.ToString(); if (xmlNodeLists != null) { //FeedData.MainTitle = rssSubNodetitle.InnerText; } XmlNode rssSubNodeCategory = xd.SelectSingleNode("//feed/category[1]"); if (rssSubNodeCategory != null) { FeedData.MainCategory = rssSubNodeCategory.InnerText; } XmlNode rssSubNodeLogo = xd.SelectSingleNode("//feed/logo"); if (rssSubNodeLogo != null) { FeedData.CoverImage = rssSubNodeLogo.InnerText; } XmlNodeList nodes = xd.SelectNodes("//entry"); if (nodes != null) { ModelFeedDetail modelfeedDetail = new ModelFeedDetail(); foreach (XmlNode item in nodes) { modelfeedDetail.FeedDetailId = item.SelectSingleNode("/dc:identifier").InnerText; modelfeedDetail.PostTitle = item.SelectSingleNode("/title").InnerText; modelfeedDetail.PublishDate = item.SelectSingleNode("/published").InnerText; //optional modelfeedDetail.RssImage = item.SelectSingleNode("").InnerText; modelfeedDetail.ShortDescription = item.SelectSingleNode("/summary").InnerText; modelfeedDetail.Category = item.SelectSingleNode("/category ").Attributes["label"].Value; //for send request on Detail page to get Post Details if (item.SelectSingleNode("/link").Attributes != null) { modelfeedDetail.DetailPageURL = item.SelectSingleNode("/link").Attributes["href"].Value; } else { modelfeedDetail.DetailPageURL = item.SelectSingleNode("/link").InnerText; } CommunicationResponse respDetailPage = CommunicationResponse.GetPage(modelfeedDetail.DetailPageURL); if (resp.IsValid) { XmlDocument xdDetailPage = respDetailPage.ToXml(); modelfeedDetail.DetailPageImage = xdDetailPage.SelectSingleNode("//*[@id='page']/div/div[2]/div/div[1]/div[1]/div[2]/figure[1]/span/img").Attributes["src"].Value; modelfeedDetail.DetailPagePostDetail = xdDetailPage.SelectSingleNode("//*[@id='page']/div/div[2]/div/div[1]/div[1]/div[2]/p").InnerText; } FeedData.FeedDetail.Add(modelfeedDetail); } } */ } } return FeedList; } catch (Exception ex) { throw ex; } }
public static void ScrappingData() { DataTable rssData = new DataTable(); BLLFeed objBLLFeed = new BLLFeed(); rssData = objBLLFeed.GetFeedUrls(0); if (rssData.Rows.Count > 0) { foreach (DataRow dataRow in rssData.Rows) { ModelFeed modelFeed = new ModelFeed(); modelFeed.FeedURL = dataRow["FeedURL"].ToString(); modelFeed.FeedTitlePath = dataRow["FeedTitlePath"].ToString(); modelFeed.FeedLastUpdateDateTimePath = dataRow["FeedLastUpdateDateTimePath"].ToString(); modelFeed.FeedCoverImagePath = dataRow["FeedCoverImagePath"].ToString(); modelFeed.FeedItemsPath = dataRow["FeedItemsPath"].ToString(); modelFeed.FeedShortDescPath = dataRow["FeedShortDescPath"].ToString(); modelFeed.FeedDetailPageURLPath = dataRow["FeedDetailPageURLPath"].ToString(); modelFeed.FeedPubDatePath = dataRow["FeedPubDatePath"].ToString(); modelFeed.FeedImagePath = dataRow["FeedImagePath"].ToString(); modelFeed.FeedDetailPageImagePath = dataRow["FeedDetailPageImagePath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedPostDetailsPath"].ToString(); modelFeed.FeedCategoryPath = dataRow["FeedCategoryPath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedSubCategoryPath"].ToString(); modelFeed.FeedURL = "http://www.espncricinfo.com/rss/content/story/feeds/6.xml"; //List<ModelFeedsData> feedsDetailList = StartScrapper(modelFeed); GetRSSFeeds(modelFeed.FeedURL); } } }
private static void StartSafeScrappedData(ModelTwitterFeeds modelTwitterFeeds, ModelFeed modelFeed) { try { CommunicationResponse resp = CommunicationResponse.GetPage(modelFeed.FeedURL); if (resp.IsValid) { //XmlDocument RSSXml = new XmlDocument(); //RSSXml.Load(url); XmlDocument xd = resp.ToXml(); //XmlDocument xd = new XmlDocument(); xd.Load(modelFeed.FeedURL); if (xd != null) { ModelFeedDetail modelfeedDetail = new ModelFeedDetail(); XmlElement xmlDoc = xd.DocumentElement; /*XmlNodeList xmlNodeLists = xd.SelectNodes(feedsSetting.FeedItemsPath); if (xmlNodeLists.Count > 0 && xmlNodeLists != null) { foreach (XmlNode item in xmlNodeLists) { if (!String.IsNullOrEmpty(feedsSetting.FeedTitlePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedTitlePath) != null) { //getting item Title from RSS Feed Page modelfeedDetail.PostTitle = item.SelectSingleNode(feedsSetting.FeedTitlePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedShortDescPath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedShortDescPath) != null) { //getting short description from RSS Feed Page modelfeedDetail.ShortDescription = item.SelectSingleNode(feedsSetting.FeedShortDescPath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedPubDatePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedPubDatePath) != null) { //getting item publish date from RSS Feed Page modelfeedDetail.PublishDate = item.SelectSingleNode(feedsSetting.FeedPubDatePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedImagePath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedImagePath) != null) { //getting item image from RSS Feed Page modelfeedDetail.RssImage = item.SelectSingleNode(feedsSetting.FeedImagePath).InnerText; } } if (!String.IsNullOrEmpty(feedsSetting.FeedDetailPageURLPath.Trim())) { if (item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath) != null) { //getting Detail Page URL from Feed modelfeedDetail.DetailPageURL = item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath).InnerText; } //If Detail page URL not empty Then if (!String.IsNullOrEmpty(modelfeedDetail.DetailPageURL)) { //web Request on Detial page URL //XmlDocument xdDetailPage = new XmlDocument(); //xdDetailPage.Load(modelfeedDetail.DetailPageURL); CommunicationResponse GetdetailPageResponse = CommunicationResponse.GetPage(modelfeedDetail.DetailPageURL); XmlDocument xdDetailPage = GetdetailPageResponse.ToXml(); if (xdDetailPage != null) { //Detail Page Will screape here } } } } }*/ } } } catch (Exception ex) { throw ex; } }
public static void ScrappingData() { //string xpath = "//div[@class='story-body'] //p[position()>=1]"; //string cssSelector = "div.story-body p:not(:first-of-type)"; //ScrapeDetailfromPage("http://www.bbc.com/arabic/scienceandtech/2015/12/151219_usa_drug_dose_deaths?ocid=socialflow_twitter", xpath); //string SoundCloudURL = "https://soundcloud.com/bbc-arabic/global-news-beat-prisoners-punishing-loaf-food"; //string cssSelector = "//div[@class='sc-type-small'] //p"; //ScrapeDetailfromPage(SoundCloudURL, cssSelector); DataTable rssData = new DataTable(); BLLFeed objBLLFeed = new BLLFeed(); rssData = objBLLFeed.GetFeedUrls(2); List<ModelFeedsDetailScrappingHelper> listScrappingHelper = new List<ModelFeedsDetailScrappingHelper>(); listScrappingHelper = objBLLFeed.GetScrapperHostsData(); /*>>> API Creadentials <<<<//// >> Under Informnation is creating from Personal Account Information API >> For any change in this credentials Login Account : [email protected] >> API Name : News Get Application >> Please don't change any word without permission from following API Admin*/ string _consumerKey = "cvRCCaqLUlM9SyolFwYEQQ2uZ"; string _consumerSecret = "sVvOgCPwuzXo37v4qjAriGbftwEefMC9xNadPQLTsoOkeqiJ8C"; string _accessToken = "65042389-YQ3jLfP1RWu9Q7So9VAj3Rc3J9oMB0Suuv0jlioAN"; string _accessTokenSecret = "yHMVHihkHAaCIOJi2M9WEOq4fpODY1a0hdS92J9OZEJEL"; // API access Data End var service = new TwitterService(_consumerKey, _consumerSecret); //var tweets = new ListTweetsOnHomeTimelineOptions(); service.AuthenticateWith(_accessToken, _accessTokenSecret); if (rssData.Rows.Count > 0) { foreach (DataRow dataRow in rssData.Rows) { ModelFeed modelFeed = new ModelFeed(); modelFeed.FeedURL = dataRow["FeedURL"].ToString(); modelFeed.FeedTitlePath = dataRow["FeedTitlePath"].ToString(); modelFeed.FeedLastUpdateDateTimePath = dataRow["FeedLastUpdateDateTimePath"].ToString(); modelFeed.FeedCoverImagePath = dataRow["FeedCoverImagePath"].ToString(); modelFeed.FeedItemsPath = dataRow["FeedItemsPath"].ToString(); modelFeed.FeedShortDescPath = dataRow["FeedShortDescPath"].ToString(); modelFeed.FeedDetailPageURLPath = dataRow["FeedDetailPageURLPath"].ToString(); modelFeed.FeedPubDatePath = dataRow["FeedPubDatePath"].ToString(); modelFeed.FeedImagePath = dataRow["FeedImagePath"].ToString(); modelFeed.FeedDetailPageImagePath = dataRow["FeedDetailPageImagePath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedPostDetailsPath"].ToString(); modelFeed.FeedCategoryPath = dataRow["FeedCategoryPath"].ToString(); modelFeed.FeedPostDetailsPath = dataRow["FeedSubCategoryPath"].ToString(); modelFeed.FeedId = Convert.ToDouble(dataRow["FeedId"].ToString()); //double TwitterPageID = Convert.ToDouble(dataRow["TwitterPageId"].ToString()); string LastMaxTweetPostedId = String.Empty; if (!String.IsNullOrEmpty(dataRow["MaxPostedId"].ToString()) && !Convert.IsDBNull(dataRow["MaxPostedId"])) { LastMaxTweetPostedId = dataRow["MaxPostedId"].ToString(); } //Screen Name modelFeed.FeedChannelName = dataRow["FeedChannelName"].ToString(); //modelFeed.FeedURL = "http://www.espncricinfo.com/rss/content/story/feeds/6.xml"; //List<ModelFeedsData> feedsDetailList = StartScrapper(modelFeed); //modelFeed.FeedChannelName = "cnnarabic"; GetTwitterTweetsWithDetails(service, modelFeed.FeedId, LastMaxTweetPostedId, modelFeed, listScrappingHelper); } } }
public static void GetTwitterTweetsWithDetails(TwitterService service, double feedId, string LastMaxTweetPostedId, ModelFeed modelFeedScrapperPath, List<ModelFeedsDetailScrappingHelper> listScrappingHelper) { ModelTwitterFeeds modelTwitterFeeds = new ModelTwitterFeeds(); ModelTwitterFeedsDetails modelTwitterFeedsDetail = new ModelTwitterFeedsDetails(); ModelFeedMultimedia modelModelFeedMultimedia = new ModelFeedMultimedia(); List<ModelFeedMultimedia> lstmodelModelFeedImage = new List<ModelFeedMultimedia>(); List<ModelTwitterFeedsDetails> lstModelTwitterFeedsDetail = new List<ModelTwitterFeedsDetails>(); BLLFeed objBLLFeed = new BLLFeed(); string DetailPageURL = String.Empty; try { //TwitterAccount user = new TwitterAccount. //TwitterUser twitterUser = //>>GET OTHER USER TIMELINE//BeginListTweetsOnHomeTimeline var User_OptionInit = new ListTweetsOnUserTimelineOptions { ScreenName = modelFeedScrapperPath.FeedChannelName, Count = 20, ExcludeReplies = true }; var User_Tweets = service.ListTweetsOnUserTimeline(User_OptionInit); /* var profilebanner = service.GetUserProfile(new GetUserProfileOptions() { IncludeEntities = true, SkipStatus = true }); if (profilebanner != null) { //profilebanner.ContributorsEnabled //profilebanner.CreatedDate modelTwitterFeeds.UserPageDesc = profilebanner.Description; //profilebanner.FavouritesCount; modelTwitterFeeds.UserPageFollowers = profilebanner.FollowersCount; //profilebanner.FollowRequestSent; //profilebanner.FriendsCount; modelTwitterFeeds.UserPageId = profilebanner.Id; //profilebanner.IsDefaultProfile; //profilebanner.IsGeoEnabled; //profilebanner.IsProfileBackgroundTiled; //profilebanner.IsProtected; //profilebanner.IsTranslator; //profilebanner.IsVerified; modelTwitterFeeds.UserPageLanguage = profilebanner.Language; //profilebanner.ListedCount; //profilebanner.Location; modelTwitterFeeds.UserPageTitle = profilebanner.Name; //profilebanner.ProfileBackgroundColor; //profilebanner.ProfileBackgroundImageUrl; //profilebanner.ProfileBackgroundImageUrlHttps; modelTwitterFeeds.UserPageLogoImage = profilebanner.ProfileImageUrl; //profilebanner.ProfileImageUrlHttps; //profilebanner.ProfileLinkColor; //profilebanner.ProfileSidebarBorderColor; //profilebanner.ProfileSidebarFillColor; //profilebanner.ProfileTextColor; //profilebanner.RawSource; modelTwitterFeeds.UserScreenName = profilebanner.ScreenName; //profilebanner.Status; //profilebanner.StatusesCount; //profilebanner.TimeZone; //JObject o = new JObject(new JProperty("Name", "John Smith"),new JProperty("BirthDate", new DateTime(1983, 3, 20))); JObject obj = (JObject)JsonConvert.DeserializeObject(profilebanner.RawSource); if (obj != null) { if (obj["user"]["profile_banner_url"] != null) { modelTwitterFeeds.UserPageCoverImageURL = (string)obj["user"]["profile_banner_url"]; } } } */ string TweetText = String.Empty; int tweetCountFirstRun = 0; foreach (var tweet in User_Tweets) { if (!String.IsNullOrEmpty(LastMaxTweetPostedId)) { if (LastMaxTweetPostedId.ToLower().Trim() == tweet.IdStr.ToLower().Trim()) { break; } } modelTwitterFeedsDetail = new ModelTwitterFeedsDetails(); modelModelFeedMultimedia = new ModelFeedMultimedia(); var userDetail = tweet.User; if (tweet != null) { if (tweetCountFirstRun == 0) { if (userDetail != null) { modelTwitterFeeds.FeedId = feedId; modelTwitterFeeds.UserPageId = tweet.User.Id; modelTwitterFeeds.UserPageTitle = tweet.User.Name; modelTwitterFeeds.UserScreenName = tweet.User.ScreenName; modelTwitterFeeds.UserPageDesc = tweet.User.Description; modelTwitterFeeds.UserPageLanguage = tweet.User.Language; modelTwitterFeeds.UserPageFollowers = tweet.User.FollowersCount; //modelTwitterFeeds.UserPageCoverImageURL = tweet.User.ProfileBackgroundImageUrl; modelTwitterFeeds.UserPageLogoImage = tweet.User.ProfileImageUrl; //JObject o = new JObject(new JProperty("Name", "John Smith"),new JProperty("BirthDate", new DateTime(1983, 3, 20))); JObject obj = (JObject)JsonConvert.DeserializeObject(tweet.RawSource); if (obj != null) { if (obj["user"]["profile_banner_url"] != null) { modelTwitterFeeds.UserPageCoverImageURL = (string)obj["user"]["profile_banner_url"]; } } } tweetCountFirstRun++; } var st1 = tweet.Text.ToSafeString(); //string TweetText = tweet.Text; //modelTwitterFeedsDetail.TwitterPageId; modelTwitterFeedsDetail.FeedPostedtId = tweet.Id; modelTwitterFeedsDetail.FeedText = String.Empty; modelTwitterFeedsDetail.FeedLanguage = tweet.Language; modelTwitterFeedsDetail.FeedPostDate = tweet.CreatedDate; modelTwitterFeedsDetail.FeedTextDetail = ""; var st2 = tweet.Source; //string var st3 = tweet.TextAsHtml; //string var st4 = tweet.TextDecoded; //string var st5 = tweet.RetweetedStatus; //TwitterStatus var st6 = tweet.RetweetCount; //int var st7 = tweet.RawSource; //string var st8 = tweet.Place; //TwitterPlace var st9 = tweet.Location; //TwitterGeoLocation var st10 = tweet.Language; //string var st11 = tweet.IsTruncated; //bool var st12 = tweet.IsRetweeted; //bool var st13 = tweet.IsPossiblySensitive; //bool is nullable var st14 = tweet.IsFavorited; //bool var st15 = tweet.InReplyToUserId; //long is nullable var st16 = tweet.InReplyToStatusId; //long is nullable var st17 = tweet.InReplyToScreenName; //string var st18 = tweet.IdStr; //string var st19 = tweet.Id; //long var st20 = tweet.FavoriteCount; //int var st21 = tweet.ExtendedEntities; //TwitterExtendedEntities var st22 = tweet.Entities; //TwitterEntities var twitterEntities = new TwitterEntities(); twitterEntities = tweet.Entities; IList<TwitterUrl> twitterUrl = twitterEntities.Urls; //List<TwitterUrl> if (twitterUrl != null && twitterUrl.Count > 0) { foreach (var url in twitterUrl) { var url1 = url.DisplayUrl; //string var url2 = url.EndIndex; //int var url3 = url.EntityType; //TwitterEntityType var entityType = url3; //4 Entity Types are Defined //TwitterEntityType.HashTag; //0 //TwitterEntityType.Mention; //1 //TwitterEntityType.Url; //2 //TwitterEntityType.Media; //3 var url4 = url.ExpandedValue; //string var url5 = url.Indices; //IList<int> var url6 = url.StartIndex; //int var url7 = url.Value; //string //MODEL DATA //Tweet Detail Page URL if (!String.IsNullOrEmpty(url.ExpandedValue)) { modelTwitterFeedsDetail.FeedDetailPageURL = url.ExpandedValue.Trim(); } else if (!String.IsNullOrEmpty(url.Value)) { if (String.IsNullOrEmpty(modelTwitterFeedsDetail.FeedDetailPageURL)) { modelTwitterFeedsDetail.FeedDetailPageURL = url.Value.Trim(); } } TweetText = TweetText.Trim().Replace(url.Value, String.Empty).Trim(); } } IList<TwitterMedia> twitterMedia = twitterEntities.Media; //List<TwitterMedia> foreach (var media in twitterMedia) { var media1 = media.DisplayUrl; //string var media2 = media.EndIndex; //int var media3 = media.EntityType; //TwitterEntity var media4 = media.ExpandedUrl; //string var media5 = media.Id; //long var media6 = media.IdAsString; //string var media7 = media.Indices; //IList<int> var media8 = media.MediaType; //TwitterMediaType var twitterMediaType = media8; /*Three Types of MediaType TwitterMediaType.Photo; //0 TwitterMediaType.Video; //1 TwitterMediaType.AnimatedGif; //2 */ var media9 = media.MediaUrl; //string var media10 = media.MediaUrlHttps; //string //var FeedImageURLhttp = media.MediaUrl; for Http Image //var FeedImageURLhttps = media.MediaUrlHttps; for Https Image var media11 = media.Sizes; //TwitterMediaSizes var twitterMediaSizes = media11; //media11.Large //media11.Medium //media11.Small //media11.Thumb var media12 = media.StartIndex; //int var media13 = media.Url; //string //MODEL DATA string tweetMultimediaURL = !String.IsNullOrEmpty(media.Url.Trim()) ? media.Url.Trim() : String.Empty; TweetText = TweetText.Trim().Replace(media.Url, String.Empty).Trim(); if (!String.IsNullOrEmpty(tweetMultimediaURL)) { modelModelFeedMultimedia.MultiMediaURL = tweetMultimediaURL; modelModelFeedMultimedia.MultiMediaType = Convert.ToInt32(media.MediaType); //Add model object to list //lstmodelModelFeedImage.Add(modelModelFeedMultimedia); modelTwitterFeedsDetail.FeedMultimediaList.Add(modelModelFeedMultimedia); } } IList<TwitterHashTag> twitterHashTag = twitterEntities.HashTags; //List<TwitterHashTag> foreach (var hashTag in twitterHashTag) { var ht1 = hashTag.EndIndex; var ht2 = hashTag.EntityType; var ht3 = hashTag.Indices; var ht4 = hashTag.StartIndex; var ht5 = hashTag.Text; // hashTagText for refine Tweet Text TweetText = TweetText.Trim().Replace("#" + hashTag.Text, string.Empty).Trim(); } TweetText = TweetText.Replace("\n", String.Empty).Trim(); var st23 = tweet.CreatedDate; //DateTime var st24 = tweet.Author; //ITweeter var ProfileImageURL = st24.ProfileImageUrl; var ss = st24.RawSource; var screenName = st24.ScreenName; modelTwitterFeedsDetail.FeedText = TweetText; //string DetailPageSelector = "div.story-body__inner p:not(:first-of-type)"; if (!String.IsNullOrEmpty(modelTwitterFeedsDetail.FeedDetailPageURL)) { Uri DetailPageURI = new Uri(modelTwitterFeedsDetail.FeedDetailPageURL); string DetailPageTextSelector = String.Empty; string DetailPageImageSelector = String.Empty; ModelFeedsDetailScrappingHelper rowDetailHelper = listScrappingHelper.Where(m => m.HostName.Contains(DetailPageURI.Host.Trim().ToLower())).FirstOrDefault(); if (rowDetailHelper != null) { DetailPageTextSelector = rowDetailHelper.DetailPagePath; DetailPageImageSelector = rowDetailHelper.DetailPageImagePath; } /* if (DetailPageURI.Host.Trim().ToLower().Contains("cnn")) { //bbc.com Text Details= //div[@class='story-body'] //p[position()>=1] //bbc.com img = //div[@class='story-body__inner'] //img[@class='js-image-replace'] //cnn = ////p[@dir='RTL']js-image-replace DetailPageTextSelector = "//div[contains(@class, 'article-left')] //p"; DetailPageImageSelector = "//div[contains(@class, 'gallery-big-images')] //img"; } else if (DetailPageURI.Host.Trim().ToLower().Contains("soundcloud")) { DetailPageTextSelector = "//div[contains(@class, 'sc-type-small')] //p"; DetailPageImageSelector = "//div[@class='article-left'] //p"; } */ HtmlDocument requestedDoc = ScrapeDetailfromPage(DetailPageURI.AbsoluteUri); if (requestedDoc != null) { //if Details Page Selector Text not found then not send request if (!String.IsNullOrEmpty(DetailPageTextSelector)) { string detailTextFromURL = ScrapeTextfromDetailPage(requestedDoc, DetailPageTextSelector); modelTwitterFeedsDetail.FeedTextDetail = detailTextFromURL; } //if default twitter post not have any multimedia type then if (modelTwitterFeedsDetail.FeedMultimediaList.Count <= 0) { //if Details Page Selector Image not found then not send request if (!String.IsNullOrEmpty(DetailPageImageSelector)) { modelTwitterFeedsDetail.FeedMultimediaList = ScrapeImageDetailfromPage(requestedDoc, DetailPageImageSelector); //if details page not found on detected tag & Tweet having Image and Embeded Video on Details page // Then we make a new request to scrape twitter create image from sperate static created request if (modelTwitterFeedsDetail.FeedMultimediaList.Count <= 0) { //Sample URI for Video Image get://https://twitter.com/cnnarabic/status/678548429530906624 //https://twitter.com/i/cards/tfw/v1/678585688783257600?cardname=summary_large_image //https://twitter.com/i/cards/tfw/v1/678585688783257600 //string StaticImageURI = "https://twitter.com/" + modelTwitterFeeds.UserScreenName.Trim() + "//status/" + modelTwitterFeedsDetail.FeedPostId; string StaticImageURI = "https://twitter.com/i/cards/tfw/v1/" + tweet.Id; HtmlDocument VideoImageRequestedDoc = ScrapeDetailfromPage(StaticImageURI); string staticImageSelector = "//div[contains(@class, 'SummaryCard-image')] //img"; modelTwitterFeedsDetail.FeedMultimediaList = ScrapeImageDetailfromPage(VideoImageRequestedDoc, staticImageSelector); } } } } } else { modelTwitterFeedsDetail.FeedTextDetail = String.Empty; } } lstModelTwitterFeedsDetail.Add(modelTwitterFeedsDetail); } modelTwitterFeeds.TwitterFeedDetails = lstModelTwitterFeedsDetail; objBLLFeed.InsertTwitterScrappedData(modelTwitterFeeds); //StartSafeScrappedData(modelTwitterFeeds, modelFeed); } catch (Exception ex) { throw ex; } }