public decimal InsertTwitterScrappedData(ModelTwitterFeeds objModelTwitterFeeds)
{
decimal TwitterPageId = 0;
decimal twitterFeedDetailId = 0;
try
{
SqlParameter[] paramTwitterFeeds = new SqlParameter[9];
paramTwitterFeeds[0] = new SqlParameter("@FeedId", objModelTwitterFeeds.FeedId);
paramTwitterFeeds[1] = new SqlParameter("@UserPageId", objModelTwitterFeeds.UserPageId);
paramTwitterFeeds[2] = new SqlParameter("@UserPageTitle", objModelTwitterFeeds.UserPageTitle);
paramTwitterFeeds[3] = new SqlParameter("@UserScreenName", objModelTwitterFeeds.UserScreenName);
paramTwitterFeeds[4] = new SqlParameter("@UserPageDesc", objModelTwitterFeeds.UserPageDesc);
paramTwitterFeeds[5] = new SqlParameter("@UserPageLanguage", objModelTwitterFeeds.UserPageLanguage);
paramTwitterFeeds[6] = new SqlParameter("@UserPageFollowers", objModelTwitterFeeds.UserPageFollowers);
paramTwitterFeeds[7] = new SqlParameter("@UserPageCoverImageURL", objModelTwitterFeeds.UserPageCoverImageURL);
paramTwitterFeeds[8] = new SqlParameter("@UserPageLogoImage", objModelTwitterFeeds.UserPageLogoImage);
TwitterPageId = DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedsData]", paramTwitterFeeds);
if (TwitterPageId > 0)
{
if (objModelTwitterFeeds.TwitterFeedDetails.Count > 0)
{
foreach (ModelTwitterFeedsDetails Tweet in objModelTwitterFeeds.TwitterFeedDetails)
{
SqlParameter[] paramFeedsDetails = new SqlParameter[7];
paramFeedsDetails[0] = new SqlParameter("@TwitterPageId", TwitterPageId);
paramFeedsDetails[1] = new SqlParameter("@TweetPostedId", Tweet.FeedPostedtId);
paramFeedsDetails[2] = new SqlParameter("@TweetPostedDate", Tweet.FeedPostDate);
paramFeedsDetails[3] = new SqlParameter("@TweetFeedDetailPageURL", Tweet.FeedDetailPageURL);
paramFeedsDetails[4] = new SqlParameter("@TweetShortText", Tweet.FeedText);
paramFeedsDetails[5] = new SqlParameter("@TweetDetailText", Tweet.FeedTextDetail);
if (Tweet.FeedMultimediaList != null && Tweet.FeedMultimediaList.Count > 0)
{
paramFeedsDetails[6] = new SqlParameter("@IsMultiMedia", true);
}
else
{
paramFeedsDetails[6] = new SqlParameter("@IsMultiMedia", false);
}
twitterFeedDetailId = DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedDataDetails]", paramFeedsDetails);
if (Tweet.FeedMultimediaList.Count > 0 && twitterFeedDetailId > 0)
{
foreach (ModelFeedMultimedia multimediaFile in Tweet.FeedMultimediaList)
{
SqlParameter[] parammultimediaFile = new SqlParameter[4];
parammultimediaFile[0] = new SqlParameter("@TwitterFeedDetailId", twitterFeedDetailId);
parammultimediaFile[1] = new SqlParameter("@MultiMediaType", multimediaFile.MultiMediaType);
parammultimediaFile[2] = new SqlParameter("@MultiMediaURL", multimediaFile.MultiMediaURL);
parammultimediaFile[3] = new SqlParameter("@MultiMediaExtension", multimediaFile.MultiMediaExtension);
DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedMultiMedia]", parammultimediaFile);
}
}
}
}
}
return twitterFeedDetailId;
}
catch (Exception ex)
{
DALUtility.ErrorLog(ex.Message, "BLLFeed, InsertTwitterScrappedData");
}
return twitterFeedDetailId;
}
public decimal InsertTwitterScrappedData(ModelTwitterFeeds objModelTwitterFeeds)
{
decimal TwitterPageId = 0;
decimal twitterFeedDetailId = 0;
try
{
SqlParameter[] paramTwitterFeeds = new SqlParameter[9];
paramTwitterFeeds[0] = new SqlParameter("@FeedId", objModelTwitterFeeds.FeedId);
paramTwitterFeeds[1] = new SqlParameter("@UserPageId", objModelTwitterFeeds.UserPageId);
paramTwitterFeeds[2] = new SqlParameter("@UserPageTitle", objModelTwitterFeeds.UserPageTitle);
paramTwitterFeeds[3] = new SqlParameter("@UserScreenName", objModelTwitterFeeds.UserScreenName);
paramTwitterFeeds[4] = new SqlParameter("@UserPageDesc", objModelTwitterFeeds.UserPageDesc);
paramTwitterFeeds[5] = new SqlParameter("@UserPageLanguage", objModelTwitterFeeds.UserPageLanguage);
paramTwitterFeeds[6] = new SqlParameter("@UserPageFollowers", objModelTwitterFeeds.UserPageFollowers);
paramTwitterFeeds[7] = new SqlParameter("@UserPageCoverImageURL", objModelTwitterFeeds.UserPageCoverImageURL);
paramTwitterFeeds[8] = new SqlParameter("@UserPageLogoImage", objModelTwitterFeeds.UserPageLogoImage);
TwitterPageId = DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedsData]", paramTwitterFeeds);
if (TwitterPageId > 0)
{
if (objModelTwitterFeeds.TwitterFeedDetails.Count > 0)
{
foreach (ModelTwitterFeedsDetails Tweet in objModelTwitterFeeds.TwitterFeedDetails)
{
SqlParameter[] paramFeedsDetails = new SqlParameter[7];
paramFeedsDetails[0] = new SqlParameter("@TwitterPageId", TwitterPageId);
paramFeedsDetails[1] = new SqlParameter("@TweetPostedId", Tweet.FeedPostedtId);
paramFeedsDetails[2] = new SqlParameter("@TweetPostedDate", Tweet.FeedPostDate);
paramFeedsDetails[3] = new SqlParameter("@TweetFeedDetailPageURL", Tweet.FeedDetailPageURL);
paramFeedsDetails[4] = new SqlParameter("@TweetShortText", Tweet.FeedText);
paramFeedsDetails[5] = new SqlParameter("@TweetDetailText", Tweet.FeedTextDetail);
if (Tweet.FeedMultimediaList != null && Tweet.FeedMultimediaList.Count > 0)
{
paramFeedsDetails[6] = new SqlParameter("@IsMultiMedia", true);
}
else
{
paramFeedsDetails[6] = new SqlParameter("@IsMultiMedia", false);
}
twitterFeedDetailId = DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedDataDetails]", paramFeedsDetails);
if (Tweet.FeedMultimediaList.Count > 0 && twitterFeedDetailId > 0)
{
foreach (ModelFeedMultimedia multimediaFile in Tweet.FeedMultimediaList)
{
SqlParameter[] parammultimediaFile = new SqlParameter[4];
parammultimediaFile[0] = new SqlParameter("@TwitterFeedDetailId", twitterFeedDetailId);
parammultimediaFile[1] = new SqlParameter("@MultiMediaType", multimediaFile.MultiMediaType);
parammultimediaFile[2] = new SqlParameter("@MultiMediaURL", multimediaFile.MultiMediaURL);
parammultimediaFile[3] = new SqlParameter("@MultiMediaExtension", multimediaFile.MultiMediaExtension);
DALCommon.ExecuteNonQueryReturnIdentity("[sp_Admin_InsertTwitterFeedMultiMedia]", parammultimediaFile);
}
}
}
}
}
return(twitterFeedDetailId);
}
catch (Exception ex)
{
DALUtility.ErrorLog(ex.Message, "BLLFeed, InsertTwitterScrappedData");
}
return(twitterFeedDetailId);
}
public static List<ModelTwitterFeedsDetails> GetTwitterTweets(string ScreenName)
{
ModelTwitterFeeds modelTwitterFeeds = new ModelTwitterFeeds();
List<ModelTwitterFeedsDetails> lstModelTwitterFeedsDetail = new List<ModelTwitterFeedsDetails>();
try
{
/*>>> API Creadentials <<<<////
>> Under Informnation is creating from Personal Account Information API
>> For any change in this credentials Login Account : [email protected]
>> API Name : News Get Application
>> Please don't change any word without permission from following API Admin*/
string _consumerKey = "cvRCCaqLUlM9SyolFwYEQQ2uZ";
string _consumerSecret = "sVvOgCPwuzXo37v4qjAriGbftwEefMC9xNadPQLTsoOkeqiJ8C";
string _accessToken = "65042389-YQ3jLfP1RWu9Q7So9VAj3Rc3J9oMB0Suuv0jlioAN";
string _accessTokenSecret = "yHMVHihkHAaCIOJi2M9WEOq4fpODY1a0hdS92J9OZEJEL";
// API access Data End
var service = new TwitterService(_consumerKey, _consumerSecret);
//var tweets = new ListTweetsOnHomeTimelineOptions();
service.AuthenticateWith(_accessToken, _accessTokenSecret);
IAsyncResult result = service.BeginListTweetsOnHomeTimeline(new ListTweetsOnHomeTimelineOptions());
IEnumerable<TwitterStatus> tweets = service.EndListTweetsOnHomeTimeline(result);
foreach (var tweet in tweets)
{
if (tweets != null)
{
modelTwitterFeeds.FeedId = 0;
modelTwitterFeeds.UserPageId = 0;
modelTwitterFeeds.UserPageTitle = String.Empty;
modelTwitterFeeds.UserScreenName = String.Empty;
modelTwitterFeeds.UserPageLanguage = String.Empty;
modelTwitterFeeds.UserPageFollowers = 0;
modelTwitterFeeds.UserPageCoverImageURL = String.Empty;
modelTwitterFeeds.UserPageLogoImage = String.Empty;
}
var st1 = tweet.Text; //string
var st2 = tweet.Source; //string
var st3 = tweet.TextAsHtml; //string
var st4 = tweet.TextDecoded; //string
var st5 = tweet.RetweetedStatus; //TwitterStatus
var st6 = tweet.RetweetCount; //int
var st7 = tweet.RawSource; //string
var st8 = tweet.Place; //TwitterPlace
var st9 = tweet.Location; //TwitterGeoLocation
var st10 = tweet.Language; //string
var st11 = tweet.IsTruncated; //bool
var st12 = tweet.IsRetweeted; //bool
var st13 = tweet.IsPossiblySensitive; //bool is nullable
var st14 = tweet.IsFavorited; //bool
var st15 = tweet.InReplyToUserId; //long is nullable
var st16 = tweet.InReplyToStatusId; //long is nullable
var st17 = tweet.InReplyToScreenName; //string
var st18 = tweet.IdStr; //string
var st19 = tweet.Id; //long
var st20 = tweet.FavoriteCount; //int
var st21 = tweet.ExtendedEntities; //TwitterExtendedEntities
var st22 = tweet.Entities; //TwitterEntities
var st23 = tweet.CreatedDate; //DateTime
var st24 = tweet.Author; //ITweeter
}
//TwitterAccount user = new TwitterAccount.
//TwitterUser twitterUser =
//>>GET OTHER USER TIMELINE//BeginListTweetsOnHomeTimeline
TwitterService t_service = new TwitterService(_consumerKey, _consumerSecret);
t_service.AuthenticateWith(_accessToken, _accessTokenSecret);
var t_options = new ListTweetsOnHomeTimelineOptions();
t_options.ExcludeReplies = true;
var t_tweets = t_service.ListTweetsOnHomeTimeline(t_options);
//ListTweetsOnSpecifiedUserTimeline
string ScreenNameBBCArabic = "BBCArabic";
string ScreenNameCNNArabic = "cnnarabic";
string ScreenName1 = "garbo_speaks";
var User_OptionInit = new ListTweetsOnUserTimelineOptions { ScreenName = ScreenNameCNNArabic, Count = 200, ExcludeReplies = true };
var User_Tweets = service.ListTweetsOnUserTimeline(User_OptionInit);
string TweetText = String.Empty;
foreach (var tweet in User_Tweets)
{
var userDetail = tweet.User;
if (false)
{
var us1 = tweet.User.ContributorsEnabled; //bool?
var us2 = tweet.User.CreatedDate;
var us3 = tweet.User.Description;
var us4 = tweet.User.FavouritesCount;
var us5 = tweet.User.FollowersCount;
var us6 = tweet.User.FollowRequestSent;
var us7 = tweet.User.FriendsCount;
var us8 = tweet.User.Id;
var us9 = tweet.User.IsDefaultProfile;
var us10 = tweet.User.IsGeoEnabled;
var us11 = tweet.User.IsProfileBackgroundTiled;
var us12 = tweet.User.IsProtected;
var us13 = tweet.User.IsTranslator;
var us14 = tweet.User.IsVerified;
var us15 = tweet.User.Language;
var us16 = tweet.User.ListedCount;
var us17 = tweet.User.Location;
var us18 = tweet.User.Name;
var us19 = tweet.User.ProfileBackgroundColor;
var us20 = tweet.User.ProfileBackgroundImageUrl;
var us21 = tweet.User.ProfileBackgroundImageUrlHttps;
var us22 = tweet.User.ProfileImageUrl;
var us23 = tweet.User.ProfileImageUrlHttps;
var us24 = tweet.User.ProfileLinkColor;
var us25 = tweet.User.ProfileSidebarBorderColor;
var us26 = tweet.User.ProfileSidebarFillColor;
var us27 = tweet.User.ProfileTextColor;
var us28 = tweet.User.RawSource;
var us29 = tweet.User.ScreenName;
var us30 = tweet.User.Status;
var us31 = tweet.User.StatusesCount;
var us32 = tweet.User.TimeZone;
var us33 = tweet.User.Url;
var us34 = tweet.User.UtcOffset;
}
var st1 = tweet.Text.ToSafeString(); //string
TweetText = tweet.Text;
var st2 = tweet.Source; //string
var st3 = tweet.TextAsHtml; //string
var st4 = tweet.TextDecoded; //string
var st5 = tweet.RetweetedStatus; //TwitterStatus
var st6 = tweet.RetweetCount; //int
var st7 = tweet.RawSource; //string
var st8 = tweet.Place; //TwitterPlace
var st9 = tweet.Location; //TwitterGeoLocation
var st10 = tweet.Language; //string
var st11 = tweet.IsTruncated; //bool
var st12 = tweet.IsRetweeted; //bool
var st13 = tweet.IsPossiblySensitive; //bool is nullable
var st14 = tweet.IsFavorited; //bool
var st15 = tweet.InReplyToUserId; //long is nullable
var st16 = tweet.InReplyToStatusId; //long is nullable
var st17 = tweet.InReplyToScreenName; //string
var st18 = tweet.IdStr; //string
var st19 = tweet.Id; //long
var st20 = tweet.FavoriteCount; //int
var st21 = tweet.ExtendedEntities; //TwitterExtendedEntities
var st22 = tweet.Entities; //TwitterEntities
var twitterEntities = new TwitterEntities();
twitterEntities = st22;
IList<TwitterUrl> twitterUrl = twitterEntities.Urls; //List<TwitterUrl>
foreach (var url in twitterUrl)
{
var url1 = url.DisplayUrl; //string
var url2 = url.EndIndex; //int
var url3 = url.EntityType; //TwitterEntityType
var entityType = url3;
//4 Entity Types are Defined
//TwitterEntityType.HashTag; //0
//TwitterEntityType.Mention; //1
//TwitterEntityType.Url; //2
//TwitterEntityType.Media; //3
var url4 = url.ExpandedValue; //string
var url5 = url.Indices; //IList<int>
var url6 = url.StartIndex; //int
var url7 = url.Value; //string
TweetText = TweetText.Trim().Replace(url.Value, String.Empty).Trim();
}
IList<TwitterMention> twitterMention = twitterEntities.Mentions; //List<TwitterMention>
foreach (var mention in twitterMention)
{
var url1 = mention.EndIndex; //int
var url2 = mention.EntityType; //int
var entityType = url2;
//4 Entity Types are Defined
//TwitterEntityType.HashTag; //0
//TwitterEntityType.Mention; //1
//TwitterEntityType.Url; //2
//TwitterEntityType.Media; //3
var url3 = mention.Id; //long
var url4 = mention.Indices; //IList<int>
var url5 = mention.Name; //string
var url6 = mention.ScreenName; //string
var url7 = mention.StartIndex; //int
}
IList<TwitterMedia> twitterMedia = twitterEntities.Media; //List<TwitterMedia>
foreach (var media in twitterMedia)
{
var media1 = media.DisplayUrl; //string
var media2 = media.EndIndex; //int
var media3 = media.EntityType; //TwitterEntity
var media4 = media.ExpandedUrl; //string
var media5 = media.Id; //long
var media6 = media.IdAsString; //string
var media7 = media.Indices; //IList<int>
var media8 = media.MediaType; //TwitterMediaType
var twitterMediaType = media8;
/*Three Types of MediaType
TwitterMediaType.Photo; //0
TwitterMediaType.Video; //1
TwitterMediaType.AnimatedGif; //2
*/
var media9 = media.MediaUrl; //string
var media10 = media.MediaUrlHttps; //string
//var FeedImageURLhttp = media.MediaUrl; for Http Image
//var FeedImageURLhttps = media.MediaUrlHttps; for Https Image
var media11 = media.Sizes; //TwitterMediaSizes
var twitterMediaSizes = media11;
//media11.Large
//media11.Medium
//media11.Small
//media11.Thumb
var media12 = media.StartIndex; //int
var media13 = media.Url; //string
TweetText = TweetText.Trim().Replace(media.Url, String.Empty).Trim();
}
IList<TwitterHashTag> twitterHashTag = twitterEntities.HashTags; //List<TwitterHashTag>
foreach (var hashTag in twitterHashTag)
{
var ht1 = hashTag.EndIndex;
var ht2 = hashTag.EntityType;
var ht3 = hashTag.Indices;
var ht4 = hashTag.StartIndex;
var ht5 = hashTag.Text;
// hashTagText for refine Tweet Text
TweetText = TweetText.Trim().Replace("#" + hashTag.Text, string.Empty).Trim();
}
TweetText = TweetText.Replace("\n", String.Empty).Trim();
var st23 = tweet.CreatedDate; //DateTime
var st24 = tweet.Author; //ITweeter
var ProfileImageURL = st24.ProfileImageUrl;
var ss = st24.RawSource;
var screenName = st24.ScreenName;
}
return lstModelTwitterFeedsDetail;
}
catch (Exception ex)
{
throw ex;
}
}
private static void StartSafeScrappedData(ModelTwitterFeeds modelTwitterFeeds, ModelFeed modelFeed)
{
try
{
CommunicationResponse resp = CommunicationResponse.GetPage(modelFeed.FeedURL);
if (resp.IsValid)
{
//XmlDocument RSSXml = new XmlDocument();
//RSSXml.Load(url);
XmlDocument xd = resp.ToXml();
//XmlDocument xd = new XmlDocument();
xd.Load(modelFeed.FeedURL);
if (xd != null)
{
ModelFeedDetail modelfeedDetail = new ModelFeedDetail();
XmlElement xmlDoc = xd.DocumentElement;
/*XmlNodeList xmlNodeLists = xd.SelectNodes(feedsSetting.FeedItemsPath);
if (xmlNodeLists.Count > 0 && xmlNodeLists != null)
{
foreach (XmlNode item in xmlNodeLists)
{
if (!String.IsNullOrEmpty(feedsSetting.FeedTitlePath.Trim()))
{
if (item.SelectSingleNode(feedsSetting.FeedTitlePath) != null)
{
//getting item Title from RSS Feed Page
modelfeedDetail.PostTitle = item.SelectSingleNode(feedsSetting.FeedTitlePath).InnerText;
}
}
if (!String.IsNullOrEmpty(feedsSetting.FeedShortDescPath.Trim()))
{
if (item.SelectSingleNode(feedsSetting.FeedShortDescPath) != null)
{
//getting short description from RSS Feed Page
modelfeedDetail.ShortDescription = item.SelectSingleNode(feedsSetting.FeedShortDescPath).InnerText;
}
}
if (!String.IsNullOrEmpty(feedsSetting.FeedPubDatePath.Trim()))
{
if (item.SelectSingleNode(feedsSetting.FeedPubDatePath) != null)
{
//getting item publish date from RSS Feed Page
modelfeedDetail.PublishDate = item.SelectSingleNode(feedsSetting.FeedPubDatePath).InnerText;
}
}
if (!String.IsNullOrEmpty(feedsSetting.FeedImagePath.Trim()))
{
if (item.SelectSingleNode(feedsSetting.FeedImagePath) != null)
{
//getting item image from RSS Feed Page
modelfeedDetail.RssImage = item.SelectSingleNode(feedsSetting.FeedImagePath).InnerText;
}
}
if (!String.IsNullOrEmpty(feedsSetting.FeedDetailPageURLPath.Trim()))
{
if (item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath) != null)
{
//getting Detail Page URL from Feed
modelfeedDetail.DetailPageURL = item.SelectSingleNode(feedsSetting.FeedDetailPageURLPath).InnerText;
}
//If Detail page URL not empty Then
if (!String.IsNullOrEmpty(modelfeedDetail.DetailPageURL))
{
//web Request on Detial page URL
//XmlDocument xdDetailPage = new XmlDocument();
//xdDetailPage.Load(modelfeedDetail.DetailPageURL);
CommunicationResponse GetdetailPageResponse = CommunicationResponse.GetPage(modelfeedDetail.DetailPageURL);
XmlDocument xdDetailPage = GetdetailPageResponse.ToXml();
if (xdDetailPage != null)
{
//Detail Page Will screape here
}
}
}
}
}*/
}
}
}
catch (Exception ex)
{
throw ex;
}
}
public static void GetTwitterTweetsWithDetails(TwitterService service, double feedId, string LastMaxTweetPostedId, ModelFeed modelFeedScrapperPath, List<ModelFeedsDetailScrappingHelper> listScrappingHelper)
{
ModelTwitterFeeds modelTwitterFeeds = new ModelTwitterFeeds();
ModelTwitterFeedsDetails modelTwitterFeedsDetail = new ModelTwitterFeedsDetails();
ModelFeedMultimedia modelModelFeedMultimedia = new ModelFeedMultimedia();
List<ModelFeedMultimedia> lstmodelModelFeedImage = new List<ModelFeedMultimedia>();
List<ModelTwitterFeedsDetails> lstModelTwitterFeedsDetail = new List<ModelTwitterFeedsDetails>();
BLLFeed objBLLFeed = new BLLFeed();
string DetailPageURL = String.Empty;
try
{
//TwitterAccount user = new TwitterAccount.
//TwitterUser twitterUser =
//>>GET OTHER USER TIMELINE//BeginListTweetsOnHomeTimeline
var User_OptionInit = new ListTweetsOnUserTimelineOptions { ScreenName = modelFeedScrapperPath.FeedChannelName, Count = 20, ExcludeReplies = true };
var User_Tweets = service.ListTweetsOnUserTimeline(User_OptionInit);
/*
var profilebanner = service.GetUserProfile(new GetUserProfileOptions() { IncludeEntities = true, SkipStatus = true });
if (profilebanner != null)
{
//profilebanner.ContributorsEnabled
//profilebanner.CreatedDate
modelTwitterFeeds.UserPageDesc = profilebanner.Description;
//profilebanner.FavouritesCount;
modelTwitterFeeds.UserPageFollowers = profilebanner.FollowersCount;
//profilebanner.FollowRequestSent;
//profilebanner.FriendsCount;
modelTwitterFeeds.UserPageId = profilebanner.Id;
//profilebanner.IsDefaultProfile;
//profilebanner.IsGeoEnabled;
//profilebanner.IsProfileBackgroundTiled;
//profilebanner.IsProtected;
//profilebanner.IsTranslator;
//profilebanner.IsVerified;
modelTwitterFeeds.UserPageLanguage = profilebanner.Language;
//profilebanner.ListedCount;
//profilebanner.Location;
modelTwitterFeeds.UserPageTitle = profilebanner.Name;
//profilebanner.ProfileBackgroundColor;
//profilebanner.ProfileBackgroundImageUrl;
//profilebanner.ProfileBackgroundImageUrlHttps;
modelTwitterFeeds.UserPageLogoImage = profilebanner.ProfileImageUrl;
//profilebanner.ProfileImageUrlHttps;
//profilebanner.ProfileLinkColor;
//profilebanner.ProfileSidebarBorderColor;
//profilebanner.ProfileSidebarFillColor;
//profilebanner.ProfileTextColor;
//profilebanner.RawSource;
modelTwitterFeeds.UserScreenName = profilebanner.ScreenName;
//profilebanner.Status;
//profilebanner.StatusesCount;
//profilebanner.TimeZone;
//JObject o = new JObject(new JProperty("Name", "John Smith"),new JProperty("BirthDate", new DateTime(1983, 3, 20)));
JObject obj = (JObject)JsonConvert.DeserializeObject(profilebanner.RawSource);
if (obj != null)
{
if (obj["user"]["profile_banner_url"] != null)
{
modelTwitterFeeds.UserPageCoverImageURL = (string)obj["user"]["profile_banner_url"];
}
}
}
*/
string TweetText = String.Empty;
int tweetCountFirstRun = 0;
foreach (var tweet in User_Tweets)
{
if (!String.IsNullOrEmpty(LastMaxTweetPostedId))
{
if (LastMaxTweetPostedId.ToLower().Trim() == tweet.IdStr.ToLower().Trim())
{
break;
}
}
modelTwitterFeedsDetail = new ModelTwitterFeedsDetails();
modelModelFeedMultimedia = new ModelFeedMultimedia();
var userDetail = tweet.User;
if (tweet != null)
{
if (tweetCountFirstRun == 0)
{
if (userDetail != null)
{
modelTwitterFeeds.FeedId = feedId;
modelTwitterFeeds.UserPageId = tweet.User.Id;
modelTwitterFeeds.UserPageTitle = tweet.User.Name;
modelTwitterFeeds.UserScreenName = tweet.User.ScreenName;
modelTwitterFeeds.UserPageDesc = tweet.User.Description;
modelTwitterFeeds.UserPageLanguage = tweet.User.Language;
modelTwitterFeeds.UserPageFollowers = tweet.User.FollowersCount;
//modelTwitterFeeds.UserPageCoverImageURL = tweet.User.ProfileBackgroundImageUrl;
modelTwitterFeeds.UserPageLogoImage = tweet.User.ProfileImageUrl;
//JObject o = new JObject(new JProperty("Name", "John Smith"),new JProperty("BirthDate", new DateTime(1983, 3, 20)));
JObject obj = (JObject)JsonConvert.DeserializeObject(tweet.RawSource);
if (obj != null)
{
if (obj["user"]["profile_banner_url"] != null)
{
modelTwitterFeeds.UserPageCoverImageURL = (string)obj["user"]["profile_banner_url"];
}
}
}
tweetCountFirstRun++;
}
var st1 = tweet.Text.ToSafeString(); //string
TweetText = tweet.Text;
//modelTwitterFeedsDetail.TwitterPageId;
modelTwitterFeedsDetail.FeedPostedtId = tweet.Id;
modelTwitterFeedsDetail.FeedText = String.Empty;
modelTwitterFeedsDetail.FeedLanguage = tweet.Language;
modelTwitterFeedsDetail.FeedPostDate = tweet.CreatedDate;
modelTwitterFeedsDetail.FeedTextDetail = "";
var st2 = tweet.Source; //string
var st3 = tweet.TextAsHtml; //string
var st4 = tweet.TextDecoded; //string
var st5 = tweet.RetweetedStatus; //TwitterStatus
var st6 = tweet.RetweetCount; //int
var st7 = tweet.RawSource; //string
var st8 = tweet.Place; //TwitterPlace
var st9 = tweet.Location; //TwitterGeoLocation
var st10 = tweet.Language; //string
var st11 = tweet.IsTruncated; //bool
var st12 = tweet.IsRetweeted; //bool
var st13 = tweet.IsPossiblySensitive; //bool is nullable
var st14 = tweet.IsFavorited; //bool
var st15 = tweet.InReplyToUserId; //long is nullable
var st16 = tweet.InReplyToStatusId; //long is nullable
var st17 = tweet.InReplyToScreenName; //string
var st18 = tweet.IdStr; //string
var st19 = tweet.Id; //long
var st20 = tweet.FavoriteCount; //int
var st21 = tweet.ExtendedEntities; //TwitterExtendedEntities
var st22 = tweet.Entities; //TwitterEntities
var twitterEntities = new TwitterEntities();
twitterEntities = tweet.Entities;
IList<TwitterUrl> twitterUrl = twitterEntities.Urls; //List<TwitterUrl>
if (twitterUrl != null && twitterUrl.Count > 0)
{
foreach (var url in twitterUrl)
{
var url1 = url.DisplayUrl; //string
var url2 = url.EndIndex; //int
var url3 = url.EntityType; //TwitterEntityType
var entityType = url3;
//4 Entity Types are Defined
//TwitterEntityType.HashTag; //0
//TwitterEntityType.Mention; //1
//TwitterEntityType.Url; //2
//TwitterEntityType.Media; //3
var url4 = url.ExpandedValue; //string
var url5 = url.Indices; //IList<int>
var url6 = url.StartIndex; //int
var url7 = url.Value; //string
//MODEL DATA //Tweet Detail Page URL
if (!String.IsNullOrEmpty(url.ExpandedValue))
{
modelTwitterFeedsDetail.FeedDetailPageURL = url.ExpandedValue.Trim();
}
else if (!String.IsNullOrEmpty(url.Value))
{
if (String.IsNullOrEmpty(modelTwitterFeedsDetail.FeedDetailPageURL))
{
modelTwitterFeedsDetail.FeedDetailPageURL = url.Value.Trim();
}
}
TweetText = TweetText.Trim().Replace(url.Value, String.Empty).Trim();
}
}
IList<TwitterMedia> twitterMedia = twitterEntities.Media; //List<TwitterMedia>
foreach (var media in twitterMedia)
{
var media1 = media.DisplayUrl; //string
var media2 = media.EndIndex; //int
var media3 = media.EntityType; //TwitterEntity
var media4 = media.ExpandedUrl; //string
var media5 = media.Id; //long
var media6 = media.IdAsString; //string
var media7 = media.Indices; //IList<int>
var media8 = media.MediaType; //TwitterMediaType
var twitterMediaType = media8;
/*Three Types of MediaType
TwitterMediaType.Photo; //0
TwitterMediaType.Video; //1
TwitterMediaType.AnimatedGif; //2
*/
var media9 = media.MediaUrl; //string
var media10 = media.MediaUrlHttps; //string
//var FeedImageURLhttp = media.MediaUrl; for Http Image
//var FeedImageURLhttps = media.MediaUrlHttps; for Https Image
var media11 = media.Sizes; //TwitterMediaSizes
var twitterMediaSizes = media11;
//media11.Large
//media11.Medium
//media11.Small
//media11.Thumb
var media12 = media.StartIndex; //int
var media13 = media.Url; //string
//MODEL DATA
string tweetMultimediaURL = !String.IsNullOrEmpty(media.Url.Trim()) ? media.Url.Trim() : String.Empty;
TweetText = TweetText.Trim().Replace(media.Url, String.Empty).Trim();
if (!String.IsNullOrEmpty(tweetMultimediaURL))
{
modelModelFeedMultimedia.MultiMediaURL = tweetMultimediaURL;
modelModelFeedMultimedia.MultiMediaType = Convert.ToInt32(media.MediaType);
//Add model object to list
//lstmodelModelFeedImage.Add(modelModelFeedMultimedia);
modelTwitterFeedsDetail.FeedMultimediaList.Add(modelModelFeedMultimedia);
}
}
IList<TwitterHashTag> twitterHashTag = twitterEntities.HashTags; //List<TwitterHashTag>
foreach (var hashTag in twitterHashTag)
{
var ht1 = hashTag.EndIndex;
var ht2 = hashTag.EntityType;
var ht3 = hashTag.Indices;
var ht4 = hashTag.StartIndex;
var ht5 = hashTag.Text;
// hashTagText for refine Tweet Text
TweetText = TweetText.Trim().Replace("#" + hashTag.Text, string.Empty).Trim();
}
TweetText = TweetText.Replace("\n", String.Empty).Trim();
var st23 = tweet.CreatedDate; //DateTime
var st24 = tweet.Author; //ITweeter
var ProfileImageURL = st24.ProfileImageUrl;
var ss = st24.RawSource;
var screenName = st24.ScreenName;
modelTwitterFeedsDetail.FeedText = TweetText;
//string DetailPageSelector = "div.story-body__inner p:not(:first-of-type)";
if (!String.IsNullOrEmpty(modelTwitterFeedsDetail.FeedDetailPageURL))
{
Uri DetailPageURI = new Uri(modelTwitterFeedsDetail.FeedDetailPageURL);
string DetailPageTextSelector = String.Empty;
string DetailPageImageSelector = String.Empty;
ModelFeedsDetailScrappingHelper rowDetailHelper = listScrappingHelper.Where(m => m.HostName.Contains(DetailPageURI.Host.Trim().ToLower())).FirstOrDefault();
if (rowDetailHelper != null)
{
DetailPageTextSelector = rowDetailHelper.DetailPagePath;
DetailPageImageSelector = rowDetailHelper.DetailPageImagePath;
}
/*
if (DetailPageURI.Host.Trim().ToLower().Contains("cnn"))
{
//bbc.com Text Details= //div[@class='story-body'] //p[position()>=1]
//bbc.com img = //div[@class='story-body__inner'] //img[@class='js-image-replace']
//cnn = ////p[@dir='RTL']js-image-replace
DetailPageTextSelector = "//div[contains(@class, 'article-left')] //p";
DetailPageImageSelector = "//div[contains(@class, 'gallery-big-images')] //img";
}
else if (DetailPageURI.Host.Trim().ToLower().Contains("soundcloud"))
{
DetailPageTextSelector = "//div[contains(@class, 'sc-type-small')] //p";
DetailPageImageSelector = "//div[@class='article-left'] //p";
}
*/
HtmlDocument requestedDoc = ScrapeDetailfromPage(DetailPageURI.AbsoluteUri);
if (requestedDoc != null)
{
//if Details Page Selector Text not found then not send request
if (!String.IsNullOrEmpty(DetailPageTextSelector))
{
string detailTextFromURL = ScrapeTextfromDetailPage(requestedDoc, DetailPageTextSelector);
modelTwitterFeedsDetail.FeedTextDetail = detailTextFromURL;
}
//if default twitter post not have any multimedia type then
if (modelTwitterFeedsDetail.FeedMultimediaList.Count <= 0)
{
//if Details Page Selector Image not found then not send request
if (!String.IsNullOrEmpty(DetailPageImageSelector))
{
modelTwitterFeedsDetail.FeedMultimediaList = ScrapeImageDetailfromPage(requestedDoc, DetailPageImageSelector);
//if details page not found on detected tag & Tweet having Image and Embeded Video on Details page
// Then we make a new request to scrape twitter create image from sperate static created request
if (modelTwitterFeedsDetail.FeedMultimediaList.Count <= 0)
{
//Sample URI for Video Image get://https://twitter.com/cnnarabic/status/678548429530906624
//https://twitter.com/i/cards/tfw/v1/678585688783257600?cardname=summary_large_image
//https://twitter.com/i/cards/tfw/v1/678585688783257600
//string StaticImageURI = "https://twitter.com/" + modelTwitterFeeds.UserScreenName.Trim() + "//status/" + modelTwitterFeedsDetail.FeedPostId;
string StaticImageURI = "https://twitter.com/i/cards/tfw/v1/" + tweet.Id;
HtmlDocument VideoImageRequestedDoc = ScrapeDetailfromPage(StaticImageURI);
string staticImageSelector = "//div[contains(@class, 'SummaryCard-image')] //img";
modelTwitterFeedsDetail.FeedMultimediaList = ScrapeImageDetailfromPage(VideoImageRequestedDoc, staticImageSelector);
}
}
}
}
}
else
{
modelTwitterFeedsDetail.FeedTextDetail = String.Empty;
}
}
lstModelTwitterFeedsDetail.Add(modelTwitterFeedsDetail);
}
modelTwitterFeeds.TwitterFeedDetails = lstModelTwitterFeedsDetail;
objBLLFeed.InsertTwitterScrappedData(modelTwitterFeeds);
//StartSafeScrappedData(modelTwitterFeeds, modelFeed);
}
catch (Exception ex)
{
throw ex;
}
}