public override void Init(string screenname, System.IO.TextWriter log)
{
base.Init(screenname, log);
User = UsersCollection.PrimaryUsers().Where(x => x.TwitterScreenName == screenname).First();
repoPage = new SimpleRepository<ArticleStubPage>(screenname);
repoIndex = new SimpleRepository<ArticleStubIndex>(screenname);
}
private static PostworthyUser Add(string ScreenName)
{
var users = HttpRuntime.Cache["PostworthyUsers"] as List<PostworthyUser>;
var newUser = new PostworthyUser() { TwitterScreenName = ScreenName, SiteName = ScreenName };
users.Add(newUser);
return newUser;
}
protected override void Initialize(System.Web.Routing.RequestContext requestContext)
{
base.Initialize(requestContext);
PrimaryUser = UsersCollection.PrimaryUsers().Where(u => u.IsPrimaryUser).FirstOrDefault();
if (PrimaryUser != null)
ViewBag.Brand = PrimaryUser.SiteName;
}
private static PostworthyUser Add(string ScreenName)
{
var users = HttpRuntime.Cache["PostworthyUsers"] as List <PostworthyUser>;
var newUser = new PostworthyUser()
{
TwitterScreenName = ScreenName, SiteName = ScreenName
};
users.Add(newUser);
return(newUser);
}
public DashboardModel(PostworthyUser user)
{
User = user;
PotentialTweets = new List<Tweet>();
PotentialReTweets = new List<Tweet>();
Tweeted = new List<Tweet>();
PotentialFriendRequests = new List<KeyValuePair<Tweep, int>>();
LastTweetTime = DateTime.MaxValue;
KeywordSuggestions = new List<KeyValuePair<string, int>>();
TweetsLastThirtyDays = new int[31];
TopFriendTweetCounts = new List<KeyValuePair<Tweep, int>>();
KeywordsWithOccurrenceCount = new List<KeyValuePair<string, int>>();
PotentialKeywordsWithOccurrenceCount = new List<KeyValuePair<string, int>>();
TwitterFollowSuggestions = new List<Tweep>();
PendingKeywordAdd = new List<string>();
PendingKeywordIgnore = new List<string>();
PendingTweetRemoval = new List<string>();
LoadFromRepository();
}
private TwitterModel(string screenname)
{
PrimaryUser = UsersCollection.Single(screenname);
if (PrimaryUser == null)
throw new Exception(screenname + " user not found!");
}
public PostworthyArticleModel(PostworthyUser primaryUser)
{
PrimaryUser = primaryUser;
}
public DualStreamMonitor(PostworthyUser primaryUser, TextWriter log)
{
this.log = log;
PrimaryUser = primaryUser;
}
private static List<TweetGroup> CreateGroups(PostworthyUser user, DateTime? day)
{
var repoTweets = new SimpleRepository<Tweet>(user.TwitterScreenName);
List<string> screenNames = null;
screenNames = TwitterModel.Instance(user.TwitterScreenName).GetRelevantScreenNames(user.TwitterScreenName);
int RetweetThreshold = user.RetweetThreshold;
DateTime start = day == null ? DateTime.Now.AddHours(-48) : day.Value.StartOfDay();
DateTime end = day == null ? DateTime.Now : day.Value.EndOfDay();
Func<Tweet, bool> where = t =>
t != null &&
//Should everything be displayed or do you only want content
(user.OnlyTweetsWithLinks == false || (t.Links != null && t.Links.Count > 0)) &&
//Minumum threshold applied so we get results worth seeing (if it is your own tweet it gets a pass on this step)
((t.RetweetCount > RetweetThreshold || t.User.ScreenName.ToLower() == user.TwitterScreenName.ToLower()) &&
//Apply Date Range
(t.CreatedAt >= start && t.CreatedAt <= end));
var startGrouping = DateTime.Now;
Console.WriteLine("{0}: Starting grouping procedure for {1}", startGrouping, user.TwitterScreenName);
Console.WriteLine("{0}: Fetching tweets for {1}", startGrouping, user.TwitterScreenName);
var tweets = screenNames
//For each screen name (i.e. - you and your friends if included) select the most recent tweets
.SelectMany(x => repoTweets.Query(x + TwitterModel.Instance(user.TwitterScreenName).TWEETS, where: where) ?? new List<Tweet>())
//Order all tweets based on rank (TweetRank takes into acount many important factors, i.e. - time, mentions, hotness, ect.)
.OrderByDescending(t => t.TweetRank)
//Just to make sure we are not trying to group a very very large number of items
.Take(5000)
.ToList();
Console.WriteLine("{0}: Grouping tweets by similarity for {1}", DateTime.Now, user.TwitterScreenName);
var groups = tweets
//Group similar tweets
.GroupSimilar2()
//Convert groups into something we can display
.Select(g => new TweetGroup(g) { RepositoryKey = TwitterModel.Instance(user.TwitterScreenName).CONTENT })
//Order by TweetRank
.OrderByDescending(g => g.TweetRank)
//Only the top 500
.Take(500);
List<TweetGroup> results = null;
if (groups != null && groups.Count() > 0)
{
//Get Standard Deviation
double stdev = 0;
var values = groups.Select(x => x.TweetRank);
double avg = values.Average();
stdev = Math.Sqrt(values.Sum(d => (d - avg) * (d - avg)) / values.Count());
//Filter groups that are way high...
groups = groups.Where(x => x.TweetRank < (avg + stdev));
results = groups.ToList();
}
var endGrouping = DateTime.Now;
Console.WriteLine("{0}: Grouping procedure for {1} completed and it took {2} minutes to complete", endGrouping, user.TwitterScreenName, (endGrouping - startGrouping).TotalMinutes);
return results ?? new List<TweetGroup>();
}
private static async Task<List<ArticleStub>> CreateContent(PostworthyUser user, List<TweetGroup> groupingResults, ArticleStubPage existing)
{
var startContent = DateTime.Now;
Console.WriteLine("{0}: Starting content procedure for {1}", startContent, user.TwitterScreenName);
List<ArticleStub> results = new List<ArticleStub>();
var contentItems = new List<object>();
foreach (var result in groupingResults)
{
if (contentItems.Count >= MAX_CONTENT)
break;
var existingItem = existing != null ?
existing.ArticleStubs.Where(x => result.Links.Select(y => y.Uri).Contains(x.Link)).FirstOrDefault() : null;
if (existingItem != null)
{
contentItems.Add(existingItem);
continue;
}
var imageUris = new List<Uri>();
imageUris = result.Links.Where(l => l.Image != null).Select(l => l.Image).ToList();
var links = result.Links.OrderByDescending(x => x.ShareCount);
foreach (var uriex in links)
{
if (uriex.IsHtmlContentUrl)
{
var doc = new HtmlAgilityPack.HtmlDocument();
try
{
var req = uriex.Uri.GetWebRequest(15000, 15000);
using (var resp = await req.GetResponseAsync())
{
using (var reader = new StreamReader(resp.GetResponseStream(), true))
{
doc.Load(reader);
}
}
}
catch (Exception ex) { }
if (doc.DocumentNode != null)
{
imageUris.AddRange(ExtractImageUris(uriex, doc));
var content = new
{
Title = uriex.Title,
SubTitle = uriex.Description,
Link = uriex.Uri,
//Image = image == null ? null : ImageManipulation.EncodeImage(image, width, height),
Summary = ExtractSummary(uriex.Title + " " + uriex.Description, doc),
Video = uriex.Video,
Images = imageUris
};
contentItems.Add(content);
break;
}
}
}
}
var newImages = contentItems
.Where(x => x.GetType() != typeof(ArticleStub))
.Select(x => (dynamic)x)
.SelectMany(x => ((List<Uri>)x.Images).Select(y => new { ID = ((object)x.Title).GetHashCode(), Image = y }))
.ToList();
var stubImages = contentItems
.Where(x => x.GetType() == typeof(ArticleStub))
.Where(x => ((ArticleStub)x).OriginalImageUri != null)
.Select(x => new { ID = ((ArticleStub)x).Title.GetHashCode(), Image = ((ArticleStub)x).OriginalImageUri })
.ToArray();
if (stubImages != null && stubImages.Length > 0)
newImages.AddRange(stubImages);
var allImages = newImages.ToArray();
var excludedImages = new List<Uri>();
for (int i = 0; i < allImages.Length - 1; i++)
{
var img = allImages[i];
if (!excludedImages.Contains(img.Image))
{
for (int j = i + 1; j < allImages.Length; j++)
{
var img2 = allImages[j];
if (img.Image == img2.Image && img.ID != img2.ID)
{
excludedImages.Add(img2.Image);
break;
}
}
}
}
foreach (var obj in contentItems)
{
if (obj.GetType() != typeof(ArticleStub))
{
dynamic item = obj;
var image = await GetBestImage(((List<Uri>)item.Images ?? new List<Uri>()).Where(y => !excludedImages.Contains(y)));
results.Add(new ArticleStub
{
Title = item.Title,
SubTitle = item.SubTitle,
Link = item.Link,
Image = image != null ? image.Item1 : null,
Summary = item.Summary,
Video = item.Video,
OriginalImageUri = image != null ? image.Item2 : null
});
}
else if (excludedImages.Contains(((ArticleStub)obj).OriginalImageUri))
{
var item = (ArticleStub)obj;
item.Image = null;
results.Add(item);
}
else
results.Add(obj as ArticleStub);
}
var endContent = DateTime.Now;
Console.WriteLine("{0}: Content procedure for {1} completed and it took {2} minutes to complete", endContent, user.TwitterScreenName, (endContent - startContent).TotalMinutes);
return results;
}