public async Task <TwitterData> GetLatestTwitterData(int searchRequestID)
{
TwitterData lastTwitterData = null;
try {
using (var db = new BEMainDBContext()) {
var execsBySreq = db.BEExecutions
.Where(exec => (exec.SearchRequestID == searchRequestID))
.Include(exec => exec.TheMinerData)
.Where(exec => exec.TheMinerData.Count != 0)
.OrderBy(exec => exec.StartedOn)
.ToList();
var lastMinerData = execsBySreq.Select(exec => exec.TheMinerData
.Where(md => md.TheSource == SourceOption.Twitter)
.Last())
.Where(md => md != null)
.ToList()
.Last();
lastTwitterData = db.TwitterData
.Where(td => td.TheExecutionID == lastMinerData.TheExecutionID)
.ToList()
.FirstOrDefault();
}
} catch {
lastTwitterData = null;
}
return(lastTwitterData);
}
public void TwitterUpdateSnapshotsInDb()
{
List <TwitterAccount> twitterAccounts = new List <TwitterAccount>();
twitterAccounts.Add(new TwitterAccount {
Id = "MapleLeafs", FriendlyName = "Toronto Maple Leafs"
});
twitterAccounts.Add(new TwitterAccount {
Id = "phoenixcoyotes", FriendlyName = "Phoenix Coyotes"
});
List <TwitterSnapshot> twitterAccountSnapshots = TwitterData.UpdateSnapshotsInDb(twitterAccounts);
Assert.AreEqual(2, twitterAccountSnapshots.Count, "There are 2 snapshots");
// Call Update again to make sure dupes are added
twitterAccountSnapshots = TwitterData.UpdateSnapshotsInDb(twitterAccounts);
// Make sure we have the right items in the db
using (SportsDataContext db = new SportsDataContext())
{
List <TwitterSnapshot> snapshotsFromToday = (from s in db.TwitterSnapshot_DbSet.Include(x => x.TwitterAccount)
where DbFunctions.TruncateTime(s.DateOfSnapshot) == DbFunctions.TruncateTime(DateTime.UtcNow)
orderby s.TwitterAccount.Id
select s).ToList();
Assert.AreEqual(2, snapshotsFromToday.Count, "There are 2 snapshots, not 4");
Assert.AreEqual(DateTime.UtcNow.Date, snapshotsFromToday[0].DateOfSnapshot.Date, "The snapshots are from today");
Assert.AreEqual(snapshotsFromToday[0].DateOfSnapshot.Date, snapshotsFromToday[1].DateOfSnapshot.Date, "The snapshots are equal");
Assert.AreEqual("MapleLeafs", snapshotsFromToday[0].TwitterAccount.Id, "The first snapshot is from MapleLeafs");
Assert.AreEqual("phoenixcoyotes", snapshotsFromToday[1].TwitterAccount.Id, "The first snapshot is from phoenixcoyotes");
}
}
public async Task PostTwitterData(TwitterData newTwitterData)
{
using (var db = new BEMainDBContext()) {
db.TwitterData.Add(newTwitterData);
await db.SaveChangesAsync();
}
}
private void TagHandler(object sender, IrcEventArgs e)
{
string sendto = (string.IsNullOrEmpty(e.Data.Channel)) ? e.Data.Nick : e.Data.Channel;
if (e.Data.MessageArray[0].ToLower() == "!+tag")
{
var tag = new TwitterTag {
Tag = e.Data.MessageArray[1]
};
TwitterData.TwitterTags.InsertOnSubmit(tag);
TwitterData.SubmitChanges();
BotMethods.SendMessage(SendType.Message, sendto, "Automatic Search activated: " + "http://search.twitter.com/search?q=" + HttpUtility.UrlEncode(e.Data.MessageArray[1]));
}
if (e.Data.MessageArray[0].ToLower() == "!-tag")
{
var tag = TwitterData.TwitterTags.Where(t => t.Tag == e.Data.MessageArray[1]).FirstOrDefault();
if (tag != null)
{
TwitterData.TwitterTags.DeleteOnSubmit(tag);
TwitterData.SubmitChanges();
}
}
if (e.Data.MessageArray[0].ToLower() == "!tags")
{
foreach (string line in TwitterData.TwitterTags.Select(t => t.Tag).ToLines(350))
{
BotMethods.SendMessage(SendType.Message, sendto, line);
}
}
}
public async Task SaveTwitterDataAsync()
{
TwitterData data = new TwitterData();
data.AccountList = new List <TwitterAccount>(this.AccountList);
data.TimelineTabList = new List <TimelineTab>(this.TimelineListTab);
await SaveData.SaveTwitterDataAsync(data);
}
public async Task <TwitterData> GetTwitterData(int executionID)
{
TwitterData twitterData = null;
using (var db = new BEMainDBContext()) {
twitterData = db.TwitterData
.Where(td => td.TheExecutionID == executionID)
.ToList()
.FirstOrDefault();
}
return(twitterData);
}
public ActionResult Social()
{
Dictionary <string, Exception> results = new Dictionary <string, Exception>();
if (this.IsKeyValid())
{
results.Add("FacebookData", this.Update(delegate() { FacebookData.UpdateAllSnapshotsInDb(); }));
results.Add("TwitterData", this.Update(delegate() { TwitterData.UpdateAllSnapshotsInDb(); }));
}
return(Json(results, JsonRequestBehavior.AllowGet));
}
public List <TwitterObject> GetTwitter(string param)
{
string url = $"https://api.twitter.com/1.1/search/tweets.json?q={param}&result_type=recent&tweet_mode=extended&count=100";
Auth.SetUserCredentials(ConfigurationManager.AppSettings["CustomerKey"],
ConfigurationManager.AppSettings["CustomerKeySecret"],
ConfigurationManager.AppSettings["AcessToken"],
ConfigurationManager.AppSettings["AcessTokenSecret"]);
var results = TwitterAccessor.GetQueryableJsonObjectFromGETQuery(url).ToString();
TwitterData twitterData = JsonConvert.DeserializeObject <TwitterData>(results);
return(twitterData.twitterData.FindAll(x => x.retweeted == null));
}
/*prepares a string from the CSV(scrub and split), turns it into object for mongo */
private TwitterData ConvertToTwitterData(string indata)
{
TwitterData twitterData = new TwitterData();
var lineFromFile = indata.Replace("\"", "").Split(',');
twitterData.Standford_id = long.Parse((lineFromFile[1]));
twitterData.Polarity = int.Parse(lineFromFile[0]);
twitterData.LongDate = lineFromFile[2];
twitterData.Query = lineFromFile[3];
twitterData.UserName = lineFromFile[4];
for (int i = 5; i < lineFromFile.Length; i++)
{
twitterData.Text += lineFromFile[i];
}
return(twitterData);
}
/// <summary>
/// キー作成
/// </summary>
private void createKey()
{
var twitter = new TwitterData();
twitter.Open();
twitter.GetAllTimeLineData("irof");
localKeyA = twitter.GetRandomStatus();
localKeyB = twitter.GetRandomStatus(localKeyA.StatusID);
//画面描画
SetWebBrowserA("https://twitter.com/irof/statuses/" + localKeyA.StatusID);
SetWebBrowserB("https://twitter.com/irof/statuses/" + localKeyB.StatusID);
//公開キー作成
this.createPublicKey();
}
public TwitterData GetMyTimeline(string url, OAuthData oAuthData)
{
timelineUrl = url;
var authenticate = new Authenticate();
TwitAuthenticateResponse twitAuthResponse = authenticate.AuthenticateMe(oAuthData.OAuthConsumerKey, oAuthData.OAuthConsumerSecret, OAuthUrl);
// Do the timeline
var timeLineJson = new Utility().RequstJson(timelineUrl, twitAuthResponse.token_type, twitAuthResponse.access_token);
TwitterData response = new TwitterData();
response.xmls = new List <XmlDocument>();
if (timeLineJson.StartsWith("Error:"))
{
response.error = timeLineJson;
return(response);
}
//List<XmlDocument> xmls = new List<XmlDocument>();
if (timeLineJson == string.Empty || timeLineJson == "[]")
{
return(response);
}
timeLineJson = timeLineJson.Substring(3, timeLineJson.Length - 6);
var timeLines = timeLineJson.Split(new [] { "\"},{\"" }, System.StringSplitOptions.None);
try
{
foreach (string line in timeLines)
{
response.xmls.Add(JsonConvert.DeserializeXmlNode("{\"root\":[{\"" + line + "\"}]}"));
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
return(response);
}
public async Task <List <TwitterUser> > PrepareUserInsert(List <String> twitterUserList)
{
List <TwitterUser> twitterUserData = new List <TwitterUser>();
foreach (var user in twitterUserList)
{
TwitterData twitterData = await scrapeTwitterUser(user);
TwitterUser newTwitterUser = new TwitterUser
{
Id_str = twitterData.User.Id_str,
Screen_name = twitterData.User.Screen_name,
Description = twitterData.User.Description,
Followers_count = twitterData.User.Followers_count
};
twitterUserData.Add(newTwitterUser);
}
;
return(twitterUserData);
}
/******************** Actor Logic Implementation Methods ********************/
// Called before the Miner starts its job (mainMineAsync)
protected override async Task onMineBeginAsync()
{
// Initialize the TwitterData for this Execution, starting from where the previous execution finished
BESearchRequest theSearchRequest = await GetTheSearchRequest();
TwitterData theTwitterData = await dbHandlerService.GetLatestTwitterData(theSearchRequest.ID);
if (theTwitterData == null)
{
miningForFirstTime = true;
theTwitterData = new TwitterData(theSearchRequest.ActiveExecutionID);
}
else
{
miningForFirstTime = false;
theTwitterData.TheTextsNum = 0;
theTwitterData.TheExecutionID = theSearchRequest.ActiveExecutionID;
}
await SaveTheMinerData(theTwitterData);
}
public async Task RepeatTwitterApiCall(String user, String max_id)
{
List <TwitterData> twitterData = await scrapeTwitterData(user, max_id);
//Twitter API will return only one tweet (the max_id tweet) once reach limit (3200) of tweets available
if (twitterData != null && twitterData.Count > 1)
{
//The max_id tweet is fetched on previous call so don't include.
twitterData = twitterData.Where(x => x.Id_str != max_id).ToList();
max_id = twitterData.LastOrDefault().Id_str;
foreach (var tweet in twitterData)
{
foreach (var item in _searchTerms)
{
if (tweet.Full_text.ToLower().Contains(item))
{
TwitterData newTwitterData = new TwitterData
{
Id_str = tweet.Id_str,
Created_at = tweet.Created_at,
Full_text = tweet.Full_text,
Retweet_count = tweet.Retweet_count,
Favorite_count = tweet.Favorite_count,
Screen_name = tweet.User.Screen_name
};
_userTweetsAll.Add(newTwitterData);
break;
}
}
}
await Task.Delay(5000);
await RepeatTwitterApiCall(user, max_id);
}
else
{
Console.WriteLine("Done.");
}
}
public void ReadTweets(string screenName, bool isFirstTime)
{
bool multipleFlag = false;
string maxId = "";
string sinceId = "";
while (!finishedToProcessScreenName && pageCounter < 16)
{
pageCounter++;
if (isFirstTime)
{
string maxIdInDB = DataContext.GetMinMessageIdForUser(screenName);
decimal maxIdInt = 0;
try
{
maxIdInt = maxIdInDB == null ? 0 : decimal.Parse(maxIdInDB) - 1;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
maxId = pageCounter == 1 ? "" : "&max_id=" + maxIdInt;
timelineUrl = string.Format(TimelineFormatFirstTime, screenName, IncludeRts, ExcludeReplies, maxId, Count);
}
else
{ //if first 200 rows get sinceid from DB alse use the original and the id of the oldest new tweet as maxid
string id = multipleFlag == false ? (sinceId = DataContext.GetMaxMessageIdForUser(screenName)) : sinceId + "&max_id=" + maxId;
timelineUrl = string.Format(TimelineFormatNotFirstTime, screenName, IncludeRts, ExcludeReplies, id, Count);
}
OAuthTwitterWrapper.OAuthTwitterWrapper oAuthT = new OAuthTwitterWrapper.OAuthTwitterWrapper();
TwitterData tData = oAuthT.GetMyTimeline(timelineUrl, oAuthData);
List <TwitterItem> items = new List <TwitterItem>();
// convert xml to twitter item
foreach (XmlDocument status in tData.xmls)
{
XElement xml = XElement.Load(new XmlNodeReader(status));
items.Add(new TwitterItem(xml));
statusCounter++;
}
if (items.Count < 200)
{
finishedToProcessScreenName = true;
//insert rows to DB
if (items.Count > 0)
{
DataContext.InsertTwitterItems(items);
}
DataContext.UpdateScreenNames_LastUpdated(currentScreenName);
}
//insert rows to DB
else
{
Console.WriteLine(String.Format("sinceid = {0, 2} maxid = {1, 2} ", sinceId, maxId));
DataContext.InsertTwitterItems(items);
multipleFlag = true;
//get the oldest new tweet's id (maxid)
maxId = items[items.Count - 1].MessageId.ToString();
DataContext.UpdateScreenNames_LastUpdated(currentScreenName);
}
Console.WriteLine("stutus counter = " + items.Count);
authenticateMessageCounter++;
if (authenticateMessageCounter >= 120)
{
Console.WriteLine(" authenticate Message Counter = " + authenticateMessageCounter);
//DataContext.UpdateScreenNames_LastUpdated(currentScreenName);
DataContext.UpdateQueriesTracking(screenName, tData.error, runId, items.Count);
done = true;
break;
}
if (tData.error == null)
{
tData.error = "HTTP/1.1 200 OK";
}
DataContext.UpdateQueriesTracking(screenName, tData.error, runId, items.Count);
}
finishedFlag = true;
}
protected async Task SaveTheMinerData(TwitterData theTwitterData)
{
await StateManager.SetStateAsync(NewStateNames.TheMinerData, theTwitterData);
await SaveStateAsync();
}
/*---------- TwitterData Management ----------*/
public async Task StoreTwitterData(TwitterData newTwitterData)
{
await TheTwitterDataContr.PostTwitterData(newTwitterData);
}
public async Task InsertTwitterData(TwitterData twitterData, String tableName)
{
string sql = $@"insert into {tableName} (Id_str, Created_at, Full_text, Retweet_Count, Favorite_count, Screen_name)
values (@Id_str, @Created_at, @Full_text, @Retweet_Count, @Favorite_count, @Screen_name)";
await _sqlAccess.SaveData(sql, twitterData);
}
public async Task Start()
{
var client = new TwitterClient(_credentials);
// Using the sample stream
var stream = client.Streams.CreateSampleStream();
try
{
//setting stream filter levels
stream.AddLanguageFilter(LanguageFilter.English);
stream.FilterLevel = Tweetinvi.Streaming.Parameters.StreamFilterLevel.Low;
DateTime streamingStartedOn = DateTime.UtcNow;
TwitterData _data = new TwitterData();
//set file for save twitter data
if (File.Exists(_filePath))
{
File.Delete(_filePath);
}
File.Create(_filePath).Dispose();
stream.TweetReceived += (sender, t) =>
{
//exclude retweets
if (t.Tweet.IsRetweet)
{
return;
}
//setting total tweets
_data.TotalTweets++;
//setting average tweet per hour / minute / second
var timeDiff = DateTime.UtcNow - streamingStartedOn;
_data.AverageTweetPerHour = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalHours);
_data.AverageTweetPerMinute = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalMinutes);
_data.AverageTweetPerSecond = Convert.ToInt32(_data.TotalTweets / timeDiff.TotalSeconds);
if (_data.TotalTweets == 1)
{
_data.TopDomains = new List <string>();
_data.TopHashtags = new List <string>();
_data.TopEmojis = new List <string>();
}
//setting top domains and url percentage
if (t.Tweet.Urls.Count > 0)
{
_data.TotalTweetWithUrl++;
_data.TopDomains.AddRange(t.Tweet.Urls.Select(u => u.DisplayedURL.Split("/")[0]).Distinct().ToList());
}
_data.UrlPercentage = Math.Round((decimal)(_data.TotalTweetWithUrl * 100 / _data.TotalTweets), 2);
//setting photo percentage
if (t.Tweet.Entities.Medias.Count > 0)
{
_data.TotalTweetWithPhoto++;
}
_data.PhotoPercentage = Math.Round((decimal)(_data.TotalTweetWithPhoto * 100 / _data.TotalTweets), 2);
//setting top emoji and emoji percentage
var emojis = t.Tweet.FullText.Where(c => c > 255).Select(emoji => Utils.Utf8Encoder.GetString(Utils.Utf8Encoder.GetBytes(emoji.ToString()))).Distinct().ToList();
if (emojis != null)
{
if (emojis.Count > 0)
{
_data.TotalTweetWithEmoji++;
_data.TopEmojis.AddRange(emojis);
//_data.TopEmojis.AddRange(t.Tweet.Entities.Symbols.Select(s => s.Text).Distinct().ToList());
}
_data.EmojiPercentage = Math.Round((decimal)(_data.TotalTweetWithEmoji * 100 / _data.TotalTweets), 2);
}
//setting top hashtags
if (t.Tweet.Hashtags.Count > 0)
{
_data.TopHashtags.AddRange(t.Tweet.Hashtags.Select(h => h.Text).Distinct().ToList());
}
//group and order by numbers
_data.TopDomains = _data.TopDomains.GroupBy(domain => domain).OrderByDescending(domain => domain.Count()).Select(domain => domain.Key).Take(5).ToList();
_data.TopHashtags = _data.TopHashtags.GroupBy(hash => hash).OrderByDescending(hash => hash.Count()).Select(hash => hash.Key).Take(5).ToList();
_data.TopEmojis = _data.TopEmojis.GroupBy(emoji => emoji).OrderByDescending(emoji => emoji.Count()).Select(emoji => emoji.Key).Take(5).ToList();
string json = JsonConvert.SerializeObject(_data);
if (File.Exists(_filePath))
{
//write data in file
File.WriteAllText(_filePath, json);
}
else
{
//stop streaming service
stream.Stop();
}
};
// Start
await stream.StartAsync();
}
catch (Exception ex)
{
//stop streaming service on exception
stream.Stop();
throw ex;
}
}
public async Task SaveTwitterDataAsync()
{
TwitterData data = new TwitterData();
data.AccountList = new List<TwitterAccount>(this.AccountList);
data.TimelineTabList = new List<TimelineTab>(this.TimelineListTab);
await SaveData.SaveTwitterDataAsync(data);
}
public string GetTwitterData(string screenName)
{
TweetDAL tDAL = new TweetDAL();
string data = tDAL.GetData(screenName);
var jsonObject = JArray.Parse(data);
if (jsonObject.HasValues)
{
string temp = (string)jsonObject[0]["user"]["profile_image_url_https"];
ProfileInfo tProfileInfo = new ProfileInfo((string)jsonObject[0]["user"]["name"], (string)jsonObject[0]["user"]["screen_name"],
temp.Substring(0, (temp.Length - ((temp.Substring(temp.IndexOf("_normal."))).Length))) + temp.Substring(temp.IndexOf("_normal.") + 7),
(string)jsonObject[0]["user"]["description"], (string)jsonObject[0]["user"]["followers_count"], (string)jsonObject[0]["user"]["friends_count"]);
Tweets[] tData = new Tweets[10];
int count = 0;
foreach (var item in jsonObject)
{
string Const_TwitterDateTemplate = "ddd MMM dd HH:mm:ss +ffff yyyy";
DateTime createdAt = DateTime.ParseExact((string)item["created_at"], Const_TwitterDateTemplate, new System.Globalization.CultureInfo("en-US"));
string mediaUrl = "", mediaType = "", retweet = "";
if (item.SelectToken("retweeted_status") != null)
{
retweet = (string)item["retweeted_status"]["full_text"];
}
if (item["entities"].HasValues)
{
if (item.SelectToken("extended_entities") != null)
{
if (item["extended_entities"].SelectToken("media") != null)
{
if ((string)item["extended_entities"]["media"][0]["type"] == "photo")
{
mediaType = "photo";
mediaUrl = (string)item["extended_entities"]["media"][0]["media_url_https"];
}
else if ((string)item["extended_entities"]["media"][0]["type"] == "video")
{
mediaType = "video";
string[] url = new string[4];
int rate = 0, j = 0;
JArray tempArr = (JArray)item["extended_entities"]["media"][0]["video_info"]["variants"];
for (int i = 0; i < tempArr.Count; i++)
{
if ((string)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["content_type"] == "video/mp4")
{
url[i] = (string)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["url"];
if (rate == 0)
{
rate = (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"];
j = i;
}
else
{
if (rate > (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"])
{
rate = (int)item["extended_entities"]["media"][0]["video_info"]["variants"][i]["bitrate"];
j = i;
}
}
}
}
mediaUrl = (string)item["extended_entities"]["media"][0]["video_info"]["variants"][j]["url"];
}
}
}
}
if (retweet != "")
{
tData[count] = new Tweets(retweet, createdAt.ToString(), (string)item["retweet_count"], mediaType, mediaUrl);
}
else
{
tData[count] = new Tweets((string)item["full_text"], createdAt.ToString(), (string)item["retweet_count"], mediaType, mediaUrl);
}
count++;
}
TwitterData tProfile = new TwitterData(tProfileInfo, tData);
var tProfileJson = new JavaScriptSerializer().Serialize(tProfile);
return(tProfileJson);
}
else
{
return("{}");
}
}
