public static void UpdateGame(Game game)
{
List<ActivityPoint> points = null;
using (var repository = new TweetRepository())
{
points = (from tweet in repository.GetAll().AsParallel()
where tweet.GameId == game.Id
group tweet by tweet.Timestamp.AddSeconds(-tweet.Timestamp.Second) into minute
select new ActivityPoint()
{
Time = minute.Key,
GameId = game.Id,
TweetCount = minute.Count()
}).ToList();
}
using (var repository = new ActivityRepository())
{
if (!repository.AddRange(points))
{
Console.WriteLine("Error occurred with saving points");
}
}
}
public void SaveNewGameAndExistingTeamsTest()
{
// setup
Game saved = null;
string homeTeam = "TestTeam1", awayTeam = "TestTeam2";
long homeId = -1, awayId = -1, gameId = -1;
using (var repository = new TeamRepository())
{
homeId = repository.Add(new Team() { Name = homeTeam });
awayId = repository.Add(new Team() { Name = awayTeam });
}
Game game = new Game()
{
Start = new DateTime(2014, 1, 1),
HomeTeamId = homeId,
AwayTeamId = awayId,
TweetsRetrieved = RetrievalStatus.NONE
};
// execute
using (var repository = new GameRepository())
{
gameId = repository.Save(game).Id;
saved = (from g in repository.DataSet.Include("HomeTeam").Include("AwayTeam")
where g.Id == gameId
select g).FirstOrDefault();
}
// validate
Assert.IsNotNull(saved);
Assert.IsNotNull(saved.HomeTeam);
Assert.IsNotNull(saved.AwayTeam);
Assert.AreEqual(homeTeam, saved.HomeTeam.Name);
Assert.AreEqual(awayTeam, saved.AwayTeam.Name);
}
public void EventFromJsonEventSecondHalfTest()
{
// set up
DateTime start = new DateTime(2014, 1, 1);
GameEventMiner miner = new GameEventMiner();
GameEvent evnt = null;
Game game = new Game()
{
Id = 1,
Start = start,
HalftimeStart = start.AddMinutes(45)
};
JsonGameEvent jsonEvnt = new JsonGameEvent()
{
Comment = "Goal",
CommentTypeId = 1,
MatchPeriodId = 2,
Minute = 55,
Second = 2,
Time = "55'"
};
// exercise
evnt = miner.EventFromJsonEvent(jsonEvnt, game);
// validate
Assert.IsNotNull(evnt);
Assert.AreEqual(GameEventType.GOAL, evnt.Type);
Assert.AreEqual(new DateTime(2014, 1, 1, 1, 10, 2), evnt.Timestamp);
Assert.AreEqual("55'", evnt.MatchTime);
Assert.AreEqual("Goal", evnt.Comment);
}
public void EventFromJsonEventFailTest()
{
// set up
GameEventMiner miner = new GameEventMiner();
GameEvent evnt = null;
Game game = new Game()
{
Id = 1,
Start = new DateTime(2014, 1, 1)
};
JsonGameEvent jsonEvnt = new JsonGameEvent()
{
Comment = "Invalid",
CommentTypeId = 99,
MatchPeriodId = 1,
Minute = 40,
Second = 2,
Time = "40'"
};
// exercise
evnt = miner.EventFromJsonEvent(jsonEvnt, game);
// validate
Assert.IsNull(evnt);
}
public Game GameFromArray(string[] line)
{
// set up a new game
Game game = new Game();
// set up the teams
using (var repository = new TeamRepository())
{
var teams = (repository.GetAll().Where(t => t.Name == line[iHome])).ToList();
if (teams.Count > 0)
{
game.HomeTeamId = teams[0].Id;
}
else
{
var home = new Team() { Name = line[iHome] };
home.Id = repository.Add(home);
game.HomeTeamId = home.Id;
}
teams = (repository.GetAll().Where(t => t.Name == line[iAway])).ToList();
if (teams.Count > 0)
{
game.AwayTeamId = teams[0].Id;
}
else
{
var away = new Team() { Name = line[iAway] };
away.Id = repository.Add(away);
game.AwayTeamId = away.Id;
}
}
// get the start time
DateTime start = DateTime.Parse(line[iDate]);
TimeSpan time = TimeSpan.Parse(line[iTime]);
game.Start = start + time;
// get the score
string score = line[iScore];
var scores = Regex.Matches(score, @"\d");
game.HomeGoals = Int32.Parse(scores[0].Value);
game.AwayGoals = Int32.Parse(scores[1].Value);
// set the status of the tweet mining
game.TweetsRetrieved = RetrievalStatus.NONE;
return game;
}
public void BuildQueryTest()
{
string expected = "#chelsea OR #cfc OR #coys OR #thfc OR #CFCTHFC";
// Set up teams
Team chelsea = new Team()
{
Name = "Chelsea",
ShortName = "CFC",
Hashtags = new List<Hashtag>()
};
foreach (var tag in new List<string>() { "#chelsea", "#cfc" })
chelsea.Hashtags.Add(new Hashtag() { Value = tag });
Team spurs = new Team()
{
Name = "Tottenham Hotspur",
ShortName = "THFC",
Hashtags = new List<Hashtag>()
};
foreach (var tag in new List<string>() { "#coys", "#thfc" })
chelsea.Hashtags.Add(new Hashtag() { Value = tag });
// Set up the Game
Game game = new Game()
{
HomeTeam = chelsea,
AwayTeam = spurs,
Start = new DateTime(2014, 3, 8, 17, 30, 0),
};
TweetMiner miner = new TweetMiner();
// Execute
string actual = miner.BuildQuery(game);
// Validate
Assert.IsNotNull(actual);
Assert.AreEqual(expected, actual);
}
/// <summary>
/// translates a JsonGameEvent to a GameEvent. returns null if the event
/// cannot be translated
/// </summary>
/// <param name="jsonEvent">the event to translate</param>
/// <param name="game">the game the event belongs to</param>
/// <returns>the new game event</returns>
public GameEvent EventFromJsonEvent(JsonGameEvent jsonEvent, Game game)
{
GameEvent evnt = new GameEvent();
// set up the easy fields first
evnt.MatchTime = jsonEvent.Time;
evnt.Comment = jsonEvent.Comment;
evnt.GameId = game.Id;
// calculate the timestamp
if (jsonEvent.MatchPeriodId == 1)
{
evnt.Timestamp = game.Start.AddMinutes(jsonEvent.Minute).AddSeconds(jsonEvent.Second);
}
else if (jsonEvent.MatchPeriodId == 2)
{
TimeSpan extra = game.HalftimeStart.Value.AddMinutes(-45) - game.Start;
// add 15 minutes for half time and any extra time
evnt.Timestamp = game.Start.AddMinutes(jsonEvent.Minute + 15)
.AddSeconds(jsonEvent.Second).AddSeconds(extra.TotalSeconds);
}
else
{
return null;
}
// determine what time of event this is
switch (jsonEvent.CommentTypeId)
{
case GoalType:
evnt.Type = GameEventType.GOAL;
break;
case SubType:
evnt.Type = GameEventType.SUB;
break;
case YellowType:
evnt.Type = GameEventType.YELLOW;
break;
case RedType:
evnt.Type = GameEventType.RED;
break;
case StartOfHalfType:
if (jsonEvent.MatchPeriodId == 1)
{
evnt.Type = GameEventType.FIRST_HALF_START;
evnt.MatchTime = "0'";
}
else if (jsonEvent.MatchPeriodId == 2)
{
evnt.Type = GameEventType.SECOND_HALF_START;
evnt.MatchTime = "45'";
game.HalftimeEnd = evnt.Timestamp;
}
else
return null;
break;
case EndFirstHalfType:
evnt.Type = GameEventType.FIRST_HALF_END;
game.HalftimeStart = evnt.Timestamp;
break;
case EndSecondHalfType:
evnt.Type = GameEventType.SECOND_HALF_END;
game.End = evnt.Timestamp;
break;
default:
return null;
}
return evnt;
}
/// <summary>
/// From the timeline data scraped identify what game it corresponds.
/// returns null if the game is not found
/// </summary>
/// <param name="data">the scraped timeline data</param>
/// <returns>the corresponding game object</returns>
public Game GetGameFromDetailsData(string data)
{
Game game = new Game();
var allJson = JsonConvert.DeserializeObject<JObject>(data);
string homeStr = allJson["Data"]["HomeTeam"].ToString();
JsonTeam homeTeam = JsonConvert.DeserializeObject<JsonTeam>(homeStr);
string awayStr = allJson["Data"]["AwayTeam"].ToString();
JsonTeam awayTeam = JsonConvert.DeserializeObject<JsonTeam>(awayStr);
string startStr = allJson["Data"]["DateTime"].ToString();
DateTime start = Convert.ToDateTime(startStr);
game.Start = start;
if (homeTeam == null || awayTeam == null ||
homeTeam.Name == null || awayTeam.Name == null)
{
return null;
}
// set up the teams
using (var repository = new TeamRepository())
{
// home team
var teams = (repository.GetAll().Where(t => t.ShortName == homeTeam.Code)).ToList();
if (teams.Count > 0)
{
game.HomeTeamId = teams[0].Id;
}
else
{
var home = new Team() { Name = homeTeam.Name };
home.Id = repository.Add(home);
game.HomeTeamId = home.Id;
game.HomeTeam = home;
}
// away team
teams = (repository.GetAll().Where(t => t.ShortName == awayTeam.Code)).ToList();
if (teams.Count > 0)
{
game.AwayTeamId = teams[0].Id;
}
else
{
var away = new Team() { Name = awayTeam.Name };
away.Id = repository.Add(away);
game.AwayTeamId = away.Id;
}
}
// get the score
game.HomeGoals = homeTeam.Score;
game.AwayGoals = awayTeam.Score;
// set the status of the tweet mining
game.TweetsRetrieved = RetrievalStatus.NONE;
return game;
}
public List<GameEvent> GetGameEvents(Game game)
{
// cannot do this without this info
if (game.MatchDay == null || game.LiveMatchId == null)
return null;
List<GameEvent> events = new List<GameEvent>();
// get the commentary data
string data = GetUrlData(commentaryUrl, game.MatchDay.Value, game.LiveMatchId.Value);
IEnumerable<JsonGameEvent> jsonEvents = GetJsonEventsFromCommentary(data).OrderBy(e => e.Order);
foreach (var jsonEvent in jsonEvents)
{
GameEvent evnt = EventFromJsonEvent(jsonEvent, game);
if (evnt != null)
events.Add(evnt);
}
return events;
}
public void EventFromJsonEventTest()
{
// set up
GameEventMiner miner = new GameEventMiner();
GameEvent evnt = null;
Game game = new Game()
{
Id = 1,
Start = new DateTime(2014, 1, 1)
};
JsonGameEvent jsonEvnt = new JsonGameEvent()
{
Comment = "Goal",
CommentTypeId = 5,
MatchPeriodId = 1,
Minute = 40,
Second = 2,
Time = "40'"
};
// exercise
evnt = miner.EventFromJsonEvent(jsonEvnt, game);
// validate
Assert.IsNotNull(evnt);
Assert.AreEqual(GameEventType.RED, evnt.Type);
Assert.AreEqual(new DateTime(2014, 1, 1, 0, 40, 2), evnt.Timestamp);
Assert.AreEqual("40'", evnt.MatchTime);
Assert.AreEqual("Goal", evnt.Comment);
}
public string BuildQuery(Game game)
{
// build the query based on offical team hashtags
IEnumerable<Hashtag> hashtags = game.HomeTeam.Hashtags.Union(game.AwayTeam.Hashtags);
StringBuilder hashtagQuery = new StringBuilder();
foreach (var hashtag in hashtags)
{
hashtagQuery.Append(hashtag.Value + " OR ");
}
hashtagQuery.AppendFormat("#{0}{1}", game.HomeTeam.ShortName, game.AwayTeam.ShortName);
string query = hashtagQuery.ToString();
return query;
}
private async void ScrapeTweets(Game game)
{
DateTime pointerTime, lastPrintTime, adjustedStart;
ulong maxId;
long counter = 0;
string query = BuildQuery(game);
// get the id of the closest tweet to full time
if (game.TweetsRetrieved == RetrievalStatus.IN_PROGRESS)
{
using (var repository = new TweetRepository())
{
var lastTweet = repository.GetAll()
.Where(tweet => tweet.GameId == game.Id)
.OrderBy(tweet => tweet.TwitterId)
.Take(1).First();
maxId = (ulong)lastTweet.TwitterId;
pointerTime = lastTweet.Timestamp;
lastPrintTime = lastTweet.Timestamp;
}
}
else
{
Status lastStatus = null;
while (true) {
try
{
// need to wrap this in try as this method can exceed rate limit as well
lastStatus = FindClosestTweetToTime(BSTDateTime.ToBSTDateTime(game.End.Value), query);
break;
}
catch (AggregateException e)
{
if (e.InnerException is TwitterQueryException)
{
// wait 15 mins and try again
Console.Out.WriteLine(String.Format("Rate limit hit. Sleeping for 15 mins from {0}{1}Zzz Zzz",
DateTime.Now, Environment.NewLine));
Thread.Sleep(1000 * 60 * 15);
Console.Out.WriteLine("Awake now, continuing...");
}
}
}
if (lastStatus == null)
{
Console.Out.WriteLine(String.Format("Unable to find starting point for game.. Skipping..{0}",
Environment.NewLine));
_completed = true;
return;
}
maxId = lastStatus.StatusID;
pointerTime = lastStatus.CreatedAt;
lastPrintTime = lastStatus.CreatedAt;
}
// adjust times for british daylight savings
pointerTime = BSTDateTime.ToBSTDateTime(pointerTime);
lastPrintTime = BSTDateTime.ToBSTDateTime(lastPrintTime);
adjustedStart = BSTDateTime.ToBSTDateTime(game.Start);
while (adjustedStart < pointerTime)
{
try {
var resp = await (from search in Twitter.Search
where search.Type == SearchType.Search
&& search.ResultType == ResultType.Recent
&& search.Query == query
&& search.Count == 100
&& search.MaxID == maxId
select search).SingleOrDefaultAsync();
if (resp != null && resp.Statuses != null)
{
// update loop
Status earliest = resp.Statuses[resp.Statuses.Count - 1];
maxId = (ulong)earliest.StatusID;
counter += resp.Statuses.Count;
pointerTime = earliest.CreatedAt;
// add the statuses to the queue
_statuses.Enqueue(resp.Statuses);
}
else
{
break;
}
}
catch (TwitterQueryException)
{
Console.Out.WriteLine(String.Format("Rate limit hit. Sleeping for 15 mins from {0}{1}Zzz Zzz",
DateTime.Now, Environment.NewLine));
Thread.Sleep(1000 * 60 * 15);
Console.Out.WriteLine("Awake now, continuing...");
continue;
}
if ((lastPrintTime - pointerTime).Minutes > 1)
{
Console.Out.WriteLine(String.Format("pointerTime: {0} tweets: {1}", BSTDateTime.FromBSTDateTime(pointerTime), counter));
lastPrintTime = pointerTime;
}
}
// done at this point
_completed = true;
}
