public static void UpdateGame(Game game) { List<ActivityPoint> points = null; using (var repository = new TweetRepository()) { points = (from tweet in repository.GetAll().AsParallel() where tweet.GameId == game.Id group tweet by tweet.Timestamp.AddSeconds(-tweet.Timestamp.Second) into minute select new ActivityPoint() { Time = minute.Key, GameId = game.Id, TweetCount = minute.Count() }).ToList(); } using (var repository = new ActivityRepository()) { if (!repository.AddRange(points)) { Console.WriteLine("Error occurred with saving points"); } } }
public void SaveNewGameAndExistingTeamsTest() { // setup Game saved = null; string homeTeam = "TestTeam1", awayTeam = "TestTeam2"; long homeId = -1, awayId = -1, gameId = -1; using (var repository = new TeamRepository()) { homeId = repository.Add(new Team() { Name = homeTeam }); awayId = repository.Add(new Team() { Name = awayTeam }); } Game game = new Game() { Start = new DateTime(2014, 1, 1), HomeTeamId = homeId, AwayTeamId = awayId, TweetsRetrieved = RetrievalStatus.NONE }; // execute using (var repository = new GameRepository()) { gameId = repository.Save(game).Id; saved = (from g in repository.DataSet.Include("HomeTeam").Include("AwayTeam") where g.Id == gameId select g).FirstOrDefault(); } // validate Assert.IsNotNull(saved); Assert.IsNotNull(saved.HomeTeam); Assert.IsNotNull(saved.AwayTeam); Assert.AreEqual(homeTeam, saved.HomeTeam.Name); Assert.AreEqual(awayTeam, saved.AwayTeam.Name); }
public void EventFromJsonEventSecondHalfTest() { // set up DateTime start = new DateTime(2014, 1, 1); GameEventMiner miner = new GameEventMiner(); GameEvent evnt = null; Game game = new Game() { Id = 1, Start = start, HalftimeStart = start.AddMinutes(45) }; JsonGameEvent jsonEvnt = new JsonGameEvent() { Comment = "Goal", CommentTypeId = 1, MatchPeriodId = 2, Minute = 55, Second = 2, Time = "55'" }; // exercise evnt = miner.EventFromJsonEvent(jsonEvnt, game); // validate Assert.IsNotNull(evnt); Assert.AreEqual(GameEventType.GOAL, evnt.Type); Assert.AreEqual(new DateTime(2014, 1, 1, 1, 10, 2), evnt.Timestamp); Assert.AreEqual("55'", evnt.MatchTime); Assert.AreEqual("Goal", evnt.Comment); }
public void EventFromJsonEventFailTest() { // set up GameEventMiner miner = new GameEventMiner(); GameEvent evnt = null; Game game = new Game() { Id = 1, Start = new DateTime(2014, 1, 1) }; JsonGameEvent jsonEvnt = new JsonGameEvent() { Comment = "Invalid", CommentTypeId = 99, MatchPeriodId = 1, Minute = 40, Second = 2, Time = "40'" }; // exercise evnt = miner.EventFromJsonEvent(jsonEvnt, game); // validate Assert.IsNull(evnt); }
public Game GameFromArray(string[] line) { // set up a new game Game game = new Game(); // set up the teams using (var repository = new TeamRepository()) { var teams = (repository.GetAll().Where(t => t.Name == line[iHome])).ToList(); if (teams.Count > 0) { game.HomeTeamId = teams[0].Id; } else { var home = new Team() { Name = line[iHome] }; home.Id = repository.Add(home); game.HomeTeamId = home.Id; } teams = (repository.GetAll().Where(t => t.Name == line[iAway])).ToList(); if (teams.Count > 0) { game.AwayTeamId = teams[0].Id; } else { var away = new Team() { Name = line[iAway] }; away.Id = repository.Add(away); game.AwayTeamId = away.Id; } } // get the start time DateTime start = DateTime.Parse(line[iDate]); TimeSpan time = TimeSpan.Parse(line[iTime]); game.Start = start + time; // get the score string score = line[iScore]; var scores = Regex.Matches(score, @"\d"); game.HomeGoals = Int32.Parse(scores[0].Value); game.AwayGoals = Int32.Parse(scores[1].Value); // set the status of the tweet mining game.TweetsRetrieved = RetrievalStatus.NONE; return game; }
public void BuildQueryTest() { string expected = "#chelsea OR #cfc OR #coys OR #thfc OR #CFCTHFC"; // Set up teams Team chelsea = new Team() { Name = "Chelsea", ShortName = "CFC", Hashtags = new List<Hashtag>() }; foreach (var tag in new List<string>() { "#chelsea", "#cfc" }) chelsea.Hashtags.Add(new Hashtag() { Value = tag }); Team spurs = new Team() { Name = "Tottenham Hotspur", ShortName = "THFC", Hashtags = new List<Hashtag>() }; foreach (var tag in new List<string>() { "#coys", "#thfc" }) chelsea.Hashtags.Add(new Hashtag() { Value = tag }); // Set up the Game Game game = new Game() { HomeTeam = chelsea, AwayTeam = spurs, Start = new DateTime(2014, 3, 8, 17, 30, 0), }; TweetMiner miner = new TweetMiner(); // Execute string actual = miner.BuildQuery(game); // Validate Assert.IsNotNull(actual); Assert.AreEqual(expected, actual); }
/// <summary> /// translates a JsonGameEvent to a GameEvent. returns null if the event /// cannot be translated /// </summary> /// <param name="jsonEvent">the event to translate</param> /// <param name="game">the game the event belongs to</param> /// <returns>the new game event</returns> public GameEvent EventFromJsonEvent(JsonGameEvent jsonEvent, Game game) { GameEvent evnt = new GameEvent(); // set up the easy fields first evnt.MatchTime = jsonEvent.Time; evnt.Comment = jsonEvent.Comment; evnt.GameId = game.Id; // calculate the timestamp if (jsonEvent.MatchPeriodId == 1) { evnt.Timestamp = game.Start.AddMinutes(jsonEvent.Minute).AddSeconds(jsonEvent.Second); } else if (jsonEvent.MatchPeriodId == 2) { TimeSpan extra = game.HalftimeStart.Value.AddMinutes(-45) - game.Start; // add 15 minutes for half time and any extra time evnt.Timestamp = game.Start.AddMinutes(jsonEvent.Minute + 15) .AddSeconds(jsonEvent.Second).AddSeconds(extra.TotalSeconds); } else { return null; } // determine what time of event this is switch (jsonEvent.CommentTypeId) { case GoalType: evnt.Type = GameEventType.GOAL; break; case SubType: evnt.Type = GameEventType.SUB; break; case YellowType: evnt.Type = GameEventType.YELLOW; break; case RedType: evnt.Type = GameEventType.RED; break; case StartOfHalfType: if (jsonEvent.MatchPeriodId == 1) { evnt.Type = GameEventType.FIRST_HALF_START; evnt.MatchTime = "0'"; } else if (jsonEvent.MatchPeriodId == 2) { evnt.Type = GameEventType.SECOND_HALF_START; evnt.MatchTime = "45'"; game.HalftimeEnd = evnt.Timestamp; } else return null; break; case EndFirstHalfType: evnt.Type = GameEventType.FIRST_HALF_END; game.HalftimeStart = evnt.Timestamp; break; case EndSecondHalfType: evnt.Type = GameEventType.SECOND_HALF_END; game.End = evnt.Timestamp; break; default: return null; } return evnt; }
/// <summary> /// From the timeline data scraped identify what game it corresponds. /// returns null if the game is not found /// </summary> /// <param name="data">the scraped timeline data</param> /// <returns>the corresponding game object</returns> public Game GetGameFromDetailsData(string data) { Game game = new Game(); var allJson = JsonConvert.DeserializeObject<JObject>(data); string homeStr = allJson["Data"]["HomeTeam"].ToString(); JsonTeam homeTeam = JsonConvert.DeserializeObject<JsonTeam>(homeStr); string awayStr = allJson["Data"]["AwayTeam"].ToString(); JsonTeam awayTeam = JsonConvert.DeserializeObject<JsonTeam>(awayStr); string startStr = allJson["Data"]["DateTime"].ToString(); DateTime start = Convert.ToDateTime(startStr); game.Start = start; if (homeTeam == null || awayTeam == null || homeTeam.Name == null || awayTeam.Name == null) { return null; } // set up the teams using (var repository = new TeamRepository()) { // home team var teams = (repository.GetAll().Where(t => t.ShortName == homeTeam.Code)).ToList(); if (teams.Count > 0) { game.HomeTeamId = teams[0].Id; } else { var home = new Team() { Name = homeTeam.Name }; home.Id = repository.Add(home); game.HomeTeamId = home.Id; game.HomeTeam = home; } // away team teams = (repository.GetAll().Where(t => t.ShortName == awayTeam.Code)).ToList(); if (teams.Count > 0) { game.AwayTeamId = teams[0].Id; } else { var away = new Team() { Name = awayTeam.Name }; away.Id = repository.Add(away); game.AwayTeamId = away.Id; } } // get the score game.HomeGoals = homeTeam.Score; game.AwayGoals = awayTeam.Score; // set the status of the tweet mining game.TweetsRetrieved = RetrievalStatus.NONE; return game; }
public List<GameEvent> GetGameEvents(Game game) { // cannot do this without this info if (game.MatchDay == null || game.LiveMatchId == null) return null; List<GameEvent> events = new List<GameEvent>(); // get the commentary data string data = GetUrlData(commentaryUrl, game.MatchDay.Value, game.LiveMatchId.Value); IEnumerable<JsonGameEvent> jsonEvents = GetJsonEventsFromCommentary(data).OrderBy(e => e.Order); foreach (var jsonEvent in jsonEvents) { GameEvent evnt = EventFromJsonEvent(jsonEvent, game); if (evnt != null) events.Add(evnt); } return events; }
public void EventFromJsonEventTest() { // set up GameEventMiner miner = new GameEventMiner(); GameEvent evnt = null; Game game = new Game() { Id = 1, Start = new DateTime(2014, 1, 1) }; JsonGameEvent jsonEvnt = new JsonGameEvent() { Comment = "Goal", CommentTypeId = 5, MatchPeriodId = 1, Minute = 40, Second = 2, Time = "40'" }; // exercise evnt = miner.EventFromJsonEvent(jsonEvnt, game); // validate Assert.IsNotNull(evnt); Assert.AreEqual(GameEventType.RED, evnt.Type); Assert.AreEqual(new DateTime(2014, 1, 1, 0, 40, 2), evnt.Timestamp); Assert.AreEqual("40'", evnt.MatchTime); Assert.AreEqual("Goal", evnt.Comment); }
public string BuildQuery(Game game) { // build the query based on offical team hashtags IEnumerable<Hashtag> hashtags = game.HomeTeam.Hashtags.Union(game.AwayTeam.Hashtags); StringBuilder hashtagQuery = new StringBuilder(); foreach (var hashtag in hashtags) { hashtagQuery.Append(hashtag.Value + " OR "); } hashtagQuery.AppendFormat("#{0}{1}", game.HomeTeam.ShortName, game.AwayTeam.ShortName); string query = hashtagQuery.ToString(); return query; }
private async void ScrapeTweets(Game game) { DateTime pointerTime, lastPrintTime, adjustedStart; ulong maxId; long counter = 0; string query = BuildQuery(game); // get the id of the closest tweet to full time if (game.TweetsRetrieved == RetrievalStatus.IN_PROGRESS) { using (var repository = new TweetRepository()) { var lastTweet = repository.GetAll() .Where(tweet => tweet.GameId == game.Id) .OrderBy(tweet => tweet.TwitterId) .Take(1).First(); maxId = (ulong)lastTweet.TwitterId; pointerTime = lastTweet.Timestamp; lastPrintTime = lastTweet.Timestamp; } } else { Status lastStatus = null; while (true) { try { // need to wrap this in try as this method can exceed rate limit as well lastStatus = FindClosestTweetToTime(BSTDateTime.ToBSTDateTime(game.End.Value), query); break; } catch (AggregateException e) { if (e.InnerException is TwitterQueryException) { // wait 15 mins and try again Console.Out.WriteLine(String.Format("Rate limit hit. Sleeping for 15 mins from {0}{1}Zzz Zzz", DateTime.Now, Environment.NewLine)); Thread.Sleep(1000 * 60 * 15); Console.Out.WriteLine("Awake now, continuing..."); } } } if (lastStatus == null) { Console.Out.WriteLine(String.Format("Unable to find starting point for game.. Skipping..{0}", Environment.NewLine)); _completed = true; return; } maxId = lastStatus.StatusID; pointerTime = lastStatus.CreatedAt; lastPrintTime = lastStatus.CreatedAt; } // adjust times for british daylight savings pointerTime = BSTDateTime.ToBSTDateTime(pointerTime); lastPrintTime = BSTDateTime.ToBSTDateTime(lastPrintTime); adjustedStart = BSTDateTime.ToBSTDateTime(game.Start); while (adjustedStart < pointerTime) { try { var resp = await (from search in Twitter.Search where search.Type == SearchType.Search && search.ResultType == ResultType.Recent && search.Query == query && search.Count == 100 && search.MaxID == maxId select search).SingleOrDefaultAsync(); if (resp != null && resp.Statuses != null) { // update loop Status earliest = resp.Statuses[resp.Statuses.Count - 1]; maxId = (ulong)earliest.StatusID; counter += resp.Statuses.Count; pointerTime = earliest.CreatedAt; // add the statuses to the queue _statuses.Enqueue(resp.Statuses); } else { break; } } catch (TwitterQueryException) { Console.Out.WriteLine(String.Format("Rate limit hit. Sleeping for 15 mins from {0}{1}Zzz Zzz", DateTime.Now, Environment.NewLine)); Thread.Sleep(1000 * 60 * 15); Console.Out.WriteLine("Awake now, continuing..."); continue; } if ((lastPrintTime - pointerTime).Minutes > 1) { Console.Out.WriteLine(String.Format("pointerTime: {0} tweets: {1}", BSTDateTime.FromBSTDateTime(pointerTime), counter)); lastPrintTime = pointerTime; } } // done at this point _completed = true; }