public void CrawWhoScored() { if (isRunning) { return; } GBCommon.DataFolder = @"c:\tests\GBData"; if (!Directory.Exists(GBCommon.DataFolder)) { Directory.CreateDirectory(GBCommon.DataFolder); } while (true) { isRunning = true; GBCommon.LogInfo("Starting a new round..."); int startFrom = -1; int.TryParse(ConfigurationManager.AppSettings["startfrom"], out startFrom); if (startFrom <= 0) { GBCollector.Collect(); } else { GBCollector.Collect(startFrom); } int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval); Thread.Sleep(TimeSpan.FromSeconds(sleepInterval)); } }
public void UpdateResults() { var response1 = GBCommon.SendRequest( "{\"$or\":[{\"BetItem.Result\":{\"$not\":{\"$exists\":\"true\"}}}, {\"BetItem.Result\":\"null\"}, {\"BetItem.Result\":\"Unknown\"}]}", ConfigurationManager.AppSettings["datastore"], ConfigurationManager.AppSettings["apikey"], ConfigurationManager.AppSettings["passwd"], "application/json", "POST" ); List <Stake> stakes = new List <Stake>(); using (StreamReader streamReader = new StreamReader(response1.GetResponseStream())) { var text = streamReader.ReadToEnd(); stakes = JsonConvert.DeserializeObject <List <Stake> >(text); } var matchesToQuery = from stake in stakes where stake.BetItem.Result == GameResult.Unknown select stake.BetItem.GameName; foreach (string matchIndex in matchesToQuery) { var result = this.QueryResultOfOneMatch(matchIndex); if (result != GameResult.Unknown) { var response2 = GBCommon.SendRequest( "[{\"BetItem.GameName\":\"" + matchIndex + "\"}, {" + "\"BetItem.Result\":\"" + result.ToString() + "\"}]", ConfigurationManager.AppSettings["datastore"], ConfigurationManager.AppSettings["apikey"], ConfigurationManager.AppSettings["passwd"], "application/json", "PATCH" ); using (StreamReader streamReader = new StreamReader(response2.GetResponseStream())) { var text = streamReader.ReadToEnd(); dynamic responseResult = JsonConvert.DeserializeObject(text); if (responseResult.msg == "success") { GBCommon.LogInfo("Successfully updated result of {0} to {1}", matchIndex, result); } else { GBCommon.LogInfo("Failed to update result of {0}", matchIndex); } } } } }
static void Main(string[] args) { GBResultCollector collector = new GBResultCollector(ConfigurationManager.AppSettings["baseuri"]); int sleepInterval = 3600; // Default value in secs while (true) { GBCommon.LogInfo("Starting a new result query round..."); GBCommon.LogInfo("Current time: {0}", DateTime.Now); collector.UpdateResults(); int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval); Thread.Sleep(TimeSpan.FromSeconds(sleepInterval)); } }
static void Main(string[] args) { int sleepInterval = 60; // In secs while (true) { GBCommon.LogInfo("Starting a new reporting round..."); GBCommon.LogInfo("Current time: {0}", DateTime.Now); // Dynamically change data folder if (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["datafolder"])) { GBCommon.DataFolder = ConfigurationManager.AppSettings["datafolder"]; if (!Directory.Exists(GBCommon.DataFolder)) { Directory.CreateDirectory(GBCommon.DataFolder); } } GBReporter.Report(); int.TryParse(ConfigurationManager.AppSettings["reportinterval"], out sleepInterval); Thread.Sleep(TimeSpan.FromSeconds(sleepInterval)); } }
public GameResult QueryResultOfOneMatch(string matchIndex) { Uri uri = new Uri(baseuri, string.Format("Matches/{0}", matchIndex)); WebRequest getRequest = WebRequest.Create(uri); WebResponse response = GBCommon.GetResponseWithRetries(getRequest, GBCommon.DefaultRetries); using (Stream stream = response.GetResponseStream()) { using (StreamReader reader = new StreamReader(stream)) { HtmlDocument doc = new HtmlDocument(); doc.Load(reader); if (null == doc.DocumentNode) { throw new WebException(matchIndex + ": Error when parsing the root node"); } HtmlNode bodyNode = doc.DocumentNode.SelectSingleNode("//body"); if (null == bodyNode) { throw new WebException(matchIndex + ": Error when parsing the body node"); } var scripts = bodyNode.SelectNodes("//script[@type='text/javascript']"); string pattern = @"var\s+matchHeaderJson\s*=\s*JSON.parse\(\'(.*)\'\);"; foreach (var script in scripts) { Match match = Regex.Match(script.InnerText, pattern); if (match.Success) { string matchHeaderJson = match.Groups[1].Value; if (!string.IsNullOrEmpty(matchHeaderJson)) { dynamic matchHeader = JsonConvert.DeserializeObject(matchHeaderJson); string resultStr = matchHeader.FullTimeResult; if (!string.IsNullOrEmpty(resultStr)) { int score1 = int.Parse(resultStr.Split(':')[0].Trim(' ')); int score2 = int.Parse(resultStr.Split(':')[1].Trim(' ')); if (score1 < score2) { return(GameResult.Lose); } else if (score1 == score2) { return(GameResult.Draw); } else { return(GameResult.Win); } } } } } } } return(GameResult.Unknown); }
static void Main(string[] args) { if (args.Length >= 1 && args[0] == "/?") { Usage(); return; } double threshold = 0; if (args.Length >= 1) { string thresholdString = args[0]; if (!double.TryParse(thresholdString, out threshold)) { Console.WriteLine("Invalid threshold"); return; } } var response = GBCommon.SendRequest( "{\"$and\":[{\"BetItem.Result\":{\"$exists\":\"true\"}}, {\"BetItem.Result\":{\"$ne\":\"Unknown\"}}, {\"ROI\": {\"$gte\":" + (threshold / 100).ToString() + "}}]}", ConfigurationManager.AppSettings["datastore"], ConfigurationManager.AppSettings["apikey"], ConfigurationManager.AppSettings["passwd"], "application/json", "POST" ); List <Stake> stakes = new List <Stake>(); using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) { string responseString = streamReader.ReadToEnd(); stakes = JsonConvert.DeserializeObject <List <Stake> > (responseString); } double actualBet = 0; double actualReturn = 0; foreach (var stake in stakes) { actualBet++; if (stake.Decision.ToString() == stake.BetItem.Result.ToString()) { if (stake.BetItem.Odds != null && stake.BetItem.Odds is ThreeWayOdds) { if (stake.BetItem.Result == GameResult.Win) { actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Win; } if (stake.BetItem.Result == GameResult.Draw) { actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Draw; } if (stake.BetItem.Result == GameResult.Lose) { actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Lose; } } } } if (actualBet == 0) { Console.WriteLine("N/A"); } else { Console.WriteLine(actualReturn / actualBet); } }
protected void CollectOneMatch(string matchIndex) { Uri uri = new Uri(baseuri, string.Format("Matches/{0}/Betting", matchIndex)); WebRequest getRequest = WebRequest.Create(uri); string matchAlias = gameType.ToString() + "-" + matchIndex; AveragedNonWeightedBetItemManager mgr = new AveragedNonWeightedBetItemManager(); Guid matchGuid = Guid.NewGuid(); string team1 = "", team2 = "", dateStr = ""; WebResponse response = GBCommon.GetResponseWithRetries(getRequest, GBCommon.DefaultRetries); using (Stream stream = response.GetResponseStream()) { using (StreamReader reader = new StreamReader(stream)) { HtmlDocument doc = new HtmlDocument(); doc.Load(reader); if (null == doc.DocumentNode) { throw new WebException(matchAlias + ": Error when parsing the root node"); } HtmlNode bodyNode = doc.DocumentNode.SelectSingleNode("//body"); if (null == bodyNode) { throw new WebException(matchAlias + ": Error when parsing the body node"); } var scripts = bodyNode.SelectNodes("//script[@type='text/javascript']"); string pattern = @"matchHeader.load.*\d+,\d+,\'([A-Za-z0-9\. ]+)\',\'([A-Za-z0-9\. ]+)\',\'([0-9:/ ]+)\'"; foreach (var script in scripts) { Match match = Regex.Match(script.InnerText, pattern); if (match.Success) { team1 = match.Groups[1].Value.Replace(' ', '_'); team2 = match.Groups[2].Value.Replace(' ', '_'); dateStr = match.Groups[3].Value; } } if (string.IsNullOrEmpty(dateStr) || string.IsNullOrEmpty(team1) || string.IsNullOrEmpty(team2)) { throw new ApplicationException(matchAlias + ": No match time or team name is found"); } var bookMakerNameNodes = bodyNode.SelectNodes("//div[@id='ThreeWay-OrdinaryTime']//a[@class='bm-name']"); if (null == bookMakerNameNodes) { throw new ApplicationException(matchAlias + ": No target odds found"); } foreach (var bookMakerNameNode in bookMakerNameNodes) { var node = bookMakerNameNode.ParentNode.ParentNode; string bookMaker = node.ChildNodes[1].SelectSingleNode(".//a[@class='bm-name']").InnerText; string win = node.ChildNodes[3].SelectSingleNode(".//a/span").InnerText.Trim(); string draw = node.ChildNodes[5].SelectSingleNode(".//a/span").InnerText.Trim(); string lose = node.ChildNodes[7].SelectSingleNode(".//a/span").InnerText.Trim(); BetItem bet = new BetItem( matchGuid.ToString(), matchIndex, gameType, new System.Collections.Generic.List <Team>() { new Team(team1, teamType), new Team(team2, teamType) }, new ThreeWayOdds(Convert.ToDouble(win), Convert.ToDouble(lose), Convert.ToDouble(draw)), false, DateTime.UtcNow, Convert.ToDateTime(dateStr), bookMaker ); mgr.CurrentBets.Add(bet); } string fileName = GBCommon.ConstructRecordFileName(gameType, team1, team2, dateStr); if (File.Exists(fileName)) { File.Delete(fileName); } GBCommon.LogInfo("{0}: Collected {1}", DateTime.Now, fileName); mgr.Serialize(fileName); } } }
/// <summary> /// Collect all available records from last index or a particular index /// </summary> /// <param name="startFrom">Specified starting index</param> public void Collect(int startFrom) { lock (collectorLock) { if (inProgress) { GBCommon.LogInfo("{0}: Another collection in progress", DateTime.Now); return; } else { inProgress = true; } } try { int lastProcessed = GBCommon.ReadContinuationIndex(GBCommon.ConstructCollectContinuationFileName(gameType)); int startIndex = startFrom > 0 ? startFrom : -99 + lastProcessed; Stopwatch watch = new Stopwatch(); watch.Start(); int count = 0; int lastValid = -1; int currentIndex = startIndex; for (; ; currentIndex++) { try { CollectOneMatch(currentIndex.ToString()); lastValid = currentIndex; // Reset the count for valid index count = 0; } catch (ApplicationException e) { if (currentIndex < lastProcessed) { // In this case, a past match or past empty slot is being processed continue; } if (++count < toleranceCount) { // In this case, a possible future empty slot is being processed } else { GBCommon.LogInfo("Stops at {0}", e.Message); GBCommon.LogInfo("Elapsed time: {0}", watch.Elapsed); break; } } catch (WebException e) { GBCommon.LogInfo("Stops at {0}", e.Message); GBCommon.LogInfo("Elapsed time: {0}", watch.Elapsed); break; } } if (lastValid >= lastProcessed) { GBCommon.WriteContinuationIndex(GBCommon.ConstructCollectContinuationFileName(gameType), lastValid); } Console.WriteLine("Collection Completed."); } finally { lock (collectorLock) { inProgress = false; } } }
static void Main(string[] args) { int startFrom = -1; string gameType = string.Empty; if (!args.Any()) { PrintUsage(); return; } if (args.Any()) { if (args[0].Contains("/?") || args[0].Contains("/h")) { PrintUsage(); return; } else { // Check the commandline arg first gameType = args[0]; if (args.Count() >= 2) { int.TryParse(args[1], out startFrom); } } } // Check the app config second if (startFrom <= 0) { int.TryParse(ConfigurationManager.AppSettings["startfrom"], out startFrom); } //GBCommon.DataFolder = @"c:\tests\GBData"; if (!Directory.Exists(GBCommon.DataFolder)) { Directory.CreateDirectory(GBCommon.DataFolder); } GBCollector collector = new GBCollector( ConfigurationManager.AppSettings["baseuri"], gameType, ConfigurationManager.AppSettings["teamtype"], ConfigurationManager.AppSettings["tolerancecount"]); int sleepInterval = 60; // In secs while (true) { GBCommon.LogInfo("Starting a new collection round..."); GBCommon.LogInfo("Current time: {0}", DateTime.Now); string dataFolder = ConfigurationManager.AppSettings["datafolder"]; if (!string.IsNullOrEmpty(dataFolder) && !string.IsNullOrEmpty(gameType)) { GBCommon.DataFolder = dataFolder; if (!Directory.Exists(Path.Combine(dataFolder, gameType))) { Directory.CreateDirectory(Path.Combine(dataFolder, gameType)); } } // Check the ini third if (startFrom <= 0) { collector.Collect(); } else { collector.Collect(startFrom); startFrom = 0; } int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval); Thread.Sleep(TimeSpan.FromSeconds(sleepInterval)); } }
public static void Report() { foreach (string folder in Directory.GetDirectories(GBCommon.DataFolder)) { GameType gameType = (GameType)Enum.Parse(typeof(GameType), folder.Remove(0, GBCommon.DataFolder.Length).TrimStart('\\')); DateTime continuationStartDate = GBCommon.ReadContinuationDate(GBCommon.ConstructReportContinuationFileName(gameType)); DateTime latest = continuationStartDate; var files = new DirectoryInfo(folder).GetFiles("*.json", SearchOption.TopDirectoryOnly).Where(file => file.LastWriteTime > continuationStartDate + TimeSpan.FromSeconds(1)); foreach (var file in files) { AveragedNonWeightedBetItemManager mgr = new AveragedNonWeightedBetItemManager(); mgr.Deserialize(file.FullName, OddsType.ThreeWay); if (mgr.CurrentBets.Count < 1) { throw new FileLoadException(string.Format("Record file {0} not valid", file.Name)); } BetItem firstBet = mgr.CurrentBets[0]; Stake bestStake = mgr.BestStake(firstBet.GameName, firstBet.GameType, firstBet.Odds.Type); string output = bestStake.ROI.ToString("p2") + " " + bestStake.BetItem.MatchDate.ToShortDateString() + " " + bestStake.BetItem.Teams[0].Name + " " + bestStake.BetItem.Teams[1].Name + " " + bestStake.BetItem.BookMaker.Replace(' ', '_') + " " + bestStake.BetItem.Odds.Type.ToString() + " " + bestStake.Decision; GBCommon.Report(output, gameType); if (file.LastWriteTime > latest) { latest = file.LastWriteTime; } // If the game stake exists, delete the old one var deleteStakeResponse = GBCommon.SendRequest( "{\"BetItem.GameName\": \"" + bestStake.BetItem.GameName + "\"}", ConfigurationManager.AppSettings["datastore"], ConfigurationManager.AppSettings["apikey"], ConfigurationManager.AppSettings["passwd"], "application/json", "DELETE" ); using (StreamReader streamReader = new StreamReader(deleteStakeResponse.GetResponseStream())) { var text = streamReader.ReadToEnd(); Console.WriteLine(text); } // Add the new stake var addStakeResponse = GBCommon.SendRequest( JsonConvert.SerializeObject(bestStake), ConfigurationManager.AppSettings["datastore"], ConfigurationManager.AppSettings["apikey"], ConfigurationManager.AppSettings["passwd"], "application/json", "PUT" ); using (StreamReader streamReader = new StreamReader(addStakeResponse.GetResponseStream())) { var text = streamReader.ReadToEnd(); Console.WriteLine(text); } } if (latest > continuationStartDate) { GBCommon.WriteContinuationDate(GBCommon.ConstructReportContinuationFileName(gameType), latest); } } GBCommon.LogInfo("Reporting Completed."); }