public void CrawWhoScored()
{
if (isRunning)
{
return;
}
GBCommon.DataFolder = @"c:\tests\GBData";
if (!Directory.Exists(GBCommon.DataFolder))
{
Directory.CreateDirectory(GBCommon.DataFolder);
}
while (true)
{
isRunning = true;
GBCommon.LogInfo("Starting a new round...");
int startFrom = -1;
int.TryParse(ConfigurationManager.AppSettings["startfrom"], out startFrom);
if (startFrom <= 0)
{
GBCollector.Collect();
}
else
{
GBCollector.Collect(startFrom);
}
int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval);
Thread.Sleep(TimeSpan.FromSeconds(sleepInterval));
}
}
public void UpdateResults()
{
var response1 = GBCommon.SendRequest(
"{\"$or\":[{\"BetItem.Result\":{\"$not\":{\"$exists\":\"true\"}}}, {\"BetItem.Result\":\"null\"}, {\"BetItem.Result\":\"Unknown\"}]}",
ConfigurationManager.AppSettings["datastore"],
ConfigurationManager.AppSettings["apikey"],
ConfigurationManager.AppSettings["passwd"],
"application/json",
"POST"
);
List <Stake> stakes = new List <Stake>();
using (StreamReader streamReader = new StreamReader(response1.GetResponseStream()))
{
var text = streamReader.ReadToEnd();
stakes = JsonConvert.DeserializeObject <List <Stake> >(text);
}
var matchesToQuery =
from stake in stakes
where stake.BetItem.Result == GameResult.Unknown
select stake.BetItem.GameName;
foreach (string matchIndex in matchesToQuery)
{
var result = this.QueryResultOfOneMatch(matchIndex);
if (result != GameResult.Unknown)
{
var response2 = GBCommon.SendRequest(
"[{\"BetItem.GameName\":\"" + matchIndex + "\"}, {" + "\"BetItem.Result\":\"" + result.ToString() + "\"}]",
ConfigurationManager.AppSettings["datastore"],
ConfigurationManager.AppSettings["apikey"],
ConfigurationManager.AppSettings["passwd"],
"application/json",
"PATCH"
);
using (StreamReader streamReader = new StreamReader(response2.GetResponseStream()))
{
var text = streamReader.ReadToEnd();
dynamic responseResult = JsonConvert.DeserializeObject(text);
if (responseResult.msg == "success")
{
GBCommon.LogInfo("Successfully updated result of {0} to {1}", matchIndex, result);
}
else
{
GBCommon.LogInfo("Failed to update result of {0}", matchIndex);
}
}
}
}
}
static void Main(string[] args)
{
GBResultCollector collector = new GBResultCollector(ConfigurationManager.AppSettings["baseuri"]);
int sleepInterval = 3600; // Default value in secs
while (true)
{
GBCommon.LogInfo("Starting a new result query round...");
GBCommon.LogInfo("Current time: {0}", DateTime.Now);
collector.UpdateResults();
int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval);
Thread.Sleep(TimeSpan.FromSeconds(sleepInterval));
}
}
static void Main(string[] args)
{
int sleepInterval = 60; // In secs
while (true)
{
GBCommon.LogInfo("Starting a new reporting round...");
GBCommon.LogInfo("Current time: {0}", DateTime.Now);
// Dynamically change data folder
if (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["datafolder"]))
{
GBCommon.DataFolder = ConfigurationManager.AppSettings["datafolder"];
if (!Directory.Exists(GBCommon.DataFolder))
{
Directory.CreateDirectory(GBCommon.DataFolder);
}
}
GBReporter.Report();
int.TryParse(ConfigurationManager.AppSettings["reportinterval"], out sleepInterval);
Thread.Sleep(TimeSpan.FromSeconds(sleepInterval));
}
}
public GameResult QueryResultOfOneMatch(string matchIndex)
{
Uri uri = new Uri(baseuri, string.Format("Matches/{0}", matchIndex));
WebRequest getRequest = WebRequest.Create(uri);
WebResponse response = GBCommon.GetResponseWithRetries(getRequest, GBCommon.DefaultRetries);
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
HtmlDocument doc = new HtmlDocument();
doc.Load(reader);
if (null == doc.DocumentNode)
{
throw new WebException(matchIndex + ": Error when parsing the root node");
}
HtmlNode bodyNode = doc.DocumentNode.SelectSingleNode("//body");
if (null == bodyNode)
{
throw new WebException(matchIndex + ": Error when parsing the body node");
}
var scripts = bodyNode.SelectNodes("//script[@type='text/javascript']");
string pattern = @"var\s+matchHeaderJson\s*=\s*JSON.parse\(\'(.*)\'\);";
foreach (var script in scripts)
{
Match match = Regex.Match(script.InnerText, pattern);
if (match.Success)
{
string matchHeaderJson = match.Groups[1].Value;
if (!string.IsNullOrEmpty(matchHeaderJson))
{
dynamic matchHeader = JsonConvert.DeserializeObject(matchHeaderJson);
string resultStr = matchHeader.FullTimeResult;
if (!string.IsNullOrEmpty(resultStr))
{
int score1 = int.Parse(resultStr.Split(':')[0].Trim(' '));
int score2 = int.Parse(resultStr.Split(':')[1].Trim(' '));
if (score1 < score2)
{
return(GameResult.Lose);
}
else if (score1 == score2)
{
return(GameResult.Draw);
}
else
{
return(GameResult.Win);
}
}
}
}
}
}
}
return(GameResult.Unknown);
}
static void Main(string[] args)
{
if (args.Length >= 1 && args[0] == "/?")
{
Usage();
return;
}
double threshold = 0;
if (args.Length >= 1)
{
string thresholdString = args[0];
if (!double.TryParse(thresholdString, out threshold))
{
Console.WriteLine("Invalid threshold");
return;
}
}
var response = GBCommon.SendRequest(
"{\"$and\":[{\"BetItem.Result\":{\"$exists\":\"true\"}}, {\"BetItem.Result\":{\"$ne\":\"Unknown\"}}, {\"ROI\": {\"$gte\":" + (threshold / 100).ToString() + "}}]}",
ConfigurationManager.AppSettings["datastore"],
ConfigurationManager.AppSettings["apikey"],
ConfigurationManager.AppSettings["passwd"],
"application/json",
"POST"
);
List <Stake> stakes = new List <Stake>();
using (StreamReader streamReader = new StreamReader(response.GetResponseStream()))
{
string responseString = streamReader.ReadToEnd();
stakes = JsonConvert.DeserializeObject <List <Stake> > (responseString);
}
double actualBet = 0;
double actualReturn = 0;
foreach (var stake in stakes)
{
actualBet++;
if (stake.Decision.ToString() == stake.BetItem.Result.ToString())
{
if (stake.BetItem.Odds != null && stake.BetItem.Odds is ThreeWayOdds)
{
if (stake.BetItem.Result == GameResult.Win)
{
actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Win;
}
if (stake.BetItem.Result == GameResult.Draw)
{
actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Draw;
}
if (stake.BetItem.Result == GameResult.Lose)
{
actualReturn += ((ThreeWayOdds)stake.BetItem.Odds).Lose;
}
}
}
}
if (actualBet == 0)
{
Console.WriteLine("N/A");
}
else
{
Console.WriteLine(actualReturn / actualBet);
}
}
protected void CollectOneMatch(string matchIndex)
{
Uri uri = new Uri(baseuri, string.Format("Matches/{0}/Betting", matchIndex));
WebRequest getRequest = WebRequest.Create(uri);
string matchAlias = gameType.ToString() + "-" + matchIndex;
AveragedNonWeightedBetItemManager mgr = new AveragedNonWeightedBetItemManager();
Guid matchGuid = Guid.NewGuid();
string team1 = "", team2 = "", dateStr = "";
WebResponse response = GBCommon.GetResponseWithRetries(getRequest, GBCommon.DefaultRetries);
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
HtmlDocument doc = new HtmlDocument();
doc.Load(reader);
if (null == doc.DocumentNode)
{
throw new WebException(matchAlias + ": Error when parsing the root node");
}
HtmlNode bodyNode = doc.DocumentNode.SelectSingleNode("//body");
if (null == bodyNode)
{
throw new WebException(matchAlias + ": Error when parsing the body node");
}
var scripts = bodyNode.SelectNodes("//script[@type='text/javascript']");
string pattern = @"matchHeader.load.*\d+,\d+,\'([A-Za-z0-9\. ]+)\',\'([A-Za-z0-9\. ]+)\',\'([0-9:/ ]+)\'";
foreach (var script in scripts)
{
Match match = Regex.Match(script.InnerText, pattern);
if (match.Success)
{
team1 = match.Groups[1].Value.Replace(' ', '_');
team2 = match.Groups[2].Value.Replace(' ', '_');
dateStr = match.Groups[3].Value;
}
}
if (string.IsNullOrEmpty(dateStr) || string.IsNullOrEmpty(team1) || string.IsNullOrEmpty(team2))
{
throw new ApplicationException(matchAlias + ": No match time or team name is found");
}
var bookMakerNameNodes = bodyNode.SelectNodes("//div[@id='ThreeWay-OrdinaryTime']//a[@class='bm-name']");
if (null == bookMakerNameNodes)
{
throw new ApplicationException(matchAlias + ": No target odds found");
}
foreach (var bookMakerNameNode in bookMakerNameNodes)
{
var node = bookMakerNameNode.ParentNode.ParentNode;
string bookMaker = node.ChildNodes[1].SelectSingleNode(".//a[@class='bm-name']").InnerText;
string win = node.ChildNodes[3].SelectSingleNode(".//a/span").InnerText.Trim();
string draw = node.ChildNodes[5].SelectSingleNode(".//a/span").InnerText.Trim();
string lose = node.ChildNodes[7].SelectSingleNode(".//a/span").InnerText.Trim();
BetItem bet = new BetItem(
matchGuid.ToString(),
matchIndex,
gameType,
new System.Collections.Generic.List <Team>()
{
new Team(team1, teamType), new Team(team2, teamType)
},
new ThreeWayOdds(Convert.ToDouble(win), Convert.ToDouble(lose), Convert.ToDouble(draw)),
false,
DateTime.UtcNow,
Convert.ToDateTime(dateStr),
bookMaker
);
mgr.CurrentBets.Add(bet);
}
string fileName = GBCommon.ConstructRecordFileName(gameType, team1, team2, dateStr);
if (File.Exists(fileName))
{
File.Delete(fileName);
}
GBCommon.LogInfo("{0}: Collected {1}", DateTime.Now, fileName);
mgr.Serialize(fileName);
}
}
}
/// <summary>
/// Collect all available records from last index or a particular index
/// </summary>
/// <param name="startFrom">Specified starting index</param>
public void Collect(int startFrom)
{
lock (collectorLock)
{
if (inProgress)
{
GBCommon.LogInfo("{0}: Another collection in progress", DateTime.Now);
return;
}
else
{
inProgress = true;
}
}
try
{
int lastProcessed = GBCommon.ReadContinuationIndex(GBCommon.ConstructCollectContinuationFileName(gameType));
int startIndex = startFrom > 0 ? startFrom : -99 + lastProcessed;
Stopwatch watch = new Stopwatch();
watch.Start();
int count = 0;
int lastValid = -1;
int currentIndex = startIndex;
for (; ; currentIndex++)
{
try
{
CollectOneMatch(currentIndex.ToString());
lastValid = currentIndex;
// Reset the count for valid index
count = 0;
}
catch (ApplicationException e)
{
if (currentIndex < lastProcessed)
{
// In this case, a past match or past empty slot is being processed
continue;
}
if (++count < toleranceCount)
{
// In this case, a possible future empty slot is being processed
}
else
{
GBCommon.LogInfo("Stops at {0}", e.Message);
GBCommon.LogInfo("Elapsed time: {0}", watch.Elapsed);
break;
}
}
catch (WebException e)
{
GBCommon.LogInfo("Stops at {0}", e.Message);
GBCommon.LogInfo("Elapsed time: {0}", watch.Elapsed);
break;
}
}
if (lastValid >= lastProcessed)
{
GBCommon.WriteContinuationIndex(GBCommon.ConstructCollectContinuationFileName(gameType), lastValid);
}
Console.WriteLine("Collection Completed.");
}
finally
{
lock (collectorLock)
{
inProgress = false;
}
}
}
static void Main(string[] args)
{
int startFrom = -1;
string gameType = string.Empty;
if (!args.Any())
{
PrintUsage();
return;
}
if (args.Any())
{
if (args[0].Contains("/?") || args[0].Contains("/h"))
{
PrintUsage();
return;
}
else
{
// Check the commandline arg first
gameType = args[0];
if (args.Count() >= 2)
{
int.TryParse(args[1], out startFrom);
}
}
}
// Check the app config second
if (startFrom <= 0)
{
int.TryParse(ConfigurationManager.AppSettings["startfrom"], out startFrom);
}
//GBCommon.DataFolder = @"c:\tests\GBData";
if (!Directory.Exists(GBCommon.DataFolder))
{
Directory.CreateDirectory(GBCommon.DataFolder);
}
GBCollector collector = new GBCollector(
ConfigurationManager.AppSettings["baseuri"],
gameType,
ConfigurationManager.AppSettings["teamtype"],
ConfigurationManager.AppSettings["tolerancecount"]);
int sleepInterval = 60; // In secs
while (true)
{
GBCommon.LogInfo("Starting a new collection round...");
GBCommon.LogInfo("Current time: {0}", DateTime.Now);
string dataFolder = ConfigurationManager.AppSettings["datafolder"];
if (!string.IsNullOrEmpty(dataFolder) && !string.IsNullOrEmpty(gameType))
{
GBCommon.DataFolder = dataFolder;
if (!Directory.Exists(Path.Combine(dataFolder, gameType)))
{
Directory.CreateDirectory(Path.Combine(dataFolder, gameType));
}
}
// Check the ini third
if (startFrom <= 0)
{
collector.Collect();
}
else
{
collector.Collect(startFrom);
startFrom = 0;
}
int.TryParse(ConfigurationManager.AppSettings["collectinterval"], out sleepInterval);
Thread.Sleep(TimeSpan.FromSeconds(sleepInterval));
}
}
public static void Report()
{
foreach (string folder in Directory.GetDirectories(GBCommon.DataFolder))
{
GameType gameType = (GameType)Enum.Parse(typeof(GameType), folder.Remove(0, GBCommon.DataFolder.Length).TrimStart('\\'));
DateTime continuationStartDate = GBCommon.ReadContinuationDate(GBCommon.ConstructReportContinuationFileName(gameType));
DateTime latest = continuationStartDate;
var files = new DirectoryInfo(folder).GetFiles("*.json", SearchOption.TopDirectoryOnly).Where(file => file.LastWriteTime > continuationStartDate + TimeSpan.FromSeconds(1));
foreach (var file in files)
{
AveragedNonWeightedBetItemManager mgr = new AveragedNonWeightedBetItemManager();
mgr.Deserialize(file.FullName, OddsType.ThreeWay);
if (mgr.CurrentBets.Count < 1)
{
throw new FileLoadException(string.Format("Record file {0} not valid", file.Name));
}
BetItem firstBet = mgr.CurrentBets[0];
Stake bestStake = mgr.BestStake(firstBet.GameName, firstBet.GameType, firstBet.Odds.Type);
string output = bestStake.ROI.ToString("p2")
+ " "
+ bestStake.BetItem.MatchDate.ToShortDateString()
+ " "
+ bestStake.BetItem.Teams[0].Name
+ " "
+ bestStake.BetItem.Teams[1].Name
+ " "
+ bestStake.BetItem.BookMaker.Replace(' ', '_')
+ " "
+ bestStake.BetItem.Odds.Type.ToString()
+ " "
+ bestStake.Decision;
GBCommon.Report(output, gameType);
if (file.LastWriteTime > latest)
{
latest = file.LastWriteTime;
}
// If the game stake exists, delete the old one
var deleteStakeResponse = GBCommon.SendRequest(
"{\"BetItem.GameName\": \"" + bestStake.BetItem.GameName + "\"}",
ConfigurationManager.AppSettings["datastore"],
ConfigurationManager.AppSettings["apikey"],
ConfigurationManager.AppSettings["passwd"],
"application/json",
"DELETE"
);
using (StreamReader streamReader = new StreamReader(deleteStakeResponse.GetResponseStream()))
{
var text = streamReader.ReadToEnd();
Console.WriteLine(text);
}
// Add the new stake
var addStakeResponse = GBCommon.SendRequest(
JsonConvert.SerializeObject(bestStake),
ConfigurationManager.AppSettings["datastore"],
ConfigurationManager.AppSettings["apikey"],
ConfigurationManager.AppSettings["passwd"],
"application/json",
"PUT"
);
using (StreamReader streamReader = new StreamReader(addStakeResponse.GetResponseStream()))
{
var text = streamReader.ReadToEnd();
Console.WriteLine(text);
}
}
if (latest > continuationStartDate)
{
GBCommon.WriteContinuationDate(GBCommon.ConstructReportContinuationFileName(gameType), latest);
}
}
GBCommon.LogInfo("Reporting Completed.");
}
