private static int WriteUserTeamSeasonToDB(UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
UserTeamSeasonRepository userTeamSeasonRepository = new UserTeamSeasonRepository();
int userTeamSeasonRowsInserted = userTeamSeasonRepository.InsertUserTeamSeason(userTeamSeasonsInsert, db);
return(userTeamSeasonRowsInserted);
}
catch (Exception ex)
{
Logger.Error("Program WriteUserTeamSeasonToDB error: " + ex.Message);
return(0);
}
}
public static void WriteUserTeamSeasonToDB(int pageId, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
UserTeamSeasonRepository userTeamSeasonRepository = new UserTeamSeasonRepository();
//Write UserTeamSeason to the db
Globals.UserTeamSeasonRowsInserted = userTeamSeasonRepository.InsertUserTeamSeason(userTeamSeasonsInsert, db);
Logger.Out("UserTeamSeason bulk insert complete (PageId: " + Convert.ToString(pageId) + ")");
userTeamSeasonsInsert.Clear();
}
catch (Exception ex)
{
Logger.Error("Program WriteUserTeamSeasonToDB error: " + ex.Message);
}
}
public int InsertUserTeamSeason(UserTeamSeasons userTeamSeasons, SqlConnection db)
{
try
{
int rowsAffected = 0;
using (IDataReader reader = userTeamSeasons.GetDataReader())
//using (var db = new SqlConnection(ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam"].ConnectionString))
{
//db.Open();
using (var bulkCopy = new SqlBulkCopy(db))
{
bulkCopy.BulkCopyTimeout = 1000;
bulkCopy.BatchSize = 500;
bulkCopy.DestinationTableName = "UserTeamSeason";
bulkCopy.EnableStreaming = true;
// Add your column mappings here
bulkCopy.ColumnMappings.Add("season_name", "season_name");
bulkCopy.ColumnMappings.Add("total_points", "total_points");
bulkCopy.ColumnMappings.Add("rank", "userteam_rank");
bulkCopy.ColumnMappings.Add("userteamid", "userteamid");
//using (var dataReader = userTeamSeasons.ToDataReader())
//{
bulkCopy.WriteToServer(reader);
rowsAffected = SqlBulkCopyExtension.RowsCopiedCount(bulkCopy);
//}
}
}
return(rowsAffected);
}
catch (Exception ex)
{
Logger.Error("UserTeamSeason Repository (insert) error: " + ex.Message);
throw ex;
}
}
public static void GetUserTeamDataJsonLoop(int startingUserTeamId, int userTeamId, List <int> toDoUserTeamIds, List <int> existingUserTeamIds, List <int> userTeamIdsWithSeasons, string userTeamUrl, UserTeams userTeamInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamChips userTeamChipsInsert, UserTeamSeasons userTeamSeasonsInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, int userTeamRetries, SqlConnection db)
{
//int userTeamRowsProcessed = 0;
try
{
if (db.State == ConnectionState.Closed)
{
db.ConnectionString = ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam202021"].ConnectionString;
db.Open();
}
//userTeamSeasonNames = userTeamSeasonRepository.GetAllUserTeamSeasonNamesForUserTeamId(userTeamId, db);
//if (userTeamSeasonNames.Count == 0)
//{
// Get the fantasyPremierLeaguePl1ayerData using JSON.NET
//FantasyPremierLeagueAPIClient.GetUserTeamDataJson(userTeamId, existingUserTeamIds, userTeamIdsWithSeasons, userTeamUrl, userTeamInsert, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamRetries, userTeamRowsAdded, db);
//Process UserTeam, UserTeamClassicLeague, UserTeamH2hLeague, UserTeamCup
FantasyPremierLeagueAPIUserTeam.GetUserTeamData(userTeamId, existingUserTeamIds, userTeamUrl, userTeamInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, db);
//Process UserTeamSeasonHistory
if (!userTeamIdsWithSeasons.Contains(userTeamId))
{
FantasyPremierLeagueAPIGameweekHistory.GetUserTeamHistoryDataJson(userTeamId, userTeamIdsWithSeasons, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, db);
}
//}
}
catch (Exception ex)
{
Logger.Error("GetUserTeamDataJsonLoop data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
}
}
static void Main(string[] args)
{
XmlConfigurator.Configure();
//string LogName = GetType().Assembly.GetName().Name + ".log";
//log4net.GlobalContext.Properties["LogName"] = LogName;
UserTeamRepository userTeamRepository = new UserTeamRepository();
UserTeamSeasonRepository userTeamSeasonRepository = new UserTeamSeasonRepository();
UserTeamClassicLeagueRepository userTeamClassicLeagueRepository = new UserTeamClassicLeagueRepository();
UserTeams userTeamInsert = new UserTeams();
UserTeamSeasons userTeamSeasonsInsert = new UserTeamSeasons();
UserTeamGameweekHistories userTeamGameweekHistoriesInsert = new UserTeamGameweekHistories();
UserTeamChips userTeamChipsInsert = new UserTeamChips();
UserTeamClassicLeagues userTeamClassicLeaguesInsert = new UserTeamClassicLeagues();
UserTeamH2hLeagues userTeamH2hLeaguesInsert = new UserTeamH2hLeagues();
SqlConnection db = new SqlConnection(ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam202021"].ConnectionString);
try
{
Logger.Out("Starting...");
Logger.Out("");
int userTeamRetries = 0;
int startingUserTeamId = 1;
bool test = true;
if (args.Length == 0)
{
System.Console.WriteLine("No arguments passed into program");
}
else
{
test = int.TryParse(args[0], out startingUserTeamId);
if (test == false)
{
System.Console.WriteLine("Arguments passed into program were not integer");
return;
}
}
Logger.Out("Starting UserTeams data load");
Logger.Out("");
FantasyPremierLeagueAPI_Bootstrap.GetPlayerBootstrapDataJson();
string userTeamUrl = ConfigSettings.ReadSetting("userTeamURL");
//string userTeamLeaguesUrl = ConfigSettings.ReadSetting("userTeamLeaguesURL");
using (db)
{
db.Open();
SetLatestGameweek("UserTeamGameweekHistory");
SetActualGameweek("UserTeamGameweekPick");
//List<string> userTeamSeasonNames;
Logger.Out("Starting GetAllUserTeamIds call");
List <int> existingUserTeamIds = userTeamRepository.GetAllUserTeamIds(startingUserTeamId, db);
Logger.Out("Starting GetAllUserTeamIdsWithLeagues call");
List <int> userTeamIdsWithLeagues = userTeamClassicLeagueRepository.GetAllUserTeamIdsWithLeagues(startingUserTeamId, db);
Logger.Out("Starting GetAllUserTeamIdsWithSeasons call");
List <int> userTeamIdsWithSeasons = userTeamSeasonRepository.GetAllUserTeamIdsWithSeasons(startingUserTeamId, db);
int userTeamsToProcess = Globals.bootstrapUserTeamCount - startingUserTeamId;
if (userTeamsToProcess < 0)
{
userTeamsToProcess = 0;
}
List <int> userTeamIds = Enumerable.Range(startingUserTeamId, userTeamsToProcess).ToList();
//List<int> toDoUserTeamIds = userTeamIds.Except(userTeamIdsWithLeagues).ToList();
List <int> toDoUserTeamIds = userTeamIds.Except(existingUserTeamIds).ToList();
Logger.Out("");
Logger.Out("Starting UserTeamId" + startingUserTeamId.ToString());
Logger.Out("");
Globals.apiCalls = 0;
Globals.apiUserTeamCalls = 0;
Globals.apiUserTeamHistoryCalls = 0;
Globals.userTeamInsertCount = 0;
FantasyPremierLeagueAPIClient.GetUserTeamDataJson(startingUserTeamId, toDoUserTeamIds, existingUserTeamIds, userTeamIdsWithSeasons, userTeamUrl, userTeamInsert, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamRetries, db);
db.Close();
}
Logger.Out("UserTeams data load complete");
Logger.Out("");
Logger.Out("Finished!!!");
//// Wait for user input - keep the program running
Console.ReadKey();
}
catch (Exception ex)
{
Logger.Error("Program error: " + ex.Message);
throw ex;
}
}
public static bool GetLeagueDataJson(int leagueId, List <int> userTeamIds, string userTeamLeaguesUrl, string userTeamUrl, UserTeams userTeamsUpdateInsert, UserTeamCupMatches userTeamCupInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamPicks userTeamPicksInsert, UserTeamPickAutomaticSubs userTeamPickAutomaticSubsInsert, UserTeamChips userTeamChipsInsert, UserTeamTransferHistoryData userTeamTransferHistoriesInsert, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
UserTeamRepository userTeamRepository = new UserTeamRepository();
UserTeamPickRepository userTeamPickRepository = new UserTeamPickRepository();
UserTeamGameweekHistoryRepository userTeamGameweekHistoryRepository = new UserTeamGameweekHistoryRepository();
UserTeamTransferHistoryRepository userTeamTransferHistoryRepository = new UserTeamTransferHistoryRepository();
UserTeamClassicLeagueRepository userTeamClassicLeagueRepository = new UserTeamClassicLeagueRepository();
UserTeamSeasonRepository userTeamSeasonRepository = new UserTeamSeasonRepository();
try
{
int userTeamId = 0;
bool has_next = false;
List <int> leagueUserTeamIds = new List <int>();
string url = string.Format(userTeamLeaguesUrl, leagueId, Globals.PageId);
JsonSerializer serializer = new JsonSerializer()
{
Formatting = Formatting.None
};
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
using (HttpClient client = new HttpClient())
using (Stream s = client.GetStreamAsync(url).Result)
using (StreamReader sr = new StreamReader(s))
using (JsonReader reader = new JsonTextReader(sr))
{
Globals.ApiCalls += 1;
Globals.ApiPageCalls += 1;
// read the json from a stream
// json size doesn't matter because only a small piece is read at a time from the HTTP request
var leagueStandingsData = serializer.Deserialize <LeagueStandingsData>(reader);
if (leagueStandingsData != null)
{
League league = leagueStandingsData.league;
Standings standings = leagueStandingsData.standings;
string leagueName = league.name;
has_next = standings.has_next;
Logger.Out(leagueName + " (" + Convert.ToString(leagueId) + "): Page " + Convert.ToString(Globals.PageId));
Logger.Out("");
if (db.State == ConnectionState.Closed)
{
db.ConnectionString = ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam"].ConnectionString;
db.Open();
Logger.Error("GetLeagueDataJson Info (LeagueId: " + leagueId.ToString() + ", PageId:" + Globals.PageId.ToString() + "): Reopening closed db connection");
}
List <int> pageUserTeamIds = new List <int>();
foreach (TeamLeaguePosition teamLeaguePosition in standings.results)
{
userTeamId = teamLeaguePosition.entry;
//Add each UserTeamId to a list for checking if GameweekHistory, TransferHistory, and Picks have already been processed for that GW by UserTeamId
pageUserTeamIds.Add(userTeamId);
}
using (DataTable dtUserMaxGWForUserTeamForGameweekHistory = userTeamGameweekHistoryRepository.GetMaxGameweekIdFromUserTeamGameweekHistoryForUserTeamIds(pageUserTeamIds, db))
using (DataTable dtUserMaxGWForUserTeamForPicks = userTeamPickRepository.GetMaxGameweekIdFromUserTeamPickForUserTeamIds(pageUserTeamIds, db))
using (DataTable dtUserMaxGWForUserTeamForTransferHistory = userTeamTransferHistoryRepository.GetMaxGameweekIdFromUserTeamTransferHistoryForUserTeamIds(pageUserTeamIds, db))
using (DataTable dtClassicLeagueCountForUserTeam = userTeamClassicLeagueRepository.GetClassicLeagueCountFromUserTeamClassicLeagueForUserTeamIds(pageUserTeamIds, db))
using (DataTable dtSeasonCountForUserTeam = userTeamSeasonRepository.GetSeasonCountFromUserTeamSeasonForUserTeamIds(pageUserTeamIds, db))
using (DataTable dtUserTeam = userTeamRepository.GetUserTeamForUserTeamIds(pageUserTeamIds, db))
{
foreach (TeamLeaguePosition teamLeaguePosition in standings.results)
{
userTeamId = teamLeaguePosition.entry;
Globals.MaxGWFromGameweekHistoryForUserTeamId = GetColumnNameValue(userTeamId, dtUserMaxGWForUserTeamForGameweekHistory, "gameweekid");
Globals.MaxGWFromPicksForUserTeamId = GetColumnNameValue(userTeamId, dtUserMaxGWForUserTeamForPicks, "gameweekid");
Globals.MaxGWFromTransferHistoryForUserTeamId = GetColumnNameValue(userTeamId, dtUserMaxGWForUserTeamForTransferHistory, "gameweekid");
Globals.LeagueCountFromUserTeamClassicLeagueForUserTeamId = GetColumnNameValue(userTeamId, dtClassicLeagueCountForUserTeam, "leagueCount");
Globals.SeasonCountFromUserTeamSeasonForUserTeamId = GetColumnNameValue(userTeamId, dtSeasonCountForUserTeam, "seasonCount");
Globals.ExistingUserTeamId = GetColumnNameValue(userTeamId, dtUserTeam, "userteamid");
FantasyPremierLeagueAPIClient.GetUserTeamDataJson(userTeamId, userTeamIds, userTeamUrl, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
}
}
pageUserTeamIds.Clear();
}
return(has_next);
}
}
catch (Exception ex)
{
Logger.Error("GetLeagueDataJson data exception (LeagueId: " + leagueId.ToString() + ", PageId:" + Globals.PageId.ToString() + "): " + ex.Message);
bool has_next = true;
if (Globals.LeagueRetries < 10)
{
has_next = GetLeagueDataJson(leagueId, userTeamIds, userTeamLeaguesUrl, userTeamUrl, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
Globals.LeagueRetries += 1;
}
else
{
Logger.Error("GetLeagueDataJson data exception (LeagueId: " + leagueId.ToString() + "): League/Page doesn't exist skipping to next!!!");
//Program.WriteToDB(pageId, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, db);
Globals.LeagueRetries = 0;
//throw new Exception("GetLeagueDataJson data exception (LeagueId: " + leagueId.ToString() + ", PageId:" + Globals.PageId.ToString() + "): " + ex.Message);
}
return(has_next);
}
}
public static void WriteToDB(int pageId, UserTeams userTeamsUpdateInsert, UserTeamCupMatches userTeamCupInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamPicks userTeamPicksInsert, UserTeamPickAutomaticSubs userTeamPickAutomaticSubsInsert, UserTeamChips userTeamChipsInsert, UserTeamTransferHistoryData userTeamTransferHistoriesInsert, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
Globals.UserTeamUpdateRowsInserted = 0;
Globals.UserTeamPickRowsInserted = 0;
Globals.UserTeamPickAutomaticSubsRowsInserted = 0;
Globals.UserTeamGameweekHistoryRowsInserted = 0;
Globals.UserTeamChipRowsInserted = 0;
Globals.UserTeamTransferHistoryRowsInserted = 0;
Globals.UserTeamSeasonRowsInserted = 0;
Globals.UserTeamClassicLeagueRowsInserted = 0;
Globals.UserTeamH2hLeagueRowsInserted = 0;
Globals.UserTeamCupRowsInserted = 0;
Logger.Out("");
WriteUserTeamToDB(Globals.PageId, userTeamsUpdateInsert, db);
WriteUserTeamChipToDB(Globals.PageId, userTeamChipsInsert, db);
WriteUserTeamGameweekHistoryToDB(Globals.PageId, userTeamGameweekHistoriesInsert, db);
WriteUserTeamPicksToDB(Globals.PageId, userTeamPicksInsert, db);
WriteUserTeamPickAutomaticSubsToDB(Globals.PageId, userTeamPickAutomaticSubsInsert, db);
WriteUserTeamTransferHistoryToDB(Globals.PageId, userTeamTransferHistoriesInsert, db);
WriteUserTeamSeasonToDB(Globals.PageId, userTeamSeasonsInsert, db);
WriteUserTeamClassicLeagueToDB(Globals.PageId, userTeamClassicLeaguesInsert, db);
WriteUserTeamH2hLeagueToDB(Globals.PageId, userTeamH2hLeaguesInsert, db);
WriteUserTeamCupToDB(Globals.PageId, userTeamCupInsert, db);
Logger.Out("");
Logger.Out("UserTeam: " + Convert.ToString(Globals.UserTeamInsertCount) + " rows inserted");
Logger.Out("UserTeamCup: " + Convert.ToString(Globals.UserTeamCupRowsInserted) + " rows inserted");
Logger.Out("UserTeamUpdates: " + Convert.ToString(Globals.UserTeamUpdateRowsInserted) + " rows inserted");
Logger.Out("UserTeamSeason: " + Convert.ToString(Globals.UserTeamSeasonRowsInserted) + " rows inserted");
Logger.Out("UserTeamClassicLeague: " + Convert.ToString(Globals.UserTeamClassicLeagueRowsInserted) + " rows inserted");
Logger.Out("UserTeamH2hLeague: " + Convert.ToString(Globals.UserTeamH2hLeagueRowsInserted) + " rows inserted");
Logger.Out("UserTeamGameweekHistory: " + Convert.ToString(Globals.UserTeamGameweekHistoryRowsInserted) + " rows inserted");
Logger.Out("UserTeamTransferHistory: " + Convert.ToString(Globals.UserTeamTransferHistoryRowsInserted) + " rows inserted");
Logger.Out("UserTeamChip: " + Convert.ToString(Globals.UserTeamChipRowsInserted) + " rows inserted");
Logger.Out("UserTeamPick: " + Convert.ToString(Globals.UserTeamPickRowsInserted) + " rows inserted");
Logger.Out("UserTeamPickAutomaticSub: " + Convert.ToString(Globals.UserTeamPickAutomaticSubsRowsInserted) + " rows inserted");
Logger.Out("");
Logger.Out("API Page calls: " + Convert.ToString(Globals.ApiPageCalls));
Logger.Out("API UserTeam calls: " + Convert.ToString(Globals.ApiUserTeamCalls));
Logger.Out("API UserTeamHistory calls: " + Convert.ToString(Globals.ApiUserTeamHistoryCalls));
Logger.Out("API UserTeamTransferHistory calls: " + Convert.ToString(Globals.ApiUserTeamTransferHistoryCalls));
Logger.Out("API UserTeamPick calls: " + Convert.ToString(Globals.ApiUserTeamPickCalls));
Logger.Out("API calls: " + Convert.ToString(Globals.ApiCalls));
Logger.Out("");
}
catch (Exception ex)
{
Logger.Error("Program WriteToDB error: " + ex.Message);
//WriteToDB(userTeamRowsInserted, userTeamSeasonRowsInserted, userTeamInsert, userTeamSeasonsInsert, db);
}
}
static void Main(string[] args)
{
XmlConfigurator.Configure();
//int pageId;
int pageCnt;
int leagueId = 314;
UserTeams userTeamsUpdateInsert = new UserTeams();
UserTeamPicks userTeamPicksInsert = new UserTeamPicks();
UserTeamPickAutomaticSubs userTeamPickAutomaticSubsInsert = new UserTeamPickAutomaticSubs();
UserTeamGameweekHistories userTeamGameweekHistoriesInsert = new UserTeamGameweekHistories();
UserTeamChips userTeamChipsInsert = new UserTeamChips();
UserTeamTransferHistoryData userTeamTransferHistoriesInsert = new UserTeamTransferHistoryData();
UserTeamSeasons userTeamSeasonsInsert = new UserTeamSeasons();
UserTeamClassicLeagues userTeamClassicLeaguesInsert = new UserTeamClassicLeagues();
UserTeamH2hLeagues userTeamH2hLeaguesInsert = new UserTeamH2hLeagues();
UserTeamCupMatches userTeamCupInsert = new UserTeamCupMatches();
UserTeamRepository userTeamRepository = new UserTeamRepository();
SqlConnection db = new SqlConnection(ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam"].ConnectionString);
try
{
Logger.Out("Starting...");
Logger.Out("");
int pageId;
Globals.PageId = 1;
int insertInterval;
Globals.InsertInterval = 10;
bool test = true;
if (args.Length == 0)
{
//System.Console.WriteLine("No arguments passed into program");
Logger.Out("No arguments passed into program");
}
else
{
test = int.TryParse(args[0], out pageId);
if (test == false)
{
Logger.Out("Argument[0] passed into program doesnn't exist or was not integer");
return;
}
else
{
Globals.PageId = pageId;
}
test = int.TryParse(args[1], out insertInterval);
if (test == false)
{
//System.Console.WriteLine("Arguments passed into program were not integer");
Logger.Out("Argument[1] InsertInterval passed into program doesnn't exist or were not integer");
return;
}
else
{
Globals.InsertInterval = insertInterval;
}
}
Logger.Out("Starting UserTeams data load");
Logger.Out("");
string userTeamUrl = ConfigSettings.ReadSetting("userTeamURL");
string userTeamLeaguesUrl = ConfigSettings.ReadSetting("userTeamLeaguesURL");
using (db)
{
db.Open();
bool has_next = true;
Globals.UserTeamRetries = 0;
pageCnt = 0;
List <int> userTeamIds = userTeamRepository.GetAllUserTeamIds(db);
Globals.ApiCalls = 0;
Globals.ApiPageCalls = 0;
Globals.ApiUserTeamCalls = 0;
Globals.ApiUserTeamHistoryCalls = 0;
Globals.ApiUserTeamTransferHistoryCalls = 0;
Globals.ApiUserTeamPickCalls = 0;
Globals.UserTeamInsertCount = 0;
while (has_next == true)
{
SetLatestGameweek("UserTeamGameweekHistory");
SetActualGameweek("UserTeamGameweekPick");
pageCnt += 1;
if (db.State == ConnectionState.Closed)
{
db.ConnectionString = ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam"].ConnectionString;
db.Open();
Logger.Error("Program Info (LeagueId: " + leagueId.ToString() + ", PageId:" + Globals.PageId.ToString() + "): Reopening closed db connection");
}
has_next = FantasyPremierLeagueLeaguesAPIClient.GetLeagueDataJson(leagueId, userTeamIds, userTeamLeaguesUrl, userTeamUrl, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
if (pageCnt >= Globals.InsertInterval)
{
WriteToDB(Globals.PageId, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
Globals.ApiCalls = 0;
Globals.ApiPageCalls = 0;
Globals.ApiUserTeamCalls = 0;
Globals.ApiUserTeamHistoryCalls = 0;
Globals.ApiUserTeamTransferHistoryCalls = 0;
Globals.ApiUserTeamPickCalls = 0;
Globals.UserTeamInsertCount = 0;
pageCnt = 0;
}
Globals.PageId += 1;
}
WriteToDB(Globals.PageId, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
db.Close();
}
Logger.Out("UserTeams data load complete");
Logger.Out("");
Logger.Out("Finished!!!");
//// Wait for user input - keep the program running
Console.ReadKey();
}
catch (Exception ex)
{
Logger.Error(ex.Message);
//Logger.Error(userTeamName + " caused error!!!");
if (db.State == ConnectionState.Closed)
{
db.ConnectionString = ConfigurationManager.ConnectionStrings["FantasyPremierLeagueUserTeam"].ConnectionString;
db.Open();
Logger.Error("Program Exception Info (LeagueId: " + leagueId.ToString() + ", PageId:" + Globals.PageId.ToString() + "): Reopening closed db connection");
}
//If error is thrown from sub program write existing records to the DB
WriteToDB(Globals.PageId, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
//db.Close();
//throw ex;
}
}
public static void GetUserTeamSeasonJson(int userTeamId, UserTeamGameweekHistoryData userTeamHistoryData, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
//Load UserTeamSeason data
UserTeamSeasonRepository userTeamSeasonRepository = new UserTeamSeasonRepository();
List <string> UserTeamSeasonNames = userTeamSeasonRepository.GetAllUserTeamSeasonNamesForUserTeamId(userTeamId, db);
foreach (UserTeamSeason userTeamSeason in userTeamHistoryData.past)
{
//needed if want to assign value from parent to add into db table
userTeamSeason.userteamid = userTeamId;
//userTeamSeason.season = userTeamSeasonRepository.GetSeasonIdFromSeasonName(userTeamSeason.season_name);
if (!UserTeamSeasonNames.Contains(userTeamSeason.season_name) && !userTeamSeasonsInsert.Contains(userTeamSeason))
{
userTeamSeasonsInsert.Add(userTeamSeason);
}
//else
//{
// userTeamSeasonRepository.UpdateUserTeamSeason(userTeamSeason);
//}
}
}
catch (Exception ex)
{
Logger.Error("GetUserTeamSeasonJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
throw new Exception("GetUserTeamSeasonJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
}
}
public static void GetUserTeamHistoryDataJson(int userTeamId, List <int> userTeamIdsWithSeasons, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamChips userTeamChipsInsert, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
string userTeamHistoryUrl = ConfigSettings.ReadSetting("userTeamHistoryURL");
userTeamHistoryUrl = string.Format(userTeamHistoryUrl, userTeamId);
HttpClient client = new HttpClient();
JsonSerializer serializer = new JsonSerializer();
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
using (Stream s = client.GetStreamAsync(userTeamHistoryUrl).Result)
using (StreamReader sr = new StreamReader(s))
using (JsonReader reader = new JsonTextReader(sr))
{
Globals.apiCalls += 1;
Globals.apiUserTeamHistoryCalls += 1;
// read the json from a stream
// json size doesn't matter because only a small piece is read at a time from the HTTP request
var userTeamHistoryData = serializer.Deserialize <UserTeamGameweekHistoryData>(reader);
//Process UserTeamSeasonHistory
if (userTeamHistoryData.past != null && !userTeamIdsWithSeasons.Contains(userTeamId))
{
GetUserTeamSeasonJson(userTeamId, userTeamHistoryData, userTeamSeasonsInsert, db);
}
//Process UserTeamGameweekHistory
if (userTeamHistoryData.current != null && !userTeamIdsWithSeasons.Contains(userTeamId))
{
GetUserTeamGameweekHistoryJson(userTeamId, userTeamHistoryData, userTeamGameweekHistoriesInsert, db);
}
//Process UserTeamChip
if (userTeamHistoryData.chips != null && !userTeamIdsWithSeasons.Contains(userTeamId))
{
GetUserTeamChipJson(userTeamId, userTeamChipsInsert, userTeamHistoryData, db);
}
}
}
catch (Exception ex)
{
Logger.Error("GetUserTeamHistoryDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
throw new Exception("GetUserTeamHistoryDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
}
}
public static void GetUserTeamDataJson(int userTeamId, List <int> userTeamIds, string userTeamUrl, UserTeams userTeamsUpdateInsert, UserTeamCupMatches userTeamCupInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamPicks userTeamPicksInsert, UserTeamPickAutomaticSubs userTeamPickAutomaticSubsInsert, UserTeamChips userTeamChipsInsert, UserTeamTransferHistoryData userTeamTransferHistoriesInsert, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
//Process UserTeam, UserTeamClassicLeague, UserTeamH2hLeague, UserTeamCup
////if (Globals.ExistingUserTeamId == 0 || Globals.LeagueCountFromUserTeamClassicLeagueForUserTeamId == 0)
//{
FantasyPremierLeagueAPIUserTeamLeagueAndCup.GetUserTeamLeagueAndCupJson(userTeamId, userTeamIds, userTeamUrl, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, db);
//}
//else
//{
// Logger.Out("UserTeam: " + Convert.ToString(userTeamId) + " - skipped (already exists with leagues)");
//}
//Process UserTeamGameweekHistory, UserTeamChip, UserTeamSeasonHistory
if (Globals.MaxGWFromGameweekHistoryForUserTeamId < Globals.LatestGameweek)
{
FantasyPremierLeagueAPIGameweekHistory.GetUserTeamHistoryDataJson(userTeamId, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, db);
}
//Process UserTeamPick, UserTeamPickAutomaticSub
//Doesn't check UserTeamPickAutomaticSub - which will be missing if run during an active Gameweek - matches are in progress or before have been played)
if (Globals.MaxGWFromPicksForUserTeamId < Globals.ActualGameweek && Globals.ActualGameweek > 0)
{
FantasyPremierLeagueAPIPick.GetUserTeamPickDataJson(userTeamId, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, db);
}
//Process UserTeamTransferHistory
if (Globals.MaxGWFromTransferHistoryForUserTeamId < Globals.ActualGameweek && Globals.ActualGameweek > 0)
{
FantasyPremierLeagueAPITransferHistory.GetUserTeamTransferHistoryDataJson(userTeamId, userTeamTransferHistoriesInsert, db);
}
Logger.Out("");
}
catch (Exception ex)
{
Logger.Error("GetUserTeamDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
//throw new Exception("GetUserTeamDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
if (Globals.UserTeamRetries < 10)
{
Globals.UserTeamRetries += 1;
GetUserTeamDataJson(userTeamId, userTeamIds, userTeamUrl, userTeamsUpdateInsert, userTeamCupInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamGameweekHistoriesInsert, userTeamPicksInsert, userTeamPickAutomaticSubsInsert, userTeamChipsInsert, userTeamTransferHistoriesInsert, userTeamSeasonsInsert, db);
}
else
{
Logger.Error("GetUserTeamDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): Issue processing User Team skipping to next!!!");
Thread.Sleep(20000);
}
}
}
public static void GetUserTeamDataJson(int startingUserTeamId, List <int> toDoUserTeamIds, List <int> existingUserTeamIds, List <int> userTeamIdsWithSeasons, string userTeamUrl, UserTeams userTeamInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamChips userTeamChipsInsert, UserTeamSeasons userTeamSeasonsInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, int userTeamRetries, SqlConnection db)
{
int userTeamId = 0;
Globals.userTeamRowsProcessed = 0;
try
{
foreach (int loopUserTeamId in toDoUserTeamIds)
{
userTeamId = loopUserTeamId;
//url = string.Format(userTeamUrl, userTeamId);
FantasyPremierLeagueAPIClientLoop.GetUserTeamDataJsonLoop(startingUserTeamId, userTeamId, toDoUserTeamIds, existingUserTeamIds, userTeamIdsWithSeasons, userTeamUrl, userTeamInsert, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, userTeamRetries, db);
if (Globals.userTeamRowsProcessed >= 500)
{
WriteToDB(userTeamInsert, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, db);
Globals.userTeamRowsProcessed = 0;
Logger.Out(userTeamId.ToString());
Logger.Out("");
}
else
{
Globals.userTeamRowsProcessed += 1;
}
existingUserTeamIds.Add(userTeamId);
//Globals.processedUserTeamIds.Add(userTeamId);
//var itemToRemove = toDoUserTeamIds.SingleOrDefault(r => r == userTeamId);
//toDoUserTeamIds.Remove(itemToRemove);
}
WriteToDB(userTeamInsert, userTeamGameweekHistoriesInsert, userTeamChipsInsert, userTeamSeasonsInsert, userTeamClassicLeaguesInsert, userTeamH2hLeaguesInsert, db);
}
catch (Exception ex)
{
Logger.Error("GetUserTeamDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
Globals.userTeamRowsProcessed += 1;
}
}
public static void WriteToDB(UserTeams userTeamInsert, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamChips userTeamChipsInsert, UserTeamSeasons userTeamSeasonsInsert, UserTeamClassicLeagues userTeamClassicLeaguesInsert, UserTeamH2hLeagues userTeamH2hLeaguesInsert, SqlConnection db)
{
try
{
int userTeamRowsInserted = 0;
int userTeamGameweekHistoryRowsInserted = 0;
int userTeamChipRowsInserted = 0;
int userTeamSeasonRowsInserted = 0;
int userTeamClassicLeagueRowsInserted = 0;
int userTeamH2hLeagueRowsInserted = 0;
//Write UserTeam to the db
userTeamRowsInserted = WriteUserTeamToDB(userTeamInsert, db);
//userTeamRowsInserted = userTeamRepository.InsertUserTeam(userTeamInsert, db);
//Write UserTeamChip to the db
userTeamChipRowsInserted = WriteUserTeamChipToDB(userTeamChipsInsert, db);
//userTeamChipRowsInserted = userTeamChipRepository.InsertUserTeamChip(userTeamChipsInsert, db);
//Write UserTeamGameweekHistory to the db
userTeamGameweekHistoryRowsInserted = WriteUserTeamGameweekHistoryToDB(userTeamGameweekHistoriesInsert, db);
//userTeamGameweekHistoryRowsInserted = userTeamGameweekHistoryRepository.InsertUserTeamGameweekHistories(userTeamGameweekHistoriesInsert, db);
//Write UserTeamSeason to the db
userTeamSeasonRowsInserted = WriteUserTeamSeasonToDB(userTeamSeasonsInsert, db);
//userTeamSeasonRowsInserted = userTeamSeasonRepository.InsertUserTeamSeason(userTeamSeasonsInsert, db);
//Write UserTeamClassicLeague to the db
userTeamClassicLeagueRowsInserted = WriteUserTeamClassicLeagueToDB(userTeamClassicLeaguesInsert, db);
//userTeamClassicLeagueRowsInserted = userTeamClassicLeagueRepository.InsertUserTeamClassicLeague(userTeamClassicLeaguesInsert, db);
//Write UserTeamH2hLeague to the db
userTeamH2hLeagueRowsInserted = WriteUserTeamH2hLeagueToDB(userTeamH2hLeaguesInsert, db);
//userTeamH2hLeagueRowsInserted = userTeamH2hLeagueRepository.InsertUserTeamH2hLeague(userTeamH2hLeaguesInsert, db);
Logger.Out("");
Logger.Out("UserTeam bulk insert complete (UserTeam rows inserted: " + Convert.ToString(userTeamRowsInserted) + ")");
Logger.Out("UserTeamGameweekHistory bulk insert complete (UserTeamGameweekHistory rows inserted: " + Convert.ToString(userTeamGameweekHistoryRowsInserted) + ")");
Logger.Out("UserTeamChip bulk insert complete (UserTeamChip rows inserted: " + Convert.ToString(userTeamChipRowsInserted) + ")");
Logger.Out("UserTeamSeason bulk insert complete (UserTeamSeason rows inserted: " + Convert.ToString(userTeamSeasonRowsInserted) + ")");
Logger.Out("UserTeamClassicLeague bulk insert complete (UserTeamClassicLeague rows inserted: " + Convert.ToString(userTeamClassicLeagueRowsInserted) + ")");
Logger.Out("UserTeamH2hLeague bulk insert complete (UserTeamH2hLeague rows inserted: " + Convert.ToString(userTeamH2hLeagueRowsInserted) + ")");
Logger.Out("");
Logger.Out("API UserTeam calls: " + Convert.ToString(Globals.apiUserTeamCalls));
Logger.Out("API UserTeamHistory calls: " + Convert.ToString(Globals.apiUserTeamHistoryCalls));
Logger.Out("API calls: " + Convert.ToString(Globals.apiCalls));
Logger.Out("");
userTeamInsert.Clear();
userTeamGameweekHistoriesInsert.Clear();
userTeamChipsInsert.Clear();
userTeamSeasonsInsert.Clear();
userTeamClassicLeaguesInsert.Clear();
userTeamH2hLeaguesInsert.Clear();
Globals.apiCalls = 0;
Globals.apiUserTeamCalls = 0;
Globals.apiUserTeamHistoryCalls = 0;
Globals.userTeamInsertCount = 0;
}
catch (Exception ex)
{
Logger.Error("Program WriteToDB error: " + ex.Message);
//WriteToDB(userTeamRowsInserted, userTeamSeasonRowsInserted, userTeamInsert, userTeamSeasonsInsert, db);
}
}
public static void GetUserTeamHistoryDataJson(int userTeamId, UserTeamGameweekHistories userTeamGameweekHistoriesInsert, UserTeamChips userTeamChipsInsert, UserTeamSeasons userTeamSeasonsInsert, SqlConnection db)
{
try
{
string userTeamHistoryUrl = ConfigSettings.ReadSetting("userTeamHistoryURL");
userTeamHistoryUrl = string.Format(userTeamHistoryUrl, userTeamId);
JsonSerializer serializer = new JsonSerializer()
{
Formatting = Formatting.None
};
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
using (HttpClient client = new HttpClient())
using (Stream s = client.GetStreamAsync(userTeamHistoryUrl).Result)
using (StreamReader sr = new StreamReader(s))
using (JsonReader reader = new JsonTextReader(sr))
{
Globals.ApiCalls += 1;
Globals.ApiUserTeamHistoryCalls += 1;
// read the json from a stream
// json size doesn't matter because only a small piece is read at a time from the HTTP request
var userTeamHistoryData = serializer.Deserialize <UserTeamGameweekHistoryData>(reader);
if (userTeamHistoryData != null)
{
//Process UserTeamChip
if (userTeamHistoryData.chips != null)
{
GetUserTeamChipJson(userTeamId, userTeamChipsInsert, userTeamHistoryData, db);
}
//Process UserTeamSeasonHistory
if (userTeamHistoryData.past != null)
{
GetUserTeamSeasonJson(userTeamId, userTeamHistoryData, userTeamSeasonsInsert, db);
}
//Process UserTeamGameweekHistory
if (Globals.MaxGWFromGameweekHistoryForUserTeamId < Globals.LatestGameweek && Globals.LatestGameweek > 0)
{
GetUserTeamGameweekHistoryJson(userTeamId, userTeamHistoryData, userTeamGameweekHistoriesInsert, db);
}
}
else
{
Logger.Out("UserTeamGameweekHistory: " + Convert.ToString(userTeamId) + " - skipped - null userTeamHistoryData (" + userTeamHistoryUrl + ")");
}
}
}
catch (Exception ex)
{
Logger.Error("GetUserTeamHistoryDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
//throw new Exception("GetUserTeamHistoryDataJson data exception (UserTeamId: " + userTeamId.ToString() + "): " + ex.Message);
}
}