static int queryRange(StravaXApi stravaXApi, StravaXApiContext db, IList <ActivityRangeQuery> queries, bool doVerbose)
{
int ret = 0;
foreach (ActivityRangeQuery arq in queries)
{
try
{
// https://docs.microsoft.com/en-us/ef/core/saving/concurrency
try
{
// reserve query, mark it as run.
arq.Status = QueryStatus.Run;
arq.StatusChanged = DateTime.Now;
db.SaveChanges();
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"skip conflicted entry for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
continue;
}
var ActivitiesList = stravaXApi.getActivities(arq.AthleteId, $"{arq.DateFrom.Year:D4}", $"{arq.DateFrom.Month:D2}");
int EnterredActivityCount = 0;
foreach (ActivityShort ActivityShort in ActivitiesList)
{
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
try{
// do it better after foreach, but first check how it works with the concurrentcy exception
db.SaveChanges();
EnterredActivityCount++;
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"❌ skip conflicted activity {ActivityShort}");
continue;
}
}
else
{
logger.LogInformation($"❌ {ActivityShort.ActivityId} allready in database");
}
}
arq.Status = QueryStatus.Done;
arq.StatusChanged = DateTime.Now;
// should not have to save anything.
db.SaveChanges();
if (doVerbose)
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount}/{db.ActivityShortDB.Count()} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
else // '.Count()' is pretty expensive, skip it if verbose should be minimal.
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
ErrorCountConsecutive = 0;
}
catch (Exception e)
{
ErrorCountConsecutive++;
ErrorCount++;
logger.LogInformation($"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}");
arq.Status = QueryStatus.Error;
arq.StatusChanged = DateTime.Now;
arq.Message = $"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}";
if (ErrorCountConsecutive > 2)
{
// After 3 consecutive errors, I assume the selenium driver is down. Stop it all.
throw e;
}
}
if (doVerbose)
{
logger.LogInformation($"activities stored:{db.ActivityShortDB.Count()}/{QueryStatus.Created}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Created)}/{QueryStatus.Reserved}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Reserved)}");
}
Count++;
// Exist when KeepRunning is false (from the debugger),
// or the file 'QueryActivities.quit' exists.
// *Program will exit with "touch QueryActivities.quit" in /app directory in container.
Boolean KeepRunning = true;
if (!KeepRunning || File.Exists("QueryActivities.quit.immediatly"))
{
logger.LogInformation($"break {KeepRunning} {Count}");
// regular exit, container should ended.
throw new CancelExecution("break {KeepRunning} {Count}");
}
}
return(ret);
}
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete activities with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
DateTime now = DateTime.Now;
for (int year = FromYear; year <= ToYear; year++)
{
for (int month = 01; month <= 12; month++)
{
if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth))
{
continue;
}
List <ActivityShort> ActivitiesMonthList;
try
{
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
catch (StaleElementReferenceException)
{
// Wait and try again.
Thread.Sleep(2000);
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
ActivitiesList.AddRange(ActivitiesMonthList);
using (StravaXApiContext db = new StravaXApiContext())
{
foreach (ActivityShort ActivityShort in ActivitiesList)
{
Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
db.SaveChanges();
Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
}
else
{
Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
}
}
Console.WriteLine($"total read = {ActivitiesList.Count}");
Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
ActivitiesList.Clear();
}
}
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
