static int queryAthlete(string aid, string clientId, StravaXApi stravaXApi, StravaXApiContext db, bool doVerbose)
{
int ret = 0;
IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();
while (q0.Count > 0 && ret == 0)
{
// Mark all queries with "RESERVED"
IList <ActivityRangeQuery> queries = new List <ActivityRangeQuery>();
foreach (ActivityRangeQuery arq in q0)
{
try
{
arq.Status = QueryStatus.Reserved;
arq.StatusChanged = DateTime.Now;
arq.Message = $"Reserved by {clientId}";
queries.Add(arq);
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"skip: Can't reserve query {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2} for {clientId}");
continue;
}
}
db.SaveChanges();
// Run for all reserved queries
try
{
ret = queryRange(stravaXApi, db, queries, doVerbose);
}
catch (CancelExecution e)
{
// just to clarify that it may happend.
// finally will also call in that case.
throw e;
}
finally
{
// integrity check, status should not be Reserved anymore
foreach (ActivityRangeQuery arq in q0)
{
if (arq.Status == QueryStatus.Reserved)
{
logger.LogInformation($"WARN: remove reservation on {arq}");
}
}
db.SaveChanges();
}
// retrieve next queries for max one year.
q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();
}
return(ret);
}
static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete connections with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
AthleteShort AthleteMasterShort;
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "following";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "followers";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static void WriteQueriesForAthlete(StravaXApi stravaXApi, StravaXApiContext db, string AthleteId)
{
DateTime Now = DateTime.Now;
lastStravaHttpRequest = DateTime.Now;
int ToYear = Now.Year;
int ToMonth = Now.Month;
//
// Verify if queries have to be generated
//
// Retrieve all entered queries or the wanted athlete.
List <ActivityRangeQuery> queriesForAthlete = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).OrderBy(a => a.DateFrom).ToList();
int FromYear;
int FromMonth;
// If we already have an entry, we assume that the entry contains the first activity date, as it is expensive to retrieve its value with Selenium.
if (queriesForAthlete.Count == 0)
{
// Retrieve the first activity date with selenium.
// may throw PrivateAthleteException.
// be sure of a one seconde intervall between to requests
int dt = DateTime.Now.Millisecond - lastStravaHttpRequest.Millisecond;
if (dt < 2000)
{
System.Threading.Tasks.Task.Delay(dt).Wait();
}
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
lastStravaHttpRequest = DateTime.Now;
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
FromYear = FirstActivityDate.Year;
FromMonth = FirstActivityDate.Month;
}
else
{
// retrieve first and last date
DateTime minDT = queriesForAthlete.First().DateFrom;
// DateTime maxDT = queriesForPatient.Last().DateFrom;
FromYear = minDT.Year;
FromMonth = minDT.Month;
}
Console.WriteLine($"queries enterred:{queriesForAthlete.Count}/total:{db.ActivityQueriesDB.Count()}");
if (FromYear == ToYear)
{
for (int month = FromMonth; month <= ToMonth; month++)
{
AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
else
{
// first year
for (int month = FromMonth; month <= 12; month++)
{
AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
// all years after
for (int year = FromYear + 1; year <= ToYear - 1; year++)
{
for (int month = 01; month <= 12; month++)
{
AddQuery(db, AthleteId, new DateTime(year, month, 1), new DateTime(year, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
// last year
for (int month = 01; month <= ToMonth; month++)
{
// from first day of month to last day of the month
AddQuery(db, AthleteId, new DateTime(ToYear, month, 1), new DateTime(ToYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
int qCount = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).Count();
Console.WriteLine($"✅ enterred:{qCount}/total:{db.ActivityQueriesDB.Count()}");
}
static internal int WriteQueriesForAthletes(StravaXApi stravaXApi)
{
int ret = -1;
Console.WriteLine("Create range queries.");
using (StravaXApiContext db = new StravaXApiContext())
{
stravaXApi.signIn();
try
{
// https://docs.microsoft.com/en-us/ef/ef6/querying/
// First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
// Not best praxis but enought for Strava.XApi.
IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
foreach (AthleteShort Athlete in AllAthletes)
{
TimeSpan ts = stopWatch.Elapsed;
try
{
WriteQueriesForAthlete(stravaXApi, db, Athlete.AthleteId);
Console.WriteLine($"Athlete:{Athlete} run since:{ts.TotalSeconds}s");
}
catch (PrivateAthleteException e)
{
// TODO AB#27 athlete should be marked as private to avoid second visit.
// create a dummy Query to prevent next search
ActivityRangeQuery arq = new ActivityRangeQuery();
arq.AthleteId = Athlete.AthleteId;
arq.DateFrom = new DateTime(2019, 12, 01);
arq.DateTo = new DateTime(2019, 12, 31);
arq.Status = QueryStatus.Done;
arq.Message = $"private athlete {Athlete.AthleteId}";
arq.StatusChanged = DateTime.Now;
db.ActivityQueriesDB.Add(arq);
Console.WriteLine($"SKIP: private athlete {Athlete.AthleteId} {e.Message}");
}
catch (TooManyStravaRequestException e)
{
Console.WriteLine($"Too Many Queries detected {e.Message} need to wait some hours");
}
catch (Exception e)
{
Console.WriteLine($"SKIP:{Athlete.AthleteId} {e.ToString()}");
}
db.SaveChanges();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = -1;
}
finally
{
stravaXApi.Dispose();
}
}
return(ret);
}
static internal int Downloader(StravaXApi stravaXApi, string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log ImageDownloader");
bool doGpx = false;
string maxCountStr = null;
string AthleteId = null;
string ActivityTypeStr = null;
string gpxDir = "gpx";
var p = new OptionSet()
{
{ "g|gpx", v => { doGpx = true; } },
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "m|max_count=", v => { maxCountStr = v; } },
{ "d|dirname=", v => { gpxDir = v; } },
};
p.Parse(args);
int ret = -1;
try
{
List <ActivityShort> activities;
using (StravaXApiContext db = new StravaXApiContext())
{
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
// Ignore Activities without image map. They have probably been enterred without gps-track.
dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
// retrieve the activities from the database.
activities = dbs.ToList();
}
logger.LogInformation($"BEGIN GPX Download for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
int countSkipped = 0;
int countDownload = 0;
if (doGpx && activities.Count > 0)
{
bool needTracksDownload = false;
// detect if any activity needs a gpx file
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
needTracksDownload = true;
break;
}
}
// if no track is needed, we could exit without signin in Strava.
if (needTracksDownload)
{
// signin
stravaXApi.signIn();
// go throw all activities.
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
string outputFilenameGZip = $"{outputFilename}.gz";
string outputFilenameErr = $"{outputFilename}.err";
// download only if the gpx, gz or the err file do not exists.
if (!File.Exists($"{outputDir}/{outputFilename}") && !File.Exists($"{outputDir}/{outputFilenameGZip}") && !File.Exists($"{outputDir}/{outputFilenameErr}"))
{
// create directory if needed
if (!Directory.Exists(outputDir))
{
DirectoryInfo DirInfo = Directory.CreateDirectory(outputDir);
logger.LogDebug($"directory for GPX created at {DirInfo.FullName}");
}
try
{
stravaXApi.getActivityGpxSelenium(activity.ActivityId, $"{outputDir}/{outputFilenameGZip}");
countDownload++;
}
catch (PrivateGpxException e)
{
logger.LogDebug($"GPX Track private for {activity.AthleteId}: {e.ActivityId} {e.Message}");
// write error file, do prevent a second try.
using (StreamWriter outputFile = new StreamWriter(Path.Combine(outputDir, outputFilenameErr)))
{
outputFile.WriteLine($"Error while downloading GPX for athlete:{activity.AthleteId} activity{activity.ActivityId}");
outputFile.WriteLine($"{e.Message}");
outputFile.WriteLine($"{e.StackTrace}");
}
countSkipped++;
}
}
else
{
logger.LogDebug($"GPX Track already downloaded for {activity.AthleteId}");
countSkipped++;
}
}
}
}
else if (!doGpx)
{
// just retrieve without retrieving the gpx files.
int countToDownload = 0;
int countDownloaded = 0;
foreach (ActivityShort activity in activities)
{
// logger.LogInformation($"activity {activity.StatShortString} -> {Utils.extractActivityTime(activity)}.");
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
countToDownload++;
}
else
{
countDownloaded++;
}
}
logger.LogInformation($"GPX Track to download:{countToDownload} already downloaded:{countDownloaded}");
}
logger.LogInformation($"DONE GPX Download {countDownload} (skipped: {countSkipped})for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
// everything done, with return 0 we are telling the container, that there's no need to restart.
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
// return error code, container should restart.
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static internal int SendQueriesForActivities(StravaXApi stravaXApi, string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log Query Activities");
string clientId = Guid.NewGuid().ToString();
int TimerSeconds = -1;
int TimerExitCode = 2;
bool doVerbose = false;
var p = new OptionSet()
{
{ "t|timer_sec=", v => { TimerSeconds = int.Parse(v); } },
{ "e|timer_exit_code=", v => { TimerExitCode = int.Parse(v); } },
{ "v|verbose", v => { doVerbose = true; } },
};
p.Parse(args);
int ret = -1;
logger.LogInformation($"Query activities. Client:{clientId}");
using (StravaXApiContext db = new StravaXApiContext())
{
stravaXApi.signIn();
// 1) find an athlete with opened queries and without reserved queries.
// string aid =db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).First().AthleteId;
// select distinct AthleteId from [dbo].[ActivityQueriesDB] where Status=2
// intersect
// select distinct AthleteId from [dbo].[ActivityQueriesDB] where (Status<>0 AND Status<>3)
var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();
if (AthleteIdList.Count == 0)
{
logger.LogInformation($"no more athlete to search for. created:{qAthleteCreated.Count()} reserved:{qAthleteReserved.Count()}");
// TODO some queries may have been reserved, but no Thread is working on it. It should be usefull to detect and reset.
}
else
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
ret = 0;
try {
while (AthleteIdList.Count > 0 && ret == 0)
{
string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count));
logger.LogInformation($"retrieve activity for athlete {aid}");
// https://docs.microsoft.com/en-us/ef/ef6/querying/
// First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
// Not best praxis but enought for Prototype.
// https://stackoverflow.com/a/2656612/281188
// IList<ActivityRangeQuery> queries = db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(50).ToList();
ret = queryAthlete(aid, clientId, stravaXApi, db, doVerbose);
TimeSpan ts = stopWatch.Elapsed;
if (TimerSeconds > 0 && ts.TotalSeconds > TimerSeconds)
{
ret = TimerExitCode;
logger.LogInformation($"Timer reached after {ts.ToString()} now exit with {ret}.");
// exit with error code, container should restart
break;
}
Boolean KeepRunning = true;
if (!KeepRunning || File.Exists("QueryActivities.quit"))
{
logger.LogInformation($"break {KeepRunning} {Count}");
// regular exit, container should ended.
ret = 0;
break;
}
// search for a new athlete.
qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();
}
}
catch (CancelExecution e)
{
logger.LogInformation($"Execution has been canceled: {e.Message}");
}
}
}
return(ret);
}
static int queryRange(StravaXApi stravaXApi, StravaXApiContext db, IList <ActivityRangeQuery> queries, bool doVerbose)
{
int ret = 0;
foreach (ActivityRangeQuery arq in queries)
{
try
{
// https://docs.microsoft.com/en-us/ef/core/saving/concurrency
try
{
// reserve query, mark it as run.
arq.Status = QueryStatus.Run;
arq.StatusChanged = DateTime.Now;
db.SaveChanges();
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"skip conflicted entry for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
continue;
}
var ActivitiesList = stravaXApi.getActivities(arq.AthleteId, $"{arq.DateFrom.Year:D4}", $"{arq.DateFrom.Month:D2}");
int EnterredActivityCount = 0;
foreach (ActivityShort ActivityShort in ActivitiesList)
{
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
try{
// do it better after foreach, but first check how it works with the concurrentcy exception
db.SaveChanges();
EnterredActivityCount++;
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"❌ skip conflicted activity {ActivityShort}");
continue;
}
}
else
{
logger.LogInformation($"❌ {ActivityShort.ActivityId} allready in database");
}
}
arq.Status = QueryStatus.Done;
arq.StatusChanged = DateTime.Now;
// should not have to save anything.
db.SaveChanges();
if (doVerbose)
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount}/{db.ActivityShortDB.Count()} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
else // '.Count()' is pretty expensive, skip it if verbose should be minimal.
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
ErrorCountConsecutive = 0;
}
catch (Exception e)
{
ErrorCountConsecutive++;
ErrorCount++;
logger.LogInformation($"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}");
arq.Status = QueryStatus.Error;
arq.StatusChanged = DateTime.Now;
arq.Message = $"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}";
if (ErrorCountConsecutive > 2)
{
// After 3 consecutive errors, I assume the selenium driver is down. Stop it all.
throw e;
}
}
if (doVerbose)
{
logger.LogInformation($"activities stored:{db.ActivityShortDB.Count()}/{QueryStatus.Created}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Created)}/{QueryStatus.Reserved}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Reserved)}");
}
Count++;
// Exist when KeepRunning is false (from the debugger),
// or the file 'QueryActivities.quit' exists.
// *Program will exit with "touch QueryActivities.quit" in /app directory in container.
Boolean KeepRunning = true;
if (!KeepRunning || File.Exists("QueryActivities.quit.immediatly"))
{
logger.LogInformation($"break {KeepRunning} {Count}");
// regular exit, container should ended.
throw new CancelExecution("break {KeepRunning} {Count}");
}
}
return(ret);
}
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete activities with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
DateTime now = DateTime.Now;
for (int year = FromYear; year <= ToYear; year++)
{
for (int month = 01; month <= 12; month++)
{
if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth))
{
continue;
}
List <ActivityShort> ActivitiesMonthList;
try
{
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
catch (StaleElementReferenceException)
{
// Wait and try again.
Thread.Sleep(2000);
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
ActivitiesList.AddRange(ActivitiesMonthList);
using (StravaXApiContext db = new StravaXApiContext())
{
foreach (ActivityShort ActivityShort in ActivitiesList)
{
Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
db.SaveChanges();
Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
}
else
{
Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
}
}
Console.WriteLine($"total read = {ActivitiesList.Count}");
Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
ActivitiesList.Clear();
}
}
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}