static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete connections with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
AthleteShort AthleteMasterShort;
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "following";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "followers";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static internal int Downloader(StravaXApi stravaXApi, string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log ImageDownloader");
bool doGpx = false;
string maxCountStr = null;
string AthleteId = null;
string ActivityTypeStr = null;
string gpxDir = "gpx";
var p = new OptionSet()
{
{ "g|gpx", v => { doGpx = true; } },
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "m|max_count=", v => { maxCountStr = v; } },
{ "d|dirname=", v => { gpxDir = v; } },
};
p.Parse(args);
int ret = -1;
try
{
List <ActivityShort> activities;
using (StravaXApiContext db = new StravaXApiContext())
{
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
// Ignore Activities without image map. They have probably been enterred without gps-track.
dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
// retrieve the activities from the database.
activities = dbs.ToList();
}
logger.LogInformation($"BEGIN GPX Download for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
int countSkipped = 0;
int countDownload = 0;
if (doGpx && activities.Count > 0)
{
bool needTracksDownload = false;
// detect if any activity needs a gpx file
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
needTracksDownload = true;
break;
}
}
// if no track is needed, we could exit without signin in Strava.
if (needTracksDownload)
{
// signin
stravaXApi.signIn();
// go throw all activities.
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
string outputFilenameGZip = $"{outputFilename}.gz";
string outputFilenameErr = $"{outputFilename}.err";
// download only if the gpx, gz or the err file do not exists.
if (!File.Exists($"{outputDir}/{outputFilename}") && !File.Exists($"{outputDir}/{outputFilenameGZip}") && !File.Exists($"{outputDir}/{outputFilenameErr}"))
{
// create directory if needed
if (!Directory.Exists(outputDir))
{
DirectoryInfo DirInfo = Directory.CreateDirectory(outputDir);
logger.LogDebug($"directory for GPX created at {DirInfo.FullName}");
}
try
{
stravaXApi.getActivityGpxSelenium(activity.ActivityId, $"{outputDir}/{outputFilenameGZip}");
countDownload++;
}
catch (PrivateGpxException e)
{
logger.LogDebug($"GPX Track private for {activity.AthleteId}: {e.ActivityId} {e.Message}");
// write error file, do prevent a second try.
using (StreamWriter outputFile = new StreamWriter(Path.Combine(outputDir, outputFilenameErr)))
{
outputFile.WriteLine($"Error while downloading GPX for athlete:{activity.AthleteId} activity{activity.ActivityId}");
outputFile.WriteLine($"{e.Message}");
outputFile.WriteLine($"{e.StackTrace}");
}
countSkipped++;
}
}
else
{
logger.LogDebug($"GPX Track already downloaded for {activity.AthleteId}");
countSkipped++;
}
}
}
}
else if (!doGpx)
{
// just retrieve without retrieving the gpx files.
int countToDownload = 0;
int countDownloaded = 0;
foreach (ActivityShort activity in activities)
{
// logger.LogInformation($"activity {activity.StatShortString} -> {Utils.extractActivityTime(activity)}.");
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
countToDownload++;
}
else
{
countDownloaded++;
}
}
logger.LogInformation($"GPX Track to download:{countToDownload} already downloaded:{countDownloaded}");
}
logger.LogInformation($"DONE GPX Download {countDownload} (skipped: {countSkipped})for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
// everything done, with return 0 we are telling the container, that there's no need to restart.
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
// return error code, container should restart.
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static internal int WriteQueriesForAthletes(StravaXApi stravaXApi)
{
int ret = -1;
Console.WriteLine("Create range queries.");
using (StravaXApiContext db = new StravaXApiContext())
{
stravaXApi.signIn();
try
{
// https://docs.microsoft.com/en-us/ef/ef6/querying/
// First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
// Not best praxis but enought for Strava.XApi.
IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
foreach (AthleteShort Athlete in AllAthletes)
{
TimeSpan ts = stopWatch.Elapsed;
try
{
WriteQueriesForAthlete(stravaXApi, db, Athlete.AthleteId);
Console.WriteLine($"Athlete:{Athlete} run since:{ts.TotalSeconds}s");
}
catch (PrivateAthleteException e)
{
// TODO AB#27 athlete should be marked as private to avoid second visit.
// create a dummy Query to prevent next search
ActivityRangeQuery arq = new ActivityRangeQuery();
arq.AthleteId = Athlete.AthleteId;
arq.DateFrom = new DateTime(2019, 12, 01);
arq.DateTo = new DateTime(2019, 12, 31);
arq.Status = QueryStatus.Done;
arq.Message = $"private athlete {Athlete.AthleteId}";
arq.StatusChanged = DateTime.Now;
db.ActivityQueriesDB.Add(arq);
Console.WriteLine($"SKIP: private athlete {Athlete.AthleteId} {e.Message}");
}
catch (TooManyStravaRequestException e)
{
Console.WriteLine($"Too Many Queries detected {e.Message} need to wait some hours");
}
catch (Exception e)
{
Console.WriteLine($"SKIP:{Athlete.AthleteId} {e.ToString()}");
}
db.SaveChanges();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = -1;
}
finally
{
stravaXApi.Dispose();
}
}
return(ret);
}
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete activities with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
DateTime now = DateTime.Now;
for (int year = FromYear; year <= ToYear; year++)
{
for (int month = 01; month <= 12; month++)
{
if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth))
{
continue;
}
List <ActivityShort> ActivitiesMonthList;
try
{
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
catch (StaleElementReferenceException)
{
// Wait and try again.
Thread.Sleep(2000);
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
ActivitiesList.AddRange(ActivitiesMonthList);
using (StravaXApiContext db = new StravaXApiContext())
{
foreach (ActivityShort ActivityShort in ActivitiesList)
{
Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
db.SaveChanges();
Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
}
else
{
Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
}
}
Console.WriteLine($"total read = {ActivitiesList.Count}");
Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
ActivitiesList.Clear();
}
}
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
