/// <summary>
/// Convert GPX files to KML.
/// GPX File should have bee retrieved with get-gpx.
/// All GPX are merged in one KML file with subfolder for athlete and activity types.
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public static int Convert(string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log GpxToKml");
string AthleteId = null;
string ActivityTypeStr = null;
string filenameExtention = ".kml";
string exportKmlPath = "StravaXApi.kml";
string splitBlockSizeStr = null;
string exportTypeStr = ExportType.HeatMap.ToString();
string beginDateStr = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "e|export_type=", v => { exportTypeStr = v; } },
{ "bd|begin_date=", v => { beginDateStr = v; } },
{ "d|destination=", v => { exportKmlPath = v; } },
{ "s|split=", v => { splitBlockSizeStr = v; } },
};
p.Parse(args);
int ret = -1;
if (!exportKmlPath.EndsWith(filenameExtention))
{
exportKmlPath = $"{exportKmlPath}{filenameExtention}";
}
ExportType exportType = (ExportType)Enum.Parse(typeof(ExportType), exportTypeStr);
try
{
List <ActivityShort> activities;
using (StravaXApiContext db = new StravaXApiContext())
{
// Just do one query for all queries, order will be user to separate athletes and activity type.
// [note] code is not really readable, afterthere it would have be a better idea to use several queries : athletes/activity types/activities in date range.
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderBy(a => a.AthleteId).ThenBy(a => a.ActivityType).ThenByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
if (beginDateStr != null)
{
DateTime dateTime = DateTime.Parse(beginDateStr);
dbs = dbs.Where(a => a.ActivityDate >= dateTime);
}
// Ignore Activities without image map. They have probably been enterred without gps-track.
dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
// query all activities at once.
activities = dbs.ToList();
logger.LogInformation($"BEGIN GPX➡️KML {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)}/{(beginDateStr==null?"all dates":DateTime.Parse(beginDateStr).ToString())} :{activities.Count()}");
string lastAthleteId = null;
ActivityType lastActivityType = ActivityType.Other;
XElement StravaXApiFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", "Strava-X-Api")
, new XElement(kml + "open", "1")
);
XElement currentAthleteFolder = null;
XElement currentActivityTypeFolder = null;
int Count = 0;
int AthleteCount = 0;
int splitBlockSize = int.MaxValue;
if (splitBlockSizeStr != null)
{
splitBlockSize = int.Parse(splitBlockSizeStr);
}
// go throw all activities, sorted by athlete / activity type / activity date
foreach (ActivityShort activity in activities)
{
// different locations for gpx files.
string outputDir = $"gpx/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
string outputFilenameGz = $"{activity.ActivityId}_{activity.AthleteId}.gpx.gz";
string outputFilenameErr = $"{activity.ActivityId}_{activity.AthleteId}.err";
// skip activities without gpx file at the beginning to avoid node creation in kml file.
if (File.Exists($"{outputDir}/{outputFilenameErr}") ||
(!File.Exists($"{outputDir}/{outputFilenameGz}") &&
!File.Exists($"{outputDir}/{outputFilename}")
))
{
// Console.WriteLine($"Skip activity {activity}");
continue;
}
if (lastAthleteId == null)
{
// first round init athlete
AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId);
string athleteName = athlete.AthleteName;
currentAthleteFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", athleteName)
, new XElement(kml + "visibility", "0")
, new XElement(kml + "open", "0"));
StravaXApiFolder.Add(currentAthleteFolder);
// first round init activity
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastAthleteId = activity.AthleteId;
lastActivityType = activity.ActivityType;
AthleteCount = 1;
}
else
{
if (lastAthleteId == activity.AthleteId)
{
// same athlete, check activity type
if (lastActivityType != activity.ActivityType)
{
// start a new activity folder
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastActivityType = activity.ActivityType;
}
}
else
{
AthleteCount++;
// if splitBlockSize has been set, we may need to write the kml file.
if (splitBlockSize != int.MaxValue && AthleteCount % splitBlockSize == 0)
{
// end of athlete block size reached, save a kml file.
saveKmlFile(StravaXApiFolder, exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize):D2}{filenameExtention}"), logger);
// begin a new tree.
StravaXApiFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", "Strava-X-Api")
, new XElement(kml + "open", "1")
);
}
// start a new athlete folder
AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId);
string athleteName = athlete.AthleteName;
currentAthleteFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", athleteName)
, new XElement(kml + "visibility", "0")
, new XElement(kml + "open", "0"));
StravaXApiFolder.Add(currentAthleteFolder);
// start a new activity folder
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastAthleteId = activity.AthleteId;
lastActivityType = activity.ActivityType;
}
}
try
{
// read the gpx file and add it to the kml node.
if (File.Exists($"{outputDir}/{outputFilenameErr}"))
{
// Skip error file.
logger.LogWarning($"SKIP: GPX not availlable for {activity.ActivityId} {lastAthleteId} {lastActivityType}");
}
else if (File.Exists($"{outputDir}/{outputFilenameGz}"))
{
using (FileStream compressedFileStream = File.OpenRead($"{outputDir}/{outputFilenameGz}"))
{
using (GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Decompress))
{
currentActivityTypeFolder.Add(handleGpxStream(compressionStream, exportType, activity));
}
}
}
else if (File.Exists($"{outputDir}/{outputFilename}"))
{
using (FileStream fileStream = File.OpenRead($"{outputDir}/{outputFilename}"))
{
currentActivityTypeFolder.Add(handleGpxStream(fileStream, exportType, activity));
}
}
// else skip, because GPX is not availlable.
}
catch (Exception e)
{
logger.LogWarning($"SKIP: Can't parse GPX for {activity.ActivityId} {lastAthleteId} {lastActivityType} err:{e.Message}");
}
if (Count++ % 100 == 0)
{
Console.WriteLine($"{Count}/{activities.Count()}");
}
}
string exportKmlPathForSplit = (splitBlockSize == int.MaxValue)?exportKmlPath:exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize)+1:D2}{filenameExtention}");
saveKmlFile(StravaXApiFolder, exportKmlPathForSplit, logger);
}
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
ret = 1;
}
return(ret);
}
static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete connections with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
AthleteShort AthleteMasterShort;
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "following";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "followers";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
public List <AthleteShort> getConnectedAthetes(AthleteShort Athlete, string FollowType)
{
string NextPageUrl = $"https://www.strava.com/athletes/{Athlete.AthleteId}/follows?type={FollowType}";
// BrowserDriver.Navigate().GoToUrl(NextPageUrl);
List <AthleteShort> AthleteShortList = new List <AthleteShort>();
DateTime CrawlDate = DateTime.Now;
int PageCount = 0;
do
{
logger.LogInformation($"open {NextPageUrl}");
BrowserDriver.Navigate().GoToUrl(NextPageUrl);
var ConnectedAthleteElts = BrowserDriver.FindElements(By.XPath("//li[@data-athlete-id]"));
foreach (IWebElement ConnectedAthleteElt in ConnectedAthleteElts)
{
try{
var ConnectedAthleteId = ConnectedAthleteElt.GetAttribute("data-athlete-id");
logger.LogDebug($"ConnectedAthleteId {ConnectedAthleteId}");
var ConnectedAthleteName = ConnectedAthleteElt.FindElement(By.XPath("./div[@title]")).GetAttribute("title");
logger.LogDebug($"ConnectedAthleteName {ConnectedAthleteName}");
var ConnectedAthleteAvatarUrl = ConnectedAthleteElt.FindElement(By.XPath(".//img[@class='avatar-img']")).GetAttribute("src");
logger.LogDebug($"ConnectedAthleteAvatarUrl {ConnectedAthleteAvatarUrl}");
var ConnectedAthleteBadge = ConnectedAthleteElt.FindElement(By.XPath(".//div[@class='avatar-badge']/span/span")).GetAttribute("class");
logger.LogDebug($"ConnectedAthleteBadge {ConnectedAthleteBadge}");
var ConnectedAthleteLocation = ConnectedAthleteElt.FindElement(By.XPath(".//div[@class='location mt-0']")).Text;
logger.LogDebug($"ConnectedAthleteLocation {ConnectedAthleteLocation}");
var AthleteConnectionType = ConnectedAthleteElt.FindElement(By.XPath(".//button")).Text;
logger.LogDebug($"AthleteConnectionType {AthleteConnectionType}");
var AthleteShort = new ConnectedAthlete();
AthleteShort.AthleteId = ConnectedAthleteId;
AthleteShort.AthleteName = ConnectedAthleteName;
AthleteShort.AthleteAvatarUrl = ConnectedAthleteAvatarUrl;
AthleteShort.AthleteBadge = ConnectedAthleteBadge;
AthleteShort.AthleteLocation = ConnectedAthleteLocation;
AthleteShort.AthleteLastCrawled = CrawlDate;
AthleteShortList.Add(AthleteShort);
logger.LogInformation($"add {AthleteShort}");
// We also have informations about the connection state
AthleteShort.ConnectionState = AthleteConnectionType;
}
catch (Exception e) when(e is WebDriverException || e is NotFoundException)
{
if (e is InvalidElementStateException || e is StaleElementReferenceException)
{
// Page seams to be incorrect loaded. Probably need to wait more.
throw e;
}
logger.LogInformation($"Skip athlete at {NextPageUrl} Err:{e.Message}");
}
}
try
{
NextPageUrl = BrowserDriver.FindElement(By.XPath("//li[@class='next_page']/a")).GetAttribute("href");
logger.LogDebug($"next page={NextPageUrl}");
}
catch (WebDriverException)
{
NextPageUrl = "";
}
PageCount++;
}while(!string.IsNullOrEmpty(NextPageUrl));
return(AthleteShortList);
}
