/// <summary> /// Convert GPX files to KML. /// GPX File should have bee retrieved with get-gpx. /// All GPX are merged in one KML file with subfolder for athlete and activity types. /// </summary> /// <param name="args"></param> /// <returns></returns> public static int Convert(string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log GpxToKml"); string AthleteId = null; string ActivityTypeStr = null; string filenameExtention = ".kml"; string exportKmlPath = "StravaXApi.kml"; string splitBlockSizeStr = null; string exportTypeStr = ExportType.HeatMap.ToString(); string beginDateStr = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "e|export_type=", v => { exportTypeStr = v; } }, { "bd|begin_date=", v => { beginDateStr = v; } }, { "d|destination=", v => { exportKmlPath = v; } }, { "s|split=", v => { splitBlockSizeStr = v; } }, }; p.Parse(args); int ret = -1; if (!exportKmlPath.EndsWith(filenameExtention)) { exportKmlPath = $"{exportKmlPath}{filenameExtention}"; } ExportType exportType = (ExportType)Enum.Parse(typeof(ExportType), exportTypeStr); try { List <ActivityShort> activities; using (StravaXApiContext db = new StravaXApiContext()) { // Just do one query for all queries, order will be user to separate athletes and activity type. // [note] code is not really readable, afterthere it would have be a better idea to use several queries : athletes/activity types/activities in date range. IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderBy(a => a.AthleteId).ThenBy(a => a.ActivityType).ThenByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (AthleteId != null) { dbs = dbs.Where(a => a.AthleteId == AthleteId); } if (beginDateStr != null) { DateTime dateTime = DateTime.Parse(beginDateStr); dbs = dbs.Where(a => a.ActivityDate >= dateTime); } // Ignore Activities without image map. They have probably been enterred without gps-track. dbs = dbs.Where(a => a.ActivityImageMapUrl != null); // query all activities at once. activities = dbs.ToList(); logger.LogInformation($"BEGIN GPX➡️KML {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)}/{(beginDateStr==null?"all dates":DateTime.Parse(beginDateStr).ToString())} :{activities.Count()}"); string lastAthleteId = null; ActivityType lastActivityType = ActivityType.Other; XElement StravaXApiFolder = new XElement(kml + "Folder" , new XElement(kml + "name", "Strava-X-Api") , new XElement(kml + "open", "1") ); XElement currentAthleteFolder = null; XElement currentActivityTypeFolder = null; int Count = 0; int AthleteCount = 0; int splitBlockSize = int.MaxValue; if (splitBlockSizeStr != null) { splitBlockSize = int.Parse(splitBlockSizeStr); } // go throw all activities, sorted by athlete / activity type / activity date foreach (ActivityShort activity in activities) { // different locations for gpx files. string outputDir = $"gpx/{activity.AthleteId}"; string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx"; string outputFilenameGz = $"{activity.ActivityId}_{activity.AthleteId}.gpx.gz"; string outputFilenameErr = $"{activity.ActivityId}_{activity.AthleteId}.err"; // skip activities without gpx file at the beginning to avoid node creation in kml file. if (File.Exists($"{outputDir}/{outputFilenameErr}") || (!File.Exists($"{outputDir}/{outputFilenameGz}") && !File.Exists($"{outputDir}/{outputFilename}") )) { // Console.WriteLine($"Skip activity {activity}"); continue; } if (lastAthleteId == null) { // first round init athlete AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId); string athleteName = athlete.AthleteName; currentAthleteFolder = new XElement(kml + "Folder" , new XElement(kml + "name", athleteName) , new XElement(kml + "visibility", "0") , new XElement(kml + "open", "0")); StravaXApiFolder.Add(currentAthleteFolder); // first round init activity currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastAthleteId = activity.AthleteId; lastActivityType = activity.ActivityType; AthleteCount = 1; } else { if (lastAthleteId == activity.AthleteId) { // same athlete, check activity type if (lastActivityType != activity.ActivityType) { // start a new activity folder currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastActivityType = activity.ActivityType; } } else { AthleteCount++; // if splitBlockSize has been set, we may need to write the kml file. if (splitBlockSize != int.MaxValue && AthleteCount % splitBlockSize == 0) { // end of athlete block size reached, save a kml file. saveKmlFile(StravaXApiFolder, exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize):D2}{filenameExtention}"), logger); // begin a new tree. StravaXApiFolder = new XElement(kml + "Folder" , new XElement(kml + "name", "Strava-X-Api") , new XElement(kml + "open", "1") ); } // start a new athlete folder AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId); string athleteName = athlete.AthleteName; currentAthleteFolder = new XElement(kml + "Folder" , new XElement(kml + "name", athleteName) , new XElement(kml + "visibility", "0") , new XElement(kml + "open", "0")); StravaXApiFolder.Add(currentAthleteFolder); // start a new activity folder currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastAthleteId = activity.AthleteId; lastActivityType = activity.ActivityType; } } try { // read the gpx file and add it to the kml node. if (File.Exists($"{outputDir}/{outputFilenameErr}")) { // Skip error file. logger.LogWarning($"SKIP: GPX not availlable for {activity.ActivityId} {lastAthleteId} {lastActivityType}"); } else if (File.Exists($"{outputDir}/{outputFilenameGz}")) { using (FileStream compressedFileStream = File.OpenRead($"{outputDir}/{outputFilenameGz}")) { using (GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Decompress)) { currentActivityTypeFolder.Add(handleGpxStream(compressionStream, exportType, activity)); } } } else if (File.Exists($"{outputDir}/{outputFilename}")) { using (FileStream fileStream = File.OpenRead($"{outputDir}/{outputFilename}")) { currentActivityTypeFolder.Add(handleGpxStream(fileStream, exportType, activity)); } } // else skip, because GPX is not availlable. } catch (Exception e) { logger.LogWarning($"SKIP: Can't parse GPX for {activity.ActivityId} {lastAthleteId} {lastActivityType} err:{e.Message}"); } if (Count++ % 100 == 0) { Console.WriteLine($"{Count}/{activities.Count()}"); } } string exportKmlPathForSplit = (splitBlockSize == int.MaxValue)?exportKmlPath:exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize)+1:D2}{filenameExtention}"); saveKmlFile(StravaXApiFolder, exportKmlPathForSplit, logger); } ret = 0; } catch (Exception e) { logger.LogError($"ERROR:{e.ToString()}"); ret = 1; } return(ret); }
static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args) { int ret = -1; Console.WriteLine("Read athlete connections with Strava-X-API."); String AthleteId = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, }; p.Parse(args); if (AthleteId == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing athlete id"); } try { stravaXApi.signIn(); AthleteShort AthleteMasterShort; using (StravaXApiContext db = new StravaXApiContext()) { AthleteMasterShort = db.AthleteShortDB.Find(AthleteId); if (AthleteMasterShort == null) { AthleteMasterShort = new AthleteShort(); // create a dummy master AthleteMasterShort.AthleteId = AthleteId; // [TODO] other parameters should be retrieved with selenium AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity; db.SaveChanges(); } else { // Eagerly Loading prevent the list to be loaded at creation // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load(); Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}"); } string FollowType = "following"; var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType); Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections"); foreach (AthleteShort _AthleteShort in AthleteShortList) { AthleteShort AthleteShortfromDb; // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId); if (AthleteShortfromDb == null) { // add athlete to the db if need. AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity; } else { Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database"); } Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); // such the connected athlete with they id. AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId)); if (_ConnectedAthleteShort == null) { // add connection if needed. AthleteConnection ac = new AthleteConnection(); ac.FromId = AthleteMasterShort.AthleteId; ac.ToId = AthleteShortfromDb.AthleteId; ac.Type = FollowType; ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState; AthleteMasterShort.Connections.Add(ac); Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}"); } else { Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)"); } } db.SaveChanges(); Console.WriteLine($"total read = {AthleteShortList.Count}"); Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); AthleteShortList.Clear(); } using (StravaXApiContext db = new StravaXApiContext()) { AthleteMasterShort = db.AthleteShortDB.Find(AthleteId); if (AthleteMasterShort == null) { AthleteMasterShort = new AthleteShort(); // create a dummy master AthleteMasterShort.AthleteId = AthleteId; // [TODO] other parameters should be retrieved with selenium AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity; db.SaveChanges(); } else { // Eagerly Loading prevent the list to be loaded at creation // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load(); Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}"); } string FollowType = "followers"; var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType); Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections"); foreach (AthleteShort _AthleteShort in AthleteShortList) { AthleteShort AthleteShortfromDb; // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId); if (AthleteShortfromDb == null) { // add athlete to the db if need. AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity; } else { Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database"); } Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); // such the connected athlete with they id. AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId)); if (_ConnectedAthleteShort == null) { // add connection if needed. AthleteConnection ac = new AthleteConnection(); ac.FromId = AthleteMasterShort.AthleteId; ac.ToId = AthleteShortfromDb.AthleteId; ac.Type = FollowType; AthleteMasterShort.Connections.Add(ac); Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}"); } else { Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)"); } } db.SaveChanges(); Console.WriteLine($"total read = {AthleteShortList.Count}"); Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); AthleteShortList.Clear(); } ret = 0; } catch (Exception e) { Console.WriteLine($"ERROR:{e.ToString()}"); ret = 1; } finally { stravaXApi.Dispose(); } return(ret); }
public List <AthleteShort> getConnectedAthetes(AthleteShort Athlete, string FollowType) { string NextPageUrl = $"https://www.strava.com/athletes/{Athlete.AthleteId}/follows?type={FollowType}"; // BrowserDriver.Navigate().GoToUrl(NextPageUrl); List <AthleteShort> AthleteShortList = new List <AthleteShort>(); DateTime CrawlDate = DateTime.Now; int PageCount = 0; do { logger.LogInformation($"open {NextPageUrl}"); BrowserDriver.Navigate().GoToUrl(NextPageUrl); var ConnectedAthleteElts = BrowserDriver.FindElements(By.XPath("//li[@data-athlete-id]")); foreach (IWebElement ConnectedAthleteElt in ConnectedAthleteElts) { try{ var ConnectedAthleteId = ConnectedAthleteElt.GetAttribute("data-athlete-id"); logger.LogDebug($"ConnectedAthleteId {ConnectedAthleteId}"); var ConnectedAthleteName = ConnectedAthleteElt.FindElement(By.XPath("./div[@title]")).GetAttribute("title"); logger.LogDebug($"ConnectedAthleteName {ConnectedAthleteName}"); var ConnectedAthleteAvatarUrl = ConnectedAthleteElt.FindElement(By.XPath(".//img[@class='avatar-img']")).GetAttribute("src"); logger.LogDebug($"ConnectedAthleteAvatarUrl {ConnectedAthleteAvatarUrl}"); var ConnectedAthleteBadge = ConnectedAthleteElt.FindElement(By.XPath(".//div[@class='avatar-badge']/span/span")).GetAttribute("class"); logger.LogDebug($"ConnectedAthleteBadge {ConnectedAthleteBadge}"); var ConnectedAthleteLocation = ConnectedAthleteElt.FindElement(By.XPath(".//div[@class='location mt-0']")).Text; logger.LogDebug($"ConnectedAthleteLocation {ConnectedAthleteLocation}"); var AthleteConnectionType = ConnectedAthleteElt.FindElement(By.XPath(".//button")).Text; logger.LogDebug($"AthleteConnectionType {AthleteConnectionType}"); var AthleteShort = new ConnectedAthlete(); AthleteShort.AthleteId = ConnectedAthleteId; AthleteShort.AthleteName = ConnectedAthleteName; AthleteShort.AthleteAvatarUrl = ConnectedAthleteAvatarUrl; AthleteShort.AthleteBadge = ConnectedAthleteBadge; AthleteShort.AthleteLocation = ConnectedAthleteLocation; AthleteShort.AthleteLastCrawled = CrawlDate; AthleteShortList.Add(AthleteShort); logger.LogInformation($"add {AthleteShort}"); // We also have informations about the connection state AthleteShort.ConnectionState = AthleteConnectionType; } catch (Exception e) when(e is WebDriverException || e is NotFoundException) { if (e is InvalidElementStateException || e is StaleElementReferenceException) { // Page seams to be incorrect loaded. Probably need to wait more. throw e; } logger.LogInformation($"Skip athlete at {NextPageUrl} Err:{e.Message}"); } } try { NextPageUrl = BrowserDriver.FindElement(By.XPath("//li[@class='next_page']/a")).GetAttribute("href"); logger.LogDebug($"next page={NextPageUrl}"); } catch (WebDriverException) { NextPageUrl = ""; } PageCount++; }while(!string.IsNullOrEmpty(NextPageUrl)); return(AthleteShortList); }