// Retrieve the time of the activity. The value may be container in the short description. // If this information is not availlable, an exception wil be thrown. public static TimeSpan extractActivityTime(ActivityShort activity) { string description = activity.StatShortString; string hString = "0"; string mString = "0"; string sString = "0"; var match = Regex.Match(description, "([0-9]{1,2})h ([0-9]{1,2})m ([0-9]{1,2})s"); if (match.Success) { hString = match.Groups[1].Value; mString = match.Groups[2].Value; sString = match.Groups[3].Value; } else { match = Regex.Match(description, "([0-9]{1,2})h ([0-9]{1,2})m"); if (match.Success) { hString = match.Groups[1].Value; mString = match.Groups[2].Value; } else { match = Regex.Match(description, "([0-9]{1,2})m ([0-9]{1,2})s"); if (match.Success) { mString = match.Groups[1].Value; sString = match.Groups[2].Value; } else { match = Regex.Match(description, "([0-9]{1,2})s"); if (match.Success) { sString = match.Groups[1].Value; } else { // no time information availlable throw new TimeSpanNotFoundException(description); } } } } TimeSpan ret = new TimeSpan(int.Parse(hString), int.Parse(mString), int.Parse(sString)); return(ret); }
/// <summary> /// Extract all points from the gpx file and fill an XElement which could be integrate in the kml file. /// </summary> /// <param name="activity"></param> /// <param name="stream"></param> /// <param name="kml"></param> /// <returns></returns> public static XElement readGpx(ActivityShort activity, Stream stream, XNamespace kml) { XDocument gpxDoc = XDocument.Load(stream); XNamespace ns = "{http://www.topografix.com/GPX/1/1}"; var gpxElt = gpxDoc.Root; var trkptElts = gpxElt.Element($"{ns}trk").Element($"{ns}trkseg").Elements(); StringBuilder sb = new StringBuilder(); foreach (XElement element in trkptElts) { // Console.WriteLine($" {element}"); var lat = element.Attribute("lat").Value; var lon = element.Attribute("lon").Value; var ele = element.Element($"{ns}ele").Value; if (sb.Length > 0) { sb.Append(" "); } sb.Append($"{lon},{lat},{ele}"); } var kmlGpxDocument = new XElement(kml + "Document" , new XElement(kml + "name", activity.ActivityTitle) , new XElement(kml + "Style", new XAttribute("id", "lineStyle") , new XElement(kml + "LineStyle", new XElement(kml + "color", "80ffac59"), new XElement(kml + "width", "6") )) , new XElement(kml + "Placemark" , new XElement(kml + "name", "Path") , new XElement(kml + "visibility", "0") , new XElement(kml + "styleUrl", "#lineStyle") , new XElement(kml + "LineString", new XElement(kml + "tessellate", "1"), new XElement(kml + "coordinates", sb.ToString()) ))); return(kmlGpxDocument); }
/// <summary> /// Convert a gpx file in kml. /// Estimate the timestamp of each point, depending on the activity start and its duration. /// This make the timeline navigation in Google-Earth possible. /// </summary> /// <param name="activity"></param> /// <param name="stream"></param> /// <param name="kml"></param> /// <param name="gx"></param> /// <returns></returns>// public static XElement convertToKmlWithTime(ActivityShort activity, Stream stream, XNamespace kml, XNamespace gx) { XDocument gpxDoc = XDocument.Load(stream); XNamespace ns = "{http://www.topografix.com/GPX/1/1}"; var gpxElt = gpxDoc.Root; var trkptElts = gpxElt.Element($"{ns}trk").Element($"{ns}trkseg").Elements(); List <XElement> whenList = new List <XElement>(); List <XElement> coordList = new List <XElement>(); // start time of the activity var startTime = activity.ActivityDate; // end time of the activity var activityTime = Utils.extractActivityTime(activity); var endTime = activity.ActivityDate.Add(activityTime); int pointCount = trkptElts.Count(); // double deltaTimeMs = (activityTime.Milliseconds); int currentPointIndex = 1; foreach (XElement element in trkptElts) { // Console.WriteLine($" {element}"); var lat = element.Attribute("lat").Value; var lon = element.Attribute("lon").Value; var ele = element.Element($"{ns}ele").Value; // 2020-02-09T11:45:03Z var whenPoint = startTime.AddSeconds(((double)activityTime.TotalSeconds) * ((double)currentPointIndex / (double)pointCount)); XElement whendElt = new XElement(kml + "when", whenPoint.ToString("yyyy-MM-ddTHH:mm:ssZ")); whenList.Add(whendElt); XElement coordElt = new XElement(gx + "coord", $"{lon} {lat} {ele}"); coordList.Add(coordElt); currentPointIndex++; } var kmlGpxDocument = new XElement(kml + "Document" , new XElement(kml + "name", activity.ActivityTitle) , new XElement(kml + "Style", new XAttribute("id", "multiTrack_h") , new XElement(kml + "IconStyle", new XElement(kml + "scale", "1.2"), new XElement(kml + "Icon", new XElement(kml + "href", "http://earth.google.com/images/kml-icons/track-directional/track-0.png"))) , new XElement(kml + "LineStyle", new XElement(kml + "color", "99ffac59"), new XElement(kml + "width", "8") )) , new XElement(kml + "StyleMap", new XAttribute("id", "multiTrack") , new XElement(kml + "Pair", new XElement(kml + "key", "normal"), new XElement(kml + "styleUrl", "#multiTrack_n") ) , new XElement(kml + "Pair", new XElement(kml + "key", "highlight"), new XElement(kml + "styleUrl", "#multiTrack_h") ) ) , new XElement(kml + "Style", new XAttribute("id", "multiTrack_n") , new XElement(kml + "IconStyle", new XElement(kml + "Icon", new XElement(kml + "href", "http://earth.google.com/images/kml-icons/track-directional/track-0.png"))) , new XElement(kml + "LineStyle", new XElement(kml + "color", "99ffac59"), new XElement(kml + "width", "6") )) , new XElement(kml + "Placemark" , new XElement(kml + "name", activity.ActivityTitle) , new XElement(kml + "visibility", "0") , new XElement(kml + "styleUrl", "#multiTrack") , new XElement(gx + "Track", whenList, coordList ))); return(kmlGpxDocument); }
private static XElement handleGpxStream(Stream stream, ExportType exportType, ActivityShort activity) { XElement GpxToKmlElt; switch (exportType) { case ExportType.HeatMap: // for heatmap without time. GpxToKmlElt = readGpx(activity, stream, kml); break; case ExportType.TimeMap: // for heatmap with time. GpxToKmlElt = convertToKmlWithTime(activity, stream, kml, gx); break; default: throw new System.InvalidOperationException($"Export type not supported {exportType}"); } return(GpxToKmlElt); }
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args) { int ret = -1; Console.WriteLine("Read athlete activities with Strava-X-API."); String AthleteId = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, }; p.Parse(args); if (AthleteId == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing athlete id"); } try { stravaXApi.signIn(); List <ActivityShort> ActivitiesList = new List <ActivityShort>(); DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId); System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}"); int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR")); int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH")); int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR")); int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH")); DateTime now = DateTime.Now; for (int year = FromYear; year <= ToYear; year++) { for (int month = 01; month <= 12; month++) { if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth)) { continue; } List <ActivityShort> ActivitiesMonthList; try { ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}"); } catch (StaleElementReferenceException) { // Wait and try again. Thread.Sleep(2000); ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}"); } ActivitiesList.AddRange(ActivitiesMonthList); using (StravaXApiContext db = new StravaXApiContext()) { foreach (ActivityShort ActivityShort in ActivitiesList) { Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null) { db.ActivityShortDB.Add(ActivityShort); db.SaveChanges(); Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}"); } else { Console.WriteLine($"{ActivityShort.ActivityId} allready in database"); } } Console.WriteLine($"total read = {ActivitiesList.Count}"); Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}"); ActivitiesList.Clear(); } } } ret = 0; } catch (Exception e) { Console.WriteLine($"ERROR:{e.ToString()}"); ret = 1; } finally { stravaXApi.Dispose(); } return(ret); }
private List <ActivityShort> _getActivities(String AthleteId, String Year, String Month) { String url = $"https://www.strava.com/athletes/{AthleteId}#interval_type?chart_type=miles&interval_type=month&interval={Year}{Month}&year_offset=0"; BrowserDriver.Navigate().GoToUrl(url); logger.LogInformation($"open {url}"); DateTime CrawlDate = DateTime.Now; // Should wait for element. // Thread.Sleep(2000); IWait <IWebDriver> wait = new OpenQA.Selenium.Support.UI.WebDriverWait(BrowserDriver, TimeSpan.FromSeconds(30.00)); wait.Until(driver1 => ((IJavaScriptExecutor)BrowserDriver).ExecuteScript("return document.readyState").Equals("complete")); Thread.Sleep(3000); if (ScreenshotsMonthActivities) { if (!Directory.Exists("./screenshots")) { DirectoryInfo DirInfo = Directory.CreateDirectory("./screenshots"); logger.LogInformation($"directory for screenshots created at {DirInfo.FullName}"); } if (!Directory.Exists($"./screenshots/{AthleteId}")) { DirectoryInfo DirInfo = Directory.CreateDirectory($"./screenshots/{AthleteId}"); } ((ITakesScreenshot)BrowserDriver).GetScreenshot().SaveAsFile($"./screenshots/{AthleteId}/{AthleteId}_{Year}_{Month}.png"); } // Find all activity icons in thos page var Elts = BrowserDriver.FindElements(By.XPath("//div[@class='entry-type-icon']")); logger.LogDebug($"Elts count={Elts.Count} for {Year}/{Month}"); List <ActivityShort> ActivitiesList = new List <ActivityShort>(); foreach (IWebElement Elt in Elts) { try{ // locate the div for activity number var ActivityNumberElt = Elt.FindElement(By.XPath("./../../..")); var ActivityId = ActivityNumberElt.GetAttribute("id"); // for activitty with picture we have to search on step higher if (String.IsNullOrEmpty(ActivityId)) { ActivityNumberElt = ActivityNumberElt.FindElement(By.XPath("./..")); ActivityId = ActivityNumberElt.GetAttribute("id"); } ActivityId = ActivityId.Substring("Activity-".Length); // activity title var ActivityTitleElt = ActivityNumberElt.FindElement(By.XPath(".//strong/a")); var ActivityTitle = ActivityTitleElt.Text; // activity thumbnails images var ActivityImageElts = ActivityNumberElt.FindElements(By.XPath(".//img[@alt='Photo']")); List <String> ActivityThumbnailsList = new List <String>(); foreach (IWebElement ActivityImageElt in ActivityImageElts) { // https://dgtzuqphqg23d.cloudfront.net/Wz2CrhzkXF3hm99lZmgWRBRbWhHBPLxUGDja_aMJDeQ-128x72.jpg string imageUrl = ActivityImageElt.GetAttribute("src"); ActivityThumbnailsList.Add(imageUrl); logger.LogDebug($"Activity {ActivityId} url {imageUrl}"); if (DownloadThumbnailsActivities) { System.Net.WebClient webClient = new System.Net.WebClient(); string[] pathElts = imageUrl.Split('/'); string localFileName = $"./screenshots/{AthleteId}_{Year}_{Month}_{ActivityId}_{pathElts[pathElts.Length-1]}.png"; webClient.DownloadFile(imageUrl, localFileName); } } // activity big images var ActivityBigImageElts = ActivityNumberElt.FindElements(By.XPath(".//li[@str-type='photo']")); List <String> ActivityImagesList = new List <String>(); foreach (IWebElement ActivityImageElt in ActivityBigImageElts) { // https://dgtzuqphqg23d.cloudfront.net/Wz2CrhzkXF3hm99lZmgWRBRbWhHBPLxUGDja_aMJDeQ-2048x1152.jpg string imageUrl = ActivityImageElt.GetAttribute("str-target-url"); ActivityImagesList.Add(imageUrl); logger.LogDebug($"Activity {ActivityId} url {imageUrl}"); if (DownloadImagesActivities) { System.Net.WebClient webClient = new System.Net.WebClient(); string[] pathElts = imageUrl.Split('/'); string localFileName = $"./screenshots/{AthleteId}_{Year}_{Month}_{ActivityId}_{pathElts[pathElts.Length-1]}.png"; webClient.DownloadFile(imageUrl, localFileName); } } // Locate activity time information string ActivityTimeString = ""; IWebElement ActivityTimeElt; string AthleteIdInGroup = AthleteId; List <String> GroupActivityList = new List <String>(); List <String> GroupAthleteList = new List <String>(); try{ if (ActivityNumberElt.TagName == "li") { // because of group activities I need to go to parents higher. ActivityTimeElt = ActivityNumberElt.FindElement(By.XPath("./../..//time[@class='timestamp']")); // find out which Athlete Id var AthleteIdInGroupElt = ActivityNumberElt.FindElement(By.XPath(".//a[contains(@href,'/athletes/')]")); AthleteIdInGroup = AthleteIdInGroupElt.GetAttribute("href"); AthleteIdInGroup = AthleteIdInGroup.Substring(AthleteIdInGroup.LastIndexOf("/") + 1); logger.LogDebug($"Groupped activity : Activity {ActivityId} Athlete {AthleteIdInGroup}"); // for group activities we are creating groups var GroupActivityElts = ActivityNumberElt.FindElements(By.XPath("./../../..//li[@class='entity-details feed-entry']")); foreach (IWebElement GroupActivityElt in GroupActivityElts) { string GroupActivityString = GroupActivityElt.GetAttribute("id"); GroupActivityString = GroupActivityString.Substring("Activity-".Length); if (ActivityId != GroupActivityString) { // in ActivitysGroupElts we have both the original activity and the other activities in the group. We need to filter GroupActivityList.Add(GroupActivityString); // also retrieve the athlete id for the activity string GroupAthleteUrl = GroupActivityElt.FindElement(By.XPath(".//a[@class='avatar-content']")).GetAttribute("href"); string GroupAthleteId = GroupAthleteUrl.Substring(GroupAthleteUrl.LastIndexOf('/') + 1); GroupAthleteList.Add(GroupAthleteId); logger.LogDebug($"Group {ActivityId} with {GroupActivityString} from {GroupAthleteId}"); } } } else { ActivityTimeElt = ActivityNumberElt.FindElement(By.XPath(".//time[@class='timestamp']")); } ActivityTimeString = ActivityTimeElt.GetAttribute("datetime"); } catch (WebDriverException e) { throw new NotFoundException($"can't read activity time {ActivityId} at {url} Err:{e.Message}", e); } string ActivityImageMapUrl; try{ IWebElement ImageWithMapElt; if (ActivityNumberElt.TagName == "li") { // Image with map for activities with group ImageWithMapElt = ActivityNumberElt.FindElement(By.XPath("./../../div/a/div[contains(@str-type,'map')]/img")); } else { // Image with map for activities without group ImageWithMapElt = ActivityNumberElt.FindElement(By.XPath(".//a[contains(@str-type,'map')]/img")); } ActivityImageMapUrl = ImageWithMapElt.GetAttribute("src"); } catch (WebDriverException) { // activity map is not always present. ActivityImageMapUrl = null; } // get stats short description as text string ActivityStatShort; IWebElement StatShortElt; StatShortElt = ActivityNumberElt.FindElement(By.XPath(".//ul[contains(@class,'list-stats')]")); ActivityStatShort = StatShortElt.Text; // Retrieve the activity class, with that it's poosible to know the activity type var ActivityTypeElt = Elt.FindElement(By.XPath("./span/span")); ActivityType ActivityType = parseActivityType(ActivityTypeElt.GetAttribute("class")); DateTime ActivityTime = DateTime.Parse(ActivityTimeString.Substring(0, ActivityTimeString.Length - 4)); logger.LogInformation($"Id={ActivityId} Text={ActivityTitle} Type={ActivityType} Time={ActivityTime}"); var ActivityShort = new ActivityShort(); ActivityShort.AthleteId = AthleteId; ActivityShort.ActivityId = ActivityId; ActivityShort.ActivityTitle = ActivityTitle; ActivityShort.ActivityType = ActivityType; ActivityShort.StatShortString = ActivityStatShort; ActivityShort.ActivityDate = ActivityTime; ActivityShort.ActivityLastCrawled = CrawlDate; ActivityShort.ActivityImageMapUrl = ActivityImageMapUrl; ActivityShort.ActivityThumbnailsList = ActivityThumbnailsList; ActivityShort.ActivityImagesList = ActivityImagesList; ActivityShort.GroupActivityList = GroupActivityList; ActivityShort.GroupAthleteList = GroupAthleteList; ActivitiesList.Add(ActivityShort); } catch (Exception e) when(e is WebDriverException || e is NotFoundException) { if (e is InvalidElementStateException || e is StaleElementReferenceException) { // Page seams to be incorrect loaded. Probably need to wait more. throw e; } logger.LogInformation($"Skip Activity at {url} Err:{e.Message}"); } } return(ActivitiesList); }
static public int WriteState(string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.DbStats", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log Stats"); bool doGarbage = false; bool doAll = false; bool doListimages = false; bool doListMaps = false; bool doActivityList = false; string AthleteId = null; string ActivityTypeStr = null; bool doAthleteStats = false; string doRenewMonth = null; bool doQueryCounts = false; bool doGpxCounts = false; bool doActivityCounts = false; bool doAthleteCounts = false; bool doListAthletes = false; var p = new OptionSet() { { "g|garbage", v => { doGarbage = true; } }, { "a|all", v => { doAll = true; } }, { "act|listactivities", v => { doActivityList = true; } }, { "li|listimages", v => { doListimages = true; } }, { "la|listathletes", v => { doListAthletes = true; } }, { "lm|listmaps", v => { doListMaps = true; } }, { "aid|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "as|athlete-stats", v => { doAthleteStats = true; } }, { "m|renew-month=", v => { doRenewMonth = v; } }, { "cath|count-athletes", v => { doAthleteCounts = true; } }, { "cq|count-queries", v => { doQueryCounts = true; } }, { "cgpx|count-gpx", v => { doGpxCounts = true; } }, { "cact|count-activities", v => { doActivityCounts = true; } }, }; p.Parse(args); int ret = -1; try { using (StravaXApiContext db = new StravaXApiContext()) { // logger.LogInformation($"Queries stored {db.ActivityQueriesDB.Count()}"); // logger.LogInformation($"Activities stored {db.ActivityShortDB.Count()}"); // var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct(); // logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}"); /* * foreach(var aId in al) * { * AthleteShort ath = db.AthleteShortDB.Find(aId); * // for format: https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated * if (ath != null) * { * logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {ath.AthleteId,9} {ath.AthleteName,-40} {ath.AthleteLocation}"); * } * else * { * logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {aId,9}"); * } * } */ /* * // output for first query * foreach(ActivityRangeQuery arq in db.ActivityQueriesDB) * { * logger.LogInformation($"{arq}"); * break; * } */ if (doListAthletes || doAll) { foreach (AthleteShort athlete in db.AthleteShortDB) { logger.LogInformation($"Athletes {athlete.AthleteName} {athlete.AthleteId}"); } } if (doAthleteCounts || doAll) { // output status count for queries var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct(); logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}"); } if (doQueryCounts || doAll) { // output status count for queries var status = db.ActivityQueriesDB.Select(a => a.Status).Distinct(); logger.LogInformation($"Queries:"); foreach (var st in status) { logger.LogInformation($" {st} {db.ActivityQueriesDB.Count(a => a.Status==st)}"); } } if (doGpxCounts || doAll) { // Count GPX Files and sort by activity-Types. string rootPath = "gpx"; foreach (string athleteEntry in Directory.GetDirectories(rootPath)) { var match = Regex.Match(athleteEntry, ".*[/]([0-9]+)"); string AthleteIdStr = match.Groups[1].Value; // logger.LogInformation($" directory for Athlete {AthleteIdStr} found {athleteEntry}"); Hashtable errTable = new Hashtable(); Hashtable trackTable = new Hashtable(); foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.err")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.err"); string ActivityIdStr = match.Groups[1].Value; errTable.Add(ActivityIdStr, gpxActivity); } foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx.gz")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.gz"); string ActivityIdStr = match.Groups[1].Value; if (errTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous"); } if (trackTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}"); } else { trackTable.Add(ActivityIdStr, gpxActivity); } } foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx"); string ActivityIdStr = match.Groups[1].Value; if (errTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous"); } if (trackTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}"); } else { logger.LogWarning($"Found GPX without compression: track for {ActivityIdStr} already enterred {gpxActivity}"); trackTable.Add(ActivityIdStr, gpxActivity); } } Hashtable typeCount = new Hashtable(); foreach (string activity_id in trackTable.Keys) { ActivityShort activity = db.ActivityShortDB.Find(activity_id); if (activity == null) { // logger.LogWarning($"can't find activity for {activity_id}"); continue; } int tCount = 0; if (typeCount.ContainsKey(activity.ActivityType)) { tCount = (int)typeCount[activity.ActivityType]; typeCount.Remove(activity.ActivityType); } tCount++; typeCount.Add(activity.ActivityType, tCount); } // var al = db.ActivityShortDB.Where(a=>a.AthleteId==AthleteIdStr); // int actCount = al.Count(); string keys = string.Join(",", typeCount.Keys.Cast <ActivityType>().Select(x => $"{x.ToString()} {typeCount[x]}").ToArray()); int actCount = 0; logger.LogInformation($"Athlete {AthleteIdStr}, activities found in DB: {actCount} GPX: {trackTable.Keys.Count} ERR: {errTable.Keys.Count} {keys}"); } } if (doRenewMonth != null) { // set all DONE queries for the given month to Created var match = Regex.Match(doRenewMonth, "([0-9]{4})/([0-9]{2})"); int year = Int32.Parse(match.Groups[1].Value); int month = Int32.Parse(match.Groups[2].Value); var statusDone = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.DateFrom == new DateTime(year, month, 01)); foreach (ActivityRangeQuery arq in statusDone.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"reset for {year}/{month} from {QueryStatus.Run} to {QueryStatus.Created}"; logger.LogInformation($"query for {year}/{month} with {QueryStatus.Done} {arq.AthleteId} {arq.Message}"); } logger.LogInformation($"begin: save changes"); db.SaveChanges(); logger.LogInformation($"done: save changes"); } if (doGarbage) { { var statusError = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Error); foreach (ActivityRangeQuery arq in statusError.ToList()) { logger.LogInformation($"query with {QueryStatus.Error} {arq.AthleteId} {arq.Message}"); } } { var statusRun = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Run); foreach (ActivityRangeQuery arq in statusRun.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"garbage-reset from {QueryStatus.Run} to {QueryStatus.Created}"; logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}"); } db.SaveChanges(); } { var statusReserved = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Reserved); foreach (ActivityRangeQuery arq in statusReserved.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"garbage-reset from {QueryStatus.Reserved} to {QueryStatus.Created}"; logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}"); db.SaveChanges(); } } } if (doActivityCounts || doAll) { // Find out athlete with open queries: var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct(); logger.LogInformation($"{qAthleteCreated.Count()} with {QueryStatus.Created} queries"); var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct(); logger.LogInformation($"{qAthleteCreated.Count()} without {QueryStatus.Reserved} queries"); logger.LogInformation($"{qAthleteCreated.Intersect(qAthleteReserved).Count()} with {QueryStatus.Created} and without {QueryStatus.Reserved} queries"); List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(10).ToList(); logger.LogInformation($"{AthleteIdList.Count()} athletes from this list:"); foreach (string aId in AthleteIdList) { logger.LogInformation($" AthleteId={aId}"); } if (AthleteIdList.Count() > 0) { // retrieve one random athlete string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count)); logger.LogInformation($"retrieve activity for athlete {aid}"); // 5 first queries for this athlete IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(10).ToList(); foreach (ActivityRangeQuery arq in q0) { logger.LogInformation($" query={arq}"); var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo))); logger.LogInformation($" find {ActivitiesInRange.Count()} activities in it."); } } } if (doAll) { // find activities in a QueryRange foreach (ActivityRangeQuery arq in db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done).OrderByDescending(a => a.DateFrom).Take(10)) { var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo))); logger.LogInformation($" find {ActivitiesInRange.Count()} in range {arq}"); } } if (doAll) { // retrieve queries type for athlete string aId = "2754335"; var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList(); logger.LogInformation($" athl {aId}"); foreach (QueryStatus qs in qsList) { logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}"); } // IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(6).ToList(); } if (doAll) { string aId = "26319532"; var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList(); logger.LogInformation($" athl {aId}"); foreach (QueryStatus qs in qsList) { logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}"); } } if (doAll || doListimages) { List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList(); logger.LogInformation($" activity count:{activitiesWithImages.Count()}"); int imageCount = 0; foreach (ActivityShort activity in activitiesWithImages) { List <string> images = activity.ActivityImagesList; imageCount += images.Count(); logger.LogInformation($"activity {activity.ActivityTitle}"); foreach (string imageUrl in images) { logger.LogInformation($" url:{imageUrl}"); } } logger.LogInformation($" images {imageCount}"); } if (doAll || doListMaps) { List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList(); logger.LogInformation($" activity count:{activitiesWithImages.Count()}"); int imageCount = 0; foreach (ActivityShort activity in activitiesWithImages) { string ImageMapUrl = activity.ActivityImageMapUrl; if (ImageMapUrl == null) { continue; } imageCount++; logger.LogInformation($"activity {activity.ActivityTitle} {ImageMapUrl}"); WebClient webClient = new WebClient(); string outputDir = $"maps/{activity.AthleteId}"; string outputFilename = $"{activity.ActivityId}.png"; logger.LogWarning($"NOT IMPLEMENTED"); } logger.LogInformation($" images {imageCount}/{activitiesWithImages}"); } if (doAll || doAthleteStats) { IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList(); logger.LogInformation($" athletes {AllAthletes.Count()}"); logger.LogInformation($" first: [{AllAthletes.First()}] last: [{AllAthletes.Last()}]"); int AthleteCount = db.ActivityQueriesDB.Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($"athletes in queries {AthleteCount}"); int PrivateAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.Message.Contains("private")).Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($" private {PrivateAthleteCount}"); int CreatedAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($" {QueryStatus.Created} {CreatedAthleteCount}"); } if (doAll || doActivityList) { List <ActivityShort> activities; IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (AthleteId != null) { dbs = dbs.Where(a => a.AthleteId == AthleteId); } activities = dbs.ToList(); logger.LogInformation($"List activities for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}"); /* * if (activities.Count>0 && doActivityList) * { * foreach(ActivityShort activity in activities) * { * logger.LogInformation($" {activity}"); * } * } */ } /* * { * // opened queries with activities * int i=0; * int nbAthletes=db.AthleteShortDB.Count(); * foreach(AthleteShort athlete in db.AthleteShortDB) * { * i++; * // logger.LogInformation($"retrieve activity for athlete {athlete.AthleteId}"); * // 5 first queries for this athlete * IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==athlete.AthleteId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(5).ToList(); * int activitiesWithCreatedQueriesCount=0; * foreach(ActivityRangeQuery arq in q0) * { * var ActivitiesInRange = db.ActivityShortDB.Where(a=>a.AthleteId==arq.AthleteId && ((a.ActivityDate>=arq.DateFrom)&&(a.ActivityDate<=arq.DateTo))); * activitiesWithCreatedQueriesCount+=ActivitiesInRange.Count(); * } * if (activitiesWithCreatedQueriesCount>0) * { * logger.LogInformation($" athlete {athlete.AthleteId} as {activitiesWithCreatedQueriesCount} activities with created queries queries:{q0.Count()} activites{db.ActivityShortDB.Where(a=>a.AthleteId==athlete.AthleteId).Count()}."); * } * if (i%10==0) * { * logger.LogInformation($"{i}/{nbAthletes}"); * } * } * } */ /* * // output activity count for activity type * logger.LogInformation($"Activity types Σ :{db.ActivityShortDB.Count()}"); * var ActivityTypes = db.ActivityShortDB.Select(a => a.ActivityType).Distinct(); * foreach(var aType in ActivityTypes) * { * logger.LogInformation($" {aType,18} {db.ActivityShortDB.Count(a => a.ActivityType==aType),6}"); * } */ /* * // output athletes with ski activities and their ski activity count * logger.LogInformation($"All athletes with {ActivityType.BackcountrySki}"); * var Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Select(a => a.AthleteId).Distinct(); * foreach(var A4Type in Activity4Type) * { * var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Where(a => a.AthleteId==A4Type).Count(); * var athlete = db.AthleteShortDB.Find(A4Type); * logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})"); * } */ /* * // output athletes with run activities and their run activity count * logger.LogInformation($"All athletes with {ActivityType.Run}"); * Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Select(a => a.AthleteId).Distinct(); * foreach(var A4Type in Activity4Type) * { * var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Where(a => a.AthleteId==A4Type).Count(); * var athlete = db.AthleteShortDB.Find(A4Type); * logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})"); * } */ } ret = 0; } catch (Exception e) { logger.LogInformation($"ERROR:{e.ToString()}"); ret = 1; } return(ret); }