// Retrieve the time of the activity. The value may be container in the short description.
// If this information is not availlable, an exception wil be thrown.
public static TimeSpan extractActivityTime(ActivityShort activity)
{
string description = activity.StatShortString;
string hString = "0";
string mString = "0";
string sString = "0";
var match = Regex.Match(description, "([0-9]{1,2})h ([0-9]{1,2})m ([0-9]{1,2})s");
if (match.Success)
{
hString = match.Groups[1].Value;
mString = match.Groups[2].Value;
sString = match.Groups[3].Value;
}
else
{
match = Regex.Match(description, "([0-9]{1,2})h ([0-9]{1,2})m");
if (match.Success)
{
hString = match.Groups[1].Value;
mString = match.Groups[2].Value;
}
else
{
match = Regex.Match(description, "([0-9]{1,2})m ([0-9]{1,2})s");
if (match.Success)
{
mString = match.Groups[1].Value;
sString = match.Groups[2].Value;
}
else
{
match = Regex.Match(description, "([0-9]{1,2})s");
if (match.Success)
{
sString = match.Groups[1].Value;
}
else
{
// no time information availlable
throw new TimeSpanNotFoundException(description);
}
}
}
}
TimeSpan ret = new TimeSpan(int.Parse(hString), int.Parse(mString), int.Parse(sString));
return(ret);
}
/// <summary>
/// Extract all points from the gpx file and fill an XElement which could be integrate in the kml file.
/// </summary>
/// <param name="activity"></param>
/// <param name="stream"></param>
/// <param name="kml"></param>
/// <returns></returns>
public static XElement readGpx(ActivityShort activity, Stream stream, XNamespace kml)
{
XDocument gpxDoc = XDocument.Load(stream);
XNamespace ns = "{http://www.topografix.com/GPX/1/1}";
var gpxElt = gpxDoc.Root;
var trkptElts = gpxElt.Element($"{ns}trk").Element($"{ns}trkseg").Elements();
StringBuilder sb = new StringBuilder();
foreach (XElement element in trkptElts)
{
// Console.WriteLine($" {element}");
var lat = element.Attribute("lat").Value;
var lon = element.Attribute("lon").Value;
var ele = element.Element($"{ns}ele").Value;
if (sb.Length > 0)
{
sb.Append(" ");
}
sb.Append($"{lon},{lat},{ele}");
}
var kmlGpxDocument = new XElement(kml + "Document"
, new XElement(kml + "name", activity.ActivityTitle)
, new XElement(kml + "Style", new XAttribute("id", "lineStyle")
, new XElement(kml + "LineStyle",
new XElement(kml + "color", "80ffac59"),
new XElement(kml + "width", "6")
))
, new XElement(kml + "Placemark"
, new XElement(kml + "name", "Path")
, new XElement(kml + "visibility", "0")
, new XElement(kml + "styleUrl", "#lineStyle")
, new XElement(kml + "LineString",
new XElement(kml + "tessellate", "1"),
new XElement(kml + "coordinates", sb.ToString())
)));
return(kmlGpxDocument);
}
/// <summary>
/// Convert a gpx file in kml.
/// Estimate the timestamp of each point, depending on the activity start and its duration.
/// This make the timeline navigation in Google-Earth possible.
/// </summary>
/// <param name="activity"></param>
/// <param name="stream"></param>
/// <param name="kml"></param>
/// <param name="gx"></param>
/// <returns></returns>//
public static XElement convertToKmlWithTime(ActivityShort activity, Stream stream, XNamespace kml, XNamespace gx)
{
XDocument gpxDoc = XDocument.Load(stream);
XNamespace ns = "{http://www.topografix.com/GPX/1/1}";
var gpxElt = gpxDoc.Root;
var trkptElts = gpxElt.Element($"{ns}trk").Element($"{ns}trkseg").Elements();
List <XElement> whenList = new List <XElement>();
List <XElement> coordList = new List <XElement>();
// start time of the activity
var startTime = activity.ActivityDate;
// end time of the activity
var activityTime = Utils.extractActivityTime(activity);
var endTime = activity.ActivityDate.Add(activityTime);
int pointCount = trkptElts.Count();
// double deltaTimeMs = (activityTime.Milliseconds);
int currentPointIndex = 1;
foreach (XElement element in trkptElts)
{
// Console.WriteLine($" {element}");
var lat = element.Attribute("lat").Value;
var lon = element.Attribute("lon").Value;
var ele = element.Element($"{ns}ele").Value;
// 2020-02-09T11:45:03Z
var whenPoint = startTime.AddSeconds(((double)activityTime.TotalSeconds) * ((double)currentPointIndex / (double)pointCount));
XElement whendElt = new XElement(kml + "when", whenPoint.ToString("yyyy-MM-ddTHH:mm:ssZ"));
whenList.Add(whendElt);
XElement coordElt = new XElement(gx + "coord", $"{lon} {lat} {ele}");
coordList.Add(coordElt);
currentPointIndex++;
}
var kmlGpxDocument = new XElement(kml + "Document"
, new XElement(kml + "name", activity.ActivityTitle)
, new XElement(kml + "Style", new XAttribute("id", "multiTrack_h")
, new XElement(kml + "IconStyle",
new XElement(kml + "scale", "1.2"),
new XElement(kml + "Icon",
new XElement(kml + "href", "http://earth.google.com/images/kml-icons/track-directional/track-0.png")))
, new XElement(kml + "LineStyle",
new XElement(kml + "color", "99ffac59"),
new XElement(kml + "width", "8")
))
, new XElement(kml + "StyleMap", new XAttribute("id", "multiTrack")
, new XElement(kml + "Pair",
new XElement(kml + "key", "normal"),
new XElement(kml + "styleUrl", "#multiTrack_n")
)
, new XElement(kml + "Pair",
new XElement(kml + "key", "highlight"),
new XElement(kml + "styleUrl", "#multiTrack_h")
)
)
, new XElement(kml + "Style", new XAttribute("id", "multiTrack_n")
, new XElement(kml + "IconStyle",
new XElement(kml + "Icon",
new XElement(kml + "href", "http://earth.google.com/images/kml-icons/track-directional/track-0.png")))
, new XElement(kml + "LineStyle",
new XElement(kml + "color", "99ffac59"),
new XElement(kml + "width", "6")
))
, new XElement(kml + "Placemark"
, new XElement(kml + "name", activity.ActivityTitle)
, new XElement(kml + "visibility", "0")
, new XElement(kml + "styleUrl", "#multiTrack")
, new XElement(gx + "Track",
whenList,
coordList
)));
return(kmlGpxDocument);
}
private static XElement handleGpxStream(Stream stream, ExportType exportType, ActivityShort activity)
{
XElement GpxToKmlElt;
switch (exportType)
{
case ExportType.HeatMap:
// for heatmap without time.
GpxToKmlElt = readGpx(activity, stream, kml);
break;
case ExportType.TimeMap:
// for heatmap with time.
GpxToKmlElt = convertToKmlWithTime(activity, stream, kml, gx);
break;
default:
throw new System.InvalidOperationException($"Export type not supported {exportType}");
}
return(GpxToKmlElt);
}
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete activities with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
DateTime now = DateTime.Now;
for (int year = FromYear; year <= ToYear; year++)
{
for (int month = 01; month <= 12; month++)
{
if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth))
{
continue;
}
List <ActivityShort> ActivitiesMonthList;
try
{
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
catch (StaleElementReferenceException)
{
// Wait and try again.
Thread.Sleep(2000);
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
ActivitiesList.AddRange(ActivitiesMonthList);
using (StravaXApiContext db = new StravaXApiContext())
{
foreach (ActivityShort ActivityShort in ActivitiesList)
{
Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
db.SaveChanges();
Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
}
else
{
Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
}
}
Console.WriteLine($"total read = {ActivitiesList.Count}");
Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
ActivitiesList.Clear();
}
}
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
private List <ActivityShort> _getActivities(String AthleteId, String Year, String Month)
{
String url = $"https://www.strava.com/athletes/{AthleteId}#interval_type?chart_type=miles&interval_type=month&interval={Year}{Month}&year_offset=0";
BrowserDriver.Navigate().GoToUrl(url);
logger.LogInformation($"open {url}");
DateTime CrawlDate = DateTime.Now;
// Should wait for element.
// Thread.Sleep(2000);
IWait <IWebDriver> wait = new OpenQA.Selenium.Support.UI.WebDriverWait(BrowserDriver, TimeSpan.FromSeconds(30.00));
wait.Until(driver1 => ((IJavaScriptExecutor)BrowserDriver).ExecuteScript("return document.readyState").Equals("complete"));
Thread.Sleep(3000);
if (ScreenshotsMonthActivities)
{
if (!Directory.Exists("./screenshots"))
{
DirectoryInfo DirInfo = Directory.CreateDirectory("./screenshots");
logger.LogInformation($"directory for screenshots created at {DirInfo.FullName}");
}
if (!Directory.Exists($"./screenshots/{AthleteId}"))
{
DirectoryInfo DirInfo = Directory.CreateDirectory($"./screenshots/{AthleteId}");
}
((ITakesScreenshot)BrowserDriver).GetScreenshot().SaveAsFile($"./screenshots/{AthleteId}/{AthleteId}_{Year}_{Month}.png");
}
// Find all activity icons in thos page
var Elts = BrowserDriver.FindElements(By.XPath("//div[@class='entry-type-icon']"));
logger.LogDebug($"Elts count={Elts.Count} for {Year}/{Month}");
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
foreach (IWebElement Elt in Elts)
{
try{
// locate the div for activity number
var ActivityNumberElt = Elt.FindElement(By.XPath("./../../.."));
var ActivityId = ActivityNumberElt.GetAttribute("id");
// for activitty with picture we have to search on step higher
if (String.IsNullOrEmpty(ActivityId))
{
ActivityNumberElt = ActivityNumberElt.FindElement(By.XPath("./.."));
ActivityId = ActivityNumberElt.GetAttribute("id");
}
ActivityId = ActivityId.Substring("Activity-".Length);
// activity title
var ActivityTitleElt = ActivityNumberElt.FindElement(By.XPath(".//strong/a"));
var ActivityTitle = ActivityTitleElt.Text;
// activity thumbnails images
var ActivityImageElts = ActivityNumberElt.FindElements(By.XPath(".//img[@alt='Photo']"));
List <String> ActivityThumbnailsList = new List <String>();
foreach (IWebElement ActivityImageElt in ActivityImageElts)
{
// https://dgtzuqphqg23d.cloudfront.net/Wz2CrhzkXF3hm99lZmgWRBRbWhHBPLxUGDja_aMJDeQ-128x72.jpg
string imageUrl = ActivityImageElt.GetAttribute("src");
ActivityThumbnailsList.Add(imageUrl);
logger.LogDebug($"Activity {ActivityId} url {imageUrl}");
if (DownloadThumbnailsActivities)
{
System.Net.WebClient webClient = new System.Net.WebClient();
string[] pathElts = imageUrl.Split('/');
string localFileName = $"./screenshots/{AthleteId}_{Year}_{Month}_{ActivityId}_{pathElts[pathElts.Length-1]}.png";
webClient.DownloadFile(imageUrl, localFileName);
}
}
// activity big images
var ActivityBigImageElts = ActivityNumberElt.FindElements(By.XPath(".//li[@str-type='photo']"));
List <String> ActivityImagesList = new List <String>();
foreach (IWebElement ActivityImageElt in ActivityBigImageElts)
{
// https://dgtzuqphqg23d.cloudfront.net/Wz2CrhzkXF3hm99lZmgWRBRbWhHBPLxUGDja_aMJDeQ-2048x1152.jpg
string imageUrl = ActivityImageElt.GetAttribute("str-target-url");
ActivityImagesList.Add(imageUrl);
logger.LogDebug($"Activity {ActivityId} url {imageUrl}");
if (DownloadImagesActivities)
{
System.Net.WebClient webClient = new System.Net.WebClient();
string[] pathElts = imageUrl.Split('/');
string localFileName = $"./screenshots/{AthleteId}_{Year}_{Month}_{ActivityId}_{pathElts[pathElts.Length-1]}.png";
webClient.DownloadFile(imageUrl, localFileName);
}
}
// Locate activity time information
string ActivityTimeString = "";
IWebElement ActivityTimeElt;
string AthleteIdInGroup = AthleteId;
List <String> GroupActivityList = new List <String>();
List <String> GroupAthleteList = new List <String>();
try{
if (ActivityNumberElt.TagName == "li")
{
// because of group activities I need to go to parents higher.
ActivityTimeElt = ActivityNumberElt.FindElement(By.XPath("./../..//time[@class='timestamp']"));
// find out which Athlete Id
var AthleteIdInGroupElt = ActivityNumberElt.FindElement(By.XPath(".//a[contains(@href,'/athletes/')]"));
AthleteIdInGroup = AthleteIdInGroupElt.GetAttribute("href");
AthleteIdInGroup = AthleteIdInGroup.Substring(AthleteIdInGroup.LastIndexOf("/") + 1);
logger.LogDebug($"Groupped activity : Activity {ActivityId} Athlete {AthleteIdInGroup}");
// for group activities we are creating groups
var GroupActivityElts = ActivityNumberElt.FindElements(By.XPath("./../../..//li[@class='entity-details feed-entry']"));
foreach (IWebElement GroupActivityElt in GroupActivityElts)
{
string GroupActivityString = GroupActivityElt.GetAttribute("id");
GroupActivityString = GroupActivityString.Substring("Activity-".Length);
if (ActivityId != GroupActivityString)
{
// in ActivitysGroupElts we have both the original activity and the other activities in the group. We need to filter
GroupActivityList.Add(GroupActivityString);
// also retrieve the athlete id for the activity
string GroupAthleteUrl = GroupActivityElt.FindElement(By.XPath(".//a[@class='avatar-content']")).GetAttribute("href");
string GroupAthleteId = GroupAthleteUrl.Substring(GroupAthleteUrl.LastIndexOf('/') + 1);
GroupAthleteList.Add(GroupAthleteId);
logger.LogDebug($"Group {ActivityId} with {GroupActivityString} from {GroupAthleteId}");
}
}
}
else
{
ActivityTimeElt = ActivityNumberElt.FindElement(By.XPath(".//time[@class='timestamp']"));
}
ActivityTimeString = ActivityTimeElt.GetAttribute("datetime");
}
catch (WebDriverException e) {
throw new NotFoundException($"can't read activity time {ActivityId} at {url} Err:{e.Message}", e);
}
string ActivityImageMapUrl;
try{
IWebElement ImageWithMapElt;
if (ActivityNumberElt.TagName == "li")
{
// Image with map for activities with group
ImageWithMapElt = ActivityNumberElt.FindElement(By.XPath("./../../div/a/div[contains(@str-type,'map')]/img"));
}
else
{
// Image with map for activities without group
ImageWithMapElt = ActivityNumberElt.FindElement(By.XPath(".//a[contains(@str-type,'map')]/img"));
}
ActivityImageMapUrl = ImageWithMapElt.GetAttribute("src");
}
catch (WebDriverException) {
// activity map is not always present.
ActivityImageMapUrl = null;
}
// get stats short description as text
string ActivityStatShort;
IWebElement StatShortElt;
StatShortElt = ActivityNumberElt.FindElement(By.XPath(".//ul[contains(@class,'list-stats')]"));
ActivityStatShort = StatShortElt.Text;
// Retrieve the activity class, with that it's poosible to know the activity type
var ActivityTypeElt = Elt.FindElement(By.XPath("./span/span"));
ActivityType ActivityType = parseActivityType(ActivityTypeElt.GetAttribute("class"));
DateTime ActivityTime = DateTime.Parse(ActivityTimeString.Substring(0, ActivityTimeString.Length - 4));
logger.LogInformation($"Id={ActivityId} Text={ActivityTitle} Type={ActivityType} Time={ActivityTime}");
var ActivityShort = new ActivityShort();
ActivityShort.AthleteId = AthleteId;
ActivityShort.ActivityId = ActivityId;
ActivityShort.ActivityTitle = ActivityTitle;
ActivityShort.ActivityType = ActivityType;
ActivityShort.StatShortString = ActivityStatShort;
ActivityShort.ActivityDate = ActivityTime;
ActivityShort.ActivityLastCrawled = CrawlDate;
ActivityShort.ActivityImageMapUrl = ActivityImageMapUrl;
ActivityShort.ActivityThumbnailsList = ActivityThumbnailsList;
ActivityShort.ActivityImagesList = ActivityImagesList;
ActivityShort.GroupActivityList = GroupActivityList;
ActivityShort.GroupAthleteList = GroupAthleteList;
ActivitiesList.Add(ActivityShort);
}
catch (Exception e) when(e is WebDriverException || e is NotFoundException)
{
if (e is InvalidElementStateException || e is StaleElementReferenceException)
{
// Page seams to be incorrect loaded. Probably need to wait more.
throw e;
}
logger.LogInformation($"Skip Activity at {url} Err:{e.Message}");
}
}
return(ActivitiesList);
}
static public int WriteState(string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.DbStats", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log Stats");
bool doGarbage = false;
bool doAll = false;
bool doListimages = false;
bool doListMaps = false;
bool doActivityList = false;
string AthleteId = null;
string ActivityTypeStr = null;
bool doAthleteStats = false;
string doRenewMonth = null;
bool doQueryCounts = false;
bool doGpxCounts = false;
bool doActivityCounts = false;
bool doAthleteCounts = false;
bool doListAthletes = false;
var p = new OptionSet()
{
{ "g|garbage", v => { doGarbage = true; } },
{ "a|all", v => { doAll = true; } },
{ "act|listactivities", v => { doActivityList = true; } },
{ "li|listimages", v => { doListimages = true; } },
{ "la|listathletes", v => { doListAthletes = true; } },
{ "lm|listmaps", v => { doListMaps = true; } },
{ "aid|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "as|athlete-stats", v => { doAthleteStats = true; } },
{ "m|renew-month=", v => { doRenewMonth = v; } },
{ "cath|count-athletes", v => { doAthleteCounts = true; } },
{ "cq|count-queries", v => { doQueryCounts = true; } },
{ "cgpx|count-gpx", v => { doGpxCounts = true; } },
{ "cact|count-activities", v => { doActivityCounts = true; } },
};
p.Parse(args);
int ret = -1;
try
{
using (StravaXApiContext db = new StravaXApiContext())
{
// logger.LogInformation($"Queries stored {db.ActivityQueriesDB.Count()}");
// logger.LogInformation($"Activities stored {db.ActivityShortDB.Count()}");
// var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct();
// logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}");
/*
* foreach(var aId in al)
* {
* AthleteShort ath = db.AthleteShortDB.Find(aId);
* // for format: https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated
* if (ath != null)
* {
* logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {ath.AthleteId,9} {ath.AthleteName,-40} {ath.AthleteLocation}");
* }
* else
* {
* logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {aId,9}");
* }
* }
*/
/*
* // output for first query
* foreach(ActivityRangeQuery arq in db.ActivityQueriesDB)
* {
* logger.LogInformation($"{arq}");
* break;
* }
*/
if (doListAthletes || doAll)
{
foreach (AthleteShort athlete in db.AthleteShortDB)
{
logger.LogInformation($"Athletes {athlete.AthleteName} {athlete.AthleteId}");
}
}
if (doAthleteCounts || doAll)
{
// output status count for queries
var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct();
logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}");
}
if (doQueryCounts || doAll)
{
// output status count for queries
var status = db.ActivityQueriesDB.Select(a => a.Status).Distinct();
logger.LogInformation($"Queries:");
foreach (var st in status)
{
logger.LogInformation($" {st} {db.ActivityQueriesDB.Count(a => a.Status==st)}");
}
}
if (doGpxCounts || doAll)
{
// Count GPX Files and sort by activity-Types.
string rootPath = "gpx";
foreach (string athleteEntry in Directory.GetDirectories(rootPath))
{
var match = Regex.Match(athleteEntry, ".*[/]([0-9]+)");
string AthleteIdStr = match.Groups[1].Value;
// logger.LogInformation($" directory for Athlete {AthleteIdStr} found {athleteEntry}");
Hashtable errTable = new Hashtable();
Hashtable trackTable = new Hashtable();
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.err"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.err");
string ActivityIdStr = match.Groups[1].Value;
errTable.Add(ActivityIdStr, gpxActivity);
}
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx.gz"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.gz");
string ActivityIdStr = match.Groups[1].Value;
if (errTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous");
}
if (trackTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}");
}
else
{
trackTable.Add(ActivityIdStr, gpxActivity);
}
}
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx");
string ActivityIdStr = match.Groups[1].Value;
if (errTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous");
}
if (trackTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}");
}
else
{
logger.LogWarning($"Found GPX without compression: track for {ActivityIdStr} already enterred {gpxActivity}");
trackTable.Add(ActivityIdStr, gpxActivity);
}
}
Hashtable typeCount = new Hashtable();
foreach (string activity_id in trackTable.Keys)
{
ActivityShort activity = db.ActivityShortDB.Find(activity_id);
if (activity == null)
{
// logger.LogWarning($"can't find activity for {activity_id}");
continue;
}
int tCount = 0;
if (typeCount.ContainsKey(activity.ActivityType))
{
tCount = (int)typeCount[activity.ActivityType];
typeCount.Remove(activity.ActivityType);
}
tCount++;
typeCount.Add(activity.ActivityType, tCount);
}
// var al = db.ActivityShortDB.Where(a=>a.AthleteId==AthleteIdStr);
// int actCount = al.Count();
string keys = string.Join(",", typeCount.Keys.Cast <ActivityType>().Select(x => $"{x.ToString()} {typeCount[x]}").ToArray());
int actCount = 0;
logger.LogInformation($"Athlete {AthleteIdStr}, activities found in DB: {actCount} GPX: {trackTable.Keys.Count} ERR: {errTable.Keys.Count} {keys}");
}
}
if (doRenewMonth != null)
{
// set all DONE queries for the given month to Created
var match = Regex.Match(doRenewMonth, "([0-9]{4})/([0-9]{2})");
int year = Int32.Parse(match.Groups[1].Value);
int month = Int32.Parse(match.Groups[2].Value);
var statusDone = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.DateFrom == new DateTime(year, month, 01));
foreach (ActivityRangeQuery arq in statusDone.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"reset for {year}/{month} from {QueryStatus.Run} to {QueryStatus.Created}";
logger.LogInformation($"query for {year}/{month} with {QueryStatus.Done} {arq.AthleteId} {arq.Message}");
}
logger.LogInformation($"begin: save changes");
db.SaveChanges();
logger.LogInformation($"done: save changes");
}
if (doGarbage)
{
{
var statusError = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Error);
foreach (ActivityRangeQuery arq in statusError.ToList())
{
logger.LogInformation($"query with {QueryStatus.Error} {arq.AthleteId} {arq.Message}");
}
}
{
var statusRun = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Run);
foreach (ActivityRangeQuery arq in statusRun.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"garbage-reset from {QueryStatus.Run} to {QueryStatus.Created}";
logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}");
}
db.SaveChanges();
}
{
var statusReserved = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Reserved);
foreach (ActivityRangeQuery arq in statusReserved.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"garbage-reset from {QueryStatus.Reserved} to {QueryStatus.Created}";
logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}");
db.SaveChanges();
}
}
}
if (doActivityCounts || doAll)
{
// Find out athlete with open queries:
var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
logger.LogInformation($"{qAthleteCreated.Count()} with {QueryStatus.Created} queries");
var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
logger.LogInformation($"{qAthleteCreated.Count()} without {QueryStatus.Reserved} queries");
logger.LogInformation($"{qAthleteCreated.Intersect(qAthleteReserved).Count()} with {QueryStatus.Created} and without {QueryStatus.Reserved} queries");
List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(10).ToList();
logger.LogInformation($"{AthleteIdList.Count()} athletes from this list:");
foreach (string aId in AthleteIdList)
{
logger.LogInformation($" AthleteId={aId}");
}
if (AthleteIdList.Count() > 0)
{
// retrieve one random athlete
string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count));
logger.LogInformation($"retrieve activity for athlete {aid}");
// 5 first queries for this athlete
IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(10).ToList();
foreach (ActivityRangeQuery arq in q0)
{
logger.LogInformation($" query={arq}");
var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo)));
logger.LogInformation($" find {ActivitiesInRange.Count()} activities in it.");
}
}
}
if (doAll)
{
// find activities in a QueryRange
foreach (ActivityRangeQuery arq in db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done).OrderByDescending(a => a.DateFrom).Take(10))
{
var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo)));
logger.LogInformation($" find {ActivitiesInRange.Count()} in range {arq}");
}
}
if (doAll)
{
// retrieve queries type for athlete
string aId = "2754335";
var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList();
logger.LogInformation($" athl {aId}");
foreach (QueryStatus qs in qsList)
{
logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}");
}
// IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(6).ToList();
}
if (doAll)
{
string aId = "26319532";
var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList();
logger.LogInformation($" athl {aId}");
foreach (QueryStatus qs in qsList)
{
logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}");
}
}
if (doAll || doListimages)
{
List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList();
logger.LogInformation($" activity count:{activitiesWithImages.Count()}");
int imageCount = 0;
foreach (ActivityShort activity in activitiesWithImages)
{
List <string> images = activity.ActivityImagesList;
imageCount += images.Count();
logger.LogInformation($"activity {activity.ActivityTitle}");
foreach (string imageUrl in images)
{
logger.LogInformation($" url:{imageUrl}");
}
}
logger.LogInformation($" images {imageCount}");
}
if (doAll || doListMaps)
{
List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList();
logger.LogInformation($" activity count:{activitiesWithImages.Count()}");
int imageCount = 0;
foreach (ActivityShort activity in activitiesWithImages)
{
string ImageMapUrl = activity.ActivityImageMapUrl;
if (ImageMapUrl == null)
{
continue;
}
imageCount++;
logger.LogInformation($"activity {activity.ActivityTitle} {ImageMapUrl}");
WebClient webClient = new WebClient();
string outputDir = $"maps/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}.png";
logger.LogWarning($"NOT IMPLEMENTED");
}
logger.LogInformation($" images {imageCount}/{activitiesWithImages}");
}
if (doAll || doAthleteStats)
{
IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
logger.LogInformation($" athletes {AllAthletes.Count()}");
logger.LogInformation($" first: [{AllAthletes.First()}] last: [{AllAthletes.Last()}]");
int AthleteCount = db.ActivityQueriesDB.Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($"athletes in queries {AthleteCount}");
int PrivateAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.Message.Contains("private")).Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($" private {PrivateAthleteCount}");
int CreatedAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($" {QueryStatus.Created} {CreatedAthleteCount}");
}
if (doAll || doActivityList)
{
List <ActivityShort> activities;
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
activities = dbs.ToList();
logger.LogInformation($"List activities for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
/*
* if (activities.Count>0 && doActivityList)
* {
* foreach(ActivityShort activity in activities)
* {
* logger.LogInformation($" {activity}");
* }
* }
*/
}
/*
* {
* // opened queries with activities
* int i=0;
* int nbAthletes=db.AthleteShortDB.Count();
* foreach(AthleteShort athlete in db.AthleteShortDB)
* {
* i++;
* // logger.LogInformation($"retrieve activity for athlete {athlete.AthleteId}");
* // 5 first queries for this athlete
* IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==athlete.AthleteId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(5).ToList();
* int activitiesWithCreatedQueriesCount=0;
* foreach(ActivityRangeQuery arq in q0)
* {
* var ActivitiesInRange = db.ActivityShortDB.Where(a=>a.AthleteId==arq.AthleteId && ((a.ActivityDate>=arq.DateFrom)&&(a.ActivityDate<=arq.DateTo)));
* activitiesWithCreatedQueriesCount+=ActivitiesInRange.Count();
* }
* if (activitiesWithCreatedQueriesCount>0)
* {
* logger.LogInformation($" athlete {athlete.AthleteId} as {activitiesWithCreatedQueriesCount} activities with created queries queries:{q0.Count()} activites{db.ActivityShortDB.Where(a=>a.AthleteId==athlete.AthleteId).Count()}.");
* }
* if (i%10==0)
* {
* logger.LogInformation($"{i}/{nbAthletes}");
* }
* }
* }
*/
/*
* // output activity count for activity type
* logger.LogInformation($"Activity types Σ :{db.ActivityShortDB.Count()}");
* var ActivityTypes = db.ActivityShortDB.Select(a => a.ActivityType).Distinct();
* foreach(var aType in ActivityTypes)
* {
* logger.LogInformation($" {aType,18} {db.ActivityShortDB.Count(a => a.ActivityType==aType),6}");
* }
*/
/*
* // output athletes with ski activities and their ski activity count
* logger.LogInformation($"All athletes with {ActivityType.BackcountrySki}");
* var Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Select(a => a.AthleteId).Distinct();
* foreach(var A4Type in Activity4Type)
* {
* var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Where(a => a.AthleteId==A4Type).Count();
* var athlete = db.AthleteShortDB.Find(A4Type);
* logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})");
* }
*/
/*
* // output athletes with run activities and their run activity count
* logger.LogInformation($"All athletes with {ActivityType.Run}");
* Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Select(a => a.AthleteId).Distinct();
* foreach(var A4Type in Activity4Type)
* {
* var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Where(a => a.AthleteId==A4Type).Count();
* var athlete = db.AthleteShortDB.Find(A4Type);
* logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})");
* }
*/
}
ret = 0;
}
catch (Exception e)
{
logger.LogInformation($"ERROR:{e.ToString()}");
ret = 1;
}
return(ret);
}
