static public int EnsureCreated(string[] args)
{
using (StravaXApiContext db = new StravaXApiContext())
{
// https://stackoverflow.com/questions/38238043/how-and-where-to-call-database-ensurecreated-and-database-migrate
db.Database.EnsureCreated();
}
return(0);
}
static int queryAthlete(string aid, string clientId, StravaXApi stravaXApi, StravaXApiContext db, bool doVerbose)
{
int ret = 0;
IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();
while (q0.Count > 0 && ret == 0)
{
// Mark all queries with "RESERVED"
IList <ActivityRangeQuery> queries = new List <ActivityRangeQuery>();
foreach (ActivityRangeQuery arq in q0)
{
try
{
arq.Status = QueryStatus.Reserved;
arq.StatusChanged = DateTime.Now;
arq.Message = $"Reserved by {clientId}";
queries.Add(arq);
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"skip: Can't reserve query {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2} for {clientId}");
continue;
}
}
db.SaveChanges();
// Run for all reserved queries
try
{
ret = queryRange(stravaXApi, db, queries, doVerbose);
}
catch (CancelExecution e)
{
// just to clarify that it may happend.
// finally will also call in that case.
throw e;
}
finally
{
// integrity check, status should not be Reserved anymore
foreach (ActivityRangeQuery arq in q0)
{
if (arq.Status == QueryStatus.Reserved)
{
logger.LogInformation($"WARN: remove reservation on {arq}");
}
}
db.SaveChanges();
}
// retrieve next queries for max one year.
q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();
}
return(ret);
}
static private void AddQuery(StravaXApiContext db, String AthleteId, DateTime DateFrom, DateTime DateTo, List <ActivityRangeQuery> queriesForPatient)
{
if (!isDateInList(DateFrom, queriesForPatient))
{
if (db.ActivityQueriesDB.Find(AthleteId, DateFrom, DateTo) == null)
{
ActivityRangeQuery query = new ActivityRangeQuery();
query.AthleteId = AthleteId;
query.DateFrom = DateFrom;
query.DateTo = DateTo;
db.ActivityQueriesDB.Add(query);
Console.WriteLine($"add query for {query}");
}
}
}
static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete connections with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
AthleteShort AthleteMasterShort;
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "following";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
using (StravaXApiContext db = new StravaXApiContext())
{
AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
if (AthleteMasterShort == null)
{
AthleteMasterShort = new AthleteShort();
// create a dummy master
AthleteMasterShort.AthleteId = AthleteId;
// [TODO] other parameters should be retrieved with selenium
AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
db.SaveChanges();
}
else
{
// Eagerly Loading prevent the list to be loaded at creation
// https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();
Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
}
string FollowType = "followers";
var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");
foreach (AthleteShort _AthleteShort in AthleteShortList)
{
AthleteShort AthleteShortfromDb;
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
if (AthleteShortfromDb == null)
{
// add athlete to the db if need.
AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
}
else
{
Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
}
Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
// such the connected athlete with they id.
AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
if (_ConnectedAthleteShort == null)
{
// add connection if needed.
AthleteConnection ac = new AthleteConnection();
ac.FromId = AthleteMasterShort.AthleteId;
ac.ToId = AthleteShortfromDb.AthleteId;
ac.Type = FollowType;
AthleteMasterShort.Connections.Add(ac);
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
}
else
{
Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
}
}
db.SaveChanges();
Console.WriteLine($"total read = {AthleteShortList.Count}");
Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
AthleteShortList.Clear();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
/// <summary>
/// Convert GPX files to KML.
/// GPX File should have bee retrieved with get-gpx.
/// All GPX are merged in one KML file with subfolder for athlete and activity types.
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public static int Convert(string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log GpxToKml");
string AthleteId = null;
string ActivityTypeStr = null;
string filenameExtention = ".kml";
string exportKmlPath = "StravaXApi.kml";
string splitBlockSizeStr = null;
string exportTypeStr = ExportType.HeatMap.ToString();
string beginDateStr = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "e|export_type=", v => { exportTypeStr = v; } },
{ "bd|begin_date=", v => { beginDateStr = v; } },
{ "d|destination=", v => { exportKmlPath = v; } },
{ "s|split=", v => { splitBlockSizeStr = v; } },
};
p.Parse(args);
int ret = -1;
if (!exportKmlPath.EndsWith(filenameExtention))
{
exportKmlPath = $"{exportKmlPath}{filenameExtention}";
}
ExportType exportType = (ExportType)Enum.Parse(typeof(ExportType), exportTypeStr);
try
{
List <ActivityShort> activities;
using (StravaXApiContext db = new StravaXApiContext())
{
// Just do one query for all queries, order will be user to separate athletes and activity type.
// [note] code is not really readable, afterthere it would have be a better idea to use several queries : athletes/activity types/activities in date range.
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderBy(a => a.AthleteId).ThenBy(a => a.ActivityType).ThenByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
if (beginDateStr != null)
{
DateTime dateTime = DateTime.Parse(beginDateStr);
dbs = dbs.Where(a => a.ActivityDate >= dateTime);
}
// Ignore Activities without image map. They have probably been enterred without gps-track.
dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
// query all activities at once.
activities = dbs.ToList();
logger.LogInformation($"BEGIN GPX➡️KML {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)}/{(beginDateStr==null?"all dates":DateTime.Parse(beginDateStr).ToString())} :{activities.Count()}");
string lastAthleteId = null;
ActivityType lastActivityType = ActivityType.Other;
XElement StravaXApiFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", "Strava-X-Api")
, new XElement(kml + "open", "1")
);
XElement currentAthleteFolder = null;
XElement currentActivityTypeFolder = null;
int Count = 0;
int AthleteCount = 0;
int splitBlockSize = int.MaxValue;
if (splitBlockSizeStr != null)
{
splitBlockSize = int.Parse(splitBlockSizeStr);
}
// go throw all activities, sorted by athlete / activity type / activity date
foreach (ActivityShort activity in activities)
{
// different locations for gpx files.
string outputDir = $"gpx/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
string outputFilenameGz = $"{activity.ActivityId}_{activity.AthleteId}.gpx.gz";
string outputFilenameErr = $"{activity.ActivityId}_{activity.AthleteId}.err";
// skip activities without gpx file at the beginning to avoid node creation in kml file.
if (File.Exists($"{outputDir}/{outputFilenameErr}") ||
(!File.Exists($"{outputDir}/{outputFilenameGz}") &&
!File.Exists($"{outputDir}/{outputFilename}")
))
{
// Console.WriteLine($"Skip activity {activity}");
continue;
}
if (lastAthleteId == null)
{
// first round init athlete
AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId);
string athleteName = athlete.AthleteName;
currentAthleteFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", athleteName)
, new XElement(kml + "visibility", "0")
, new XElement(kml + "open", "0"));
StravaXApiFolder.Add(currentAthleteFolder);
// first round init activity
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastAthleteId = activity.AthleteId;
lastActivityType = activity.ActivityType;
AthleteCount = 1;
}
else
{
if (lastAthleteId == activity.AthleteId)
{
// same athlete, check activity type
if (lastActivityType != activity.ActivityType)
{
// start a new activity folder
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastActivityType = activity.ActivityType;
}
}
else
{
AthleteCount++;
// if splitBlockSize has been set, we may need to write the kml file.
if (splitBlockSize != int.MaxValue && AthleteCount % splitBlockSize == 0)
{
// end of athlete block size reached, save a kml file.
saveKmlFile(StravaXApiFolder, exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize):D2}{filenameExtention}"), logger);
// begin a new tree.
StravaXApiFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", "Strava-X-Api")
, new XElement(kml + "open", "1")
);
}
// start a new athlete folder
AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId);
string athleteName = athlete.AthleteName;
currentAthleteFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", athleteName)
, new XElement(kml + "visibility", "0")
, new XElement(kml + "open", "0"));
StravaXApiFolder.Add(currentAthleteFolder);
// start a new activity folder
currentActivityTypeFolder = new XElement(kml + "Folder"
, new XElement(kml + "name", activity.ActivityType.ToString())
, new XElement(kml + "open", "0"));
currentAthleteFolder.Add(currentActivityTypeFolder);
lastAthleteId = activity.AthleteId;
lastActivityType = activity.ActivityType;
}
}
try
{
// read the gpx file and add it to the kml node.
if (File.Exists($"{outputDir}/{outputFilenameErr}"))
{
// Skip error file.
logger.LogWarning($"SKIP: GPX not availlable for {activity.ActivityId} {lastAthleteId} {lastActivityType}");
}
else if (File.Exists($"{outputDir}/{outputFilenameGz}"))
{
using (FileStream compressedFileStream = File.OpenRead($"{outputDir}/{outputFilenameGz}"))
{
using (GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Decompress))
{
currentActivityTypeFolder.Add(handleGpxStream(compressionStream, exportType, activity));
}
}
}
else if (File.Exists($"{outputDir}/{outputFilename}"))
{
using (FileStream fileStream = File.OpenRead($"{outputDir}/{outputFilename}"))
{
currentActivityTypeFolder.Add(handleGpxStream(fileStream, exportType, activity));
}
}
// else skip, because GPX is not availlable.
}
catch (Exception e)
{
logger.LogWarning($"SKIP: Can't parse GPX for {activity.ActivityId} {lastAthleteId} {lastActivityType} err:{e.Message}");
}
if (Count++ % 100 == 0)
{
Console.WriteLine($"{Count}/{activities.Count()}");
}
}
string exportKmlPathForSplit = (splitBlockSize == int.MaxValue)?exportKmlPath:exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize)+1:D2}{filenameExtention}");
saveKmlFile(StravaXApiFolder, exportKmlPathForSplit, logger);
}
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
ret = 1;
}
return(ret);
}
static void WriteQueriesForAthlete(StravaXApi stravaXApi, StravaXApiContext db, string AthleteId)
{
DateTime Now = DateTime.Now;
lastStravaHttpRequest = DateTime.Now;
int ToYear = Now.Year;
int ToMonth = Now.Month;
//
// Verify if queries have to be generated
//
// Retrieve all entered queries or the wanted athlete.
List <ActivityRangeQuery> queriesForAthlete = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).OrderBy(a => a.DateFrom).ToList();
int FromYear;
int FromMonth;
// If we already have an entry, we assume that the entry contains the first activity date, as it is expensive to retrieve its value with Selenium.
if (queriesForAthlete.Count == 0)
{
// Retrieve the first activity date with selenium.
// may throw PrivateAthleteException.
// be sure of a one seconde intervall between to requests
int dt = DateTime.Now.Millisecond - lastStravaHttpRequest.Millisecond;
if (dt < 2000)
{
System.Threading.Tasks.Task.Delay(dt).Wait();
}
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
lastStravaHttpRequest = DateTime.Now;
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
FromYear = FirstActivityDate.Year;
FromMonth = FirstActivityDate.Month;
}
else
{
// retrieve first and last date
DateTime minDT = queriesForAthlete.First().DateFrom;
// DateTime maxDT = queriesForPatient.Last().DateFrom;
FromYear = minDT.Year;
FromMonth = minDT.Month;
}
Console.WriteLine($"queries enterred:{queriesForAthlete.Count}/total:{db.ActivityQueriesDB.Count()}");
if (FromYear == ToYear)
{
for (int month = FromMonth; month <= ToMonth; month++)
{
AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
else
{
// first year
for (int month = FromMonth; month <= 12; month++)
{
AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
// all years after
for (int year = FromYear + 1; year <= ToYear - 1; year++)
{
for (int month = 01; month <= 12; month++)
{
AddQuery(db, AthleteId, new DateTime(year, month, 1), new DateTime(year, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
// last year
for (int month = 01; month <= ToMonth; month++)
{
// from first day of month to last day of the month
AddQuery(db, AthleteId, new DateTime(ToYear, month, 1), new DateTime(ToYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
}
}
int qCount = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).Count();
Console.WriteLine($"✅ enterred:{qCount}/total:{db.ActivityQueriesDB.Count()}");
}
static internal int WriteQueriesForAthletes(StravaXApi stravaXApi)
{
int ret = -1;
Console.WriteLine("Create range queries.");
using (StravaXApiContext db = new StravaXApiContext())
{
stravaXApi.signIn();
try
{
// https://docs.microsoft.com/en-us/ef/ef6/querying/
// First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
// Not best praxis but enought for Strava.XApi.
IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
foreach (AthleteShort Athlete in AllAthletes)
{
TimeSpan ts = stopWatch.Elapsed;
try
{
WriteQueriesForAthlete(stravaXApi, db, Athlete.AthleteId);
Console.WriteLine($"Athlete:{Athlete} run since:{ts.TotalSeconds}s");
}
catch (PrivateAthleteException e)
{
// TODO AB#27 athlete should be marked as private to avoid second visit.
// create a dummy Query to prevent next search
ActivityRangeQuery arq = new ActivityRangeQuery();
arq.AthleteId = Athlete.AthleteId;
arq.DateFrom = new DateTime(2019, 12, 01);
arq.DateTo = new DateTime(2019, 12, 31);
arq.Status = QueryStatus.Done;
arq.Message = $"private athlete {Athlete.AthleteId}";
arq.StatusChanged = DateTime.Now;
db.ActivityQueriesDB.Add(arq);
Console.WriteLine($"SKIP: private athlete {Athlete.AthleteId} {e.Message}");
}
catch (TooManyStravaRequestException e)
{
Console.WriteLine($"Too Many Queries detected {e.Message} need to wait some hours");
}
catch (Exception e)
{
Console.WriteLine($"SKIP:{Athlete.AthleteId} {e.ToString()}");
}
db.SaveChanges();
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = -1;
}
finally
{
stravaXApi.Dispose();
}
}
return(ret);
}
static public int Downloader(string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.ImageDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log ImageDownloader");
bool doMaps = false;
bool doActivityImages = false;
string maxCountStr = null;
string AthleteId = null;
string ActivityTypeStr = null;
var p = new OptionSet()
{
{ "s|maps", v => { doMaps = true; } },
{ "ai|activity_images", v => { doActivityImages = true; } },
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "m|max_count=", v => { maxCountStr = v; } },
};
p.Parse(args);
int ret = -1;
int maxCount = int.MaxValue;
if (maxCountStr != null)
{
maxCount = int.Parse(maxCountStr);
logger.LogInformation($" limit download to {maxCount}");
}
try
{
using (StravaXApiContext db = new StravaXApiContext())
{
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (doMaps)
{
if (AthleteId != null)
{
ret = RetrieveMaps(dbs, logger, AthleteId, ActivityTypeStr, maxCount);
}
else
{
List <AthleteShort> athletes = db.AthleteShortDB.OrderBy(a => a.AthleteId).ToList();
foreach (AthleteShort athlete in athletes)
{
ret = RetrieveMaps(dbs, logger, athlete.AthleteId, ActivityTypeStr, maxCount);
}
}
}
if (doActivityImages)
{
if (AthleteId != null)
{
ret = RetrieveActivityImages(dbs, logger, AthleteId, ActivityTypeStr, maxCount);
}
else
{
List <AthleteShort> athletes = db.AthleteShortDB.OrderBy(a => a.AthleteId).ToList();
foreach (AthleteShort athlete in athletes)
{
ret = RetrieveActivityImages(dbs, logger, athlete.AthleteId, ActivityTypeStr, maxCount);
}
}
}
}
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
ret = 1;
}
return(ret);
}
static internal int Downloader(StravaXApi stravaXApi, string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log ImageDownloader");
bool doGpx = false;
string maxCountStr = null;
string AthleteId = null;
string ActivityTypeStr = null;
string gpxDir = "gpx";
var p = new OptionSet()
{
{ "g|gpx", v => { doGpx = true; } },
{ "a|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "m|max_count=", v => { maxCountStr = v; } },
{ "d|dirname=", v => { gpxDir = v; } },
};
p.Parse(args);
int ret = -1;
try
{
List <ActivityShort> activities;
using (StravaXApiContext db = new StravaXApiContext())
{
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
// Ignore Activities without image map. They have probably been enterred without gps-track.
dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
// retrieve the activities from the database.
activities = dbs.ToList();
}
logger.LogInformation($"BEGIN GPX Download for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
int countSkipped = 0;
int countDownload = 0;
if (doGpx && activities.Count > 0)
{
bool needTracksDownload = false;
// detect if any activity needs a gpx file
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
needTracksDownload = true;
break;
}
}
// if no track is needed, we could exit without signin in Strava.
if (needTracksDownload)
{
// signin
stravaXApi.signIn();
// go throw all activities.
foreach (ActivityShort activity in activities)
{
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
string outputFilenameGZip = $"{outputFilename}.gz";
string outputFilenameErr = $"{outputFilename}.err";
// download only if the gpx, gz or the err file do not exists.
if (!File.Exists($"{outputDir}/{outputFilename}") && !File.Exists($"{outputDir}/{outputFilenameGZip}") && !File.Exists($"{outputDir}/{outputFilenameErr}"))
{
// create directory if needed
if (!Directory.Exists(outputDir))
{
DirectoryInfo DirInfo = Directory.CreateDirectory(outputDir);
logger.LogDebug($"directory for GPX created at {DirInfo.FullName}");
}
try
{
stravaXApi.getActivityGpxSelenium(activity.ActivityId, $"{outputDir}/{outputFilenameGZip}");
countDownload++;
}
catch (PrivateGpxException e)
{
logger.LogDebug($"GPX Track private for {activity.AthleteId}: {e.ActivityId} {e.Message}");
// write error file, do prevent a second try.
using (StreamWriter outputFile = new StreamWriter(Path.Combine(outputDir, outputFilenameErr)))
{
outputFile.WriteLine($"Error while downloading GPX for athlete:{activity.AthleteId} activity{activity.ActivityId}");
outputFile.WriteLine($"{e.Message}");
outputFile.WriteLine($"{e.StackTrace}");
}
countSkipped++;
}
}
else
{
logger.LogDebug($"GPX Track already downloaded for {activity.AthleteId}");
countSkipped++;
}
}
}
}
else if (!doGpx)
{
// just retrieve without retrieving the gpx files.
int countToDownload = 0;
int countDownloaded = 0;
foreach (ActivityShort activity in activities)
{
// logger.LogInformation($"activity {activity.StatShortString} -> {Utils.extractActivityTime(activity)}.");
string outputDir = $"{gpxDir}/{activity.AthleteId}";
// fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
if (!File.Exists($"{outputDir}/{outputFilename}"))
{
countToDownload++;
}
else
{
countDownloaded++;
}
}
logger.LogInformation($"GPX Track to download:{countToDownload} already downloaded:{countDownloaded}");
}
logger.LogInformation($"DONE GPX Download {countDownload} (skipped: {countSkipped})for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
// everything done, with return 0 we are telling the container, that there's no need to restart.
ret = 0;
}
catch (Exception e)
{
logger.LogError($"ERROR:{e.ToString()}");
// return error code, container should restart.
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static internal int SendQueriesForActivities(StravaXApi stravaXApi, string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log Query Activities");
string clientId = Guid.NewGuid().ToString();
int TimerSeconds = -1;
int TimerExitCode = 2;
bool doVerbose = false;
var p = new OptionSet()
{
{ "t|timer_sec=", v => { TimerSeconds = int.Parse(v); } },
{ "e|timer_exit_code=", v => { TimerExitCode = int.Parse(v); } },
{ "v|verbose", v => { doVerbose = true; } },
};
p.Parse(args);
int ret = -1;
logger.LogInformation($"Query activities. Client:{clientId}");
using (StravaXApiContext db = new StravaXApiContext())
{
stravaXApi.signIn();
// 1) find an athlete with opened queries and without reserved queries.
// string aid =db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).First().AthleteId;
// select distinct AthleteId from [dbo].[ActivityQueriesDB] where Status=2
// intersect
// select distinct AthleteId from [dbo].[ActivityQueriesDB] where (Status<>0 AND Status<>3)
var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();
if (AthleteIdList.Count == 0)
{
logger.LogInformation($"no more athlete to search for. created:{qAthleteCreated.Count()} reserved:{qAthleteReserved.Count()}");
// TODO some queries may have been reserved, but no Thread is working on it. It should be usefull to detect and reset.
}
else
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
ret = 0;
try {
while (AthleteIdList.Count > 0 && ret == 0)
{
string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count));
logger.LogInformation($"retrieve activity for athlete {aid}");
// https://docs.microsoft.com/en-us/ef/ef6/querying/
// First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
// Not best praxis but enought for Prototype.
// https://stackoverflow.com/a/2656612/281188
// IList<ActivityRangeQuery> queries = db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(50).ToList();
ret = queryAthlete(aid, clientId, stravaXApi, db, doVerbose);
TimeSpan ts = stopWatch.Elapsed;
if (TimerSeconds > 0 && ts.TotalSeconds > TimerSeconds)
{
ret = TimerExitCode;
logger.LogInformation($"Timer reached after {ts.ToString()} now exit with {ret}.");
// exit with error code, container should restart
break;
}
Boolean KeepRunning = true;
if (!KeepRunning || File.Exists("QueryActivities.quit"))
{
logger.LogInformation($"break {KeepRunning} {Count}");
// regular exit, container should ended.
ret = 0;
break;
}
// search for a new athlete.
qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();
}
}
catch (CancelExecution e)
{
logger.LogInformation($"Execution has been canceled: {e.Message}");
}
}
}
return(ret);
}
static int queryRange(StravaXApi stravaXApi, StravaXApiContext db, IList <ActivityRangeQuery> queries, bool doVerbose)
{
int ret = 0;
foreach (ActivityRangeQuery arq in queries)
{
try
{
// https://docs.microsoft.com/en-us/ef/core/saving/concurrency
try
{
// reserve query, mark it as run.
arq.Status = QueryStatus.Run;
arq.StatusChanged = DateTime.Now;
db.SaveChanges();
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"skip conflicted entry for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
continue;
}
var ActivitiesList = stravaXApi.getActivities(arq.AthleteId, $"{arq.DateFrom.Year:D4}", $"{arq.DateFrom.Month:D2}");
int EnterredActivityCount = 0;
foreach (ActivityShort ActivityShort in ActivitiesList)
{
// Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
try{
// do it better after foreach, but first check how it works with the concurrentcy exception
db.SaveChanges();
EnterredActivityCount++;
}
catch (DbUpdateConcurrencyException)
{
// Just skip this entry if some conflict exists.
logger.LogInformation($"❌ skip conflicted activity {ActivityShort}");
continue;
}
}
else
{
logger.LogInformation($"❌ {ActivityShort.ActivityId} allready in database");
}
}
arq.Status = QueryStatus.Done;
arq.StatusChanged = DateTime.Now;
// should not have to save anything.
db.SaveChanges();
if (doVerbose)
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount}/{db.ActivityShortDB.Count()} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
else // '.Count()' is pretty expensive, skip it if verbose should be minimal.
{
logger.LogInformation($"enterred activity count: {EnterredActivityCount} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
}
ErrorCountConsecutive = 0;
}
catch (Exception e)
{
ErrorCountConsecutive++;
ErrorCount++;
logger.LogInformation($"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}");
arq.Status = QueryStatus.Error;
arq.StatusChanged = DateTime.Now;
arq.Message = $"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}";
if (ErrorCountConsecutive > 2)
{
// After 3 consecutive errors, I assume the selenium driver is down. Stop it all.
throw e;
}
}
if (doVerbose)
{
logger.LogInformation($"activities stored:{db.ActivityShortDB.Count()}/{QueryStatus.Created}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Created)}/{QueryStatus.Reserved}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Reserved)}");
}
Count++;
// Exist when KeepRunning is false (from the debugger),
// or the file 'QueryActivities.quit' exists.
// *Program will exit with "touch QueryActivities.quit" in /app directory in container.
Boolean KeepRunning = true;
if (!KeepRunning || File.Exists("QueryActivities.quit.immediatly"))
{
logger.LogInformation($"break {KeepRunning} {Count}");
// regular exit, container should ended.
throw new CancelExecution("break {KeepRunning} {Count}");
}
}
return(ret);
}
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
{
int ret = -1;
Console.WriteLine("Read athlete activities with Strava-X-API.");
String AthleteId = null;
var p = new OptionSet()
{
{ "a|athleteid=", v => { AthleteId = v; } },
};
p.Parse(args);
if (AthleteId == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing athlete id");
}
try
{
stravaXApi.signIn();
List <ActivityShort> ActivitiesList = new List <ActivityShort>();
DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
DateTime now = DateTime.Now;
for (int year = FromYear; year <= ToYear; year++)
{
for (int month = 01; month <= 12; month++)
{
if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth))
{
continue;
}
List <ActivityShort> ActivitiesMonthList;
try
{
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
catch (StaleElementReferenceException)
{
// Wait and try again.
Thread.Sleep(2000);
ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
}
ActivitiesList.AddRange(ActivitiesMonthList);
using (StravaXApiContext db = new StravaXApiContext())
{
foreach (ActivityShort ActivityShort in ActivitiesList)
{
Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
{
db.ActivityShortDB.Add(ActivityShort);
db.SaveChanges();
Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
}
else
{
Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
}
}
Console.WriteLine($"total read = {ActivitiesList.Count}");
Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
ActivitiesList.Clear();
}
}
}
ret = 0;
}
catch (Exception e)
{
Console.WriteLine($"ERROR:{e.ToString()}");
ret = 1;
}
finally
{
stravaXApi.Dispose();
}
return(ret);
}
static public int DoClone(string[] args)
{
String SrcConnectionString = null;
String DestConnectionString = null;
var p = new OptionSet()
{
{ "s|source=", s => { SrcConnectionString = s; } },
{ "d|destination=", d => { DestConnectionString = d; } },
};
p.Parse(args);
if (SrcConnectionString == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing source");
}
if (DestConnectionString == null)
{
p.WriteOptionDescriptions(Console.Out);
throw new ArgumentException("missing destination");
}
Console.WriteLine($"SRC: {SrcConnectionString}");
Console.WriteLine($"DST: {DestConnectionString}");
DbContextOptions optionsSrc;
if (SrcConnectionString.StartsWith("Data Source"))
{
optionsSrc = new DbContextOptionsBuilder().UseSqlite(SrcConnectionString).Options;
}
else
{
optionsSrc = new DbContextOptionsBuilder().UseSqlServer(SrcConnectionString).Options;
}
DbContextOptions optionsDst;
if (DestConnectionString.StartsWith("Data Source"))
{
optionsDst = new DbContextOptionsBuilder().UseSqlite(DestConnectionString).Options;
}
else
{
optionsDst = new DbContextOptionsBuilder().UseSqlServer(DestConnectionString).Options;
}
using (StravaXApiContext DbSrc = new StravaXApiContext(optionsSrc))
{
Console.WriteLine($"SRC: Queries stored {DbSrc.ActivityQueriesDB.Count()}");
Console.WriteLine($"SRC: Activities stored {DbSrc.ActivityShortDB.Count()}");
var al = DbSrc.ActivityShortDB.Select(a => a.AthleteId).Distinct();
Console.WriteLine($"SRC: Athletes {al.Count()} from {DbSrc.AthleteShortDB.Count()}");
using (StravaXApiContext DbDst = new StravaXApiContext(optionsDst))
{
// https://stackoverflow.com/questions/38238043/how-and-where-to-call-database-ensurecreated-and-database-migrate
DbDst.Database.EnsureCreated();
Console.WriteLine($"DST: Queries stored {DbDst.ActivityQueriesDB.Count()}");
Console.WriteLine($"DST: Activities stored {DbDst.ActivityShortDB.Count()}");
var AlDst = DbDst.ActivityShortDB.Select(a => a.AthleteId).Distinct();
Console.WriteLine($"DST: Athletes {AlDst.Count()} from {DbDst.AthleteShortDB.Count()}");
Console.WriteLine("read athletes");
foreach (AthleteShort ath in DbSrc.AthleteShortDB)
{
DbDst.AthleteShortDB.Add(ath);
}
Console.WriteLine("save athletes");
DbDst.SaveChanges();
Console.WriteLine("read activities");
int i = 0;
int totalCount = DbSrc.ActivityShortDB.Count();
int destLen = DbDst.ActivityShortDB.Count();
foreach (ActivityShort act in DbSrc.ActivityShortDB)
{
// skip as much activities as already present.
// Only done because the first imports try has broked, but
// a better system should be developped in the future.
// if (i++<destLen)
// {
// continue;
// }
if (++i % 100 == 0)
{
Console.WriteLine($"activities {i}/{totalCount}");
}
DbDst.ActivityShortDB.Add(act);
}
Console.WriteLine("save activities");
DbDst.SaveChanges();
Console.WriteLine("read queries");
i = 0;
totalCount = DbSrc.ActivityQueriesDB.Count();
foreach (ActivityRangeQuery arq in DbSrc.ActivityQueriesDB)
{
DbDst.ActivityQueriesDB.Add(arq);
i++;
if (i % 100 == 0)
{
Console.WriteLine($"queries {i}/{totalCount}");
}
}
Console.WriteLine("save queries");
DbDst.SaveChanges();
}
}
return(0);
}
static public int WriteState(string[] args)
{
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
.AddFilter("Strava.XApi.Tools.DbStats", Microsoft.Extensions.Logging.LogLevel.Debug)
.AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
.AddProvider(new CustomLoggerProvider());
//.AddEventLog();
});
var logger = loggerFactory.CreateLogger <StravaXApi>();
logger.LogDebug("Log Stats");
bool doGarbage = false;
bool doAll = false;
bool doListimages = false;
bool doListMaps = false;
bool doActivityList = false;
string AthleteId = null;
string ActivityTypeStr = null;
bool doAthleteStats = false;
string doRenewMonth = null;
bool doQueryCounts = false;
bool doGpxCounts = false;
bool doActivityCounts = false;
bool doAthleteCounts = false;
bool doListAthletes = false;
var p = new OptionSet()
{
{ "g|garbage", v => { doGarbage = true; } },
{ "a|all", v => { doAll = true; } },
{ "act|listactivities", v => { doActivityList = true; } },
{ "li|listimages", v => { doListimages = true; } },
{ "la|listathletes", v => { doListAthletes = true; } },
{ "lm|listmaps", v => { doListMaps = true; } },
{ "aid|athleteid=", v => { AthleteId = v; } },
{ "at|activity_type=", v => { ActivityTypeStr = v; } },
{ "as|athlete-stats", v => { doAthleteStats = true; } },
{ "m|renew-month=", v => { doRenewMonth = v; } },
{ "cath|count-athletes", v => { doAthleteCounts = true; } },
{ "cq|count-queries", v => { doQueryCounts = true; } },
{ "cgpx|count-gpx", v => { doGpxCounts = true; } },
{ "cact|count-activities", v => { doActivityCounts = true; } },
};
p.Parse(args);
int ret = -1;
try
{
using (StravaXApiContext db = new StravaXApiContext())
{
// logger.LogInformation($"Queries stored {db.ActivityQueriesDB.Count()}");
// logger.LogInformation($"Activities stored {db.ActivityShortDB.Count()}");
// var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct();
// logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}");
/*
* foreach(var aId in al)
* {
* AthleteShort ath = db.AthleteShortDB.Find(aId);
* // for format: https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated
* if (ath != null)
* {
* logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {ath.AthleteId,9} {ath.AthleteName,-40} {ath.AthleteLocation}");
* }
* else
* {
* logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {aId,9}");
* }
* }
*/
/*
* // output for first query
* foreach(ActivityRangeQuery arq in db.ActivityQueriesDB)
* {
* logger.LogInformation($"{arq}");
* break;
* }
*/
if (doListAthletes || doAll)
{
foreach (AthleteShort athlete in db.AthleteShortDB)
{
logger.LogInformation($"Athletes {athlete.AthleteName} {athlete.AthleteId}");
}
}
if (doAthleteCounts || doAll)
{
// output status count for queries
var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct();
logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}");
}
if (doQueryCounts || doAll)
{
// output status count for queries
var status = db.ActivityQueriesDB.Select(a => a.Status).Distinct();
logger.LogInformation($"Queries:");
foreach (var st in status)
{
logger.LogInformation($" {st} {db.ActivityQueriesDB.Count(a => a.Status==st)}");
}
}
if (doGpxCounts || doAll)
{
// Count GPX Files and sort by activity-Types.
string rootPath = "gpx";
foreach (string athleteEntry in Directory.GetDirectories(rootPath))
{
var match = Regex.Match(athleteEntry, ".*[/]([0-9]+)");
string AthleteIdStr = match.Groups[1].Value;
// logger.LogInformation($" directory for Athlete {AthleteIdStr} found {athleteEntry}");
Hashtable errTable = new Hashtable();
Hashtable trackTable = new Hashtable();
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.err"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.err");
string ActivityIdStr = match.Groups[1].Value;
errTable.Add(ActivityIdStr, gpxActivity);
}
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx.gz"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.gz");
string ActivityIdStr = match.Groups[1].Value;
if (errTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous");
}
if (trackTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}");
}
else
{
trackTable.Add(ActivityIdStr, gpxActivity);
}
}
foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx"))
{
match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx");
string ActivityIdStr = match.Groups[1].Value;
if (errTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous");
}
if (trackTable.ContainsKey(ActivityIdStr))
{
logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}");
}
else
{
logger.LogWarning($"Found GPX without compression: track for {ActivityIdStr} already enterred {gpxActivity}");
trackTable.Add(ActivityIdStr, gpxActivity);
}
}
Hashtable typeCount = new Hashtable();
foreach (string activity_id in trackTable.Keys)
{
ActivityShort activity = db.ActivityShortDB.Find(activity_id);
if (activity == null)
{
// logger.LogWarning($"can't find activity for {activity_id}");
continue;
}
int tCount = 0;
if (typeCount.ContainsKey(activity.ActivityType))
{
tCount = (int)typeCount[activity.ActivityType];
typeCount.Remove(activity.ActivityType);
}
tCount++;
typeCount.Add(activity.ActivityType, tCount);
}
// var al = db.ActivityShortDB.Where(a=>a.AthleteId==AthleteIdStr);
// int actCount = al.Count();
string keys = string.Join(",", typeCount.Keys.Cast <ActivityType>().Select(x => $"{x.ToString()} {typeCount[x]}").ToArray());
int actCount = 0;
logger.LogInformation($"Athlete {AthleteIdStr}, activities found in DB: {actCount} GPX: {trackTable.Keys.Count} ERR: {errTable.Keys.Count} {keys}");
}
}
if (doRenewMonth != null)
{
// set all DONE queries for the given month to Created
var match = Regex.Match(doRenewMonth, "([0-9]{4})/([0-9]{2})");
int year = Int32.Parse(match.Groups[1].Value);
int month = Int32.Parse(match.Groups[2].Value);
var statusDone = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.DateFrom == new DateTime(year, month, 01));
foreach (ActivityRangeQuery arq in statusDone.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"reset for {year}/{month} from {QueryStatus.Run} to {QueryStatus.Created}";
logger.LogInformation($"query for {year}/{month} with {QueryStatus.Done} {arq.AthleteId} {arq.Message}");
}
logger.LogInformation($"begin: save changes");
db.SaveChanges();
logger.LogInformation($"done: save changes");
}
if (doGarbage)
{
{
var statusError = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Error);
foreach (ActivityRangeQuery arq in statusError.ToList())
{
logger.LogInformation($"query with {QueryStatus.Error} {arq.AthleteId} {arq.Message}");
}
}
{
var statusRun = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Run);
foreach (ActivityRangeQuery arq in statusRun.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"garbage-reset from {QueryStatus.Run} to {QueryStatus.Created}";
logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}");
}
db.SaveChanges();
}
{
var statusReserved = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Reserved);
foreach (ActivityRangeQuery arq in statusReserved.ToList())
{
arq.Status = QueryStatus.Created;
arq.StatusChanged = DateTime.Now;
arq.Message = $"garbage-reset from {QueryStatus.Reserved} to {QueryStatus.Created}";
logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}");
db.SaveChanges();
}
}
}
if (doActivityCounts || doAll)
{
// Find out athlete with open queries:
var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
logger.LogInformation($"{qAthleteCreated.Count()} with {QueryStatus.Created} queries");
var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
logger.LogInformation($"{qAthleteCreated.Count()} without {QueryStatus.Reserved} queries");
logger.LogInformation($"{qAthleteCreated.Intersect(qAthleteReserved).Count()} with {QueryStatus.Created} and without {QueryStatus.Reserved} queries");
List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(10).ToList();
logger.LogInformation($"{AthleteIdList.Count()} athletes from this list:");
foreach (string aId in AthleteIdList)
{
logger.LogInformation($" AthleteId={aId}");
}
if (AthleteIdList.Count() > 0)
{
// retrieve one random athlete
string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count));
logger.LogInformation($"retrieve activity for athlete {aid}");
// 5 first queries for this athlete
IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(10).ToList();
foreach (ActivityRangeQuery arq in q0)
{
logger.LogInformation($" query={arq}");
var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo)));
logger.LogInformation($" find {ActivitiesInRange.Count()} activities in it.");
}
}
}
if (doAll)
{
// find activities in a QueryRange
foreach (ActivityRangeQuery arq in db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done).OrderByDescending(a => a.DateFrom).Take(10))
{
var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo)));
logger.LogInformation($" find {ActivitiesInRange.Count()} in range {arq}");
}
}
if (doAll)
{
// retrieve queries type for athlete
string aId = "2754335";
var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList();
logger.LogInformation($" athl {aId}");
foreach (QueryStatus qs in qsList)
{
logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}");
}
// IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(6).ToList();
}
if (doAll)
{
string aId = "26319532";
var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList();
logger.LogInformation($" athl {aId}");
foreach (QueryStatus qs in qsList)
{
logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}");
}
}
if (doAll || doListimages)
{
List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList();
logger.LogInformation($" activity count:{activitiesWithImages.Count()}");
int imageCount = 0;
foreach (ActivityShort activity in activitiesWithImages)
{
List <string> images = activity.ActivityImagesList;
imageCount += images.Count();
logger.LogInformation($"activity {activity.ActivityTitle}");
foreach (string imageUrl in images)
{
logger.LogInformation($" url:{imageUrl}");
}
}
logger.LogInformation($" images {imageCount}");
}
if (doAll || doListMaps)
{
List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList();
logger.LogInformation($" activity count:{activitiesWithImages.Count()}");
int imageCount = 0;
foreach (ActivityShort activity in activitiesWithImages)
{
string ImageMapUrl = activity.ActivityImageMapUrl;
if (ImageMapUrl == null)
{
continue;
}
imageCount++;
logger.LogInformation($"activity {activity.ActivityTitle} {ImageMapUrl}");
WebClient webClient = new WebClient();
string outputDir = $"maps/{activity.AthleteId}";
string outputFilename = $"{activity.ActivityId}.png";
logger.LogWarning($"NOT IMPLEMENTED");
}
logger.LogInformation($" images {imageCount}/{activitiesWithImages}");
}
if (doAll || doAthleteStats)
{
IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
logger.LogInformation($" athletes {AllAthletes.Count()}");
logger.LogInformation($" first: [{AllAthletes.First()}] last: [{AllAthletes.Last()}]");
int AthleteCount = db.ActivityQueriesDB.Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($"athletes in queries {AthleteCount}");
int PrivateAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.Message.Contains("private")).Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($" private {PrivateAthleteCount}");
int CreatedAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct().Count();
logger.LogInformation($" {QueryStatus.Created} {CreatedAthleteCount}");
}
if (doAll || doActivityList)
{
List <ActivityShort> activities;
IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
if (ActivityTypeStr != null)
{
ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
dbs = dbs.Where(a => a.ActivityType == ActivityType);
}
if (AthleteId != null)
{
dbs = dbs.Where(a => a.AthleteId == AthleteId);
}
activities = dbs.ToList();
logger.LogInformation($"List activities for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
/*
* if (activities.Count>0 && doActivityList)
* {
* foreach(ActivityShort activity in activities)
* {
* logger.LogInformation($" {activity}");
* }
* }
*/
}
/*
* {
* // opened queries with activities
* int i=0;
* int nbAthletes=db.AthleteShortDB.Count();
* foreach(AthleteShort athlete in db.AthleteShortDB)
* {
* i++;
* // logger.LogInformation($"retrieve activity for athlete {athlete.AthleteId}");
* // 5 first queries for this athlete
* IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==athlete.AthleteId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(5).ToList();
* int activitiesWithCreatedQueriesCount=0;
* foreach(ActivityRangeQuery arq in q0)
* {
* var ActivitiesInRange = db.ActivityShortDB.Where(a=>a.AthleteId==arq.AthleteId && ((a.ActivityDate>=arq.DateFrom)&&(a.ActivityDate<=arq.DateTo)));
* activitiesWithCreatedQueriesCount+=ActivitiesInRange.Count();
* }
* if (activitiesWithCreatedQueriesCount>0)
* {
* logger.LogInformation($" athlete {athlete.AthleteId} as {activitiesWithCreatedQueriesCount} activities with created queries queries:{q0.Count()} activites{db.ActivityShortDB.Where(a=>a.AthleteId==athlete.AthleteId).Count()}.");
* }
* if (i%10==0)
* {
* logger.LogInformation($"{i}/{nbAthletes}");
* }
* }
* }
*/
/*
* // output activity count for activity type
* logger.LogInformation($"Activity types Σ :{db.ActivityShortDB.Count()}");
* var ActivityTypes = db.ActivityShortDB.Select(a => a.ActivityType).Distinct();
* foreach(var aType in ActivityTypes)
* {
* logger.LogInformation($" {aType,18} {db.ActivityShortDB.Count(a => a.ActivityType==aType),6}");
* }
*/
/*
* // output athletes with ski activities and their ski activity count
* logger.LogInformation($"All athletes with {ActivityType.BackcountrySki}");
* var Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Select(a => a.AthleteId).Distinct();
* foreach(var A4Type in Activity4Type)
* {
* var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Where(a => a.AthleteId==A4Type).Count();
* var athlete = db.AthleteShortDB.Find(A4Type);
* logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})");
* }
*/
/*
* // output athletes with run activities and their run activity count
* logger.LogInformation($"All athletes with {ActivityType.Run}");
* Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Select(a => a.AthleteId).Distinct();
* foreach(var A4Type in Activity4Type)
* {
* var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Where(a => a.AthleteId==A4Type).Count();
* var athlete = db.AthleteShortDB.Find(A4Type);
* logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})");
* }
*/
}
ret = 0;
}
catch (Exception e)
{
logger.LogInformation($"ERROR:{e.ToString()}");
ret = 1;
}
return(ret);
}
