static public int EnsureCreated(string[] args) { using (StravaXApiContext db = new StravaXApiContext()) { // https://stackoverflow.com/questions/38238043/how-and-where-to-call-database-ensurecreated-and-database-migrate db.Database.EnsureCreated(); } return(0); }
static int queryAthlete(string aid, string clientId, StravaXApi stravaXApi, StravaXApiContext db, bool doVerbose) { int ret = 0; IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList(); while (q0.Count > 0 && ret == 0) { // Mark all queries with "RESERVED" IList <ActivityRangeQuery> queries = new List <ActivityRangeQuery>(); foreach (ActivityRangeQuery arq in q0) { try { arq.Status = QueryStatus.Reserved; arq.StatusChanged = DateTime.Now; arq.Message = $"Reserved by {clientId}"; queries.Add(arq); } catch (DbUpdateConcurrencyException) { // Just skip this entry if some conflict exists. logger.LogInformation($"skip: Can't reserve query {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2} for {clientId}"); continue; } } db.SaveChanges(); // Run for all reserved queries try { ret = queryRange(stravaXApi, db, queries, doVerbose); } catch (CancelExecution e) { // just to clarify that it may happend. // finally will also call in that case. throw e; } finally { // integrity check, status should not be Reserved anymore foreach (ActivityRangeQuery arq in q0) { if (arq.Status == QueryStatus.Reserved) { logger.LogInformation($"WARN: remove reservation on {arq}"); } } db.SaveChanges(); } // retrieve next queries for max one year. q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList(); } return(ret); }
static private void AddQuery(StravaXApiContext db, String AthleteId, DateTime DateFrom, DateTime DateTo, List <ActivityRangeQuery> queriesForPatient) { if (!isDateInList(DateFrom, queriesForPatient)) { if (db.ActivityQueriesDB.Find(AthleteId, DateFrom, DateTo) == null) { ActivityRangeQuery query = new ActivityRangeQuery(); query.AthleteId = AthleteId; query.DateFrom = DateFrom; query.DateTo = DateTo; db.ActivityQueriesDB.Add(query); Console.WriteLine($"add query for {query}"); } } }
static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args) { int ret = -1; Console.WriteLine("Read athlete connections with Strava-X-API."); String AthleteId = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, }; p.Parse(args); if (AthleteId == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing athlete id"); } try { stravaXApi.signIn(); AthleteShort AthleteMasterShort; using (StravaXApiContext db = new StravaXApiContext()) { AthleteMasterShort = db.AthleteShortDB.Find(AthleteId); if (AthleteMasterShort == null) { AthleteMasterShort = new AthleteShort(); // create a dummy master AthleteMasterShort.AthleteId = AthleteId; // [TODO] other parameters should be retrieved with selenium AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity; db.SaveChanges(); } else { // Eagerly Loading prevent the list to be loaded at creation // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load(); Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}"); } string FollowType = "following"; var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType); Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections"); foreach (AthleteShort _AthleteShort in AthleteShortList) { AthleteShort AthleteShortfromDb; // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId); if (AthleteShortfromDb == null) { // add athlete to the db if need. AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity; } else { Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database"); } Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); // such the connected athlete with they id. AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId)); if (_ConnectedAthleteShort == null) { // add connection if needed. AthleteConnection ac = new AthleteConnection(); ac.FromId = AthleteMasterShort.AthleteId; ac.ToId = AthleteShortfromDb.AthleteId; ac.Type = FollowType; ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState; AthleteMasterShort.Connections.Add(ac); Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}"); } else { Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)"); } } db.SaveChanges(); Console.WriteLine($"total read = {AthleteShortList.Count}"); Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); AthleteShortList.Clear(); } using (StravaXApiContext db = new StravaXApiContext()) { AthleteMasterShort = db.AthleteShortDB.Find(AthleteId); if (AthleteMasterShort == null) { AthleteMasterShort = new AthleteShort(); // create a dummy master AthleteMasterShort.AthleteId = AthleteId; // [TODO] other parameters should be retrieved with selenium AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity; db.SaveChanges(); } else { // Eagerly Loading prevent the list to be loaded at creation // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load(); Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}"); } string FollowType = "followers"; var AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType); Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections"); foreach (AthleteShort _AthleteShort in AthleteShortList) { AthleteShort AthleteShortfromDb; // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId); if (AthleteShortfromDb == null) { // add athlete to the db if need. AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity; } else { Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database"); } Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); // such the connected athlete with they id. AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId)); if (_ConnectedAthleteShort == null) { // add connection if needed. AthleteConnection ac = new AthleteConnection(); ac.FromId = AthleteMasterShort.AthleteId; ac.ToId = AthleteShortfromDb.AthleteId; ac.Type = FollowType; AthleteMasterShort.Connections.Add(ac); Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}"); } else { Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)"); } } db.SaveChanges(); Console.WriteLine($"total read = {AthleteShortList.Count}"); Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}"); AthleteShortList.Clear(); } ret = 0; } catch (Exception e) { Console.WriteLine($"ERROR:{e.ToString()}"); ret = 1; } finally { stravaXApi.Dispose(); } return(ret); }
/// <summary> /// Convert GPX files to KML. /// GPX File should have bee retrieved with get-gpx. /// All GPX are merged in one KML file with subfolder for athlete and activity types. /// </summary> /// <param name="args"></param> /// <returns></returns> public static int Convert(string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log GpxToKml"); string AthleteId = null; string ActivityTypeStr = null; string filenameExtention = ".kml"; string exportKmlPath = "StravaXApi.kml"; string splitBlockSizeStr = null; string exportTypeStr = ExportType.HeatMap.ToString(); string beginDateStr = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "e|export_type=", v => { exportTypeStr = v; } }, { "bd|begin_date=", v => { beginDateStr = v; } }, { "d|destination=", v => { exportKmlPath = v; } }, { "s|split=", v => { splitBlockSizeStr = v; } }, }; p.Parse(args); int ret = -1; if (!exportKmlPath.EndsWith(filenameExtention)) { exportKmlPath = $"{exportKmlPath}{filenameExtention}"; } ExportType exportType = (ExportType)Enum.Parse(typeof(ExportType), exportTypeStr); try { List <ActivityShort> activities; using (StravaXApiContext db = new StravaXApiContext()) { // Just do one query for all queries, order will be user to separate athletes and activity type. // [note] code is not really readable, afterthere it would have be a better idea to use several queries : athletes/activity types/activities in date range. IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderBy(a => a.AthleteId).ThenBy(a => a.ActivityType).ThenByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (AthleteId != null) { dbs = dbs.Where(a => a.AthleteId == AthleteId); } if (beginDateStr != null) { DateTime dateTime = DateTime.Parse(beginDateStr); dbs = dbs.Where(a => a.ActivityDate >= dateTime); } // Ignore Activities without image map. They have probably been enterred without gps-track. dbs = dbs.Where(a => a.ActivityImageMapUrl != null); // query all activities at once. activities = dbs.ToList(); logger.LogInformation($"BEGIN GPX➡️KML {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)}/{(beginDateStr==null?"all dates":DateTime.Parse(beginDateStr).ToString())} :{activities.Count()}"); string lastAthleteId = null; ActivityType lastActivityType = ActivityType.Other; XElement StravaXApiFolder = new XElement(kml + "Folder" , new XElement(kml + "name", "Strava-X-Api") , new XElement(kml + "open", "1") ); XElement currentAthleteFolder = null; XElement currentActivityTypeFolder = null; int Count = 0; int AthleteCount = 0; int splitBlockSize = int.MaxValue; if (splitBlockSizeStr != null) { splitBlockSize = int.Parse(splitBlockSizeStr); } // go throw all activities, sorted by athlete / activity type / activity date foreach (ActivityShort activity in activities) { // different locations for gpx files. string outputDir = $"gpx/{activity.AthleteId}"; string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx"; string outputFilenameGz = $"{activity.ActivityId}_{activity.AthleteId}.gpx.gz"; string outputFilenameErr = $"{activity.ActivityId}_{activity.AthleteId}.err"; // skip activities without gpx file at the beginning to avoid node creation in kml file. if (File.Exists($"{outputDir}/{outputFilenameErr}") || (!File.Exists($"{outputDir}/{outputFilenameGz}") && !File.Exists($"{outputDir}/{outputFilename}") )) { // Console.WriteLine($"Skip activity {activity}"); continue; } if (lastAthleteId == null) { // first round init athlete AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId); string athleteName = athlete.AthleteName; currentAthleteFolder = new XElement(kml + "Folder" , new XElement(kml + "name", athleteName) , new XElement(kml + "visibility", "0") , new XElement(kml + "open", "0")); StravaXApiFolder.Add(currentAthleteFolder); // first round init activity currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastAthleteId = activity.AthleteId; lastActivityType = activity.ActivityType; AthleteCount = 1; } else { if (lastAthleteId == activity.AthleteId) { // same athlete, check activity type if (lastActivityType != activity.ActivityType) { // start a new activity folder currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastActivityType = activity.ActivityType; } } else { AthleteCount++; // if splitBlockSize has been set, we may need to write the kml file. if (splitBlockSize != int.MaxValue && AthleteCount % splitBlockSize == 0) { // end of athlete block size reached, save a kml file. saveKmlFile(StravaXApiFolder, exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize):D2}{filenameExtention}"), logger); // begin a new tree. StravaXApiFolder = new XElement(kml + "Folder" , new XElement(kml + "name", "Strava-X-Api") , new XElement(kml + "open", "1") ); } // start a new athlete folder AthleteShort athlete = db.AthleteShortDB.Find(activity.AthleteId); string athleteName = athlete.AthleteName; currentAthleteFolder = new XElement(kml + "Folder" , new XElement(kml + "name", athleteName) , new XElement(kml + "visibility", "0") , new XElement(kml + "open", "0")); StravaXApiFolder.Add(currentAthleteFolder); // start a new activity folder currentActivityTypeFolder = new XElement(kml + "Folder" , new XElement(kml + "name", activity.ActivityType.ToString()) , new XElement(kml + "open", "0")); currentAthleteFolder.Add(currentActivityTypeFolder); lastAthleteId = activity.AthleteId; lastActivityType = activity.ActivityType; } } try { // read the gpx file and add it to the kml node. if (File.Exists($"{outputDir}/{outputFilenameErr}")) { // Skip error file. logger.LogWarning($"SKIP: GPX not availlable for {activity.ActivityId} {lastAthleteId} {lastActivityType}"); } else if (File.Exists($"{outputDir}/{outputFilenameGz}")) { using (FileStream compressedFileStream = File.OpenRead($"{outputDir}/{outputFilenameGz}")) { using (GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Decompress)) { currentActivityTypeFolder.Add(handleGpxStream(compressionStream, exportType, activity)); } } } else if (File.Exists($"{outputDir}/{outputFilename}")) { using (FileStream fileStream = File.OpenRead($"{outputDir}/{outputFilename}")) { currentActivityTypeFolder.Add(handleGpxStream(fileStream, exportType, activity)); } } // else skip, because GPX is not availlable. } catch (Exception e) { logger.LogWarning($"SKIP: Can't parse GPX for {activity.ActivityId} {lastAthleteId} {lastActivityType} err:{e.Message}"); } if (Count++ % 100 == 0) { Console.WriteLine($"{Count}/{activities.Count()}"); } } string exportKmlPathForSplit = (splitBlockSize == int.MaxValue)?exportKmlPath:exportKmlPath.Replace(filenameExtention, $"_{(AthleteCount/splitBlockSize)+1:D2}{filenameExtention}"); saveKmlFile(StravaXApiFolder, exportKmlPathForSplit, logger); } ret = 0; } catch (Exception e) { logger.LogError($"ERROR:{e.ToString()}"); ret = 1; } return(ret); }
static void WriteQueriesForAthlete(StravaXApi stravaXApi, StravaXApiContext db, string AthleteId) { DateTime Now = DateTime.Now; lastStravaHttpRequest = DateTime.Now; int ToYear = Now.Year; int ToMonth = Now.Month; // // Verify if queries have to be generated // // Retrieve all entered queries or the wanted athlete. List <ActivityRangeQuery> queriesForAthlete = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).OrderBy(a => a.DateFrom).ToList(); int FromYear; int FromMonth; // If we already have an entry, we assume that the entry contains the first activity date, as it is expensive to retrieve its value with Selenium. if (queriesForAthlete.Count == 0) { // Retrieve the first activity date with selenium. // may throw PrivateAthleteException. // be sure of a one seconde intervall between to requests int dt = DateTime.Now.Millisecond - lastStravaHttpRequest.Millisecond; if (dt < 2000) { System.Threading.Tasks.Task.Delay(dt).Wait(); } DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId); lastStravaHttpRequest = DateTime.Now; System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}"); FromYear = FirstActivityDate.Year; FromMonth = FirstActivityDate.Month; } else { // retrieve first and last date DateTime minDT = queriesForAthlete.First().DateFrom; // DateTime maxDT = queriesForPatient.Last().DateFrom; FromYear = minDT.Year; FromMonth = minDT.Month; } Console.WriteLine($"queries enterred:{queriesForAthlete.Count}/total:{db.ActivityQueriesDB.Count()}"); if (FromYear == ToYear) { for (int month = FromMonth; month <= ToMonth; month++) { AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete); } } else { // first year for (int month = FromMonth; month <= 12; month++) { AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete); } // all years after for (int year = FromYear + 1; year <= ToYear - 1; year++) { for (int month = 01; month <= 12; month++) { AddQuery(db, AthleteId, new DateTime(year, month, 1), new DateTime(year, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete); } } // last year for (int month = 01; month <= ToMonth; month++) { // from first day of month to last day of the month AddQuery(db, AthleteId, new DateTime(ToYear, month, 1), new DateTime(ToYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete); } } int qCount = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).Count(); Console.WriteLine($"✅ enterred:{qCount}/total:{db.ActivityQueriesDB.Count()}"); }
static internal int WriteQueriesForAthletes(StravaXApi stravaXApi) { int ret = -1; Console.WriteLine("Create range queries."); using (StravaXApiContext db = new StravaXApiContext()) { stravaXApi.signIn(); try { // https://docs.microsoft.com/en-us/ef/ef6/querying/ // First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session." // Not best praxis but enought for Strava.XApi. IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList(); Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); foreach (AthleteShort Athlete in AllAthletes) { TimeSpan ts = stopWatch.Elapsed; try { WriteQueriesForAthlete(stravaXApi, db, Athlete.AthleteId); Console.WriteLine($"Athlete:{Athlete} run since:{ts.TotalSeconds}s"); } catch (PrivateAthleteException e) { // TODO AB#27 athlete should be marked as private to avoid second visit. // create a dummy Query to prevent next search ActivityRangeQuery arq = new ActivityRangeQuery(); arq.AthleteId = Athlete.AthleteId; arq.DateFrom = new DateTime(2019, 12, 01); arq.DateTo = new DateTime(2019, 12, 31); arq.Status = QueryStatus.Done; arq.Message = $"private athlete {Athlete.AthleteId}"; arq.StatusChanged = DateTime.Now; db.ActivityQueriesDB.Add(arq); Console.WriteLine($"SKIP: private athlete {Athlete.AthleteId} {e.Message}"); } catch (TooManyStravaRequestException e) { Console.WriteLine($"Too Many Queries detected {e.Message} need to wait some hours"); } catch (Exception e) { Console.WriteLine($"SKIP:{Athlete.AthleteId} {e.ToString()}"); } db.SaveChanges(); } ret = 0; } catch (Exception e) { Console.WriteLine($"ERROR:{e.ToString()}"); ret = -1; } finally { stravaXApi.Dispose(); } } return(ret); }
static public int Downloader(string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.ImageDownloader", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log ImageDownloader"); bool doMaps = false; bool doActivityImages = false; string maxCountStr = null; string AthleteId = null; string ActivityTypeStr = null; var p = new OptionSet() { { "s|maps", v => { doMaps = true; } }, { "ai|activity_images", v => { doActivityImages = true; } }, { "a|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "m|max_count=", v => { maxCountStr = v; } }, }; p.Parse(args); int ret = -1; int maxCount = int.MaxValue; if (maxCountStr != null) { maxCount = int.Parse(maxCountStr); logger.LogInformation($" limit download to {maxCount}"); } try { using (StravaXApiContext db = new StravaXApiContext()) { IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (doMaps) { if (AthleteId != null) { ret = RetrieveMaps(dbs, logger, AthleteId, ActivityTypeStr, maxCount); } else { List <AthleteShort> athletes = db.AthleteShortDB.OrderBy(a => a.AthleteId).ToList(); foreach (AthleteShort athlete in athletes) { ret = RetrieveMaps(dbs, logger, athlete.AthleteId, ActivityTypeStr, maxCount); } } } if (doActivityImages) { if (AthleteId != null) { ret = RetrieveActivityImages(dbs, logger, AthleteId, ActivityTypeStr, maxCount); } else { List <AthleteShort> athletes = db.AthleteShortDB.OrderBy(a => a.AthleteId).ToList(); foreach (AthleteShort athlete in athletes) { ret = RetrieveActivityImages(dbs, logger, athlete.AthleteId, ActivityTypeStr, maxCount); } } } } ret = 0; } catch (Exception e) { logger.LogError($"ERROR:{e.ToString()}"); ret = 1; } return(ret); }
static internal int Downloader(StravaXApi stravaXApi, string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log ImageDownloader"); bool doGpx = false; string maxCountStr = null; string AthleteId = null; string ActivityTypeStr = null; string gpxDir = "gpx"; var p = new OptionSet() { { "g|gpx", v => { doGpx = true; } }, { "a|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "m|max_count=", v => { maxCountStr = v; } }, { "d|dirname=", v => { gpxDir = v; } }, }; p.Parse(args); int ret = -1; try { List <ActivityShort> activities; using (StravaXApiContext db = new StravaXApiContext()) { IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (AthleteId != null) { dbs = dbs.Where(a => a.AthleteId == AthleteId); } // Ignore Activities without image map. They have probably been enterred without gps-track. dbs = dbs.Where(a => a.ActivityImageMapUrl != null); // retrieve the activities from the database. activities = dbs.ToList(); } logger.LogInformation($"BEGIN GPX Download for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}"); int countSkipped = 0; int countDownload = 0; if (doGpx && activities.Count > 0) { bool needTracksDownload = false; // detect if any activity needs a gpx file foreach (ActivityShort activity in activities) { string outputDir = $"{gpxDir}/{activity.AthleteId}"; string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx"; if (!File.Exists($"{outputDir}/{outputFilename}")) { needTracksDownload = true; break; } } // if no track is needed, we could exit without signin in Strava. if (needTracksDownload) { // signin stravaXApi.signIn(); // go throw all activities. foreach (ActivityShort activity in activities) { string outputDir = $"{gpxDir}/{activity.AthleteId}"; // fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}"); string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx"; string outputFilenameGZip = $"{outputFilename}.gz"; string outputFilenameErr = $"{outputFilename}.err"; // download only if the gpx, gz or the err file do not exists. if (!File.Exists($"{outputDir}/{outputFilename}") && !File.Exists($"{outputDir}/{outputFilenameGZip}") && !File.Exists($"{outputDir}/{outputFilenameErr}")) { // create directory if needed if (!Directory.Exists(outputDir)) { DirectoryInfo DirInfo = Directory.CreateDirectory(outputDir); logger.LogDebug($"directory for GPX created at {DirInfo.FullName}"); } try { stravaXApi.getActivityGpxSelenium(activity.ActivityId, $"{outputDir}/{outputFilenameGZip}"); countDownload++; } catch (PrivateGpxException e) { logger.LogDebug($"GPX Track private for {activity.AthleteId}: {e.ActivityId} {e.Message}"); // write error file, do prevent a second try. using (StreamWriter outputFile = new StreamWriter(Path.Combine(outputDir, outputFilenameErr))) { outputFile.WriteLine($"Error while downloading GPX for athlete:{activity.AthleteId} activity{activity.ActivityId}"); outputFile.WriteLine($"{e.Message}"); outputFile.WriteLine($"{e.StackTrace}"); } countSkipped++; } } else { logger.LogDebug($"GPX Track already downloaded for {activity.AthleteId}"); countSkipped++; } } } } else if (!doGpx) { // just retrieve without retrieving the gpx files. int countToDownload = 0; int countDownloaded = 0; foreach (ActivityShort activity in activities) { // logger.LogInformation($"activity {activity.StatShortString} -> {Utils.extractActivityTime(activity)}."); string outputDir = $"{gpxDir}/{activity.AthleteId}"; // fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}"); string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx"; if (!File.Exists($"{outputDir}/{outputFilename}")) { countToDownload++; } else { countDownloaded++; } } logger.LogInformation($"GPX Track to download:{countToDownload} already downloaded:{countDownloaded}"); } logger.LogInformation($"DONE GPX Download {countDownload} (skipped: {countSkipped})for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}"); // everything done, with return 0 we are telling the container, that there's no need to restart. ret = 0; } catch (Exception e) { logger.LogError($"ERROR:{e.ToString()}"); // return error code, container should restart. ret = 1; } finally { stravaXApi.Dispose(); } return(ret); }
static internal int SendQueriesForActivities(StravaXApi stravaXApi, string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log Query Activities"); string clientId = Guid.NewGuid().ToString(); int TimerSeconds = -1; int TimerExitCode = 2; bool doVerbose = false; var p = new OptionSet() { { "t|timer_sec=", v => { TimerSeconds = int.Parse(v); } }, { "e|timer_exit_code=", v => { TimerExitCode = int.Parse(v); } }, { "v|verbose", v => { doVerbose = true; } }, }; p.Parse(args); int ret = -1; logger.LogInformation($"Query activities. Client:{clientId}"); using (StravaXApiContext db = new StravaXApiContext()) { stravaXApi.signIn(); // 1) find an athlete with opened queries and without reserved queries. // string aid =db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).First().AthleteId; // select distinct AthleteId from [dbo].[ActivityQueriesDB] where Status=2 // intersect // select distinct AthleteId from [dbo].[ActivityQueriesDB] where (Status<>0 AND Status<>3) var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct(); var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct(); List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList(); if (AthleteIdList.Count == 0) { logger.LogInformation($"no more athlete to search for. created:{qAthleteCreated.Count()} reserved:{qAthleteReserved.Count()}"); // TODO some queries may have been reserved, but no Thread is working on it. It should be usefull to detect and reset. } else { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); ret = 0; try { while (AthleteIdList.Count > 0 && ret == 0) { string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count)); logger.LogInformation($"retrieve activity for athlete {aid}"); // https://docs.microsoft.com/en-us/ef/ef6/querying/ // First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session." // Not best praxis but enought for Prototype. // https://stackoverflow.com/a/2656612/281188 // IList<ActivityRangeQuery> queries = db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(50).ToList(); ret = queryAthlete(aid, clientId, stravaXApi, db, doVerbose); TimeSpan ts = stopWatch.Elapsed; if (TimerSeconds > 0 && ts.TotalSeconds > TimerSeconds) { ret = TimerExitCode; logger.LogInformation($"Timer reached after {ts.ToString()} now exit with {ret}."); // exit with error code, container should restart break; } Boolean KeepRunning = true; if (!KeepRunning || File.Exists("QueryActivities.quit")) { logger.LogInformation($"break {KeepRunning} {Count}"); // regular exit, container should ended. ret = 0; break; } // search for a new athlete. qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct(); qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct(); AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList(); } } catch (CancelExecution e) { logger.LogInformation($"Execution has been canceled: {e.Message}"); } } } return(ret); }
static int queryRange(StravaXApi stravaXApi, StravaXApiContext db, IList <ActivityRangeQuery> queries, bool doVerbose) { int ret = 0; foreach (ActivityRangeQuery arq in queries) { try { // https://docs.microsoft.com/en-us/ef/core/saving/concurrency try { // reserve query, mark it as run. arq.Status = QueryStatus.Run; arq.StatusChanged = DateTime.Now; db.SaveChanges(); } catch (DbUpdateConcurrencyException) { // Just skip this entry if some conflict exists. logger.LogInformation($"skip conflicted entry for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}"); continue; } var ActivitiesList = stravaXApi.getActivities(arq.AthleteId, $"{arq.DateFrom.Year:D4}", $"{arq.DateFrom.Month:D2}"); int EnterredActivityCount = 0; foreach (ActivityShort ActivityShort in ActivitiesList) { // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null) { db.ActivityShortDB.Add(ActivityShort); try{ // do it better after foreach, but first check how it works with the concurrentcy exception db.SaveChanges(); EnterredActivityCount++; } catch (DbUpdateConcurrencyException) { // Just skip this entry if some conflict exists. logger.LogInformation($"❌ skip conflicted activity {ActivityShort}"); continue; } } else { logger.LogInformation($"❌ {ActivityShort.ActivityId} allready in database"); } } arq.Status = QueryStatus.Done; arq.StatusChanged = DateTime.Now; // should not have to save anything. db.SaveChanges(); if (doVerbose) { logger.LogInformation($"enterred activity count: {EnterredActivityCount}/{db.ActivityShortDB.Count()} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}"); } else // '.Count()' is pretty expensive, skip it if verbose should be minimal. { logger.LogInformation($"enterred activity count: {EnterredActivityCount} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}"); } ErrorCountConsecutive = 0; } catch (Exception e) { ErrorCountConsecutive++; ErrorCount++; logger.LogInformation($"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}"); arq.Status = QueryStatus.Error; arq.StatusChanged = DateTime.Now; arq.Message = $"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}"; if (ErrorCountConsecutive > 2) { // After 3 consecutive errors, I assume the selenium driver is down. Stop it all. throw e; } } if (doVerbose) { logger.LogInformation($"activities stored:{db.ActivityShortDB.Count()}/{QueryStatus.Created}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Created)}/{QueryStatus.Reserved}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Reserved)}"); } Count++; // Exist when KeepRunning is false (from the debugger), // or the file 'QueryActivities.quit' exists. // *Program will exit with "touch QueryActivities.quit" in /app directory in container. Boolean KeepRunning = true; if (!KeepRunning || File.Exists("QueryActivities.quit.immediatly")) { logger.LogInformation($"break {KeepRunning} {Count}"); // regular exit, container should ended. throw new CancelExecution("break {KeepRunning} {Count}"); } } return(ret); }
static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args) { int ret = -1; Console.WriteLine("Read athlete activities with Strava-X-API."); String AthleteId = null; var p = new OptionSet() { { "a|athleteid=", v => { AthleteId = v; } }, }; p.Parse(args); if (AthleteId == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing athlete id"); } try { stravaXApi.signIn(); List <ActivityShort> ActivitiesList = new List <ActivityShort>(); DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId); System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}"); int FromYear = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR")); int FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH")); int ToYear = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR")); int ToMonth = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH")); DateTime now = DateTime.Now; for (int year = FromYear; year <= ToYear; year++) { for (int month = 01; month <= 12; month++) { if ((year <= FromYear && month < FromMonth) || (year >= ToYear && month > ToMonth)) { continue; } List <ActivityShort> ActivitiesMonthList; try { ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}"); } catch (StaleElementReferenceException) { // Wait and try again. Thread.Sleep(2000); ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}"); } ActivitiesList.AddRange(ActivitiesMonthList); using (StravaXApiContext db = new StravaXApiContext()) { foreach (ActivityShort ActivityShort in ActivitiesList) { Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}"); if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null) { db.ActivityShortDB.Add(ActivityShort); db.SaveChanges(); Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}"); } else { Console.WriteLine($"{ActivityShort.ActivityId} allready in database"); } } Console.WriteLine($"total read = {ActivitiesList.Count}"); Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}"); ActivitiesList.Clear(); } } } ret = 0; } catch (Exception e) { Console.WriteLine($"ERROR:{e.ToString()}"); ret = 1; } finally { stravaXApi.Dispose(); } return(ret); }
static public int DoClone(string[] args) { String SrcConnectionString = null; String DestConnectionString = null; var p = new OptionSet() { { "s|source=", s => { SrcConnectionString = s; } }, { "d|destination=", d => { DestConnectionString = d; } }, }; p.Parse(args); if (SrcConnectionString == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing source"); } if (DestConnectionString == null) { p.WriteOptionDescriptions(Console.Out); throw new ArgumentException("missing destination"); } Console.WriteLine($"SRC: {SrcConnectionString}"); Console.WriteLine($"DST: {DestConnectionString}"); DbContextOptions optionsSrc; if (SrcConnectionString.StartsWith("Data Source")) { optionsSrc = new DbContextOptionsBuilder().UseSqlite(SrcConnectionString).Options; } else { optionsSrc = new DbContextOptionsBuilder().UseSqlServer(SrcConnectionString).Options; } DbContextOptions optionsDst; if (DestConnectionString.StartsWith("Data Source")) { optionsDst = new DbContextOptionsBuilder().UseSqlite(DestConnectionString).Options; } else { optionsDst = new DbContextOptionsBuilder().UseSqlServer(DestConnectionString).Options; } using (StravaXApiContext DbSrc = new StravaXApiContext(optionsSrc)) { Console.WriteLine($"SRC: Queries stored {DbSrc.ActivityQueriesDB.Count()}"); Console.WriteLine($"SRC: Activities stored {DbSrc.ActivityShortDB.Count()}"); var al = DbSrc.ActivityShortDB.Select(a => a.AthleteId).Distinct(); Console.WriteLine($"SRC: Athletes {al.Count()} from {DbSrc.AthleteShortDB.Count()}"); using (StravaXApiContext DbDst = new StravaXApiContext(optionsDst)) { // https://stackoverflow.com/questions/38238043/how-and-where-to-call-database-ensurecreated-and-database-migrate DbDst.Database.EnsureCreated(); Console.WriteLine($"DST: Queries stored {DbDst.ActivityQueriesDB.Count()}"); Console.WriteLine($"DST: Activities stored {DbDst.ActivityShortDB.Count()}"); var AlDst = DbDst.ActivityShortDB.Select(a => a.AthleteId).Distinct(); Console.WriteLine($"DST: Athletes {AlDst.Count()} from {DbDst.AthleteShortDB.Count()}"); Console.WriteLine("read athletes"); foreach (AthleteShort ath in DbSrc.AthleteShortDB) { DbDst.AthleteShortDB.Add(ath); } Console.WriteLine("save athletes"); DbDst.SaveChanges(); Console.WriteLine("read activities"); int i = 0; int totalCount = DbSrc.ActivityShortDB.Count(); int destLen = DbDst.ActivityShortDB.Count(); foreach (ActivityShort act in DbSrc.ActivityShortDB) { // skip as much activities as already present. // Only done because the first imports try has broked, but // a better system should be developped in the future. // if (i++<destLen) // { // continue; // } if (++i % 100 == 0) { Console.WriteLine($"activities {i}/{totalCount}"); } DbDst.ActivityShortDB.Add(act); } Console.WriteLine("save activities"); DbDst.SaveChanges(); Console.WriteLine("read queries"); i = 0; totalCount = DbSrc.ActivityQueriesDB.Count(); foreach (ActivityRangeQuery arq in DbSrc.ActivityQueriesDB) { DbDst.ActivityQueriesDB.Add(arq); i++; if (i % 100 == 0) { Console.WriteLine($"queries {i}/{totalCount}"); } } Console.WriteLine("save queries"); DbDst.SaveChanges(); } } return(0); }
static public int WriteState(string[] args) { var loggerFactory = LoggerFactory.Create(builder => { builder .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) .AddFilter("Strava.XApi.Tools.DbStats", Microsoft.Extensions.Logging.LogLevel.Debug) .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information) .AddProvider(new CustomLoggerProvider()); //.AddEventLog(); }); var logger = loggerFactory.CreateLogger <StravaXApi>(); logger.LogDebug("Log Stats"); bool doGarbage = false; bool doAll = false; bool doListimages = false; bool doListMaps = false; bool doActivityList = false; string AthleteId = null; string ActivityTypeStr = null; bool doAthleteStats = false; string doRenewMonth = null; bool doQueryCounts = false; bool doGpxCounts = false; bool doActivityCounts = false; bool doAthleteCounts = false; bool doListAthletes = false; var p = new OptionSet() { { "g|garbage", v => { doGarbage = true; } }, { "a|all", v => { doAll = true; } }, { "act|listactivities", v => { doActivityList = true; } }, { "li|listimages", v => { doListimages = true; } }, { "la|listathletes", v => { doListAthletes = true; } }, { "lm|listmaps", v => { doListMaps = true; } }, { "aid|athleteid=", v => { AthleteId = v; } }, { "at|activity_type=", v => { ActivityTypeStr = v; } }, { "as|athlete-stats", v => { doAthleteStats = true; } }, { "m|renew-month=", v => { doRenewMonth = v; } }, { "cath|count-athletes", v => { doAthleteCounts = true; } }, { "cq|count-queries", v => { doQueryCounts = true; } }, { "cgpx|count-gpx", v => { doGpxCounts = true; } }, { "cact|count-activities", v => { doActivityCounts = true; } }, }; p.Parse(args); int ret = -1; try { using (StravaXApiContext db = new StravaXApiContext()) { // logger.LogInformation($"Queries stored {db.ActivityQueriesDB.Count()}"); // logger.LogInformation($"Activities stored {db.ActivityShortDB.Count()}"); // var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct(); // logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}"); /* * foreach(var aId in al) * { * AthleteShort ath = db.AthleteShortDB.Find(aId); * // for format: https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated * if (ath != null) * { * logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {ath.AthleteId,9} {ath.AthleteName,-40} {ath.AthleteLocation}"); * } * else * { * logger.LogInformation($" Activities:{db.ActivityShortDB.Count(a => a.AthleteId==aId),6} for {aId,9}"); * } * } */ /* * // output for first query * foreach(ActivityRangeQuery arq in db.ActivityQueriesDB) * { * logger.LogInformation($"{arq}"); * break; * } */ if (doListAthletes || doAll) { foreach (AthleteShort athlete in db.AthleteShortDB) { logger.LogInformation($"Athletes {athlete.AthleteName} {athlete.AthleteId}"); } } if (doAthleteCounts || doAll) { // output status count for queries var al = db.ActivityShortDB.Select(a => a.AthleteId).Distinct(); logger.LogInformation($"Athletes {al.Count()} from {db.AthleteShortDB.Count()}"); } if (doQueryCounts || doAll) { // output status count for queries var status = db.ActivityQueriesDB.Select(a => a.Status).Distinct(); logger.LogInformation($"Queries:"); foreach (var st in status) { logger.LogInformation($" {st} {db.ActivityQueriesDB.Count(a => a.Status==st)}"); } } if (doGpxCounts || doAll) { // Count GPX Files and sort by activity-Types. string rootPath = "gpx"; foreach (string athleteEntry in Directory.GetDirectories(rootPath)) { var match = Regex.Match(athleteEntry, ".*[/]([0-9]+)"); string AthleteIdStr = match.Groups[1].Value; // logger.LogInformation($" directory for Athlete {AthleteIdStr} found {athleteEntry}"); Hashtable errTable = new Hashtable(); Hashtable trackTable = new Hashtable(); foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.err")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.err"); string ActivityIdStr = match.Groups[1].Value; errTable.Add(ActivityIdStr, gpxActivity); } foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx.gz")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx.gz"); string ActivityIdStr = match.Groups[1].Value; if (errTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous"); } if (trackTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}"); } else { trackTable.Add(ActivityIdStr, gpxActivity); } } foreach (string gpxActivity in Directory.EnumerateFiles(athleteEntry, "*.gpx")) { match = Regex.Match(gpxActivity, $".*[/]([0-9]+)_{AthleteIdStr}.gpx"); string ActivityIdStr = match.Groups[1].Value; if (errTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} found, be marked as erroneous"); } if (trackTable.ContainsKey(ActivityIdStr)) { logger.LogWarning($"Track for {ActivityIdStr} already enterred {trackTable[ActivityIdStr]} <-> {gpxActivity}"); } else { logger.LogWarning($"Found GPX without compression: track for {ActivityIdStr} already enterred {gpxActivity}"); trackTable.Add(ActivityIdStr, gpxActivity); } } Hashtable typeCount = new Hashtable(); foreach (string activity_id in trackTable.Keys) { ActivityShort activity = db.ActivityShortDB.Find(activity_id); if (activity == null) { // logger.LogWarning($"can't find activity for {activity_id}"); continue; } int tCount = 0; if (typeCount.ContainsKey(activity.ActivityType)) { tCount = (int)typeCount[activity.ActivityType]; typeCount.Remove(activity.ActivityType); } tCount++; typeCount.Add(activity.ActivityType, tCount); } // var al = db.ActivityShortDB.Where(a=>a.AthleteId==AthleteIdStr); // int actCount = al.Count(); string keys = string.Join(",", typeCount.Keys.Cast <ActivityType>().Select(x => $"{x.ToString()} {typeCount[x]}").ToArray()); int actCount = 0; logger.LogInformation($"Athlete {AthleteIdStr}, activities found in DB: {actCount} GPX: {trackTable.Keys.Count} ERR: {errTable.Keys.Count} {keys}"); } } if (doRenewMonth != null) { // set all DONE queries for the given month to Created var match = Regex.Match(doRenewMonth, "([0-9]{4})/([0-9]{2})"); int year = Int32.Parse(match.Groups[1].Value); int month = Int32.Parse(match.Groups[2].Value); var statusDone = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.DateFrom == new DateTime(year, month, 01)); foreach (ActivityRangeQuery arq in statusDone.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"reset for {year}/{month} from {QueryStatus.Run} to {QueryStatus.Created}"; logger.LogInformation($"query for {year}/{month} with {QueryStatus.Done} {arq.AthleteId} {arq.Message}"); } logger.LogInformation($"begin: save changes"); db.SaveChanges(); logger.LogInformation($"done: save changes"); } if (doGarbage) { { var statusError = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Error); foreach (ActivityRangeQuery arq in statusError.ToList()) { logger.LogInformation($"query with {QueryStatus.Error} {arq.AthleteId} {arq.Message}"); } } { var statusRun = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Run); foreach (ActivityRangeQuery arq in statusRun.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"garbage-reset from {QueryStatus.Run} to {QueryStatus.Created}"; logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}"); } db.SaveChanges(); } { var statusReserved = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Reserved); foreach (ActivityRangeQuery arq in statusReserved.ToList()) { arq.Status = QueryStatus.Created; arq.StatusChanged = DateTime.Now; arq.Message = $"garbage-reset from {QueryStatus.Reserved} to {QueryStatus.Created}"; logger.LogInformation($"Reset {arq.AthleteId} {arq.DateFrom} {arq.Message}"); db.SaveChanges(); } } } if (doActivityCounts || doAll) { // Find out athlete with open queries: var qAthleteCreated = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct(); logger.LogInformation($"{qAthleteCreated.Count()} with {QueryStatus.Created} queries"); var qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct(); logger.LogInformation($"{qAthleteCreated.Count()} without {QueryStatus.Reserved} queries"); logger.LogInformation($"{qAthleteCreated.Intersect(qAthleteReserved).Count()} with {QueryStatus.Created} and without {QueryStatus.Reserved} queries"); List <string> AthleteIdList = qAthleteCreated.Intersect(qAthleteReserved).Take(10).ToList(); logger.LogInformation($"{AthleteIdList.Count()} athletes from this list:"); foreach (string aId in AthleteIdList) { logger.LogInformation($" AthleteId={aId}"); } if (AthleteIdList.Count() > 0) { // retrieve one random athlete string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count)); logger.LogInformation($"retrieve activity for athlete {aid}"); // 5 first queries for this athlete IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(10).ToList(); foreach (ActivityRangeQuery arq in q0) { logger.LogInformation($" query={arq}"); var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo))); logger.LogInformation($" find {ActivitiesInRange.Count()} activities in it."); } } } if (doAll) { // find activities in a QueryRange foreach (ActivityRangeQuery arq in db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done).OrderByDescending(a => a.DateFrom).Take(10)) { var ActivitiesInRange = db.ActivityShortDB.Where(a => a.AthleteId == arq.AthleteId && ((a.ActivityDate >= arq.DateFrom) && (a.ActivityDate <= arq.DateTo))); logger.LogInformation($" find {ActivitiesInRange.Count()} in range {arq}"); } } if (doAll) { // retrieve queries type for athlete string aId = "2754335"; var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList(); logger.LogInformation($" athl {aId}"); foreach (QueryStatus qs in qsList) { logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}"); } // IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(6).ToList(); } if (doAll) { string aId = "26319532"; var qsList = db.ActivityQueriesDB.Where(a => a.AthleteId == aId).Select(aId => aId.Status).Distinct().ToList(); logger.LogInformation($" athl {aId}"); foreach (QueryStatus qs in qsList) { logger.LogInformation($" query {qs} count {db.ActivityQueriesDB.Where(a => a.AthleteId==aId && a.Status==qs).Count()}"); } } if (doAll || doListimages) { List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList(); logger.LogInformation($" activity count:{activitiesWithImages.Count()}"); int imageCount = 0; foreach (ActivityShort activity in activitiesWithImages) { List <string> images = activity.ActivityImagesList; imageCount += images.Count(); logger.LogInformation($"activity {activity.ActivityTitle}"); foreach (string imageUrl in images) { logger.LogInformation($" url:{imageUrl}"); } } logger.LogInformation($" images {imageCount}"); } if (doAll || doListMaps) { List <ActivityShort> activitiesWithImages = db.ActivityShortDB.Where(a => a.ActivityImagesListAsString.Length > 0 && a.ActivityType == ActivityType.BackcountrySki).ToList(); logger.LogInformation($" activity count:{activitiesWithImages.Count()}"); int imageCount = 0; foreach (ActivityShort activity in activitiesWithImages) { string ImageMapUrl = activity.ActivityImageMapUrl; if (ImageMapUrl == null) { continue; } imageCount++; logger.LogInformation($"activity {activity.ActivityTitle} {ImageMapUrl}"); WebClient webClient = new WebClient(); string outputDir = $"maps/{activity.AthleteId}"; string outputFilename = $"{activity.ActivityId}.png"; logger.LogWarning($"NOT IMPLEMENTED"); } logger.LogInformation($" images {imageCount}/{activitiesWithImages}"); } if (doAll || doAthleteStats) { IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList(); logger.LogInformation($" athletes {AllAthletes.Count()}"); logger.LogInformation($" first: [{AllAthletes.First()}] last: [{AllAthletes.Last()}]"); int AthleteCount = db.ActivityQueriesDB.Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($"athletes in queries {AthleteCount}"); int PrivateAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Done && a.Message.Contains("private")).Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($" private {PrivateAthleteCount}"); int CreatedAthleteCount = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct().Count(); logger.LogInformation($" {QueryStatus.Created} {CreatedAthleteCount}"); } if (doAll || doActivityList) { List <ActivityShort> activities; IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate); if (ActivityTypeStr != null) { ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr); dbs = dbs.Where(a => a.ActivityType == ActivityType); } if (AthleteId != null) { dbs = dbs.Where(a => a.AthleteId == AthleteId); } activities = dbs.ToList(); logger.LogInformation($"List activities for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}"); /* * if (activities.Count>0 && doActivityList) * { * foreach(ActivityShort activity in activities) * { * logger.LogInformation($" {activity}"); * } * } */ } /* * { * // opened queries with activities * int i=0; * int nbAthletes=db.AthleteShortDB.Count(); * foreach(AthleteShort athlete in db.AthleteShortDB) * { * i++; * // logger.LogInformation($"retrieve activity for athlete {athlete.AthleteId}"); * // 5 first queries for this athlete * IList<ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId==athlete.AthleteId && a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(5).ToList(); * int activitiesWithCreatedQueriesCount=0; * foreach(ActivityRangeQuery arq in q0) * { * var ActivitiesInRange = db.ActivityShortDB.Where(a=>a.AthleteId==arq.AthleteId && ((a.ActivityDate>=arq.DateFrom)&&(a.ActivityDate<=arq.DateTo))); * activitiesWithCreatedQueriesCount+=ActivitiesInRange.Count(); * } * if (activitiesWithCreatedQueriesCount>0) * { * logger.LogInformation($" athlete {athlete.AthleteId} as {activitiesWithCreatedQueriesCount} activities with created queries queries:{q0.Count()} activites{db.ActivityShortDB.Where(a=>a.AthleteId==athlete.AthleteId).Count()}."); * } * if (i%10==0) * { * logger.LogInformation($"{i}/{nbAthletes}"); * } * } * } */ /* * // output activity count for activity type * logger.LogInformation($"Activity types Σ :{db.ActivityShortDB.Count()}"); * var ActivityTypes = db.ActivityShortDB.Select(a => a.ActivityType).Distinct(); * foreach(var aType in ActivityTypes) * { * logger.LogInformation($" {aType,18} {db.ActivityShortDB.Count(a => a.ActivityType==aType),6}"); * } */ /* * // output athletes with ski activities and their ski activity count * logger.LogInformation($"All athletes with {ActivityType.BackcountrySki}"); * var Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Select(a => a.AthleteId).Distinct(); * foreach(var A4Type in Activity4Type) * { * var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.BackcountrySki).Where(a => a.AthleteId==A4Type).Count(); * var athlete = db.AthleteShortDB.Find(A4Type); * logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})"); * } */ /* * // output athletes with run activities and their run activity count * logger.LogInformation($"All athletes with {ActivityType.Run}"); * Activity4Type = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Select(a => a.AthleteId).Distinct(); * foreach(var A4Type in Activity4Type) * { * var count = db.ActivityShortDB.Where(a => a.ActivityType==ActivityType.Run).Where(a => a.AthleteId==A4Type).Count(); * var athlete = db.AthleteShortDB.Find(A4Type); * logger.LogInformation($" {athlete?.AthleteName,30} : {A4Type,8} ({count})"); * } */ } ret = 0; } catch (Exception e) { logger.LogInformation($"ERROR:{e.ToString()}"); ret = 1; } return(ret); }