Пример #1
0
        static int queryAthlete(string aid, string clientId, StravaXApi stravaXApi, StravaXApiContext db, bool doVerbose)
        {
            int ret = 0;
            IList <ActivityRangeQuery> q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();

            while (q0.Count > 0 && ret == 0)
            {
                // Mark all queries with "RESERVED"
                IList <ActivityRangeQuery> queries = new List <ActivityRangeQuery>();
                foreach (ActivityRangeQuery arq in q0)
                {
                    try
                    {
                        arq.Status        = QueryStatus.Reserved;
                        arq.StatusChanged = DateTime.Now;
                        arq.Message       = $"Reserved by {clientId}";
                        queries.Add(arq);
                    }
                    catch (DbUpdateConcurrencyException)
                    {
                        // Just skip this entry if some conflict exists.
                        logger.LogInformation($"skip: Can't reserve query {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2} for {clientId}");
                        continue;
                    }
                }
                db.SaveChanges();
                // Run for all reserved queries
                try
                {
                    ret = queryRange(stravaXApi, db, queries, doVerbose);
                }
                catch (CancelExecution e)
                {
                    // just to clarify that it may happend.
                    // finally will also call in that case.
                    throw e;
                }
                finally
                {
                    // integrity check, status should not be Reserved anymore
                    foreach (ActivityRangeQuery arq in q0)
                    {
                        if (arq.Status == QueryStatus.Reserved)
                        {
                            logger.LogInformation($"WARN: remove reservation on {arq}");
                        }
                    }
                    db.SaveChanges();
                }
                // retrieve next queries for max one year.
                q0 = db.ActivityQueriesDB.Where(a => a.AthleteId == aid && a.Status == QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(12).ToList();
            }
            return(ret);
        }
Пример #2
0
        static internal int ReadAthleteConnectionsForAthlete(StravaXApi stravaXApi, string[] args)
        {
            int ret = -1;

            Console.WriteLine("Read athlete connections with Strava-X-API.");
            String AthleteId = null;
            var    p         = new OptionSet()
            {
                { "a|athleteid=", v => { AthleteId = v; } },
            };

            p.Parse(args);
            if (AthleteId == null)
            {
                p.WriteOptionDescriptions(Console.Out);
                throw new ArgumentException("missing athlete id");
            }

            try
            {
                stravaXApi.signIn();
                AthleteShort AthleteMasterShort;
                using (StravaXApiContext db = new StravaXApiContext())
                {
                    AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
                    if (AthleteMasterShort == null)
                    {
                        AthleteMasterShort = new AthleteShort();
                        // create a dummy master
                        AthleteMasterShort.AthleteId = AthleteId;
                        // [TODO] other parameters should be retrieved with selenium
                        AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
                        db.SaveChanges();
                    }
                    else
                    {
                        // Eagerly Loading prevent the list to be loaded at creation
                        // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
                        db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();

                        Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
                    }

                    string FollowType       = "following";
                    var    AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
                    Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");

                    foreach (AthleteShort _AthleteShort in AthleteShortList)
                    {
                        AthleteShort AthleteShortfromDb;
                        // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
                        AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
                        if (AthleteShortfromDb == null)
                        {
                            // add athlete to the db if need.
                            AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
                        }
                        else
                        {
                            Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
                        }
                        Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
                        // such the connected athlete with they id.
                        AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
                        if (_ConnectedAthleteShort == null)
                        {
                            // add connection if needed.
                            AthleteConnection ac = new AthleteConnection();
                            ac.FromId          = AthleteMasterShort.AthleteId;
                            ac.ToId            = AthleteShortfromDb.AthleteId;
                            ac.Type            = FollowType;
                            ac.ConnectionState = ((ConnectedAthlete)_AthleteShort).ConnectionState;

                            AthleteMasterShort.Connections.Add(ac);
                            Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
                        }
                        else
                        {
                            Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
                        }
                    }
                    db.SaveChanges();
                    Console.WriteLine($"total read = {AthleteShortList.Count}");
                    Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
                    AthleteShortList.Clear();
                }
                using (StravaXApiContext db = new StravaXApiContext())
                {
                    AthleteMasterShort = db.AthleteShortDB.Find(AthleteId);
                    if (AthleteMasterShort == null)
                    {
                        AthleteMasterShort = new AthleteShort();
                        // create a dummy master
                        AthleteMasterShort.AthleteId = AthleteId;
                        // [TODO] other parameters should be retrieved with selenium
                        AthleteMasterShort = db.AthleteShortDB.Add(AthleteMasterShort).Entity;
                        db.SaveChanges();
                    }
                    else
                    {
                        // Eagerly Loading prevent the list to be loaded at creation
                        // https://docs.microsoft.com/de-de/ef/ef6/querying/related-data
                        db.Entry(AthleteMasterShort).Collection(p => p.Connections).Load();

                        Console.WriteLine($"Athlete {AthleteMasterShort.AthleteId} allready enterred with {AthleteMasterShort.Connections.Count} connections {string.Join(',',AthleteMasterShort.Connections)}");
                    }

                    string FollowType       = "followers";
                    var    AthleteShortList = stravaXApi.getConnectedAthetes(AthleteMasterShort, FollowType);
                    Console.WriteLine($"Athlete {AthleteId} has {AthleteShortList.Count} connections");

                    foreach (AthleteShort _AthleteShort in AthleteShortList)
                    {
                        AthleteShort AthleteShortfromDb;
                        // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
                        AthleteShortfromDb = db.AthleteShortDB.Find(_AthleteShort.AthleteId);
                        if (AthleteShortfromDb == null)
                        {
                            // add athlete to the db if need.
                            AthleteShortfromDb = db.AthleteShortDB.Add(_AthleteShort).Entity;
                        }
                        else
                        {
                            Console.WriteLine($"{AthleteShortfromDb.AthleteId} allready in database");
                        }
                        Console.WriteLine($"Enterred Activities: {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
                        // such the connected athlete with they id.
                        AthleteConnection _ConnectedAthleteShort = AthleteMasterShort.Connections.FirstOrDefault(a => a.ToId.Equals(_AthleteShort.AthleteId));
                        if (_ConnectedAthleteShort == null)
                        {
                            // add connection if needed.
                            AthleteConnection ac = new AthleteConnection();
                            ac.FromId = AthleteMasterShort.AthleteId;
                            ac.ToId   = AthleteShortfromDb.AthleteId;
                            ac.Type   = FollowType;

                            AthleteMasterShort.Connections.Add(ac);
                            Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} has {AthleteMasterShort.Connections.Count} connection(s). Added: {_AthleteShort.AthleteId}");
                        }
                        else
                        {
                            Console.WriteLine($"athlete {AthleteMasterShort.AthleteId} already connected to {_AthleteShort.AthleteId} with {AthleteMasterShort.Connections.Count} connection(s)");
                        }
                    }
                    db.SaveChanges();
                    Console.WriteLine($"total read = {AthleteShortList.Count}");
                    Console.WriteLine($"total stored = {db.AthleteShortDB.OrderBy(b => b.AthleteId).Count()}");
                    AthleteShortList.Clear();
                }
                ret = 0;
            }
            catch (Exception e)
            {
                Console.WriteLine($"ERROR:{e.ToString()}");
                ret = 1;
            }
            finally
            {
                stravaXApi.Dispose();
            }
            return(ret);
        }
Пример #3
0
        static void WriteQueriesForAthlete(StravaXApi stravaXApi, StravaXApiContext db, string AthleteId)
        {
            DateTime Now = DateTime.Now;

            lastStravaHttpRequest = DateTime.Now;
            int ToYear  = Now.Year;
            int ToMonth = Now.Month;
            //
            // Verify if queries have to be generated
            //

            // Retrieve all entered queries or the wanted athlete.
            List <ActivityRangeQuery> queriesForAthlete = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).OrderBy(a => a.DateFrom).ToList();
            int FromYear;
            int FromMonth;

            // If we already have an entry, we assume that the entry contains the first activity date, as it is expensive to retrieve its value with Selenium.
            if (queriesForAthlete.Count == 0)
            {
                // Retrieve the first activity date with selenium.
                // may throw PrivateAthleteException.

                // be sure of a one seconde intervall between to requests
                int dt = DateTime.Now.Millisecond - lastStravaHttpRequest.Millisecond;
                if (dt < 2000)
                {
                    System.Threading.Tasks.Task.Delay(dt).Wait();
                }

                DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
                lastStravaHttpRequest = DateTime.Now;

                System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");
                FromYear  = FirstActivityDate.Year;
                FromMonth = FirstActivityDate.Month;
            }
            else
            {
                // retrieve first and last date
                DateTime minDT = queriesForAthlete.First().DateFrom;
                // DateTime maxDT = queriesForPatient.Last().DateFrom;
                FromYear  = minDT.Year;
                FromMonth = minDT.Month;
            }
            Console.WriteLine($"queries enterred:{queriesForAthlete.Count}/total:{db.ActivityQueriesDB.Count()}");
            if (FromYear == ToYear)
            {
                for (int month = FromMonth; month <= ToMonth; month++)
                {
                    AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
                }
            }
            else
            {
                // first year
                for (int month = FromMonth; month <= 12; month++)
                {
                    AddQuery(db, AthleteId, new DateTime(FromYear, month, 1), new DateTime(FromYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
                }
                // all years after
                for (int year = FromYear + 1; year <= ToYear - 1; year++)
                {
                    for (int month = 01; month <= 12; month++)
                    {
                        AddQuery(db, AthleteId, new DateTime(year, month, 1), new DateTime(year, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
                    }
                }
                // last year
                for (int month = 01; month <= ToMonth; month++)
                {
                    // from first day of month to last day of the month
                    AddQuery(db, AthleteId, new DateTime(ToYear, month, 1), new DateTime(ToYear, month, 1).AddMonths(1).AddDays(-1), queriesForAthlete);
                }
            }

            int qCount = db.ActivityQueriesDB.Where(a => a.AthleteId == AthleteId).Count();

            Console.WriteLine($"✅ enterred:{qCount}/total:{db.ActivityQueriesDB.Count()}");
        }
Пример #4
0
        static internal int WriteQueriesForAthletes(StravaXApi stravaXApi)
        {
            int ret = -1;

            Console.WriteLine("Create range queries.");
            using (StravaXApiContext db = new StravaXApiContext())
            {
                stravaXApi.signIn();
                try
                {
                    // https://docs.microsoft.com/en-us/ef/ef6/querying/
                    // First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
                    // Not best praxis but enought for Strava.XApi.
                    IList <AthleteShort> AllAthletes = db.AthleteShortDB.ToList();
                    Stopwatch            stopWatch   = new Stopwatch();
                    stopWatch.Start();
                    foreach (AthleteShort Athlete in AllAthletes)
                    {
                        TimeSpan ts = stopWatch.Elapsed;
                        try
                        {
                            WriteQueriesForAthlete(stravaXApi, db, Athlete.AthleteId);
                            Console.WriteLine($"Athlete:{Athlete} run since:{ts.TotalSeconds}s");
                        }
                        catch (PrivateAthleteException e)
                        {
                            // TODO AB#27 athlete should be marked as private to avoid second visit.
                            // create a dummy Query to prevent next search
                            ActivityRangeQuery arq = new ActivityRangeQuery();
                            arq.AthleteId     = Athlete.AthleteId;
                            arq.DateFrom      = new DateTime(2019, 12, 01);
                            arq.DateTo        = new DateTime(2019, 12, 31);
                            arq.Status        = QueryStatus.Done;
                            arq.Message       = $"private athlete {Athlete.AthleteId}";
                            arq.StatusChanged = DateTime.Now;
                            db.ActivityQueriesDB.Add(arq);
                            Console.WriteLine($"SKIP: private athlete {Athlete.AthleteId} {e.Message}");
                        }
                        catch (TooManyStravaRequestException e)
                        {
                            Console.WriteLine($"Too Many Queries detected {e.Message} need to wait some hours");
                        }
                        catch (Exception e)
                        {
                            Console.WriteLine($"SKIP:{Athlete.AthleteId} {e.ToString()}");
                        }
                        db.SaveChanges();
                    }
                    ret = 0;
                }
                catch (Exception e)
                {
                    Console.WriteLine($"ERROR:{e.ToString()}");
                    ret = -1;
                }
                finally
                {
                    stravaXApi.Dispose();
                }
            }
            return(ret);
        }
Пример #5
0
        static internal int Downloader(StravaXApi stravaXApi, string[] args)
        {
            var loggerFactory = LoggerFactory.Create(builder =>
            {
                builder
                .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
                .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
                .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
                .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
                .AddProvider(new CustomLoggerProvider());
                //.AddEventLog();
            });
            var logger = loggerFactory.CreateLogger <StravaXApi>();

            logger.LogDebug("Log ImageDownloader");
            bool   doGpx           = false;
            string maxCountStr     = null;
            string AthleteId       = null;
            string ActivityTypeStr = null;
            string gpxDir          = "gpx";
            var    p = new OptionSet()
            {
                { "g|gpx", v => { doGpx = true; } },
                { "a|athleteid=", v => { AthleteId = v; } },
                { "at|activity_type=", v => { ActivityTypeStr = v; } },
                { "m|max_count=", v => { maxCountStr = v; } },
                { "d|dirname=", v => { gpxDir = v; } },
            };

            p.Parse(args);
            int ret = -1;

            try
            {
                List <ActivityShort> activities;
                using (StravaXApiContext db = new StravaXApiContext())
                {
                    IQueryable <ActivityShort> dbs = db.ActivityShortDB.OrderByDescending(b => b.ActivityDate);
                    if (ActivityTypeStr != null)
                    {
                        ActivityType ActivityType = (ActivityType)Enum.Parse(typeof(ActivityType), ActivityTypeStr);
                        dbs = dbs.Where(a => a.ActivityType == ActivityType);
                    }
                    if (AthleteId != null)
                    {
                        dbs = dbs.Where(a => a.AthleteId == AthleteId);
                    }
                    // Ignore Activities without image map. They have probably been enterred without gps-track.
                    dbs = dbs.Where(a => a.ActivityImageMapUrl != null);
                    // retrieve the activities from the database.
                    activities = dbs.ToList();
                }
                logger.LogInformation($"BEGIN GPX Download for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
                int countSkipped  = 0;
                int countDownload = 0;
                if (doGpx && activities.Count > 0)
                {
                    bool needTracksDownload = false;
                    // detect if any activity needs a gpx file
                    foreach (ActivityShort activity in activities)
                    {
                        string outputDir      = $"{gpxDir}/{activity.AthleteId}";
                        string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
                        if (!File.Exists($"{outputDir}/{outputFilename}"))
                        {
                            needTracksDownload = true;
                            break;
                        }
                    }
                    // if no track is needed, we could exit without signin in Strava.
                    if (needTracksDownload)
                    {
                        // signin
                        stravaXApi.signIn();
                        // go throw all activities.
                        foreach (ActivityShort activity in activities)
                        {
                            string outputDir = $"{gpxDir}/{activity.AthleteId}";
                            // fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
                            string outputFilename     = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
                            string outputFilenameGZip = $"{outputFilename}.gz";
                            string outputFilenameErr  = $"{outputFilename}.err";
                            // download only if the gpx, gz or the err file do not exists.
                            if (!File.Exists($"{outputDir}/{outputFilename}") && !File.Exists($"{outputDir}/{outputFilenameGZip}") && !File.Exists($"{outputDir}/{outputFilenameErr}"))
                            {
                                // create directory if needed
                                if (!Directory.Exists(outputDir))
                                {
                                    DirectoryInfo DirInfo = Directory.CreateDirectory(outputDir);
                                    logger.LogDebug($"directory for GPX created at {DirInfo.FullName}");
                                }
                                try
                                {
                                    stravaXApi.getActivityGpxSelenium(activity.ActivityId, $"{outputDir}/{outputFilenameGZip}");
                                    countDownload++;
                                }
                                catch (PrivateGpxException e)
                                {
                                    logger.LogDebug($"GPX Track private for {activity.AthleteId}: {e.ActivityId} {e.Message}");
                                    // write error file, do prevent a second try.
                                    using (StreamWriter outputFile = new StreamWriter(Path.Combine(outputDir, outputFilenameErr)))
                                    {
                                        outputFile.WriteLine($"Error while downloading GPX for athlete:{activity.AthleteId} activity{activity.ActivityId}");
                                        outputFile.WriteLine($"{e.Message}");
                                        outputFile.WriteLine($"{e.StackTrace}");
                                    }
                                    countSkipped++;
                                }
                            }
                            else
                            {
                                logger.LogDebug($"GPX Track already downloaded for {activity.AthleteId}");
                                countSkipped++;
                            }
                        }
                    }
                }
                else if (!doGpx)
                {
                    // just retrieve without retrieving the gpx files.
                    int countToDownload = 0;
                    int countDownloaded = 0;
                    foreach (ActivityShort activity in activities)
                    {
                        // logger.LogInformation($"activity {activity.StatShortString} -> {Utils.extractActivityTime(activity)}.");
                        string outputDir = $"{gpxDir}/{activity.AthleteId}";
                        // fi.MoveTo($"{fi.Directory.FullName}/{ActivityId}_{fi.Name}");
                        string outputFilename = $"{activity.ActivityId}_{activity.AthleteId}.gpx";
                        if (!File.Exists($"{outputDir}/{outputFilename}"))
                        {
                            countToDownload++;
                        }
                        else
                        {
                            countDownloaded++;
                        }
                    }
                    logger.LogInformation($"GPX Track to download:{countToDownload} already downloaded:{countDownloaded}");
                }
                logger.LogInformation($"DONE GPX Download {countDownload} (skipped: {countSkipped})for {(AthleteId==null?"all athletes":AthleteId)}/{(ActivityTypeStr==null?"all types":ActivityTypeStr)} :{activities.Count()}");
                // everything done, with return 0 we are telling the container, that there's no need to restart.
                ret = 0;
            }
            catch (Exception e)
            {
                logger.LogError($"ERROR:{e.ToString()}");
                // return error code, container should restart.
                ret = 1;
            }
            finally
            {
                stravaXApi.Dispose();
            }

            return(ret);
        }
Пример #6
0
        static internal int SendQueriesForActivities(StravaXApi stravaXApi, string[] args)
        {
            var loggerFactory = LoggerFactory.Create(builder =>
            {
                builder
                .AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
                .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
                .AddFilter("Strava.XApi.Tools.GpxDownloader", Microsoft.Extensions.Logging.LogLevel.Debug)
                .AddFilter("StravaXApi", Microsoft.Extensions.Logging.LogLevel.Information)
                .AddProvider(new CustomLoggerProvider());
                //.AddEventLog();
            });

            logger = loggerFactory.CreateLogger <StravaXApi>();
            logger.LogDebug("Log Query Activities");

            string clientId      = Guid.NewGuid().ToString();
            int    TimerSeconds  = -1;
            int    TimerExitCode = 2;
            bool   doVerbose     = false;
            var    p             = new OptionSet()
            {
                { "t|timer_sec=", v => { TimerSeconds = int.Parse(v); } },
                { "e|timer_exit_code=", v => { TimerExitCode = int.Parse(v); } },
                { "v|verbose", v => { doVerbose = true; } },
            };

            p.Parse(args);

            int ret = -1;

            logger.LogInformation($"Query activities. Client:{clientId}");
            using (StravaXApiContext db = new StravaXApiContext())
            {
                stravaXApi.signIn();

                // 1) find an athlete with opened queries and without reserved queries.
                // string aid =db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).First().AthleteId;
                // select distinct AthleteId from [dbo].[ActivityQueriesDB] where Status=2
                // intersect
                // select distinct AthleteId from [dbo].[ActivityQueriesDB] where (Status<>0 AND Status<>3)
                var           qAthleteCreated  = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
                var           qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
                List <string> AthleteIdList    = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();

                if (AthleteIdList.Count == 0)
                {
                    logger.LogInformation($"no more athlete to search for. created:{qAthleteCreated.Count()} reserved:{qAthleteReserved.Count()}");
                    // TODO some queries may have been reserved, but no Thread is working on it. It should be usefull to detect and reset.
                }
                else
                {
                    Stopwatch stopWatch = new Stopwatch();
                    stopWatch.Start();
                    ret = 0;
                    try {
                        while (AthleteIdList.Count > 0 && ret == 0)
                        {
                            string aid = AthleteIdList.ElementAt(new Random().Next(AthleteIdList.Count));
                            logger.LogInformation($"retrieve activity for athlete {aid}");

                            // https://docs.microsoft.com/en-us/ef/ef6/querying/
                            // First retrieve all query objects to avoid "New transaction is not allowed because there are other threads running in the session."
                            // Not best praxis but enought for Prototype.
                            // https://stackoverflow.com/a/2656612/281188
                            // IList<ActivityRangeQuery> queries = db.ActivityQueriesDB.Where(a => a.Status==QueryStatus.Created).OrderByDescending(a => a.DateFrom).Take(50).ToList();
                            ret = queryAthlete(aid, clientId, stravaXApi, db, doVerbose);

                            TimeSpan ts = stopWatch.Elapsed;
                            if (TimerSeconds > 0 && ts.TotalSeconds > TimerSeconds)
                            {
                                ret = TimerExitCode;
                                logger.LogInformation($"Timer reached after {ts.ToString()} now exit with {ret}.");
                                // exit with error code, container should restart
                                break;
                            }
                            Boolean KeepRunning = true;
                            if (!KeepRunning || File.Exists("QueryActivities.quit"))
                            {
                                logger.LogInformation($"break {KeepRunning} {Count}");
                                // regular exit, container should ended.
                                ret = 0;
                                break;
                            }
                            // search for a new athlete.
                            qAthleteCreated  = db.ActivityQueriesDB.Where(a => a.Status == QueryStatus.Created).Select(a => a.AthleteId).Distinct();
                            qAthleteReserved = db.ActivityQueriesDB.Where(a => a.Status != QueryStatus.Reserved).Select(a => a.AthleteId).Distinct();
                            AthleteIdList    = qAthleteCreated.Intersect(qAthleteReserved).Take(100).ToList();
                        }
                    }
                    catch (CancelExecution e)
                    {
                        logger.LogInformation($"Execution has been canceled: {e.Message}");
                    }
                }
            }
            return(ret);
        }
Пример #7
0
        static int queryRange(StravaXApi stravaXApi, StravaXApiContext db, IList <ActivityRangeQuery> queries, bool doVerbose)
        {
            int ret = 0;

            foreach (ActivityRangeQuery arq in queries)
            {
                try
                {
                    // https://docs.microsoft.com/en-us/ef/core/saving/concurrency
                    try
                    {
                        // reserve query, mark it as run.
                        arq.Status        = QueryStatus.Run;
                        arq.StatusChanged = DateTime.Now;
                        db.SaveChanges();
                    }
                    catch (DbUpdateConcurrencyException)
                    {
                        // Just skip this entry if some conflict exists.
                        logger.LogInformation($"skip conflicted entry for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
                        continue;
                    }
                    var ActivitiesList        = stravaXApi.getActivities(arq.AthleteId, $"{arq.DateFrom.Year:D4}", $"{arq.DateFrom.Month:D2}");
                    int EnterredActivityCount = 0;
                    foreach (ActivityShort ActivityShort in ActivitiesList)
                    {
                        // Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
                        if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
                        {
                            db.ActivityShortDB.Add(ActivityShort);
                            try{
                                // do it better after foreach, but first check how it works with the concurrentcy exception
                                db.SaveChanges();
                                EnterredActivityCount++;
                            }
                            catch (DbUpdateConcurrencyException)
                            {
                                // Just skip this entry if some conflict exists.
                                logger.LogInformation($"❌ skip conflicted activity {ActivityShort}");
                                continue;
                            }
                        }
                        else
                        {
                            logger.LogInformation($"❌ {ActivityShort.ActivityId} allready in database");
                        }
                    }
                    arq.Status        = QueryStatus.Done;
                    arq.StatusChanged = DateTime.Now;
                    // should not have to save anything.
                    db.SaveChanges();
                    if (doVerbose)
                    {
                        logger.LogInformation($"enterred activity count: {EnterredActivityCount}/{db.ActivityShortDB.Count()} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
                    }
                    else // '.Count()' is pretty expensive, skip it if verbose should be minimal.
                    {
                        logger.LogInformation($"enterred activity count: {EnterredActivityCount} for {arq.AthleteId} at {arq.DateFrom.Year:D4}/{arq.DateFrom.Month:D2}");
                    }
                    ErrorCountConsecutive = 0;
                }
                catch (Exception e)
                {
                    ErrorCountConsecutive++;
                    ErrorCount++;
                    logger.LogInformation($"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}");
                    arq.Status        = QueryStatus.Error;
                    arq.StatusChanged = DateTime.Now;
                    arq.Message       = $"Error: {ErrorCountConsecutive}/3 total:{ErrorCount} -> skip:{arq} {e.Message}";
                    if (ErrorCountConsecutive > 2)
                    {
                        // After 3 consecutive errors, I assume the selenium driver is down. Stop it all.
                        throw e;
                    }
                }

                if (doVerbose)
                {
                    logger.LogInformation($"activities stored:{db.ActivityShortDB.Count()}/{QueryStatus.Created}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Created)}/{QueryStatus.Reserved}:{db.ActivityQueriesDB.Count(a => a.Status==QueryStatus.Reserved)}");
                }

                Count++;

                // Exist when KeepRunning is false (from the debugger),
                // or the file 'QueryActivities.quit' exists.
                //   *Program will exit with "touch QueryActivities.quit" in /app directory in container.
                Boolean KeepRunning = true;
                if (!KeepRunning || File.Exists("QueryActivities.quit.immediatly"))
                {
                    logger.LogInformation($"break {KeepRunning} {Count}");
                    // regular exit, container should ended.
                    throw new CancelExecution("break {KeepRunning} {Count}");
                }
            }
            return(ret);
        }
Пример #8
0
        static internal int ReadActivitiesForAthlete(StravaXApi stravaXApi, string[] args)
        {
            int ret = -1;

            Console.WriteLine("Read athlete activities with Strava-X-API.");

            String AthleteId = null;
            var    p         = new OptionSet()
            {
                { "a|athleteid=", v => { AthleteId = v; } },
            };

            p.Parse(args);
            if (AthleteId == null)
            {
                p.WriteOptionDescriptions(Console.Out);
                throw new ArgumentException("missing athlete id");
            }

            try
            {
                stravaXApi.signIn();
                List <ActivityShort> ActivitiesList = new List <ActivityShort>();

                DateTime FirstActivityDate = stravaXApi.getActivityRange(AthleteId);
                System.Console.WriteLine($"First activity at {FirstActivityDate.Year}/{FirstActivityDate.Month}");

                int      FromYear  = int.Parse(Environment.GetEnvironmentVariable("FROM_YEAR"));
                int      FromMonth = int.Parse(Environment.GetEnvironmentVariable("FROM_MONTH"));
                int      ToYear    = int.Parse(Environment.GetEnvironmentVariable("TO_YEAR"));
                int      ToMonth   = int.Parse(Environment.GetEnvironmentVariable("TO_MONTH"));
                DateTime now       = DateTime.Now;
                for (int year = FromYear; year <= ToYear; year++)
                {
                    for (int month = 01; month <= 12; month++)
                    {
                        if ((year <= FromYear && month < FromMonth) ||  (year >= ToYear && month > ToMonth))
                        {
                            continue;
                        }
                        List <ActivityShort> ActivitiesMonthList;
                        try
                        {
                            ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
                        }
                        catch (StaleElementReferenceException)
                        {
                            // Wait and try again.
                            Thread.Sleep(2000);
                            ActivitiesMonthList = stravaXApi.getActivities(AthleteId, $"{year:D4}", $"{month:D2}");
                        }
                        ActivitiesList.AddRange(ActivitiesMonthList);
                        using (StravaXApiContext db = new StravaXApiContext())
                        {
                            foreach (ActivityShort ActivityShort in ActivitiesList)
                            {
                                Console.WriteLine($"JSON={ActivityShort.SerializePrettyPrint(ActivityShort)}");
                                if (db.ActivityShortDB.Find(ActivityShort.ActivityId) == null)
                                {
                                    db.ActivityShortDB.Add(ActivityShort);
                                    db.SaveChanges();
                                    Console.WriteLine($"Enterred Activities: {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
                                }
                                else
                                {
                                    Console.WriteLine($"{ActivityShort.ActivityId} allready in database");
                                }
                            }
                            Console.WriteLine($"total read = {ActivitiesList.Count}");
                            Console.WriteLine($"total stored = {db.ActivityShortDB.OrderBy(b => b.ActivityId).Count()}");
                            ActivitiesList.Clear();
                        }
                    }
                }
                ret = 0;
            }
            catch (Exception e)
            {
                Console.WriteLine($"ERROR:{e.ToString()}");
                ret = 1;
            }
            finally
            {
                stravaXApi.Dispose();
            }
            return(ret);
        }