protected override void Context()
 {
     _databaseMediator = A.Fake <IDatabaseMediator>();
     _journalSession   = A.Fake <IJournalSession>();
     _diagramFactory   = A.Fake <IJournalDiagramFactory>();
     sut = new JournalLoader(_journalSession, _databaseMediator, _diagramFactory);
 }
 public override void Refresh()
 {
     if (Dates != null && Items != null)
     {
         var loader = new JournalLoader(this);
         loader.Loaded += delegate { OnRefreshed(); };
         loader.LoadAsync();
     }
 }
 public override void Load()
 {
     if (Dates == null && Items == null)
     {
         var loader = new JournalLoader(this);
         loader.Loaded += delegate { OnLoaded(); };
         loader.LoadAsync();
     }
 }
        public static async Task UploadAsync(Guid userIdentifier, PerformContext context)
        {
            using (var rlock = new RedisJobLock($"EDSMUserUploader.UploadAsync.{userIdentifier}"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

                    var _minioClient  = scope.ServiceProvider.GetRequiredService <MinioClient>();
                    var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();

                    var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

                    var hc = SharedSettings.GetHttpClient(scope);

                    var user = await db.ExecuteSingleRowAsync <Profile>(
                        @"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.EDSM.enabled'), 'false') = 'true'
AND JSON_VALUE(up.integration_settings, '$.EDSM.apiKey') IS NOT NULL
AND JSON_VALUE(up.integration_settings, '$.EDSM.cmdrName') IS NOT NULL",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    if (user == null)
                    {
                        return;
                    }

                    var edsmSettings = user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>();

                    if (!edsmSettings.Enabled || string.IsNullOrWhiteSpace(edsmSettings.CommanderName) || string.IsNullOrWhiteSpace(edsmSettings.ApiKey))
                    {
                        return;
                    }

                    var userJournals = await db.ExecuteListAsync <UserJournal>(
                        "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    context.WriteLine($"Found {userJournals.Count} to send to EDSM!");

                    (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDSM", context);

                    string lastLine = string.Empty;

                    bool disableIntegration = false;

                    foreach (var journalItem in userJournals.WithProgress(context))
                    {
                        IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDSM");

                        try
                        {
                            using (MemoryStream outFile = new MemoryStream())
                            {
                                var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                                int line_number = ijd.LastSentLineNumber;
                                int delay_time  = 200;

                                var restOfTheLines = journalRows.Skip(line_number).ToList();

                                bool breakJournal = false;

                                foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                                {
                                    lastLine = row;
                                    try
                                    {
                                        if (!string.IsNullOrWhiteSpace(row))
                                        {
                                            var res = await UploadJournalItemToEDSM(hc, row, userIdentifier, edsmSettings, ijd.CurrentGameState, starSystemChecker);


                                            if (res.sentData)
                                            {
                                                delay_time = 1000;
                                            }
                                            else
                                            {
                                                delay_time = 1;
                                            }

                                            switch (res.errorCode)
                                            {
                                            // This is an error from the server, stop working on journals now
                                            case -1:
                                                breakJournal = true;
                                                await discordClient.SendMessageAsync("**[EDSM Upload]** Error code from API", new List <DiscordWebhookEmbed>
                                                {
                                                    new DiscordWebhookEmbed
                                                    {
                                                        Description = res.resultContent,
                                                        Fields      = new Dictionary <string, string>()
                                                        {
                                                            { "User identifier", userIdentifier.ToString() },
                                                            { "Last line", lastLine },
                                                            { "Journal", journalItem.S3Path }
                                                        }.Select(k => new DiscordWebhookEmbedField {
                                                            Name = k.Key, Value = k.Value
                                                        }).ToList()
                                                    }
                                                });

                                                break;

                                            // These codes are OK
                                            case 100:     // OK
                                            case 101:     // Message already stored
                                            case 102:     // Message older than the stored one
                                            case 103:     // Duplicate event request
                                            case 104:     // Crew session
                                                break;

                                            // For these codes we break
                                            case 201:     // Missing commander name
                                            case 202:     // Missing API key
                                            case 203:     // Commander name/API Key not found
                                            case 204:     // Software/Software version not found
                                            case 205:     // Blacklisted software
                                                disableIntegration = true;
                                                breakJournal       = true;
                                                await discordClient.SendMessageAsync("**[EDSM Upload]** Disabled integration for user", new List <DiscordWebhookEmbed>
                                                {
                                                    new DiscordWebhookEmbed
                                                    {
                                                        Description = res.resultContent,
                                                        Fields      = new Dictionary <string, string>()
                                                        {
                                                            { "Status code", res.errorCode.ToString() },
                                                            { "User identifier", userIdentifier.ToString() },
                                                            { "Last line", lastLine },
                                                            { "Journal", journalItem.S3Path }
                                                        }.Select(k => new DiscordWebhookEmbedField {
                                                            Name = k.Key, Value = k.Value
                                                        }).ToList()
                                                    }
                                                });

                                                break;

                                            case 206:     // Cannot decode JSON
                                                break;

                                            // Missing/broken things, just ignore and go ahead with the rest of the import
                                            case 301:     // Message not found
                                            case 302:     // Cannot decode message JSON
                                            case 303:     // Missing timestamp/event from message
                                            case 304:     // Discarded event
                                                break;

                                            // Other errors.. just go ahead, pretend nothing happened
                                            case 401:     // Category unknown
                                            case 402:     // Item unknown
                                            case 451:     // System probably non existant
                                            case 452:     // An entry for the same system already exists just before the visited date.
                                            case 453:     // An entry for the same system already exists just after the visited date.
                                                break;

                                            // Exceptions and debug
                                            case 500:     // Exception: %%
                                            case 501:     // %%
                                            case 502:     // Broken gateway
                                                breakJournal = true;
                                                break;
                                            }
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        if (ex.ToString().Contains("JsonReaderException"))
                                        {
                                            // Ignore rows we cannot parse
                                            context.WriteLine(lastLine);
                                        }
                                        else
                                        {
                                            context.WriteLine("Unhandled exception");
                                            context.WriteLine(lastLine);

                                            await discordClient.SendMessageAsync("**[EDSM Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                            {
                                                new DiscordWebhookEmbed
                                                {
                                                    Description = "Unhandled exception",
                                                    Fields      = new Dictionary <string, string>()
                                                    {
                                                        { "User identifier", userIdentifier.ToString() },
                                                        { "Last line", lastLine },
                                                        { "Journal", journalItem.S3Path },
                                                        { "Exception", ex.ToString() },
                                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                                WriteIndented = true
                                                            }) }
                                                    }.Select(k => new DiscordWebhookEmbedField {
                                                        Name = k.Key, Value = k.Value
                                                    }).ToList()
                                                }
                                            });

                                            throw;
                                        }
                                    }

                                    if (breakJournal)
                                    {
                                        context.WriteLine("Jumping out from the journal");
                                        context.WriteLine(lastLine);
                                        break;
                                    }

                                    line_number++;
                                    ijd.LastSentLineNumber = line_number;
                                    journalItem.IntegrationData["EDSM"] = ijd;

                                    await Task.Delay(delay_time);
                                }

                                await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                                //await db.ExecuteNonQueryAsync(
                                //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                //    new SqlParameter("journal_id", journalItem.JournalId),
                                //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                //);

                                if (breakJournal)
                                {
                                    context.WriteLine("We're breaking off here until next batch, we got told to do that.");
                                    context.WriteLine(lastLine);
                                    break;
                                }

                                if (journalItem.CompleteEntry)
                                {
                                    ijd.LastSentLineNumber = line_number;
                                    ijd.FullySent          = true;
                                    journalItem.IntegrationData["EDSM"] = ijd;

                                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                                    //await db.ExecuteNonQueryAsync(
                                    //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                    //    new SqlParameter("journal_id", journalItem.JournalId),
                                    //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                    //);
                                }
                            }
                            lastJournal = journalItem;
                        }
                        catch (Exception ex)
                        {
                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                            //await db.ExecuteNonQueryAsync(
                            //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                            //    new SqlParameter("journal_id", journalItem.JournalId),
                            //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                            //);

                            await discordClient.SendMessageAsync("**[EDSM Upload]** Problem with upload to EDSM", new List <DiscordWebhookEmbed>
                            {
                                new DiscordWebhookEmbed
                                {
                                    Description = ex.ToString(),
                                    Fields      = new Dictionary <string, string>()
                                    {
                                        { "User identifier", userIdentifier.ToString() },
                                        { "Last line", lastLine },
                                        { "Journal", journalItem.S3Path }
                                    }.Select(k => new DiscordWebhookEmbedField {
                                        Name = k.Key, Value = k.Value
                                    }).ToList()
                                }
                            });
                        }
                    }

                    if (disableIntegration)
                    {
                        edsmSettings.Enabled             = false;
                        user.IntegrationSettings["EDSM"] = edsmSettings.AsJsonElement();

                        await db.ExecuteNonQueryAsync(
                            "UPDATE user_profile SET integration_settings = @integration_settings WHERE user_identifier = @user_identifier",
                            new SqlParameter("user_identifier", userIdentifier),
                            new SqlParameter("integration_settings", JsonSerializer.Serialize(user.IntegrationSettings))
                            );
                    }
                }
            }
        }
Exemple #5
0
        public static async Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
        {
            using (var rlock = new RedisJobLock($"CanonnRDUserUploader.UploadAsync.{userIdentifier}"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

                    var _minioClient      = scope.ServiceProvider.GetRequiredService <MinioClient>();
                    var discordClient     = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
                    var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

                    var hc = SharedSettings.GetHttpClient(scope);

                    var user = await db.ExecuteSingleRowAsync <Profile>(
                        @"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.""Canonn R\u0026D"".enabled'), 'false') = 'true'",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    if (user == null)
                    {
                        return;
                    }

                    var canonnRDSettings = user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>();

                    if (!canonnRDSettings.Enabled)
                    {
                        return;
                    }

                    var userPlatform = user.UserSettings.FrontierProfile.Platform;

                    var userJournals = await db.ExecuteListAsync <UserJournal>(
                        "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    context.WriteLine($"Found {userJournals.Count} journals to send to Canonn R&D!");

                    (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "Canonn R&D", context);

                    string lastLine = string.Empty;

                    var _rdb = SharedSettings.RedisClient.GetDatabase(3);
                    var validCanonnEvents = await _rdb.StringGetAsyncWithRetriesSaveIfMissing("canonn:allowedEvents", 10, GetValidCanonnEvents);

                    var canonnEvents = JsonSerializer.Deserialize <List <CanonnEvent> >(validCanonnEvents).Select(e => e.Definition).ToList();

                    context.WriteLine("Valid Canonn events: " + validCanonnEvents);

                    foreach (var journalItem in userJournals.WithProgress(context))
                    {
                        var loggingEnabledRedis = await _rdb.StringGetAsyncWithRetries("logging:canonn:enabled");

                        bool.TryParse(loggingEnabledRedis, out bool loggingEnabled);

                        IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "Canonn R&D");

                        try
                        {
                            using (MemoryStream outFile = new MemoryStream())
                            {
                                var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                                int line_number = ijd.LastSentLineNumber;

                                var restOfTheLines = journalRows.Skip(line_number).ToList();

                                bool breakJournal = false;

                                var journalEvents = new List <Dictionary <string, object> >();

                                foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                                {
                                    lastLine = row;
                                    try
                                    {
                                        if (!string.IsNullOrWhiteSpace(row))
                                        {
                                            var canonnEvent = await UploadJournalItemToCanonnRD(row, cmdrName, ijd.CurrentGameState, canonnEvents, starSystemChecker);

                                            if (canonnEvent != null)
                                            {
                                                canonnEvent["platform"] = userPlatform;
                                                journalEvents.Add(canonnEvent);
                                            }
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);

                                        if (ex.ToString().Contains("JsonReaderException"))
                                        {
                                            context.WriteLine(lastLine);
                                            // Ignore rows we cannot parse
                                        }
                                        else
                                        {
                                            context.WriteLine("Unhandled exception");
                                            context.WriteLine(lastLine);

                                            await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                            {
                                                new DiscordWebhookEmbed
                                                {
                                                    Description = "Unhandled exception",
                                                    Fields      = new Dictionary <string, string>()
                                                    {
                                                        { "User identifier", userIdentifier.ToString() },
                                                        { "Last line", lastLine },
                                                        { "Journal", journalItem.S3Path },
                                                        { "Exception", ex.ToString() },
                                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                                WriteIndented = true
                                                            }) }
                                                    }.Select(k => new DiscordWebhookEmbedField {
                                                        Name = k.Key, Value = k.Value
                                                    }).ToList()
                                                }
                                            });

                                            throw;
                                        }
                                    }

                                    if (breakJournal)
                                    {
                                        context.WriteLine("Jumping out from the journal");
                                        context.WriteLine(lastLine);
                                        break;
                                    }

                                    line_number++;
                                    if (line_number <= journalItem.LastProcessedLineNumber)
                                    {
                                        ijd.LastSentLineNumber = line_number;
                                        journalItem.IntegrationData["Canonn R&D"] = ijd;
                                    }

                                    if (journalEvents.Count == 25)
                                    {
                                        context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
                                        breakJournal = await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);

                                        journalEvents = new List <Dictionary <string, object> >();

                                        await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                        //await db.ExecuteNonQueryAsync(
                                        //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                        //    new SqlParameter("journal_id", journalItem.JournalId),
                                        //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                        //);
                                    }
                                }

                                if (journalEvents.Any())
                                {
                                    context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
                                    await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
                                }

                                await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                //var integration_json = JsonSerializer.Serialize(journalItem.IntegrationData);

                                //await db.ExecuteNonQueryAsync(
                                //    "UPDATE user_journal SET integration_data = JSON_MODIFY(integration_data, '$.\"Canonn R\\u0026D\"', JSON_QUERY(@integration_data)) WHERE journal_id = @journal_id",
                                //    new SqlParameter("journal_id", journalItem.JournalId),
                                //    new SqlParameter("integration_data", integration_json)
                                //);

                                if (breakJournal)
                                {
                                    context.WriteLine("We're breaking off here until next batch, we got told to do that.");
                                    context.WriteLine(lastLine);
                                    break;
                                }

                                if (journalItem.CompleteEntry)
                                {
                                    ijd.LastSentLineNumber = line_number;
                                    ijd.FullySent          = true;
                                    journalItem.IntegrationData["Canonn R&D"] = ijd;

                                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                    //await db.ExecuteNonQueryAsync(
                                    //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                    //    new SqlParameter("journal_id", journalItem.JournalId),
                                    //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                    //);
                                }

                                if (breakJournal)
                                {
                                    context.WriteLine("Break out of the loop, something went wrong");
                                    break;
                                }
                            }
                            lastJournal = journalItem;
                        }
                        catch (Exception ex)
                        {
                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                            //await db.ExecuteNonQueryAsync(
                            //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                            //    new SqlParameter("journal_id", journalItem.JournalId),
                            //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                            //);

                            await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Problem with upload to Canonn R&D", new List <DiscordWebhookEmbed>
                            {
                                new DiscordWebhookEmbed
                                {
                                    Description = ex.ToString(),
                                    Fields      = new Dictionary <string, string>()
                                    {
                                        { "User identifier", userIdentifier.ToString() },
                                        { "Last line", lastLine },
                                        { "Journal", journalItem.S3Path },
                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                WriteIndented = true
                                            }) }
                                    }.Select(k => new DiscordWebhookEmbedField {
                                        Name = k.Key, Value = k.Value
                                    }).ToList()
                                }
                            });
                        }
                    }
                }
            }
        }
Exemple #6
0
        public async static Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
        {
            using var rlock = new RedisJobLock($"EDDNUserUploader.UploadAsync.{userIdentifier}");
            if (!rlock.TryTakeLock())
            {
                return;
            }

            using var scope = Startup.ServiceProvider.CreateScope();
            var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
            var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

            var _minioClient  = scope.ServiceProvider.GetRequiredService <MinioClient>();
            var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();

            var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

            var hc = SharedSettings.GetHttpClient(scope);

            var user = await db.ExecuteSingleRowAsync <Profile>(
                "SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND deleted = 0 AND send_to_eddn = 1",
                new SqlParameter("user_identifier", userIdentifier)
                );

            if (user == null)
            {
                return;
            }

            var userJournals = await db.ExecuteListAsync <UserJournal>(
                "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number > 0 ORDER BY journal_date ASC",
                new SqlParameter("user_identifier", userIdentifier)
                );

            (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDDN", context);

            context.WriteLine($"Uploading journals for user {userIdentifier}");
            string lastLine = string.Empty;

            foreach (var journalItem in userJournals.WithProgress(context))
            {
                IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDDN");

                if (ijd != null && lastJournal != null && ijd.LastSentLineNumber != lastJournal.SentToEDDNLine)
                {
                    ijd = new IntegrationJournalData
                    {
                        FullySent          = false,
                        LastSentLineNumber = 0,
                        CurrentGameState   = new EDGameState()
                    };
                }

                try
                {
                    using (MemoryStream outFile = new MemoryStream())
                    {
                        var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                        int line_number    = journalItem.SentToEDDNLine;
                        int delay_time     = 50;
                        var restOfTheLines = journalRows.Skip(line_number).ToList();

                        foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                        {
                            lastLine = row;
                            try
                            {
                                if (!string.IsNullOrWhiteSpace(row))
                                {
                                    var time = await UploadJournalItemToEDDN(hc, row, cmdrName, ijd.CurrentGameState, userIdentifier, starSystemChecker, discordClient);

                                    if (time.TotalMilliseconds > 500)
                                    {
                                        delay_time = 500;
                                    }
                                    else if (time.TotalMilliseconds > 250)
                                    {
                                        delay_time = 250;
                                    }
                                    else if (time.TotalMilliseconds > 100)
                                    {
                                        delay_time = 100;
                                    }
                                    else if (time.TotalMilliseconds < 100)
                                    {
                                        delay_time = 50;
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                if (ex.ToString().Contains("JsonReaderException"))
                                {
                                    // Ignore rows we cannot parse
                                    context.WriteLine("Error");
                                    context.WriteLine(ex.ToString());
                                    context.WriteLine(row);
                                }
                                else
                                {
                                    await discordClient.SendMessageAsync("**[EDDN Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                    {
                                        new DiscordWebhookEmbed
                                        {
                                            Description = "Unhandled exception",
                                            Fields      = new Dictionary <string, string>()
                                            {
                                                { "User identifier", userIdentifier.ToString() },
                                                { "Last line", lastLine },
                                                { "Journal", journalItem.S3Path },
                                                { "Exception", ex.ToString() },
                                                { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                        WriteIndented = true
                                                    }) }
                                            }.Select(k => new DiscordWebhookEmbedField {
                                                Name = k.Key, Value = k.Value
                                            }).ToList()
                                        }
                                    });

                                    throw;
                                }
                            }

                            line_number++;
                            ijd.LastSentLineNumber = line_number;
                            journalItem.IntegrationData["EDDN"] = ijd;

                            await Task.Delay(delay_time);
                        }

                        await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                        await db.ExecuteNonQueryAsync(
                            "UPDATE user_journal SET sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
                            new SqlParameter("journal_id", journalItem.JournalId),
                            new SqlParameter("line_number", line_number)
                            );

                        if (journalItem.CompleteEntry)
                        {
                            ijd.LastSentLineNumber = line_number;
                            ijd.FullySent          = true;
                            journalItem.IntegrationData["EDDN"] = ijd;

                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                            await db.ExecuteNonQueryAsync(
                                "UPDATE user_journal SET sent_to_eddn = 1, sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
                                new SqlParameter("journal_id", journalItem.JournalId),
                                new SqlParameter("line_number", line_number)
                                );
                        }
                    }

                    lastJournal = journalItem;
                }
                catch (Exception ex)
                {
                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                    await discordClient.SendMessageAsync("**[EDDN Upload]** Problem with upload to EDDN", new List <DiscordWebhookEmbed>
                    {
                        new DiscordWebhookEmbed
                        {
                            Description = ex.ToString(),
                            Fields      = new Dictionary <string, string>()
                            {
                                { "User identifier", userIdentifier.ToString() },
                                { "Last line", lastLine },
                                { "Journal", journalItem.S3Path }
                            }.Select(k => new DiscordWebhookEmbedField {
                                Name = k.Key, Value = k.Value
                            }).ToList()
                        }
                    });
                }
            }
        }