public static async Task InitializeJournalDownloadersAsync(PerformContext context)
        {
            context.WriteLine("Looking for journals do download!");

            using (var scope = Startup.ServiceProvider.CreateScope())
            {
                MSSQLDB db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
                IConfiguration configuration = scope.ServiceProvider.GetRequiredService<IConfiguration>();

                var usersToDownloadJournalsFor = await db.ExecuteListAsync<Shared.Models.User.Profile>(
    @"SELECT *
FROM user_profile
WHERE DATEDIFF(MINUTE, GETUTCDATE(), CAST(JSON_VALUE(user_settings, '$.TokenExpiration') as DATETIMEOFFSET)) > 10
AND last_notification_mail IS NULL
AND skip_download = 0
AND deleted = 0"
                );

                if (usersToDownloadJournalsFor.Count > 0)
                {
                    await SSEActivitySender.SendGlobalActivityAsync("Fetching journals from Elite", $"Downloading journals for {usersToDownloadJournalsFor.Count} users");
                }

                foreach (var user in usersToDownloadJournalsFor)
                {
                    if (RedisJobLock.IsLocked($"JournalDownloader.DownloadJournal.{user.UserIdentifier}")) continue;
                    BackgroundJob.Schedule(() => JournalDownloader.DownloadJournalAsync(user.UserIdentifier, null), TimeSpan.Zero);
                }
            }

            context.WriteLine("All done!");
        }
Exemple #2
0
        public static async Task UpdateEliteSystemsAsync(PerformContext context)
        {
            using (var rlock = new RedisJobLock($"EliteSystemsUpdater.UpdateEliteSystemsAsync"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }
                context.WriteLine("Checking if we need to update the starsystem");

                var _rdb             = SharedSettings.RedisClient.GetDatabase(0);
                var lastCachedUpdate = await _rdb.StringGetAsync("spansh:latestUpdate");

                long systems = 0;

                var url = "https://downloads.spansh.co.uk/galaxy_1day.json.gz";

                if (lastCachedUpdate.IsNull)
                {
                    url = "https://downloads.spansh.co.uk/galaxy_7days.json.gz";
                }

                var batchOfBatches = new List <List <string> >();
                var batchInserts   = new List <string>();

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
                    var hc            = SharedSettings.GetHttpClient(scope);

                    var lastUpdateCheck = await hc.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);

                    context.WriteLine($"Spansh dump was updated: {lastUpdateCheck.Content.Headers.LastModified}");

                    if (lastCachedUpdate.IsNull || lastCachedUpdate != lastUpdateCheck.Content.Headers.LastModified.ToString())
                    {
                        context.WriteLine($"Downloading new system dump from {url}");
                        using (var beep = await hc.GetStreamAsync(url))
                            using (var fileStream = new BufferedStream(beep))
                                using (var gzipStream = new GZipStream(fileStream, CompressionMode.Decompress))
                                    using (var textReader = new StreamReader(gzipStream))
                                    {
                                        while (!textReader.EndOfStream)
                                        {
                                            var line = textReader.ReadLine();
                                            if (!string.IsNullOrWhiteSpace(line) && line.Trim() != "[" && line.Trim() != "]")
                                            {
                                                line = line.Trim(',').Trim();
                                                if (line.StartsWith("{") && line.EndsWith("}"))
                                                {
                                                    systems++;
                                                    var sys   = JsonSerializer.Deserialize <EDSystemData>(line);
                                                    var jsPos = JsonSerializer.Serialize(sys.Coordinates);

                                                    var sql = $@"({sys.Id64}, '{sys.Name.Replace("'", "''")}', '{jsPos}')";
                                                    batchInserts.Add(sql);

                                                    if (batchInserts.Count == 1000)
                                                    {
                                                        batchOfBatches.Add(batchInserts);
                                                        batchInserts = new List <string>();
                                                        context.WriteLine($"Systems in queue to be inserted: {batchOfBatches.Sum(b => b.Count):n0}");
                                                    }
                                                }
                                            }
                                        }
                                    }

                        if (batchInserts.Count > 0)
                        {
                            batchOfBatches.Add(batchInserts);
                        }

                        context.WriteLine($"Systems in queue to be inserted: {batchOfBatches.Sum(b => b.Count):n0}");

                        context.WriteLine($"Fetched {systems} from {url}, generated {batchOfBatches.Count} batches of inserts. Running them now.");

                        foreach (var batch in batchOfBatches.WithProgress(context))
                        {
                            await ExecuteBatchInsert(batch, db);
                        }
                    }

                    await _rdb.StringSetAsyncWithRetries("spansh:latestUpdate", lastUpdateCheck.Content.Headers.LastModified.ToString(), TimeSpan.FromHours(23));
                }

                context.WriteLine("All done!");
            }
        }
        public static async Task UploadAsync(Guid userIdentifier, PerformContext context)
        {
            using (var rlock = new RedisJobLock($"EDSMUserUploader.UploadAsync.{userIdentifier}"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

                    var _minioClient  = scope.ServiceProvider.GetRequiredService <MinioClient>();
                    var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();

                    var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

                    var hc = SharedSettings.GetHttpClient(scope);

                    var user = await db.ExecuteSingleRowAsync <Profile>(
                        @"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.EDSM.enabled'), 'false') = 'true'
AND JSON_VALUE(up.integration_settings, '$.EDSM.apiKey') IS NOT NULL
AND JSON_VALUE(up.integration_settings, '$.EDSM.cmdrName') IS NOT NULL",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    if (user == null)
                    {
                        return;
                    }

                    var edsmSettings = user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>();

                    if (!edsmSettings.Enabled || string.IsNullOrWhiteSpace(edsmSettings.CommanderName) || string.IsNullOrWhiteSpace(edsmSettings.ApiKey))
                    {
                        return;
                    }

                    var userJournals = await db.ExecuteListAsync <UserJournal>(
                        "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    context.WriteLine($"Found {userJournals.Count} to send to EDSM!");

                    (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDSM", context);

                    string lastLine = string.Empty;

                    bool disableIntegration = false;

                    foreach (var journalItem in userJournals.WithProgress(context))
                    {
                        IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDSM");

                        try
                        {
                            using (MemoryStream outFile = new MemoryStream())
                            {
                                var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                                int line_number = ijd.LastSentLineNumber;
                                int delay_time  = 200;

                                var restOfTheLines = journalRows.Skip(line_number).ToList();

                                bool breakJournal = false;

                                foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                                {
                                    lastLine = row;
                                    try
                                    {
                                        if (!string.IsNullOrWhiteSpace(row))
                                        {
                                            var res = await UploadJournalItemToEDSM(hc, row, userIdentifier, edsmSettings, ijd.CurrentGameState, starSystemChecker);


                                            if (res.sentData)
                                            {
                                                delay_time = 1000;
                                            }
                                            else
                                            {
                                                delay_time = 1;
                                            }

                                            switch (res.errorCode)
                                            {
                                            // This is an error from the server, stop working on journals now
                                            case -1:
                                                breakJournal = true;
                                                await discordClient.SendMessageAsync("**[EDSM Upload]** Error code from API", new List <DiscordWebhookEmbed>
                                                {
                                                    new DiscordWebhookEmbed
                                                    {
                                                        Description = res.resultContent,
                                                        Fields      = new Dictionary <string, string>()
                                                        {
                                                            { "User identifier", userIdentifier.ToString() },
                                                            { "Last line", lastLine },
                                                            { "Journal", journalItem.S3Path }
                                                        }.Select(k => new DiscordWebhookEmbedField {
                                                            Name = k.Key, Value = k.Value
                                                        }).ToList()
                                                    }
                                                });

                                                break;

                                            // These codes are OK
                                            case 100:     // OK
                                            case 101:     // Message already stored
                                            case 102:     // Message older than the stored one
                                            case 103:     // Duplicate event request
                                            case 104:     // Crew session
                                                break;

                                            // For these codes we break
                                            case 201:     // Missing commander name
                                            case 202:     // Missing API key
                                            case 203:     // Commander name/API Key not found
                                            case 204:     // Software/Software version not found
                                            case 205:     // Blacklisted software
                                                disableIntegration = true;
                                                breakJournal       = true;
                                                await discordClient.SendMessageAsync("**[EDSM Upload]** Disabled integration for user", new List <DiscordWebhookEmbed>
                                                {
                                                    new DiscordWebhookEmbed
                                                    {
                                                        Description = res.resultContent,
                                                        Fields      = new Dictionary <string, string>()
                                                        {
                                                            { "Status code", res.errorCode.ToString() },
                                                            { "User identifier", userIdentifier.ToString() },
                                                            { "Last line", lastLine },
                                                            { "Journal", journalItem.S3Path }
                                                        }.Select(k => new DiscordWebhookEmbedField {
                                                            Name = k.Key, Value = k.Value
                                                        }).ToList()
                                                    }
                                                });

                                                break;

                                            case 206:     // Cannot decode JSON
                                                break;

                                            // Missing/broken things, just ignore and go ahead with the rest of the import
                                            case 301:     // Message not found
                                            case 302:     // Cannot decode message JSON
                                            case 303:     // Missing timestamp/event from message
                                            case 304:     // Discarded event
                                                break;

                                            // Other errors.. just go ahead, pretend nothing happened
                                            case 401:     // Category unknown
                                            case 402:     // Item unknown
                                            case 451:     // System probably non existant
                                            case 452:     // An entry for the same system already exists just before the visited date.
                                            case 453:     // An entry for the same system already exists just after the visited date.
                                                break;

                                            // Exceptions and debug
                                            case 500:     // Exception: %%
                                            case 501:     // %%
                                            case 502:     // Broken gateway
                                                breakJournal = true;
                                                break;
                                            }
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        if (ex.ToString().Contains("JsonReaderException"))
                                        {
                                            // Ignore rows we cannot parse
                                            context.WriteLine(lastLine);
                                        }
                                        else
                                        {
                                            context.WriteLine("Unhandled exception");
                                            context.WriteLine(lastLine);

                                            await discordClient.SendMessageAsync("**[EDSM Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                            {
                                                new DiscordWebhookEmbed
                                                {
                                                    Description = "Unhandled exception",
                                                    Fields      = new Dictionary <string, string>()
                                                    {
                                                        { "User identifier", userIdentifier.ToString() },
                                                        { "Last line", lastLine },
                                                        { "Journal", journalItem.S3Path },
                                                        { "Exception", ex.ToString() },
                                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                                WriteIndented = true
                                                            }) }
                                                    }.Select(k => new DiscordWebhookEmbedField {
                                                        Name = k.Key, Value = k.Value
                                                    }).ToList()
                                                }
                                            });

                                            throw;
                                        }
                                    }

                                    if (breakJournal)
                                    {
                                        context.WriteLine("Jumping out from the journal");
                                        context.WriteLine(lastLine);
                                        break;
                                    }

                                    line_number++;
                                    ijd.LastSentLineNumber = line_number;
                                    journalItem.IntegrationData["EDSM"] = ijd;

                                    await Task.Delay(delay_time);
                                }

                                await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                                //await db.ExecuteNonQueryAsync(
                                //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                //    new SqlParameter("journal_id", journalItem.JournalId),
                                //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                //);

                                if (breakJournal)
                                {
                                    context.WriteLine("We're breaking off here until next batch, we got told to do that.");
                                    context.WriteLine(lastLine);
                                    break;
                                }

                                if (journalItem.CompleteEntry)
                                {
                                    ijd.LastSentLineNumber = line_number;
                                    ijd.FullySent          = true;
                                    journalItem.IntegrationData["EDSM"] = ijd;

                                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                                    //await db.ExecuteNonQueryAsync(
                                    //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                    //    new SqlParameter("journal_id", journalItem.JournalId),
                                    //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                    //);
                                }
                            }
                            lastJournal = journalItem;
                        }
                        catch (Exception ex)
                        {
                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);

                            //await db.ExecuteNonQueryAsync(
                            //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                            //    new SqlParameter("journal_id", journalItem.JournalId),
                            //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                            //);

                            await discordClient.SendMessageAsync("**[EDSM Upload]** Problem with upload to EDSM", new List <DiscordWebhookEmbed>
                            {
                                new DiscordWebhookEmbed
                                {
                                    Description = ex.ToString(),
                                    Fields      = new Dictionary <string, string>()
                                    {
                                        { "User identifier", userIdentifier.ToString() },
                                        { "Last line", lastLine },
                                        { "Journal", journalItem.S3Path }
                                    }.Select(k => new DiscordWebhookEmbedField {
                                        Name = k.Key, Value = k.Value
                                    }).ToList()
                                }
                            });
                        }
                    }

                    if (disableIntegration)
                    {
                        edsmSettings.Enabled             = false;
                        user.IntegrationSettings["EDSM"] = edsmSettings.AsJsonElement();

                        await db.ExecuteNonQueryAsync(
                            "UPDATE user_profile SET integration_settings = @integration_settings WHERE user_identifier = @user_identifier",
                            new SqlParameter("user_identifier", userIdentifier),
                            new SqlParameter("integration_settings", JsonSerializer.Serialize(user.IntegrationSettings))
                            );
                    }
                }
            }
        }
Exemple #4
0
        public static async Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
        {
            using (var rlock = new RedisJobLock($"CanonnRDUserUploader.UploadAsync.{userIdentifier}"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

                    var _minioClient      = scope.ServiceProvider.GetRequiredService <MinioClient>();
                    var discordClient     = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
                    var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

                    var hc = SharedSettings.GetHttpClient(scope);

                    var user = await db.ExecuteSingleRowAsync <Profile>(
                        @"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.""Canonn R\u0026D"".enabled'), 'false') = 'true'",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    if (user == null)
                    {
                        return;
                    }

                    var canonnRDSettings = user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>();

                    if (!canonnRDSettings.Enabled)
                    {
                        return;
                    }

                    var userPlatform = user.UserSettings.FrontierProfile.Platform;

                    var userJournals = await db.ExecuteListAsync <UserJournal>(
                        "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    context.WriteLine($"Found {userJournals.Count} journals to send to Canonn R&D!");

                    (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "Canonn R&D", context);

                    string lastLine = string.Empty;

                    var _rdb = SharedSettings.RedisClient.GetDatabase(3);
                    var validCanonnEvents = await _rdb.StringGetAsyncWithRetriesSaveIfMissing("canonn:allowedEvents", 10, GetValidCanonnEvents);

                    var canonnEvents = JsonSerializer.Deserialize <List <CanonnEvent> >(validCanonnEvents).Select(e => e.Definition).ToList();

                    context.WriteLine("Valid Canonn events: " + validCanonnEvents);

                    foreach (var journalItem in userJournals.WithProgress(context))
                    {
                        var loggingEnabledRedis = await _rdb.StringGetAsyncWithRetries("logging:canonn:enabled");

                        bool.TryParse(loggingEnabledRedis, out bool loggingEnabled);

                        IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "Canonn R&D");

                        try
                        {
                            using (MemoryStream outFile = new MemoryStream())
                            {
                                var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                                int line_number = ijd.LastSentLineNumber;

                                var restOfTheLines = journalRows.Skip(line_number).ToList();

                                bool breakJournal = false;

                                var journalEvents = new List <Dictionary <string, object> >();

                                foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                                {
                                    lastLine = row;
                                    try
                                    {
                                        if (!string.IsNullOrWhiteSpace(row))
                                        {
                                            var canonnEvent = await UploadJournalItemToCanonnRD(row, cmdrName, ijd.CurrentGameState, canonnEvents, starSystemChecker);

                                            if (canonnEvent != null)
                                            {
                                                canonnEvent["platform"] = userPlatform;
                                                journalEvents.Add(canonnEvent);
                                            }
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);

                                        if (ex.ToString().Contains("JsonReaderException"))
                                        {
                                            context.WriteLine(lastLine);
                                            // Ignore rows we cannot parse
                                        }
                                        else
                                        {
                                            context.WriteLine("Unhandled exception");
                                            context.WriteLine(lastLine);

                                            await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                            {
                                                new DiscordWebhookEmbed
                                                {
                                                    Description = "Unhandled exception",
                                                    Fields      = new Dictionary <string, string>()
                                                    {
                                                        { "User identifier", userIdentifier.ToString() },
                                                        { "Last line", lastLine },
                                                        { "Journal", journalItem.S3Path },
                                                        { "Exception", ex.ToString() },
                                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                                WriteIndented = true
                                                            }) }
                                                    }.Select(k => new DiscordWebhookEmbedField {
                                                        Name = k.Key, Value = k.Value
                                                    }).ToList()
                                                }
                                            });

                                            throw;
                                        }
                                    }

                                    if (breakJournal)
                                    {
                                        context.WriteLine("Jumping out from the journal");
                                        context.WriteLine(lastLine);
                                        break;
                                    }

                                    line_number++;
                                    if (line_number <= journalItem.LastProcessedLineNumber)
                                    {
                                        ijd.LastSentLineNumber = line_number;
                                        journalItem.IntegrationData["Canonn R&D"] = ijd;
                                    }

                                    if (journalEvents.Count == 25)
                                    {
                                        context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
                                        breakJournal = await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);

                                        journalEvents = new List <Dictionary <string, object> >();

                                        await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                        //await db.ExecuteNonQueryAsync(
                                        //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                        //    new SqlParameter("journal_id", journalItem.JournalId),
                                        //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                        //);
                                    }
                                }

                                if (journalEvents.Any())
                                {
                                    context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
                                    await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
                                }

                                await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                //var integration_json = JsonSerializer.Serialize(journalItem.IntegrationData);

                                //await db.ExecuteNonQueryAsync(
                                //    "UPDATE user_journal SET integration_data = JSON_MODIFY(integration_data, '$.\"Canonn R\\u0026D\"', JSON_QUERY(@integration_data)) WHERE journal_id = @journal_id",
                                //    new SqlParameter("journal_id", journalItem.JournalId),
                                //    new SqlParameter("integration_data", integration_json)
                                //);

                                if (breakJournal)
                                {
                                    context.WriteLine("We're breaking off here until next batch, we got told to do that.");
                                    context.WriteLine(lastLine);
                                    break;
                                }

                                if (journalItem.CompleteEntry)
                                {
                                    ijd.LastSentLineNumber = line_number;
                                    ijd.FullySent          = true;
                                    journalItem.IntegrationData["Canonn R&D"] = ijd;

                                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                                    //await db.ExecuteNonQueryAsync(
                                    //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                                    //    new SqlParameter("journal_id", journalItem.JournalId),
                                    //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                                    //);
                                }

                                if (breakJournal)
                                {
                                    context.WriteLine("Break out of the loop, something went wrong");
                                    break;
                                }
                            }
                            lastJournal = journalItem;
                        }
                        catch (Exception ex)
                        {
                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);

                            //await db.ExecuteNonQueryAsync(
                            //    "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
                            //    new SqlParameter("journal_id", journalItem.JournalId),
                            //    new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
                            //);

                            await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Problem with upload to Canonn R&D", new List <DiscordWebhookEmbed>
                            {
                                new DiscordWebhookEmbed
                                {
                                    Description = ex.ToString(),
                                    Fields      = new Dictionary <string, string>()
                                    {
                                        { "User identifier", userIdentifier.ToString() },
                                        { "Last line", lastLine },
                                        { "Journal", journalItem.S3Path },
                                        { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                WriteIndented = true
                                            }) }
                                    }.Select(k => new DiscordWebhookEmbedField {
                                        Name = k.Key, Value = k.Value
                                    }).ToList()
                                }
                            });
                        }
                    }
                }
            }
        }
Exemple #5
0
        public async static Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
        {
            using var rlock = new RedisJobLock($"EDDNUserUploader.UploadAsync.{userIdentifier}");
            if (!rlock.TryTakeLock())
            {
                return;
            }

            using var scope = Startup.ServiceProvider.CreateScope();
            var db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
            var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

            var _minioClient  = scope.ServiceProvider.GetRequiredService <MinioClient>();
            var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();

            var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();

            var hc = SharedSettings.GetHttpClient(scope);

            var user = await db.ExecuteSingleRowAsync <Profile>(
                "SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND deleted = 0 AND send_to_eddn = 1",
                new SqlParameter("user_identifier", userIdentifier)
                );

            if (user == null)
            {
                return;
            }

            var userJournals = await db.ExecuteListAsync <UserJournal>(
                "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number > 0 ORDER BY journal_date ASC",
                new SqlParameter("user_identifier", userIdentifier)
                );

            (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDDN", context);

            context.WriteLine($"Uploading journals for user {userIdentifier}");
            string lastLine = string.Empty;

            foreach (var journalItem in userJournals.WithProgress(context))
            {
                IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDDN");

                if (ijd != null && lastJournal != null && ijd.LastSentLineNumber != lastJournal.SentToEDDNLine)
                {
                    ijd = new IntegrationJournalData
                    {
                        FullySent          = false,
                        LastSentLineNumber = 0,
                        CurrentGameState   = new EDGameState()
                    };
                }

                try
                {
                    using (MemoryStream outFile = new MemoryStream())
                    {
                        var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);

                        int line_number    = journalItem.SentToEDDNLine;
                        int delay_time     = 50;
                        var restOfTheLines = journalRows.Skip(line_number).ToList();

                        foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
                        {
                            lastLine = row;
                            try
                            {
                                if (!string.IsNullOrWhiteSpace(row))
                                {
                                    var time = await UploadJournalItemToEDDN(hc, row, cmdrName, ijd.CurrentGameState, userIdentifier, starSystemChecker, discordClient);

                                    if (time.TotalMilliseconds > 500)
                                    {
                                        delay_time = 500;
                                    }
                                    else if (time.TotalMilliseconds > 250)
                                    {
                                        delay_time = 250;
                                    }
                                    else if (time.TotalMilliseconds > 100)
                                    {
                                        delay_time = 100;
                                    }
                                    else if (time.TotalMilliseconds < 100)
                                    {
                                        delay_time = 50;
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                if (ex.ToString().Contains("JsonReaderException"))
                                {
                                    // Ignore rows we cannot parse
                                    context.WriteLine("Error");
                                    context.WriteLine(ex.ToString());
                                    context.WriteLine(row);
                                }
                                else
                                {
                                    await discordClient.SendMessageAsync("**[EDDN Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
                                    {
                                        new DiscordWebhookEmbed
                                        {
                                            Description = "Unhandled exception",
                                            Fields      = new Dictionary <string, string>()
                                            {
                                                { "User identifier", userIdentifier.ToString() },
                                                { "Last line", lastLine },
                                                { "Journal", journalItem.S3Path },
                                                { "Exception", ex.ToString() },
                                                { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
                                                        WriteIndented = true
                                                    }) }
                                            }.Select(k => new DiscordWebhookEmbedField {
                                                Name = k.Key, Value = k.Value
                                            }).ToList()
                                        }
                                    });

                                    throw;
                                }
                            }

                            line_number++;
                            ijd.LastSentLineNumber = line_number;
                            journalItem.IntegrationData["EDDN"] = ijd;

                            await Task.Delay(delay_time);
                        }

                        await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                        await db.ExecuteNonQueryAsync(
                            "UPDATE user_journal SET sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
                            new SqlParameter("journal_id", journalItem.JournalId),
                            new SqlParameter("line_number", line_number)
                            );

                        if (journalItem.CompleteEntry)
                        {
                            ijd.LastSentLineNumber = line_number;
                            ijd.FullySent          = true;
                            journalItem.IntegrationData["EDDN"] = ijd;

                            await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                            await db.ExecuteNonQueryAsync(
                                "UPDATE user_journal SET sent_to_eddn = 1, sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
                                new SqlParameter("journal_id", journalItem.JournalId),
                                new SqlParameter("line_number", line_number)
                                );
                        }
                    }

                    lastJournal = journalItem;
                }
                catch (Exception ex)
                {
                    await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);

                    await discordClient.SendMessageAsync("**[EDDN Upload]** Problem with upload to EDDN", new List <DiscordWebhookEmbed>
                    {
                        new DiscordWebhookEmbed
                        {
                            Description = ex.ToString(),
                            Fields      = new Dictionary <string, string>()
                            {
                                { "User identifier", userIdentifier.ToString() },
                                { "Last line", lastLine },
                                { "Journal", journalItem.S3Path }
                            }.Select(k => new DiscordWebhookEmbedField {
                                Name = k.Key, Value = k.Value
                            }).ToList()
                        }
                    });
                }
            }
        }
        public static async Task DownloadJournalAsync(Guid userIdentifier, PerformContext context)
        {
            using (var rlock = new RedisJobLock($"JournalDownloader.DownloadJournal.{userIdentifier}"))
            {
                if (!rlock.TryTakeLock())
                {
                    return;
                }

                context.WriteLine($"Looking for journals for user {userIdentifier}");

                using (var scope = Startup.ServiceProvider.CreateScope())
                {
                    MSSQLDB        db            = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
                    IConfiguration configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();

                    MinioClient minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();

                    var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();

                    var user = await db.ExecuteSingleRowAsync <Profile>(
                        @"SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND last_notification_mail IS NULL AND deleted = 0 AND skip_download = 0",
                        new SqlParameter("user_identifier", userIdentifier)
                        );

                    if (user == null)
                    {
                        return;
                    }

                    var authToken = user.UserSettings.AuthToken;

                    var hc = SharedSettings.GetHttpClient(scope);

                    hc.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", authToken);
                    hc.BaseAddress = new Uri("https://companion.orerve.net");

                    var profile = await GetProfileAsync(hc);

                    var profileJson = await profile.Content.ReadAsStringAsync();

                    if (!profile.IsSuccessStatusCode || string.IsNullOrWhiteSpace(profileJson))
                    {
                        context.WriteLine($"Invalid statuscode: {profile.StatusCode} {profile.ReasonPhrase}");
                        bool resetAuth = false;
                        switch (profile.StatusCode)
                        {
                        case HttpStatusCode.BadRequest:
                            // User does not potentially own the game
                            resetAuth = true;
                            break;

                        case HttpStatusCode.Unauthorized:
                            // Invalid token (or Epic games)
                            resetAuth = true;
                            break;
                        }

                        if (string.IsNullOrWhiteSpace(profileJson))
                        {
                            resetAuth = true;
                        }

                        if (resetAuth)
                        {
                            await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
                                                                          "Could not authorize you",
                                                                          "Sorry, but there seems to be something wrong with your account. Please contact us so we can try and figure out what's wrong!"
                                                                          );

                            context.WriteLine("Bailing out early, user doesn't own Elite or has issues with cAPI auth");

                            if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
                            {
                                context.WriteLine("User cannot be fetched, asking them to reauthenticate");
                                await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
                            }
                        }

                        return;
                    }

                    var profileData = JsonSerializer.Deserialize <EliteProfile>(profileJson);

                    context.WriteLine(profileJson);

                    if (profileJson == "{}")
                    {
                        if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
                        {
                            context.WriteLine("User cannot be fetched, asking them to reauthenticate");
                            await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
                        }
                        return;
                    }

                    context.WriteLine($"Downloading journals for {profileData.Commander.Name}");

                    DateTime journalDate = DateTime.Today.AddDays(-25);

                    await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
                                                                  "Downloading your journals",
                                                                  "We're beginning to download your journals now, a few notifications may pop up."
                                                                  );

                    while (journalDate.Date != DateTime.Today)
                    {
                        context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
                        var req = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);

                        if (req.shouldBail)
                        {
                            // Failed to get loop journal
                            context.WriteLine($"Bailing because of errors");
                            return;
                        }

                        journalDate = journalDate.AddDays(1);
                    }

                    context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
                    var reqOut = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);

                    if (reqOut.shouldBail)
                    {
                        // Failed to get loop journal
                        context.WriteLine($"Bailing because of errors");
                        return;
                    }

                    if (user.SendToEDDN && !RedisJobLock.IsLocked($"EDDNUserUploader.UploadAsync.{user.UserIdentifier}"))
                    {
                        var userJournals = await db.ExecuteScalarAsync <long>(
                            "SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number >= sent_to_eddn_line",
                            new SqlParameter("user_identifier", userIdentifier)
                            );

                        if (userJournals > 0)
                        {
                            context.WriteLine($"Sending {userJournals} journals to EDDN");
                            BackgroundJob.Enqueue(() => EDDNUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
                        }
                    }

                    if (user.IntegrationSettings.ContainsKey("EDSM") && user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>().Enabled)
                    {
                        var userJournals = await db.ExecuteScalarAsync <long>(
                            "SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false'",
                            new SqlParameter("user_identifier", userIdentifier)
                            );

                        if (userJournals > 0)
                        {
                            if (!RedisJobLock.IsLocked($"EDSMUserUploader.UploadAsync.{user.UserIdentifier}"))
                            {
                                context.WriteLine($"Sending {userJournals} journals to EDSM");
                                BackgroundJob.Enqueue(() => EDSMUserUploader.UploadAsync(user.UserIdentifier, null));
                            }
                        }
                    }

                    if (user.IntegrationSettings.ContainsKey("Canonn R&D") && user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>().Enabled)
                    {
                        var userJournals = await db.ExecuteScalarAsync <long>(
                            "SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false'",
                            new SqlParameter("user_identifier", userIdentifier)
                            );

                        if (userJournals > 0)
                        {
                            if (!RedisJobLock.IsLocked($"CanonnRDUserUploader.UploadAsync.{user.UserIdentifier}"))
                            {
                                context.WriteLine($"Sending {userJournals} journals to Canonn");
                                BackgroundJob.Enqueue(() => CanonnRDUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
                            }
                        }
                    }

                    context.WriteLine("All done!");
                }
            }
        }