public static async Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context) { using (var rlock = new RedisJobLock($"CanonnRDUserUploader.UploadAsync.{userIdentifier}")) { if (!rlock.TryTakeLock()) { return; } using (var scope = Startup.ServiceProvider.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>(); var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>(); var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>(); var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>(); var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>(); var hc = SharedSettings.GetHttpClient(scope); var user = await db.ExecuteSingleRowAsync <Profile>( @"SELECT * FROM user_profile up WHERE up.user_identifier = @user_identifier AND up.deleted = 0 AND ISNULL(JSON_VALUE(up.integration_settings, '$.""Canonn R\u0026D"".enabled'), 'false') = 'true'", new SqlParameter("user_identifier", userIdentifier) ); if (user == null) { return; } var canonnRDSettings = user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>(); if (!canonnRDSettings.Enabled) { return; } var userPlatform = user.UserSettings.FrontierProfile.Platform; var userJournals = await db.ExecuteListAsync <UserJournal>( "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false' ORDER BY journal_date ASC", new SqlParameter("user_identifier", userIdentifier) ); context.WriteLine($"Found {userJournals.Count} journals to send to Canonn R&D!"); (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "Canonn R&D", context); string lastLine = string.Empty; var _rdb = SharedSettings.RedisClient.GetDatabase(3); var validCanonnEvents = await _rdb.StringGetAsyncWithRetriesSaveIfMissing("canonn:allowedEvents", 10, GetValidCanonnEvents); var canonnEvents = JsonSerializer.Deserialize <List <CanonnEvent> >(validCanonnEvents).Select(e => e.Definition).ToList(); context.WriteLine("Valid Canonn events: " + validCanonnEvents); foreach (var journalItem in userJournals.WithProgress(context)) { var loggingEnabledRedis = await _rdb.StringGetAsyncWithRetries("logging:canonn:enabled"); bool.TryParse(loggingEnabledRedis, out bool loggingEnabled); IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "Canonn R&D"); try { using (MemoryStream outFile = new MemoryStream()) { var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile); int line_number = ijd.LastSentLineNumber; var restOfTheLines = journalRows.Skip(line_number).ToList(); bool breakJournal = false; var journalEvents = new List <Dictionary <string, object> >(); foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd"))) { lastLine = row; try { if (!string.IsNullOrWhiteSpace(row)) { var canonnEvent = await UploadJournalItemToCanonnRD(row, cmdrName, ijd.CurrentGameState, canonnEvents, starSystemChecker); if (canonnEvent != null) { canonnEvent["platform"] = userPlatform; journalEvents.Add(canonnEvent); } } } catch (Exception ex) { await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents); if (ex.ToString().Contains("JsonReaderException")) { context.WriteLine(lastLine); // Ignore rows we cannot parse } else { context.WriteLine("Unhandled exception"); context.WriteLine(lastLine); await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled exception", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = "Unhandled exception", Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path }, { "Exception", ex.ToString() }, { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions { WriteIndented = true }) } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); throw; } } if (breakJournal) { context.WriteLine("Jumping out from the journal"); context.WriteLine(lastLine); break; } line_number++; if (line_number <= journalItem.LastProcessedLineNumber) { ijd.LastSentLineNumber = line_number; journalItem.IntegrationData["Canonn R&D"] = ijd; } if (journalEvents.Count == 25) { context.WriteLine($"Got {journalEvents.Count} events to send to Canonn"); breakJournal = await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents); journalEvents = new List <Dictionary <string, object> >(); await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); } } if (journalEvents.Any()) { context.WriteLine($"Got {journalEvents.Count} events to send to Canonn"); await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents); } await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd); //var integration_json = JsonSerializer.Serialize(journalItem.IntegrationData); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = JSON_MODIFY(integration_data, '$.\"Canonn R\\u0026D\"', JSON_QUERY(@integration_data)) WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", integration_json) //); if (breakJournal) { context.WriteLine("We're breaking off here until next batch, we got told to do that."); context.WriteLine(lastLine); break; } if (journalItem.CompleteEntry) { ijd.LastSentLineNumber = line_number; ijd.FullySent = true; journalItem.IntegrationData["Canonn R&D"] = ijd; await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); } if (breakJournal) { context.WriteLine("Break out of the loop, something went wrong"); break; } } lastJournal = journalItem; } catch (Exception ex) { await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Problem with upload to Canonn R&D", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = ex.ToString(), Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path }, { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions { WriteIndented = true }) } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); } } } } }
public static async Task UploadAsync(Guid userIdentifier, PerformContext context) { using (var rlock = new RedisJobLock($"EDSMUserUploader.UploadAsync.{userIdentifier}")) { if (!rlock.TryTakeLock()) { return; } using (var scope = Startup.ServiceProvider.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>(); var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>(); var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>(); var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>(); var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>(); var hc = SharedSettings.GetHttpClient(scope); var user = await db.ExecuteSingleRowAsync <Profile>( @"SELECT * FROM user_profile up WHERE up.user_identifier = @user_identifier AND up.deleted = 0 AND ISNULL(JSON_VALUE(up.integration_settings, '$.EDSM.enabled'), 'false') = 'true' AND JSON_VALUE(up.integration_settings, '$.EDSM.apiKey') IS NOT NULL AND JSON_VALUE(up.integration_settings, '$.EDSM.cmdrName') IS NOT NULL", new SqlParameter("user_identifier", userIdentifier) ); if (user == null) { return; } var edsmSettings = user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>(); if (!edsmSettings.Enabled || string.IsNullOrWhiteSpace(edsmSettings.CommanderName) || string.IsNullOrWhiteSpace(edsmSettings.ApiKey)) { return; } var userJournals = await db.ExecuteListAsync <UserJournal>( "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false' ORDER BY journal_date ASC", new SqlParameter("user_identifier", userIdentifier) ); context.WriteLine($"Found {userJournals.Count} to send to EDSM!"); (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDSM", context); string lastLine = string.Empty; bool disableIntegration = false; foreach (var journalItem in userJournals.WithProgress(context)) { IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDSM"); try { using (MemoryStream outFile = new MemoryStream()) { var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile); int line_number = ijd.LastSentLineNumber; int delay_time = 200; var restOfTheLines = journalRows.Skip(line_number).ToList(); bool breakJournal = false; foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd"))) { lastLine = row; try { if (!string.IsNullOrWhiteSpace(row)) { var res = await UploadJournalItemToEDSM(hc, row, userIdentifier, edsmSettings, ijd.CurrentGameState, starSystemChecker); if (res.sentData) { delay_time = 1000; } else { delay_time = 1; } switch (res.errorCode) { // This is an error from the server, stop working on journals now case -1: breakJournal = true; await discordClient.SendMessageAsync("**[EDSM Upload]** Error code from API", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = res.resultContent, Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); break; // These codes are OK case 100: // OK case 101: // Message already stored case 102: // Message older than the stored one case 103: // Duplicate event request case 104: // Crew session break; // For these codes we break case 201: // Missing commander name case 202: // Missing API key case 203: // Commander name/API Key not found case 204: // Software/Software version not found case 205: // Blacklisted software disableIntegration = true; breakJournal = true; await discordClient.SendMessageAsync("**[EDSM Upload]** Disabled integration for user", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = res.resultContent, Fields = new Dictionary <string, string>() { { "Status code", res.errorCode.ToString() }, { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); break; case 206: // Cannot decode JSON break; // Missing/broken things, just ignore and go ahead with the rest of the import case 301: // Message not found case 302: // Cannot decode message JSON case 303: // Missing timestamp/event from message case 304: // Discarded event break; // Other errors.. just go ahead, pretend nothing happened case 401: // Category unknown case 402: // Item unknown case 451: // System probably non existant case 452: // An entry for the same system already exists just before the visited date. case 453: // An entry for the same system already exists just after the visited date. break; // Exceptions and debug case 500: // Exception: %% case 501: // %% case 502: // Broken gateway breakJournal = true; break; } } } catch (Exception ex) { if (ex.ToString().Contains("JsonReaderException")) { // Ignore rows we cannot parse context.WriteLine(lastLine); } else { context.WriteLine("Unhandled exception"); context.WriteLine(lastLine); await discordClient.SendMessageAsync("**[EDSM Upload]** Unhandled exception", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = "Unhandled exception", Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path }, { "Exception", ex.ToString() }, { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions { WriteIndented = true }) } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); throw; } } if (breakJournal) { context.WriteLine("Jumping out from the journal"); context.WriteLine(lastLine); break; } line_number++; ijd.LastSentLineNumber = line_number; journalItem.IntegrationData["EDSM"] = ijd; await Task.Delay(delay_time); } await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); if (breakJournal) { context.WriteLine("We're breaking off here until next batch, we got told to do that."); context.WriteLine(lastLine); break; } if (journalItem.CompleteEntry) { ijd.LastSentLineNumber = line_number; ijd.FullySent = true; journalItem.IntegrationData["EDSM"] = ijd; await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); } } lastJournal = journalItem; } catch (Exception ex) { await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd); //await db.ExecuteNonQueryAsync( // "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id", // new SqlParameter("journal_id", journalItem.JournalId), // new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData)) //); await discordClient.SendMessageAsync("**[EDSM Upload]** Problem with upload to EDSM", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = ex.ToString(), Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); } } if (disableIntegration) { edsmSettings.Enabled = false; user.IntegrationSettings["EDSM"] = edsmSettings.AsJsonElement(); await db.ExecuteNonQueryAsync( "UPDATE user_profile SET integration_settings = @integration_settings WHERE user_identifier = @user_identifier", new SqlParameter("user_identifier", userIdentifier), new SqlParameter("integration_settings", JsonSerializer.Serialize(user.IntegrationSettings)) ); } } } }
public async static Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context) { using var rlock = new RedisJobLock($"EDDNUserUploader.UploadAsync.{userIdentifier}"); if (!rlock.TryTakeLock()) { return; } using var scope = Startup.ServiceProvider.CreateScope(); var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>(); var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>(); var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>(); var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>(); var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>(); var hc = SharedSettings.GetHttpClient(scope); var user = await db.ExecuteSingleRowAsync <Profile>( "SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND deleted = 0 AND send_to_eddn = 1", new SqlParameter("user_identifier", userIdentifier) ); if (user == null) { return; } var userJournals = await db.ExecuteListAsync <UserJournal>( "SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number > 0 ORDER BY journal_date ASC", new SqlParameter("user_identifier", userIdentifier) ); (EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDDN", context); context.WriteLine($"Uploading journals for user {userIdentifier}"); string lastLine = string.Empty; foreach (var journalItem in userJournals.WithProgress(context)) { IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDDN"); if (ijd != null && lastJournal != null && ijd.LastSentLineNumber != lastJournal.SentToEDDNLine) { ijd = new IntegrationJournalData { FullySent = false, LastSentLineNumber = 0, CurrentGameState = new EDGameState() }; } try { using (MemoryStream outFile = new MemoryStream()) { var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile); int line_number = journalItem.SentToEDDNLine; int delay_time = 50; var restOfTheLines = journalRows.Skip(line_number).ToList(); foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd"))) { lastLine = row; try { if (!string.IsNullOrWhiteSpace(row)) { var time = await UploadJournalItemToEDDN(hc, row, cmdrName, ijd.CurrentGameState, userIdentifier, starSystemChecker, discordClient); if (time.TotalMilliseconds > 500) { delay_time = 500; } else if (time.TotalMilliseconds > 250) { delay_time = 250; } else if (time.TotalMilliseconds > 100) { delay_time = 100; } else if (time.TotalMilliseconds < 100) { delay_time = 50; } } } catch (Exception ex) { if (ex.ToString().Contains("JsonReaderException")) { // Ignore rows we cannot parse context.WriteLine("Error"); context.WriteLine(ex.ToString()); context.WriteLine(row); } else { await discordClient.SendMessageAsync("**[EDDN Upload]** Unhandled exception", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = "Unhandled exception", Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path }, { "Exception", ex.ToString() }, { "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions { WriteIndented = true }) } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); throw; } } line_number++; ijd.LastSentLineNumber = line_number; journalItem.IntegrationData["EDDN"] = ijd; await Task.Delay(delay_time); } await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd); await db.ExecuteNonQueryAsync( "UPDATE user_journal SET sent_to_eddn_line = @line_number WHERE journal_id = @journal_id", new SqlParameter("journal_id", journalItem.JournalId), new SqlParameter("line_number", line_number) ); if (journalItem.CompleteEntry) { ijd.LastSentLineNumber = line_number; ijd.FullySent = true; journalItem.IntegrationData["EDDN"] = ijd; await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd); await db.ExecuteNonQueryAsync( "UPDATE user_journal SET sent_to_eddn = 1, sent_to_eddn_line = @line_number WHERE journal_id = @journal_id", new SqlParameter("journal_id", journalItem.JournalId), new SqlParameter("line_number", line_number) ); } } lastJournal = journalItem; } catch (Exception ex) { await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd); await discordClient.SendMessageAsync("**[EDDN Upload]** Problem with upload to EDDN", new List <DiscordWebhookEmbed> { new DiscordWebhookEmbed { Description = ex.ToString(), Fields = new Dictionary <string, string>() { { "User identifier", userIdentifier.ToString() }, { "Last line", lastLine }, { "Journal", journalItem.S3Path } }.Select(k => new DiscordWebhookEmbedField { Name = k.Key, Value = k.Value }).ToList() } }); } } }