public static async Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
{
using (var rlock = new RedisJobLock($"CanonnRDUserUploader.UploadAsync.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
using (var scope = Startup.ServiceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.""Canonn R\u0026D"".enabled'), 'false') = 'true'",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var canonnRDSettings = user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>();
if (!canonnRDSettings.Enabled)
{
return;
}
var userPlatform = user.UserSettings.FrontierProfile.Platform;
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
context.WriteLine($"Found {userJournals.Count} journals to send to Canonn R&D!");
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "Canonn R&D", context);
string lastLine = string.Empty;
var _rdb = SharedSettings.RedisClient.GetDatabase(3);
var validCanonnEvents = await _rdb.StringGetAsyncWithRetriesSaveIfMissing("canonn:allowedEvents", 10, GetValidCanonnEvents);
var canonnEvents = JsonSerializer.Deserialize <List <CanonnEvent> >(validCanonnEvents).Select(e => e.Definition).ToList();
context.WriteLine("Valid Canonn events: " + validCanonnEvents);
foreach (var journalItem in userJournals.WithProgress(context))
{
var loggingEnabledRedis = await _rdb.StringGetAsyncWithRetries("logging:canonn:enabled");
bool.TryParse(loggingEnabledRedis, out bool loggingEnabled);
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "Canonn R&D");
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = ijd.LastSentLineNumber;
var restOfTheLines = journalRows.Skip(line_number).ToList();
bool breakJournal = false;
var journalEvents = new List <Dictionary <string, object> >();
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var canonnEvent = await UploadJournalItemToCanonnRD(row, cmdrName, ijd.CurrentGameState, canonnEvents, starSystemChecker);
if (canonnEvent != null)
{
canonnEvent["platform"] = userPlatform;
journalEvents.Add(canonnEvent);
}
}
}
catch (Exception ex)
{
await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
if (ex.ToString().Contains("JsonReaderException"))
{
context.WriteLine(lastLine);
// Ignore rows we cannot parse
}
else
{
context.WriteLine("Unhandled exception");
context.WriteLine(lastLine);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
if (breakJournal)
{
context.WriteLine("Jumping out from the journal");
context.WriteLine(lastLine);
break;
}
line_number++;
if (line_number <= journalItem.LastProcessedLineNumber)
{
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["Canonn R&D"] = ijd;
}
if (journalEvents.Count == 25)
{
context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
breakJournal = await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
journalEvents = new List <Dictionary <string, object> >();
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
}
if (journalEvents.Any())
{
context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//var integration_json = JsonSerializer.Serialize(journalItem.IntegrationData);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = JSON_MODIFY(integration_data, '$.\"Canonn R\\u0026D\"', JSON_QUERY(@integration_data)) WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", integration_json)
//);
if (breakJournal)
{
context.WriteLine("We're breaking off here until next batch, we got told to do that.");
context.WriteLine(lastLine);
break;
}
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["Canonn R&D"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
if (breakJournal)
{
context.WriteLine("Break out of the loop, something went wrong");
break;
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Problem with upload to Canonn R&D", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
}
}
}
public static async Task UploadAsync(Guid userIdentifier, PerformContext context)
{
using (var rlock = new RedisJobLock($"EDSMUserUploader.UploadAsync.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
using (var scope = Startup.ServiceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.EDSM.enabled'), 'false') = 'true'
AND JSON_VALUE(up.integration_settings, '$.EDSM.apiKey') IS NOT NULL
AND JSON_VALUE(up.integration_settings, '$.EDSM.cmdrName') IS NOT NULL",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var edsmSettings = user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>();
if (!edsmSettings.Enabled || string.IsNullOrWhiteSpace(edsmSettings.CommanderName) || string.IsNullOrWhiteSpace(edsmSettings.ApiKey))
{
return;
}
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
context.WriteLine($"Found {userJournals.Count} to send to EDSM!");
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDSM", context);
string lastLine = string.Empty;
bool disableIntegration = false;
foreach (var journalItem in userJournals.WithProgress(context))
{
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDSM");
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = ijd.LastSentLineNumber;
int delay_time = 200;
var restOfTheLines = journalRows.Skip(line_number).ToList();
bool breakJournal = false;
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var res = await UploadJournalItemToEDSM(hc, row, userIdentifier, edsmSettings, ijd.CurrentGameState, starSystemChecker);
if (res.sentData)
{
delay_time = 1000;
}
else
{
delay_time = 1;
}
switch (res.errorCode)
{
// This is an error from the server, stop working on journals now
case -1:
breakJournal = true;
await discordClient.SendMessageAsync("**[EDSM Upload]** Error code from API", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
// These codes are OK
case 100: // OK
case 101: // Message already stored
case 102: // Message older than the stored one
case 103: // Duplicate event request
case 104: // Crew session
break;
// For these codes we break
case 201: // Missing commander name
case 202: // Missing API key
case 203: // Commander name/API Key not found
case 204: // Software/Software version not found
case 205: // Blacklisted software
disableIntegration = true;
breakJournal = true;
await discordClient.SendMessageAsync("**[EDSM Upload]** Disabled integration for user", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "Status code", res.errorCode.ToString() },
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
case 206: // Cannot decode JSON
break;
// Missing/broken things, just ignore and go ahead with the rest of the import
case 301: // Message not found
case 302: // Cannot decode message JSON
case 303: // Missing timestamp/event from message
case 304: // Discarded event
break;
// Other errors.. just go ahead, pretend nothing happened
case 401: // Category unknown
case 402: // Item unknown
case 451: // System probably non existant
case 452: // An entry for the same system already exists just before the visited date.
case 453: // An entry for the same system already exists just after the visited date.
break;
// Exceptions and debug
case 500: // Exception: %%
case 501: // %%
case 502: // Broken gateway
breakJournal = true;
break;
}
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("JsonReaderException"))
{
// Ignore rows we cannot parse
context.WriteLine(lastLine);
}
else
{
context.WriteLine("Unhandled exception");
context.WriteLine(lastLine);
await discordClient.SendMessageAsync("**[EDSM Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
if (breakJournal)
{
context.WriteLine("Jumping out from the journal");
context.WriteLine(lastLine);
break;
}
line_number++;
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["EDSM"] = ijd;
await Task.Delay(delay_time);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
if (breakJournal)
{
context.WriteLine("We're breaking off here until next batch, we got told to do that.");
context.WriteLine(lastLine);
break;
}
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["EDSM"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
await discordClient.SendMessageAsync("**[EDSM Upload]** Problem with upload to EDSM", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
if (disableIntegration)
{
edsmSettings.Enabled = false;
user.IntegrationSettings["EDSM"] = edsmSettings.AsJsonElement();
await db.ExecuteNonQueryAsync(
"UPDATE user_profile SET integration_settings = @integration_settings WHERE user_identifier = @user_identifier",
new SqlParameter("user_identifier", userIdentifier),
new SqlParameter("integration_settings", JsonSerializer.Serialize(user.IntegrationSettings))
);
}
}
}
}
public async static Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
{
using var rlock = new RedisJobLock($"EDDNUserUploader.UploadAsync.{userIdentifier}");
if (!rlock.TryTakeLock())
{
return;
}
using var scope = Startup.ServiceProvider.CreateScope();
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
"SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND deleted = 0 AND send_to_eddn = 1",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number > 0 ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDDN", context);
context.WriteLine($"Uploading journals for user {userIdentifier}");
string lastLine = string.Empty;
foreach (var journalItem in userJournals.WithProgress(context))
{
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDDN");
if (ijd != null && lastJournal != null && ijd.LastSentLineNumber != lastJournal.SentToEDDNLine)
{
ijd = new IntegrationJournalData
{
FullySent = false,
LastSentLineNumber = 0,
CurrentGameState = new EDGameState()
};
}
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = journalItem.SentToEDDNLine;
int delay_time = 50;
var restOfTheLines = journalRows.Skip(line_number).ToList();
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var time = await UploadJournalItemToEDDN(hc, row, cmdrName, ijd.CurrentGameState, userIdentifier, starSystemChecker, discordClient);
if (time.TotalMilliseconds > 500)
{
delay_time = 500;
}
else if (time.TotalMilliseconds > 250)
{
delay_time = 250;
}
else if (time.TotalMilliseconds > 100)
{
delay_time = 100;
}
else if (time.TotalMilliseconds < 100)
{
delay_time = 50;
}
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("JsonReaderException"))
{
// Ignore rows we cannot parse
context.WriteLine("Error");
context.WriteLine(ex.ToString());
context.WriteLine(row);
}
else
{
await discordClient.SendMessageAsync("**[EDDN Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
line_number++;
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["EDDN"] = ijd;
await Task.Delay(delay_time);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await db.ExecuteNonQueryAsync(
"UPDATE user_journal SET sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
new SqlParameter("journal_id", journalItem.JournalId),
new SqlParameter("line_number", line_number)
);
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["EDDN"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await db.ExecuteNonQueryAsync(
"UPDATE user_journal SET sent_to_eddn = 1, sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
new SqlParameter("journal_id", journalItem.JournalId),
new SqlParameter("line_number", line_number)
);
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await discordClient.SendMessageAsync("**[EDDN Upload]** Problem with upload to EDDN", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
}