public static async Task InitializeJournalDownloadersAsync(PerformContext context)
{
context.WriteLine("Looking for journals do download!");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService<IConfiguration>();
var usersToDownloadJournalsFor = await db.ExecuteListAsync<Shared.Models.User.Profile>(
@"SELECT *
FROM user_profile
WHERE DATEDIFF(MINUTE, GETUTCDATE(), CAST(JSON_VALUE(user_settings, '$.TokenExpiration') as DATETIMEOFFSET)) > 10
AND last_notification_mail IS NULL
AND skip_download = 0
AND deleted = 0"
);
if (usersToDownloadJournalsFor.Count > 0)
{
await SSEActivitySender.SendGlobalActivityAsync("Fetching journals from Elite", $"Downloading journals for {usersToDownloadJournalsFor.Count} users");
}
foreach (var user in usersToDownloadJournalsFor)
{
if (RedisJobLock.IsLocked($"JournalDownloader.DownloadJournal.{user.UserIdentifier}")) continue;
BackgroundJob.Schedule(() => JournalDownloader.DownloadJournalAsync(user.UserIdentifier, null), TimeSpan.Zero);
}
}
context.WriteLine("All done!");
}
public static async Task UpdateEliteSystemsAsync(PerformContext context)
{
using (var rlock = new RedisJobLock($"EliteSystemsUpdater.UpdateEliteSystemsAsync"))
{
if (!rlock.TryTakeLock())
{
return;
}
context.WriteLine("Checking if we need to update the starsystem");
var _rdb = SharedSettings.RedisClient.GetDatabase(0);
var lastCachedUpdate = await _rdb.StringGetAsync("spansh:latestUpdate");
long systems = 0;
var url = "https://downloads.spansh.co.uk/galaxy_1day.json.gz";
if (lastCachedUpdate.IsNull)
{
url = "https://downloads.spansh.co.uk/galaxy_7days.json.gz";
}
var batchOfBatches = new List <List <string> >();
var batchInserts = new List <string>();
using (var scope = Startup.ServiceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var hc = SharedSettings.GetHttpClient(scope);
var lastUpdateCheck = await hc.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
context.WriteLine($"Spansh dump was updated: {lastUpdateCheck.Content.Headers.LastModified}");
if (lastCachedUpdate.IsNull || lastCachedUpdate != lastUpdateCheck.Content.Headers.LastModified.ToString())
{
context.WriteLine($"Downloading new system dump from {url}");
using (var beep = await hc.GetStreamAsync(url))
using (var fileStream = new BufferedStream(beep))
using (var gzipStream = new GZipStream(fileStream, CompressionMode.Decompress))
using (var textReader = new StreamReader(gzipStream))
{
while (!textReader.EndOfStream)
{
var line = textReader.ReadLine();
if (!string.IsNullOrWhiteSpace(line) && line.Trim() != "[" && line.Trim() != "]")
{
line = line.Trim(',').Trim();
if (line.StartsWith("{") && line.EndsWith("}"))
{
systems++;
var sys = JsonSerializer.Deserialize <EDSystemData>(line);
var jsPos = JsonSerializer.Serialize(sys.Coordinates);
var sql = $@"({sys.Id64}, '{sys.Name.Replace("'", "''")}', '{jsPos}')";
batchInserts.Add(sql);
if (batchInserts.Count == 1000)
{
batchOfBatches.Add(batchInserts);
batchInserts = new List <string>();
context.WriteLine($"Systems in queue to be inserted: {batchOfBatches.Sum(b => b.Count):n0}");
}
}
}
}
}
if (batchInserts.Count > 0)
{
batchOfBatches.Add(batchInserts);
}
context.WriteLine($"Systems in queue to be inserted: {batchOfBatches.Sum(b => b.Count):n0}");
context.WriteLine($"Fetched {systems} from {url}, generated {batchOfBatches.Count} batches of inserts. Running them now.");
foreach (var batch in batchOfBatches.WithProgress(context))
{
await ExecuteBatchInsert(batch, db);
}
}
await _rdb.StringSetAsyncWithRetries("spansh:latestUpdate", lastUpdateCheck.Content.Headers.LastModified.ToString(), TimeSpan.FromHours(23));
}
context.WriteLine("All done!");
}
}
public static async Task UploadAsync(Guid userIdentifier, PerformContext context)
{
using (var rlock = new RedisJobLock($"EDSMUserUploader.UploadAsync.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
using (var scope = Startup.ServiceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.EDSM.enabled'), 'false') = 'true'
AND JSON_VALUE(up.integration_settings, '$.EDSM.apiKey') IS NOT NULL
AND JSON_VALUE(up.integration_settings, '$.EDSM.cmdrName') IS NOT NULL",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var edsmSettings = user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>();
if (!edsmSettings.Enabled || string.IsNullOrWhiteSpace(edsmSettings.CommanderName) || string.IsNullOrWhiteSpace(edsmSettings.ApiKey))
{
return;
}
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
context.WriteLine($"Found {userJournals.Count} to send to EDSM!");
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDSM", context);
string lastLine = string.Empty;
bool disableIntegration = false;
foreach (var journalItem in userJournals.WithProgress(context))
{
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDSM");
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = ijd.LastSentLineNumber;
int delay_time = 200;
var restOfTheLines = journalRows.Skip(line_number).ToList();
bool breakJournal = false;
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var res = await UploadJournalItemToEDSM(hc, row, userIdentifier, edsmSettings, ijd.CurrentGameState, starSystemChecker);
if (res.sentData)
{
delay_time = 1000;
}
else
{
delay_time = 1;
}
switch (res.errorCode)
{
// This is an error from the server, stop working on journals now
case -1:
breakJournal = true;
await discordClient.SendMessageAsync("**[EDSM Upload]** Error code from API", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
// These codes are OK
case 100: // OK
case 101: // Message already stored
case 102: // Message older than the stored one
case 103: // Duplicate event request
case 104: // Crew session
break;
// For these codes we break
case 201: // Missing commander name
case 202: // Missing API key
case 203: // Commander name/API Key not found
case 204: // Software/Software version not found
case 205: // Blacklisted software
disableIntegration = true;
breakJournal = true;
await discordClient.SendMessageAsync("**[EDSM Upload]** Disabled integration for user", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = res.resultContent,
Fields = new Dictionary <string, string>()
{
{ "Status code", res.errorCode.ToString() },
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
break;
case 206: // Cannot decode JSON
break;
// Missing/broken things, just ignore and go ahead with the rest of the import
case 301: // Message not found
case 302: // Cannot decode message JSON
case 303: // Missing timestamp/event from message
case 304: // Discarded event
break;
// Other errors.. just go ahead, pretend nothing happened
case 401: // Category unknown
case 402: // Item unknown
case 451: // System probably non existant
case 452: // An entry for the same system already exists just before the visited date.
case 453: // An entry for the same system already exists just after the visited date.
break;
// Exceptions and debug
case 500: // Exception: %%
case 501: // %%
case 502: // Broken gateway
breakJournal = true;
break;
}
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("JsonReaderException"))
{
// Ignore rows we cannot parse
context.WriteLine(lastLine);
}
else
{
context.WriteLine("Unhandled exception");
context.WriteLine(lastLine);
await discordClient.SendMessageAsync("**[EDSM Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
if (breakJournal)
{
context.WriteLine("Jumping out from the journal");
context.WriteLine(lastLine);
break;
}
line_number++;
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["EDSM"] = ijd;
await Task.Delay(delay_time);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
if (breakJournal)
{
context.WriteLine("We're breaking off here until next batch, we got told to do that.");
context.WriteLine(lastLine);
break;
}
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["EDSM"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDSM, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
await discordClient.SendMessageAsync("**[EDSM Upload]** Problem with upload to EDSM", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
if (disableIntegration)
{
edsmSettings.Enabled = false;
user.IntegrationSettings["EDSM"] = edsmSettings.AsJsonElement();
await db.ExecuteNonQueryAsync(
"UPDATE user_profile SET integration_settings = @integration_settings WHERE user_identifier = @user_identifier",
new SqlParameter("user_identifier", userIdentifier),
new SqlParameter("integration_settings", JsonSerializer.Serialize(user.IntegrationSettings))
);
}
}
}
}
public static async Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
{
using (var rlock = new RedisJobLock($"CanonnRDUserUploader.UploadAsync.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
using (var scope = Startup.ServiceProvider.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT *
FROM user_profile up
WHERE up.user_identifier = @user_identifier
AND up.deleted = 0
AND ISNULL(JSON_VALUE(up.integration_settings, '$.""Canonn R\u0026D"".enabled'), 'false') = 'true'",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var canonnRDSettings = user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>();
if (!canonnRDSettings.Enabled)
{
return;
}
var userPlatform = user.UserSettings.FrontierProfile.Platform;
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false' ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
context.WriteLine($"Found {userJournals.Count} journals to send to Canonn R&D!");
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "Canonn R&D", context);
string lastLine = string.Empty;
var _rdb = SharedSettings.RedisClient.GetDatabase(3);
var validCanonnEvents = await _rdb.StringGetAsyncWithRetriesSaveIfMissing("canonn:allowedEvents", 10, GetValidCanonnEvents);
var canonnEvents = JsonSerializer.Deserialize <List <CanonnEvent> >(validCanonnEvents).Select(e => e.Definition).ToList();
context.WriteLine("Valid Canonn events: " + validCanonnEvents);
foreach (var journalItem in userJournals.WithProgress(context))
{
var loggingEnabledRedis = await _rdb.StringGetAsyncWithRetries("logging:canonn:enabled");
bool.TryParse(loggingEnabledRedis, out bool loggingEnabled);
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "Canonn R&D");
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = ijd.LastSentLineNumber;
var restOfTheLines = journalRows.Skip(line_number).ToList();
bool breakJournal = false;
var journalEvents = new List <Dictionary <string, object> >();
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var canonnEvent = await UploadJournalItemToCanonnRD(row, cmdrName, ijd.CurrentGameState, canonnEvents, starSystemChecker);
if (canonnEvent != null)
{
canonnEvent["platform"] = userPlatform;
journalEvents.Add(canonnEvent);
}
}
}
catch (Exception ex)
{
await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
if (ex.ToString().Contains("JsonReaderException"))
{
context.WriteLine(lastLine);
// Ignore rows we cannot parse
}
else
{
context.WriteLine("Unhandled exception");
context.WriteLine(lastLine);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
if (breakJournal)
{
context.WriteLine("Jumping out from the journal");
context.WriteLine(lastLine);
break;
}
line_number++;
if (line_number <= journalItem.LastProcessedLineNumber)
{
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["Canonn R&D"] = ijd;
}
if (journalEvents.Count == 25)
{
context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
breakJournal = await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
journalEvents = new List <Dictionary <string, object> >();
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
}
if (journalEvents.Any())
{
context.WriteLine($"Got {journalEvents.Count} events to send to Canonn");
await SendEventBatch(userIdentifier, context, configuration, discordClient, hc, lastLine, journalItem, loggingEnabled, ijd, journalEvents);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//var integration_json = JsonSerializer.Serialize(journalItem.IntegrationData);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = JSON_MODIFY(integration_data, '$.\"Canonn R\\u0026D\"', JSON_QUERY(@integration_data)) WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", integration_json)
//);
if (breakJournal)
{
context.WriteLine("We're breaking off here until next batch, we got told to do that.");
context.WriteLine(lastLine);
break;
}
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["Canonn R&D"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
}
if (breakJournal)
{
context.WriteLine("Break out of the loop, something went wrong");
break;
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.CanonnRD, ijd);
//await db.ExecuteNonQueryAsync(
// "UPDATE user_journal SET integration_data = @integration_data WHERE journal_id = @journal_id",
// new SqlParameter("journal_id", journalItem.JournalId),
// new SqlParameter("integration_data", JsonSerializer.Serialize(journalItem.IntegrationData))
//);
await discordClient.SendMessageAsync("**[Canonn R&D Upload]** Problem with upload to Canonn R&D", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
}
}
}
public async static Task UploadAsync(Guid userIdentifier, string cmdrName, PerformContext context)
{
using var rlock = new RedisJobLock($"EDDNUserUploader.UploadAsync.{userIdentifier}");
if (!rlock.TryTakeLock())
{
return;
}
using var scope = Startup.ServiceProvider.CreateScope();
var db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
var _minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var starSystemChecker = scope.ServiceProvider.GetRequiredService <StarSystemChecker>();
var hc = SharedSettings.GetHttpClient(scope);
var user = await db.ExecuteSingleRowAsync <Profile>(
"SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND deleted = 0 AND send_to_eddn = 1",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var userJournals = await db.ExecuteListAsync <UserJournal>(
"SELECT * FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number > 0 ORDER BY journal_date ASC",
new SqlParameter("user_identifier", userIdentifier)
);
(EDGameState previousGameState, UserJournal lastJournal) = await GameStateHandler.LoadGameState(db, userIdentifier, userJournals, "EDDN", context);
context.WriteLine($"Uploading journals for user {userIdentifier}");
string lastLine = string.Empty;
foreach (var journalItem in userJournals.WithProgress(context))
{
IntegrationJournalData ijd = GameStateHandler.GetIntegrationJournalData(journalItem, lastJournal, "EDDN");
if (ijd != null && lastJournal != null && ijd.LastSentLineNumber != lastJournal.SentToEDDNLine)
{
ijd = new IntegrationJournalData
{
FullySent = false,
LastSentLineNumber = 0,
CurrentGameState = new EDGameState()
};
}
try
{
using (MemoryStream outFile = new MemoryStream())
{
var journalRows = await JournalLoader.LoadJournal(_minioClient, journalItem, outFile);
int line_number = journalItem.SentToEDDNLine;
int delay_time = 50;
var restOfTheLines = journalRows.Skip(line_number).ToList();
foreach (var row in restOfTheLines.WithProgress(context, journalItem.JournalDate.ToString("yyyy-MM-dd")))
{
lastLine = row;
try
{
if (!string.IsNullOrWhiteSpace(row))
{
var time = await UploadJournalItemToEDDN(hc, row, cmdrName, ijd.CurrentGameState, userIdentifier, starSystemChecker, discordClient);
if (time.TotalMilliseconds > 500)
{
delay_time = 500;
}
else if (time.TotalMilliseconds > 250)
{
delay_time = 250;
}
else if (time.TotalMilliseconds > 100)
{
delay_time = 100;
}
else if (time.TotalMilliseconds < 100)
{
delay_time = 50;
}
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("JsonReaderException"))
{
// Ignore rows we cannot parse
context.WriteLine("Error");
context.WriteLine(ex.ToString());
context.WriteLine(row);
}
else
{
await discordClient.SendMessageAsync("**[EDDN Upload]** Unhandled exception", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = "Unhandled exception",
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path },
{ "Exception", ex.ToString() },
{ "Current GameState", JsonSerializer.Serialize(ijd.CurrentGameState, new JsonSerializerOptions {
WriteIndented = true
}) }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
throw;
}
}
line_number++;
ijd.LastSentLineNumber = line_number;
journalItem.IntegrationData["EDDN"] = ijd;
await Task.Delay(delay_time);
}
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await db.ExecuteNonQueryAsync(
"UPDATE user_journal SET sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
new SqlParameter("journal_id", journalItem.JournalId),
new SqlParameter("line_number", line_number)
);
if (journalItem.CompleteEntry)
{
ijd.LastSentLineNumber = line_number;
ijd.FullySent = true;
journalItem.IntegrationData["EDDN"] = ijd;
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await db.ExecuteNonQueryAsync(
"UPDATE user_journal SET sent_to_eddn = 1, sent_to_eddn_line = @line_number WHERE journal_id = @journal_id",
new SqlParameter("journal_id", journalItem.JournalId),
new SqlParameter("line_number", line_number)
);
}
}
lastJournal = journalItem;
}
catch (Exception ex)
{
await GameStateHandler.UpdateJournalIntegrationDataAsync(db, journalItem.JournalId, IntegrationNames.EDDN, ijd);
await discordClient.SendMessageAsync("**[EDDN Upload]** Problem with upload to EDDN", new List <DiscordWebhookEmbed>
{
new DiscordWebhookEmbed
{
Description = ex.ToString(),
Fields = new Dictionary <string, string>()
{
{ "User identifier", userIdentifier.ToString() },
{ "Last line", lastLine },
{ "Journal", journalItem.S3Path }
}.Select(k => new DiscordWebhookEmbedField {
Name = k.Key, Value = k.Value
}).ToList()
}
});
}
}
}
public static async Task DownloadJournalAsync(Guid userIdentifier, PerformContext context)
{
using (var rlock = new RedisJobLock($"JournalDownloader.DownloadJournal.{userIdentifier}"))
{
if (!rlock.TryTakeLock())
{
return;
}
context.WriteLine($"Looking for journals for user {userIdentifier}");
using (var scope = Startup.ServiceProvider.CreateScope())
{
MSSQLDB db = scope.ServiceProvider.GetRequiredService <MSSQLDB>();
IConfiguration configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
MinioClient minioClient = scope.ServiceProvider.GetRequiredService <MinioClient>();
var discordClient = scope.ServiceProvider.GetRequiredService <DiscordWebhook>();
var user = await db.ExecuteSingleRowAsync <Profile>(
@"SELECT * FROM user_profile WHERE user_identifier = @user_identifier AND last_notification_mail IS NULL AND deleted = 0 AND skip_download = 0",
new SqlParameter("user_identifier", userIdentifier)
);
if (user == null)
{
return;
}
var authToken = user.UserSettings.AuthToken;
var hc = SharedSettings.GetHttpClient(scope);
hc.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", authToken);
hc.BaseAddress = new Uri("https://companion.orerve.net");
var profile = await GetProfileAsync(hc);
var profileJson = await profile.Content.ReadAsStringAsync();
if (!profile.IsSuccessStatusCode || string.IsNullOrWhiteSpace(profileJson))
{
context.WriteLine($"Invalid statuscode: {profile.StatusCode} {profile.ReasonPhrase}");
bool resetAuth = false;
switch (profile.StatusCode)
{
case HttpStatusCode.BadRequest:
// User does not potentially own the game
resetAuth = true;
break;
case HttpStatusCode.Unauthorized:
// Invalid token (or Epic games)
resetAuth = true;
break;
}
if (string.IsNullOrWhiteSpace(profileJson))
{
resetAuth = true;
}
if (resetAuth)
{
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
"Could not authorize you",
"Sorry, but there seems to be something wrong with your account. Please contact us so we can try and figure out what's wrong!"
);
context.WriteLine("Bailing out early, user doesn't own Elite or has issues with cAPI auth");
if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
{
context.WriteLine("User cannot be fetched, asking them to reauthenticate");
await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
}
}
return;
}
var profileData = JsonSerializer.Deserialize <EliteProfile>(profileJson);
context.WriteLine(profileJson);
if (profileJson == "{}")
{
if (!string.IsNullOrWhiteSpace(user.NotificationEmail))
{
context.WriteLine("User cannot be fetched, asking them to reauthenticate");
await SendLoginNotificationMethod.SendLoginNotification(db, configuration, user);
}
return;
}
context.WriteLine($"Downloading journals for {profileData.Commander.Name}");
DateTime journalDate = DateTime.Today.AddDays(-25);
await SSEActivitySender.SendUserActivityAsync(user.UserIdentifier,
"Downloading your journals",
"We're beginning to download your journals now, a few notifications may pop up."
);
while (journalDate.Date != DateTime.Today)
{
context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
var req = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);
if (req.shouldBail)
{
// Failed to get loop journal
context.WriteLine($"Bailing because of errors");
return;
}
journalDate = journalDate.AddDays(1);
}
context.WriteLine($"Fetching data for {journalDate.ToString("yyyy-MM-dd")}");
var reqOut = await TryGetJournalAsync(discordClient, journalDate, user, db, hc, minioClient, context);
if (reqOut.shouldBail)
{
// Failed to get loop journal
context.WriteLine($"Bailing because of errors");
return;
}
if (user.SendToEDDN && !RedisJobLock.IsLocked($"EDDNUserUploader.UploadAsync.{user.UserIdentifier}"))
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND sent_to_eddn = 0 AND last_processed_line_number >= sent_to_eddn_line",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
context.WriteLine($"Sending {userJournals} journals to EDDN");
BackgroundJob.Enqueue(() => EDDNUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
}
}
if (user.IntegrationSettings.ContainsKey("EDSM") && user.IntegrationSettings["EDSM"].GetTypedObject <EDSMIntegrationSettings>().Enabled)
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.lastSentLineNumber'), '0') < last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.EDSM.fullySent'), 'false') = 'false'",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
if (!RedisJobLock.IsLocked($"EDSMUserUploader.UploadAsync.{user.UserIdentifier}"))
{
context.WriteLine($"Sending {userJournals} journals to EDSM");
BackgroundJob.Enqueue(() => EDSMUserUploader.UploadAsync(user.UserIdentifier, null));
}
}
}
if (user.IntegrationSettings.ContainsKey("Canonn R&D") && user.IntegrationSettings["Canonn R&D"].GetTypedObject <CanonnRDIntegrationSettings>().Enabled)
{
var userJournals = await db.ExecuteScalarAsync <long>(
"SELECT COUNT_BIG(journal_id) FROM user_journal WHERE user_identifier = @user_identifier AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".lastSentLineNumber'), '0') <= last_processed_line_number AND last_processed_line_number > 0 AND ISNULL(JSON_VALUE(integration_data, '$.\"Canonn R\\u0026D\".fullySent'), 'false') = 'false'",
new SqlParameter("user_identifier", userIdentifier)
);
if (userJournals > 0)
{
if (!RedisJobLock.IsLocked($"CanonnRDUserUploader.UploadAsync.{user.UserIdentifier}"))
{
context.WriteLine($"Sending {userJournals} journals to Canonn");
BackgroundJob.Enqueue(() => CanonnRDUserUploader.UploadAsync(user.UserIdentifier, profileData.Commander.Name, null));
}
}
}
context.WriteLine("All done!");
}
}
}