/// <summary>
/// The entrypoint method to the program.
/// </summary>
public static void Main()
{
var exitEvent = new ManualResetEvent(false);
var cancelSource = new CancellationTokenSource();
Console.CancelKeyPress += (sender, eventArgs) =>
{
// Cancel the exit, as we need to cleanup first.
eventArgs.Cancel = true;
cancelSource.Cancel();
exitEvent.Set();
};
var gameData = GameData.Parse(@"GameData.json");
// Set up configuration.
var builder = new ConfigurationBuilder()
.SetBasePath(Environment.CurrentDirectory)
.AddJsonFile("appsettings.json")
.AddApplicationInsightsSettings(developerMode: true);
var configuration = builder.Build();
var databaseSettings = new DatabaseSettings();
configuration.GetSection("Database").Bind(databaseSettings);
var databaseSettingsOptions = Options.Create(databaseSettings);
var queueClient = CloudStorageAccount.Parse(configuration["Storage:ConnectionString"]).CreateCloudQueueClient();
// Set up telemetry
var telemetryConfiguration = TelemetryConfiguration.CreateDefault();
telemetryConfiguration.InstrumentationKey = configuration["ApplicationInsights:InstrumentationKey"];
telemetryConfiguration.TelemetryProcessorChainBuilder.Use(next => new ConsoleLoggingProcessor(next)).Build();
var telemetryClient = new TelemetryClient(telemetryConfiguration);
if (ScheduleRecompute)
{
Task.Run(async() =>
{
var uploadIds = new List <int>();
const string CommandText = "SELECT Id FROM Uploads WHERE LastComputeTime < '" + LastComputeTime + "' ORDER BY LastComputeTime DESC";
var databaseCommandFactory = new DatabaseCommandFactory(databaseSettingsOptions);
using (var command = databaseCommandFactory.Create(CommandText))
using (var reader = await command.ExecuteReaderAsync())
{
while (reader.Read())
{
var uploadId = Convert.ToInt32(reader["Id"]);
uploadIds.Add(uploadId);
}
}
Console.WriteLine($"Found {uploadIds.Count} uploads to schedule");
var uploadScheduler = new AzureStorageUploadScheduler(queueClient);
for (var i = 0; i < uploadIds.Count; i++)
{
if (cancelSource.IsCancellationRequested)
{
break;
}
Console.WriteLine($"Scheduling message {i + 1} of {uploadIds.Count} - {100 * (i + 1) / uploadIds.Count}%");
var message = new UploadProcessingMessage {
UploadId = uploadIds[i], Requester = "Ad-hoc Recompute", Priority = UploadProcessingMessagePriority.Low
};
await uploadScheduler.ScheduleAsync(message);
}
Console.WriteLine("Stopped scheduling");
});
}
var processors = new ConcurrentBag <UploadProcessor>();
// Spin up one per logical core
Console.WriteLine($"Number of processors: {Environment.ProcessorCount}");
var tasks = new Task[Environment.ProcessorCount];
for (var i = 0; i < tasks.Length; i++)
{
var processor = new UploadProcessor(databaseSettingsOptions, gameData, telemetryClient, queueClient);
processors.Add(processor);
tasks[i] = processor.ProcessAsync(cancelSource.Token);
}
// Let it run until Ctrl+C
exitEvent.WaitOne();
const int DrainTimeMs = 60000;
Console.WriteLine($"Waiting for up to {DrainTimeMs}ms for processing to drain...");
if (Task.WaitAll(tasks, DrainTimeMs))
{
Console.WriteLine("Processing drained successfully");
}
else
{
Console.WriteLine("Some processing was stuck");
foreach (var processor in processors)
{
var currentUploadId = processor.CurrentUploadId;
if (currentUploadId != null)
{
var properties = new Dictionary <string, string>
{
{ "UploadId", currentUploadId.Value.ToString() },
};
telemetryClient.TrackEvent("UploadProcessor-Abandoned-Stuck", properties);
}
}
}
telemetryClient.Flush();
}
protected override void Arrange() => _sut = new DatabaseCommandFactory(_databaseCommands);
private bool ProcessMessage(CloudQueueMessage queueMessage)
{
var properties = new Dictionary <string, string>
{
{ "CloudQueueMessage-DequeueCount", queueMessage.DequeueCount.ToString() },
{ "CloudQueueMessage-InsertionTime", queueMessage.InsertionTime.ToString() },
{ "CloudQueueMessage-Id", queueMessage.Id },
};
this.telemetryClient.TrackEvent("UploadProcessor-Recieved", properties);
using (var counterProvider = new CounterProvider(this.telemetryClient))
using (var databaseCommandFactory = new DatabaseCommandFactory(
this.databaseSettingsOptions,
counterProvider))
using (counterProvider.Measure(Counter.ProcessUpload))
{
int uploadId = -1;
try
{
var message = JsonConvert.DeserializeObject <UploadProcessingMessage>(queueMessage.AsString);
if (message == null)
{
this.telemetryClient.TrackEvent("UploadProcessor-Abandoned-ParseMessage", properties);
return(false);
}
uploadId = message.UploadId;
properties.Add("UploadId", uploadId.ToString());
this.CurrentUploadId = uploadId;
GetUploadDetails(databaseCommandFactory, uploadId, out var uploadContent, out var userId, out var playStyle);
properties.Add("UserId", userId);
if (string.IsNullOrWhiteSpace(uploadContent))
{
this.telemetryClient.TrackEvent("UploadProcessor-Abandoned-FetchUpload", properties);
return(false);
}
// Handle legacy uplaods where the upload content was missing.
if (uploadContent.Equals("LEGACY", StringComparison.OrdinalIgnoreCase))
{
this.telemetryClient.TrackEvent("UploadProcessor-Complete-Legacy", properties);
using (var command = databaseCommandFactory.Create())
{
command.CommandText = "UPDATE Uploads SET LastComputeTime = getutcdate() WHERE Id = " + uploadId;
command.ExecuteNonQuery();
}
return(true);
}
var savedGame = SavedGame.Parse(uploadContent);
if (savedGame == null)
{
this.telemetryClient.TrackEvent("UploadProcessor-Abandoned-ParseUpload", properties);
return(false);
}
this.telemetryClient.TrackEvent("UploadProcessor-Processing", properties);
var ancientLevels = new AncientLevelsModel(
this.gameData,
savedGame,
this.telemetryClient);
var outsiderLevels = new OutsiderLevelsModel(
this.gameData,
savedGame,
this.telemetryClient);
var miscellaneousStatsModel = new MiscellaneousStatsModel(savedGame);
/* Build a query that looks like this:
* MERGE INTO AncientLevels WITH (HOLDLOCK)
* USING
* (VALUES (123, 1, 100), (123, 2, 100), ... )
* AS Input(UploadId, AncientId, Level)
* ON AncientLevels.UploadId = Input.UploadId
* AND AncientLevels.AncientId = Input.AncientId
* WHEN MATCHED THEN
* UPDATE
* SET
* Level = Input.Level
* WHEN NOT MATCHED THEN
* INSERT (UploadId, AncientId, Level)
* VALUES (Input.UploadId, Input.AncientId, Input.Level);
*/
var ancientLevelsCommandText = new StringBuilder();
if (ancientLevels.AncientLevels.Count > 0)
{
ancientLevelsCommandText.Append(@"
MERGE INTO AncientLevels WITH (HOLDLOCK)
USING
( VALUES ");
var isFirst = true;
foreach (var pair in ancientLevels.AncientLevels)
{
if (!isFirst)
{
ancientLevelsCommandText.Append(",");
}
// No need to sanitize, these are all just numbers
ancientLevelsCommandText.Append("(");
ancientLevelsCommandText.Append(uploadId);
ancientLevelsCommandText.Append(",");
ancientLevelsCommandText.Append(pair.Key);
ancientLevelsCommandText.Append(",");
ancientLevelsCommandText.Append(pair.Value.AncientLevel);
ancientLevelsCommandText.Append(")");
isFirst = false;
}
ancientLevelsCommandText.Append(@"
)
AS Input(UploadId, AncientId, Level)
ON AncientLevels.UploadId = Input.UploadId
AND AncientLevels.AncientId = Input.AncientId
WHEN MATCHED THEN
UPDATE
SET
Level = Input.Level
WHEN NOT MATCHED THEN
INSERT (UploadId, AncientId, Level)
VALUES (Input.UploadId, Input.AncientId, Input.Level);");
}
/* Build a query that looks like this:
* MERGE INTO OutsiderLevels WITH (HOLDLOCK)
* USING
* (VALUES (123, 1, 100), (123, 2, 100), ... )
* AS Input(UploadId, OutsiderId, Level)
* ON OutsiderLevels.UploadId = Input.UploadId
* AND OutsiderLevels.OutsiderId = Input.OutsiderId
* WHEN MATCHED THEN
* UPDATE
* SET
* Level = Input.Level
* WHEN NOT MATCHED THEN
* INSERT (UploadId, OutsiderId, Level)
* VALUES (Input.UploadId, Input.OutsiderId, Input.Level);
*/
var outsiderLevelsCommandText = new StringBuilder();
if (outsiderLevels.OutsiderLevels.Count > 0)
{
outsiderLevelsCommandText.Append(@"
MERGE INTO OutsiderLevels WITH (HOLDLOCK)
USING
( VALUES ");
var isFirst = true;
foreach (var pair in outsiderLevels.OutsiderLevels)
{
if (!isFirst)
{
outsiderLevelsCommandText.Append(",");
}
// No need to sanitize, these are all just numbers
outsiderLevelsCommandText.Append("(");
outsiderLevelsCommandText.Append(uploadId);
outsiderLevelsCommandText.Append(",");
outsiderLevelsCommandText.Append(pair.Key);
outsiderLevelsCommandText.Append(",");
outsiderLevelsCommandText.Append(pair.Value.Level);
outsiderLevelsCommandText.Append(")");
isFirst = false;
}
outsiderLevelsCommandText.Append(@"
)
AS Input(UploadId, OutsiderId, Level)
ON OutsiderLevels.UploadId = Input.UploadId
AND OutsiderLevels.OutsiderId = Input.OutsiderId
WHEN MATCHED THEN
UPDATE
SET
Level = Input.Level
WHEN NOT MATCHED THEN
INSERT (UploadId, OutsiderId, Level)
VALUES (Input.UploadId, Input.OutsiderId, Input.Level);");
}
const string ComputedStatsCommandText = @"
MERGE INTO ComputedStats WITH (HOLDLOCK)
USING
(VALUES (
@UploadId,
@TitanDamage,
@SoulsSpent,
@HeroSoulsSacrificed,
@TotalAncientSouls,
@TranscendentPower,
@Rubies,
@HighestZoneThisTranscension,
@HighestZoneLifetime,
@AscensionsThisTranscension,
@AscensionsLifetime,
@MaxTranscendentPrimalReward,
@BossLevelToTranscendentPrimalCap) )
AS Input(
UploadId,
TitanDamage,
SoulsSpent,
HeroSoulsSacrificed,
TotalAncientSouls,
TranscendentPower,
Rubies,
HighestZoneThisTranscension,
HighestZoneLifetime,
AscensionsThisTranscension,
AscensionsLifetime,
MaxTranscendentPrimalReward,
BossLevelToTranscendentPrimalCap)
ON ComputedStats.UploadId = Input.UploadId
WHEN MATCHED THEN
UPDATE
SET
TitanDamage = Input.TitanDamage,
SoulsSpent = Input.SoulsSpent,
HeroSoulsSacrificed = Input.HeroSoulsSacrificed,
TotalAncientSouls = Input.TotalAncientSouls,
TranscendentPower = Input.TranscendentPower,
Rubies = Input.Rubies,
HighestZoneThisTranscension = Input.HighestZoneThisTranscension,
HighestZoneLifetime = Input.HighestZoneLifetime,
AscensionsThisTranscension = Input.AscensionsThisTranscension,
AscensionsLifetime = Input.AscensionsLifetime,
MaxTranscendentPrimalReward = Input.MaxTranscendentPrimalReward,
BossLevelToTranscendentPrimalCap = Input.BossLevelToTranscendentPrimalCap
WHEN NOT MATCHED THEN
INSERT (
UploadId,
TitanDamage,
SoulsSpent,
HeroSoulsSacrificed,
TotalAncientSouls,
TranscendentPower,
Rubies,
HighestZoneThisTranscension,
HighestZoneLifetime,
AscensionsThisTranscension,
AscensionsLifetime,
MaxTranscendentPrimalReward,
BossLevelToTranscendentPrimalCap)
VALUES (
Input.UploadId,
Input.TitanDamage,
Input.SoulsSpent,
Input.HeroSoulsSacrificed,
Input.TotalAncientSouls,
Input.TranscendentPower,
Input.Rubies,
Input.HighestZoneThisTranscension,
Input.HighestZoneLifetime,
Input.AscensionsThisTranscension,
Input.AscensionsLifetime,
Input.MaxTranscendentPrimalReward,
Input.BossLevelToTranscendentPrimalCap);";
var computedStatsCommandParameters = new Dictionary <string, object>
{
{ "@UploadId", uploadId },
{ "@TitanDamage", miscellaneousStatsModel.TitanDamage },
{ "@SoulsSpent", miscellaneousStatsModel.HeroSoulsSpent },
{ "@HeroSoulsSacrificed", miscellaneousStatsModel.HeroSoulsSacrificed },
{ "@TotalAncientSouls", miscellaneousStatsModel.TotalAncientSouls },
{ "@TranscendentPower", miscellaneousStatsModel.TranscendentPower },
{ "@Rubies", miscellaneousStatsModel.Rubies },
{ "@HighestZoneThisTranscension", miscellaneousStatsModel.HighestZoneThisTranscension },
{ "@HighestZoneLifetime", miscellaneousStatsModel.HighestZoneLifetime },
{ "@AscensionsThisTranscension", miscellaneousStatsModel.AscensionsThisTranscension },
{ "@AscensionsLifetime", miscellaneousStatsModel.AscensionsLifetime },
{ "@MaxTranscendentPrimalReward", miscellaneousStatsModel.MaxTranscendentPrimalReward },
{ "@BossLevelToTranscendentPrimalCap", miscellaneousStatsModel.BossLevelToTranscendentPrimalCap },
};
using (var command = databaseCommandFactory.Create())
{
command.BeginTransaction();
if (ancientLevelsCommandText.Length > 0)
{
command.CommandText = ancientLevelsCommandText.ToString();
command.ExecuteNonQuery();
}
if (outsiderLevelsCommandText.Length > 0)
{
command.CommandText = outsiderLevelsCommandText.ToString();
command.ExecuteNonQuery();
}
command.CommandText = ComputedStatsCommandText;
command.Parameters = computedStatsCommandParameters;
command.ExecuteNonQuery();
command.CommandText = "UPDATE Uploads SET LastComputeTime = getutcdate() WHERE Id = " + uploadId;
command.ExecuteNonQuery();
var commited = command.CommitTransaction();
if (!commited)
{
this.telemetryClient.TrackEvent("UploadProcessor-Abandoned-CommitTransaction", properties);
return(false);
}
}
this.telemetryClient.TrackEvent("UploadProcessor-Complete", properties);
return(true);
}
catch (Exception e)
{
this.telemetryClient.TrackException(e, properties);
return(false);
}
finally
{
this.CurrentUploadId = null;
}
}
}
public SqlDatabaseGateway(DatabaseCommandFactory command_factory)
{
this.command_factory = command_factory;
}