public static void Log(Command c, string columnName, int partition)
{
var colId = String.Format("{0,15}", $"[{columnName}|{partition}]");
var summary = c switch
{
Command_Change cmd => $"{colId} CHANGE {cmd.ColumnValue} | {cmd.ChangeType}",
Command_Enter cmd => $"{colId} ENTER {cmd.ColumnValue}",
Command_Verify cmd => $"{colId} VERIFY {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
Command_Exit cmd => $"{colId} EXIT {cmd.ColumnValue}",
Command_Ack cmd => $"{colId} ACK {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
_ => $"unknown command"
};
var logLine = String.Format("{0,-60} .." + c.Correlation.Substring(c.Correlation.Length - 8), summary);
Console.WriteLine(logLine);
}
private void HandleExit(Command_Exit cmd, Offset offset)
{
if (locked.ContainsKey(cmd.ColumnValue))
{
if (locked[cmd.ColumnValue] != cmd.Correlation)
{
Logger.Log("EXIT", $"correlation doesn't match, unlocking. [{cmd.Correlation}]. expecting: [{locked[cmd.ColumnValue]}]");
System.Environment.Exit(1);
}
}
// unlock, whether aborted or not.
locked.Remove(cmd.ColumnValue);
// also, offset is free to progress.
var inProgress = inProgressState_Secondary[cmd.Correlation]; // TODO: some validation around this.
blockedForCommit.Remove(inProgress.EnterCommandOffset);
inProgressState_Secondary.Remove(cmd.Correlation);
// if the command is aborting, then no ack is expected and we're done.
if (cmd.Action == ActionType.Abort)
{
// Logger.Log("EXIT", $"Command aborted [{cmd.Correlation}]");
return;
}
// 1. commit new changes values to change logs.
var tp = new TopicPartition(
this.tableSpecification.ChangeLogTopicName(this.columnName),
Table.Partitioner(cmd.ColumnValue, this.numPartitions));
if (cmd.Action == ActionType.Set)
{
clProducer.ProduceAsync(tp, new Message <string, string> {
Key = cmd.ColumnValue, Value = JsonConvert.SerializeObject(cmd.Data, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("EXIT", $"failed to write to changelog [{cmd.Correlation}]");
if (materialized.ContainsKey(cmd.ColumnValue))
{
materialized[cmd.ColumnValue] = cmd.Data;
}
else
{
materialized.Add(cmd.ColumnValue, cmd.Data);
}
}
else if (cmd.Action == ActionType.Delete)
{
clProducer.ProduceAsync(tp, new Message <string, string> {
Key = cmd.ColumnValue, Value = null
})
.FailIfFaulted("EXIT", "failed to write tombstone to changelog");
if (materialized.ContainsKey(cmd.ColumnValue))
{
materialized.Remove(cmd.ColumnValue);
}
else
{
Logger.Log("EXIT", "Expecting value to exist in materialized table [{cmd.Correlation}]");
}
}
// 2. send ack commands back.
var ackCommand = new Command_Ack
{
Correlation = cmd.Correlation,
SourceColumnName = this.columnName,
SourceColumnValue = cmd.ColumnValue
};
tp = new TopicPartition(
this.tableSpecification.CommandTopicName(cmd.SourceColumnName),
Table.Partitioner(cmd.SourceColumnValue, this.numPartitions));
cmdProducer.ProduceAsync(tp, new Message <Null, string> {
Value = JsonConvert.SerializeObject(ackCommand, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("EXIT", "failed to write ack cmd [{cmd.Correlation}]");
}
private void HandleVerify(Command_Verify cmd, Offset offset)
{
if (!inProgressState_Active.ContainsKey(cmd.Correlation))
{
// This could occur in the case of duplicate writes and can be safely ignored.
Logger.Log("VERIFY", $"Received verify command with no corresponding waitingVerify entry, ignoring [{cmd.Correlation}]");
return;
}
// all verify results must be true for command to succeed.
if (!cmd.Verified)
{
inProgressState_Active[cmd.Correlation].VerifyFailed.Add(new NameAndValue {
Name = cmd.SourceColumnName, Value = cmd.SourceColumnValue
});
}
// remove the received column from list we're waiting on.
inProgressState_Active[cmd.Correlation].WaitingVerify = inProgressState_Active[cmd.Correlation]
.WaitingVerify.Where(a => !(a.Name == cmd.SourceColumnName && a.Value == cmd.SourceColumnValue))
.ToList();
// if we're waiting on more, then there's nothing left to do here.
if (inProgressState_Active[cmd.Correlation].WaitingVerify.Count > 0)
{
return;
}
// --- verification result has been received for all columns.
// Logger.Log("VERIFY", $"All verify commands received for: [{cmd.Correlation}]");
var inProgress = inProgressState_Active[cmd.Correlation];
var aborting = inProgress.VerifyFailed.Count > 0;
// send exit_commands for columns that are to be set (or abort).
// note: does not include the current "this" column - that is handled as a special case.
foreach (var col in inProgress.ToSet)
{
Dictionary <string, string> dataToSet = null;
if (!aborting)
{
dataToSet = new Dictionary <string, string>(inProgress.Data);
dataToSet.Add(this.columnName, inProgress.ColumnValue);
dataToSet.Remove(col.Key);
}
var exitCommand = new Command_Exit
{
Correlation = cmd.Correlation,
ColumnValue = inProgress.Data[col.Key],
Action = aborting ? ActionType.Abort : ActionType.Set,
Data = dataToSet,
SourceColumnName = this.columnName,
SourceColumnValue = inProgress.ColumnValue
};
var tp = new TopicPartition(
tableSpecification.CommandTopicName(col.Key),
Table.Partitioner(exitCommand.ColumnValue, this.numPartitions));
cmdProducer.ProduceAsync(tp, new Message <Null, string> {
Value = JsonConvert.SerializeObject(exitCommand, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("VERIFY", "produce fail");
}
// send exit_commands for column values that are to be deleted (or abort).
foreach (var col in inProgress.ToDelete)
{
var exitCommand = new Command_Exit
{
Correlation = cmd.Correlation,
ColumnValue = col.Value,
Action = aborting ? ActionType.Abort : ActionType.Delete,
Data = null,
SourceColumnName = this.columnName,
SourceColumnValue = inProgress.ColumnValue
};
var tp = new TopicPartition(
tableSpecification.CommandTopicName(col.Key),
Table.Partitioner(exitCommand.ColumnValue, this.numPartitions));
cmdProducer.ProduceAsync(tp, new Message <Null, string> {
Value = JsonConvert.SerializeObject(exitCommand, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("VERIFY", "produce fail");
}
// if aborting, clean up everything now - there will be no acks.
if (aborting)
{
locked.Remove(inProgress.ColumnValue);
blockedForCommit.Remove(inProgress.ChangeCommandOffset);
inProgressState_Active.Remove(cmd.Correlation);
CompleteChangeRequest(cmd.Correlation, new Exception("columns verify failed"));
return;
}
}