public static void Log(Command c, string columnName, int partition)
{
var colId = String.Format("{0,15}", $"[{columnName}|{partition}]");
var summary = c switch
{
Command_Change cmd => $"{colId} CHANGE {cmd.ColumnValue} | {cmd.ChangeType}",
Command_Enter cmd => $"{colId} ENTER {cmd.ColumnValue}",
Command_Verify cmd => $"{colId} VERIFY {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
Command_Exit cmd => $"{colId} EXIT {cmd.ColumnValue}",
Command_Ack cmd => $"{colId} ACK {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
_ => $"unknown command"
};
var logLine = String.Format("{0,-60} .." + c.Correlation.Substring(c.Correlation.Length - 8), summary);
Console.WriteLine(logLine);
}
private void HandleAck(Command_Ack cmd, Offset offset)
{
// always expect the state corresponding to an ack to exist.
if (!this.inProgressState_Active.ContainsKey(cmd.Correlation))
{
Logger.Log("ACK", $"ERROR!: Ack correlation not found [{cmd.Correlation}]");
// if the task is known, then that is also unexpected. fail the task with an internal error.
if (this.inProgressTasks.ContainsKey(cmd.Correlation))
{
CompleteChangeRequest(cmd.Correlation, new Exception($"Ack correlation not found [{cmd.Correlation}]"));
}
return;
}
var inProgress = this.inProgressState_Active[cmd.Correlation];
// it's ok if this doesn't exist - may reflect a de-dupe.
if (inProgress.WaitingAck.Where(a => a.Name == cmd.SourceColumnName && a.Value == cmd.SourceColumnValue).Count() == 1)
{
inProgress.WaitingAck = inProgress
.WaitingAck.Where(a => !(a.Name == cmd.SourceColumnName && a.Value == cmd.SourceColumnValue))
.ToList();
}
else
{
Logger.Log("ACK", $"ack received, but not waiting for it {cmd.SourceColumnName}, {cmd.SourceColumnValue} [{cmd.Correlation}]");
}
if (inProgress.WaitingAck.Count != 0)
{
return;
}
// finally add data to change log topic (special case for this column).
var tp = new TopicPartition(
this.tableSpecification.ChangeLogTopicName(this.columnName),
Table.Partitioner(inProgress.ColumnValue, this.numPartitions));
clProducer.ProduceAsync(tp, new Message <string, string> {
Key = inProgress.ColumnValue, Value = JsonConvert.SerializeObject(inProgress.Data, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("EXIT", "failed to write to changelog");
if (materialized.ContainsKey(inProgress.ColumnValue))
{
materialized[inProgress.ColumnValue] = inProgress.Data;
}
else
{
materialized.Add(inProgress.ColumnValue, inProgress.Data);
}
// unlock, allow commit & remove state.
locked.Remove(inProgress.ColumnValue);
blockedForCommit.Remove(inProgressState_Active[cmd.Correlation].ChangeCommandOffset);
this.inProgressState_Active.Remove(cmd.Correlation);
// finally signal that we're done.
CompleteChangeRequest(cmd.Correlation, null);
}
private void HandleExit(Command_Exit cmd, Offset offset)
{
if (locked.ContainsKey(cmd.ColumnValue))
{
if (locked[cmd.ColumnValue] != cmd.Correlation)
{
Logger.Log("EXIT", $"correlation doesn't match, unlocking. [{cmd.Correlation}]. expecting: [{locked[cmd.ColumnValue]}]");
System.Environment.Exit(1);
}
}
// unlock, whether aborted or not.
locked.Remove(cmd.ColumnValue);
// also, offset is free to progress.
var inProgress = inProgressState_Secondary[cmd.Correlation]; // TODO: some validation around this.
blockedForCommit.Remove(inProgress.EnterCommandOffset);
inProgressState_Secondary.Remove(cmd.Correlation);
// if the command is aborting, then no ack is expected and we're done.
if (cmd.Action == ActionType.Abort)
{
// Logger.Log("EXIT", $"Command aborted [{cmd.Correlation}]");
return;
}
// 1. commit new changes values to change logs.
var tp = new TopicPartition(
this.tableSpecification.ChangeLogTopicName(this.columnName),
Table.Partitioner(cmd.ColumnValue, this.numPartitions));
if (cmd.Action == ActionType.Set)
{
clProducer.ProduceAsync(tp, new Message <string, string> {
Key = cmd.ColumnValue, Value = JsonConvert.SerializeObject(cmd.Data, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("EXIT", $"failed to write to changelog [{cmd.Correlation}]");
if (materialized.ContainsKey(cmd.ColumnValue))
{
materialized[cmd.ColumnValue] = cmd.Data;
}
else
{
materialized.Add(cmd.ColumnValue, cmd.Data);
}
}
else if (cmd.Action == ActionType.Delete)
{
clProducer.ProduceAsync(tp, new Message <string, string> {
Key = cmd.ColumnValue, Value = null
})
.FailIfFaulted("EXIT", "failed to write tombstone to changelog");
if (materialized.ContainsKey(cmd.ColumnValue))
{
materialized.Remove(cmd.ColumnValue);
}
else
{
Logger.Log("EXIT", "Expecting value to exist in materialized table [{cmd.Correlation}]");
}
}
// 2. send ack commands back.
var ackCommand = new Command_Ack
{
Correlation = cmd.Correlation,
SourceColumnName = this.columnName,
SourceColumnValue = cmd.ColumnValue
};
tp = new TopicPartition(
this.tableSpecification.CommandTopicName(cmd.SourceColumnName),
Table.Partitioner(cmd.SourceColumnValue, this.numPartitions));
cmdProducer.ProduceAsync(tp, new Message <Null, string> {
Value = JsonConvert.SerializeObject(ackCommand, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("EXIT", "failed to write ack cmd [{cmd.Correlation}]");
}
