public async Task <bool> Change(ChangeType changeType, string keyValue, Dictionary <string, string> row)
{
var keyName = this.columnName;
var p = Table.Partitioner(keyValue, this.numPartitions);
if (p != partition)
{
throw new Exception("Applying change to incorrect partition.");
}
if (row.Keys.Contains("keyName"))
{
throw new Exception("Row update dict shouldn't include value for key being updated.");
}
Command_Change changeCommand = new Command_Change
{
Correlation = Guid.NewGuid().ToString(),
ChangeType = changeType,
ColumnValue = keyValue,
Data = row
};
var r = await cmdProducer.ProduceAsync(
new TopicPartition(tableSpecification.CommandTopicName(this.columnName), this.partition),
new Message <Null, string> {
Value = JsonConvert.SerializeObject(changeCommand, Formatting.Indented, jsonSettings)
}
);
return(await WaitForResult(changeCommand.Correlation).ConfigureAwait(false));
}
public static void Log(Command c, string columnName, int partition)
{
var colId = String.Format("{0,15}", $"[{columnName}|{partition}]");
var summary = c switch
{
Command_Change cmd => $"{colId} CHANGE {cmd.ColumnValue} | {cmd.ChangeType}",
Command_Enter cmd => $"{colId} ENTER {cmd.ColumnValue}",
Command_Verify cmd => $"{colId} VERIFY {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
Command_Exit cmd => $"{colId} EXIT {cmd.ColumnValue}",
Command_Ack cmd => $"{colId} ACK {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
_ => $"unknown command"
};
var logLine = String.Format("{0,-60} .." + c.Correlation.Substring(c.Correlation.Length - 8), summary);
Console.WriteLine(logLine);
}
/// <summary>
/// Handle a brand new add or update command.
/// </summary>
private void HandleChange(Command_Change cmd, Offset offset)
{
if (inProgressState_Active.ContainsKey(cmd.Correlation))
{
// if the command_change message is received more than once, this is a duplicate
// message in the log and can simply be ignored.
Logger.Log("CHANGE", $"ignoring duplicate change command [{cmd.Correlation}]");
return;
}
// if the column value is locked, then fail the command immediately.
if (locked.ContainsKey(cmd.ColumnValue))
{
CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} locked, can't change [{cmd.Correlation}]"));
return;
}
// also abort if this is an add operation and the value exists already.
if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Add &&
this.materialized.ContainsKey(cmd.ColumnValue))
{
CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} exists, can't add [{cmd.Correlation}]"));
return;
}
// also abort if this is an update operation and the value doesn't exist already.
if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Update &&
!this.materialized.ContainsKey(cmd.ColumnValue))
{
CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} doesn't exists, can't update [{cmd.Correlation}]"));
return;
}
// AddOrUpdate can be dis-ambiguated at this point, and is not considered further
// in the workflow.
var isAddCommand = !materialized.ContainsKey(cmd.ColumnValue) || cmd.ChangeType == Experiment.ChangeType.Add;
var amalgamatedData = new Dictionary <string, string>(cmd.Data);
var uniqueColumnValuesToDelete = new Dictionary <string, string>();
// if this is an update:
if (!isAddCommand)
{
// 1. amalgamate with existing data.
var existing = materialized[cmd.ColumnValue];
foreach (var e in existing)
{
if (!amalgamatedData.ContainsKey(e.Key))
{
amalgamatedData.Add(e.Key, e.Value);
}
}
// 2. work out unique values (other than this) that have changed -
// we need the old values to be removed.
//
// notes:
// 1. the current partition column value can't have changed (obviously)
// 2. even if a unique column value hasn't changed, it
// needs to get locked (be included in the operation workflow)
// because it's data is changing.
foreach (var other in this.otherUniqueColumns)
{
if (cmd.Data.ContainsKey(other.Name))
{
if (cmd.Data[other.Name] != materialized[cmd.ColumnValue][other.Name])
{
uniqueColumnValuesToDelete.Add(other.Name, this.materialized[cmd.ColumnValue][other.Name]);
}
}
}
}
// check that all unique columns have a value.
foreach (var other in this.otherUniqueColumns)
{
if (!amalgamatedData.ContainsKey(other.Name))
{
CompleteChangeRequest(cmd.Correlation, new Exception("change request does not contain value for key {other.Name} [{cmd.Correlation}]"));
return;
}
}
// lock this column's value - at this point, we're going to try and apply the change.
locked.Add(cmd.ColumnValue, cmd.Correlation);
// prevent a commit of this offset until the final write in the workflow is done.
blockedForCommit.Add(offset, cmd.Correlation);
// keep track of info related to this command including the columns we'll be waiting for a verify from.
var otherList = otherUniqueColumns
.Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name]))
.Concat(uniqueColumnValuesToDelete.ToList())
.Select(a => new NameAndValue {
Name = a.Key, Value = a.Value
})
.ToList();
inProgressState_Active.Add(cmd.Correlation, new InProgressState_Active
{
ChangeCommandOffset = offset,
ColumnValue = cmd.ColumnValue,
WaitingVerify = otherList,
WaitingAck = otherList,
Data = amalgamatedData,
ToDelete = uniqueColumnValuesToDelete,
ToSet = otherUniqueColumns.Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name])).ToDictionary(a => a.Key, a => a.Value),
VerifyFailed = new List <NameAndValue>()
});
// finally, send an enter command to the relevant key/values
// that need locking.
foreach (var cs in inProgressState_Active[cmd.Correlation].WaitingVerify)
{
var enterCommand = new Command_Enter
{
Correlation = cmd.Correlation,
IsAddCommand = isAddCommand,
ColumnValue = cs.Value,
SourceColumnName = this.columnName,
SourceColumnValue = cmd.ColumnValue
};
var tp = new TopicPartition(
tableSpecification.CommandTopicName(cs.Name),
Table.Partitioner(cs.Value, numPartitions)
);
// TODO: verify producer settings ensure in order, gapless produce.
// with a long retry.
cmdProducer.ProduceAsync(tp, new Message <Null, string> {
Value = JsonConvert.SerializeObject(enterCommand, Formatting.Indented, jsonSettings)
})
.FailIfFaulted("CHANGE", $"A fatal problem occured writing a lock command.");
}
}