public async Task <bool> Change(ChangeType changeType, string keyValue, Dictionary <string, string> row)
        {
            var keyName = this.columnName;
            var p       = Table.Partitioner(keyValue, this.numPartitions);

            if (p != partition)
            {
                throw new Exception("Applying change to incorrect partition.");
            }

            if (row.Keys.Contains("keyName"))
            {
                throw new Exception("Row update dict shouldn't include value for key being updated.");
            }

            Command_Change changeCommand = new Command_Change
            {
                Correlation = Guid.NewGuid().ToString(),
                ChangeType  = changeType,
                ColumnValue = keyValue,
                Data        = row
            };

            var r = await cmdProducer.ProduceAsync(
                new TopicPartition(tableSpecification.CommandTopicName(this.columnName), this.partition),
                new Message <Null, string> {
                Value = JsonConvert.SerializeObject(changeCommand, Formatting.Indented, jsonSettings)
            }
                );

            return(await WaitForResult(changeCommand.Correlation).ConfigureAwait(false));
        }
Exemplo n.º 2
0
        public static void Log(Command c, string columnName, int partition)
        {
            var colId = String.Format("{0,15}", $"[{columnName}|{partition}]");

            var summary = c switch
            {
                Command_Change cmd => $"{colId} CHANGE  {cmd.ColumnValue} | {cmd.ChangeType}",
                Command_Enter cmd => $"{colId} ENTER   {cmd.ColumnValue}",
                Command_Verify cmd => $"{colId} VERIFY  {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
                Command_Exit cmd => $"{colId} EXIT    {cmd.ColumnValue}",
                Command_Ack cmd => $"{colId} ACK     {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
                _ => $"unknown command"
            };

            var logLine = String.Format("{0,-60} .." + c.Correlation.Substring(c.Correlation.Length - 8), summary);

            Console.WriteLine(logLine);
        }
        /// <summary>
        ///     Handle a brand new add or update command.
        /// </summary>
        private void HandleChange(Command_Change cmd, Offset offset)
        {
            if (inProgressState_Active.ContainsKey(cmd.Correlation))
            {
                // if the command_change message is received more than once, this is a duplicate
                // message in the log and can simply be ignored.
                Logger.Log("CHANGE", $"ignoring duplicate change command [{cmd.Correlation}]");
                return;
            }

            // if the column value is locked, then fail the command immediately.
            if (locked.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} locked, can't change [{cmd.Correlation}]"));
                return;
            }

            // also abort if this is an add operation and the value exists already.
            if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Add &&
                this.materialized.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} exists, can't add [{cmd.Correlation}]"));
                return;
            }

            // also abort if this is an update operation and the value doesn't exist already.
            if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Update &&
                !this.materialized.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} doesn't exists, can't update [{cmd.Correlation}]"));
                return;
            }

            // AddOrUpdate can be dis-ambiguated at this point, and is not considered further
            // in the workflow.
            var isAddCommand = !materialized.ContainsKey(cmd.ColumnValue) || cmd.ChangeType == Experiment.ChangeType.Add;

            var amalgamatedData            = new Dictionary <string, string>(cmd.Data);
            var uniqueColumnValuesToDelete = new Dictionary <string, string>();

            // if this is an update:
            if (!isAddCommand)
            {
                // 1. amalgamate with existing data.
                var existing = materialized[cmd.ColumnValue];
                foreach (var e in existing)
                {
                    if (!amalgamatedData.ContainsKey(e.Key))
                    {
                        amalgamatedData.Add(e.Key, e.Value);
                    }
                }

                // 2. work out unique values (other than this) that have changed -
                //    we need the old values to be removed.
                //
                //    notes:
                //     1. the current partition column value can't have changed (obviously)
                //     2. even if a unique column value hasn't changed, it
                //        needs to get locked (be included in the operation workflow)
                //        because it's data is changing.
                foreach (var other in this.otherUniqueColumns)
                {
                    if (cmd.Data.ContainsKey(other.Name))
                    {
                        if (cmd.Data[other.Name] != materialized[cmd.ColumnValue][other.Name])
                        {
                            uniqueColumnValuesToDelete.Add(other.Name, this.materialized[cmd.ColumnValue][other.Name]);
                        }
                    }
                }
            }

            // check that all unique columns have a value.
            foreach (var other in this.otherUniqueColumns)
            {
                if (!amalgamatedData.ContainsKey(other.Name))
                {
                    CompleteChangeRequest(cmd.Correlation, new Exception("change request does not contain value for key {other.Name} [{cmd.Correlation}]"));
                    return;
                }
            }

            // lock this column's value - at this point, we're going to try and apply the change.
            locked.Add(cmd.ColumnValue, cmd.Correlation);

            // prevent a commit of this offset until the final write in the workflow is done.
            blockedForCommit.Add(offset, cmd.Correlation);

            // keep track of info related to this command including the columns we'll be waiting for a verify from.
            var otherList = otherUniqueColumns
                            .Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name]))
                            .Concat(uniqueColumnValuesToDelete.ToList())
                            .Select(a => new NameAndValue {
                Name = a.Key, Value = a.Value
            })
                            .ToList();

            inProgressState_Active.Add(cmd.Correlation, new InProgressState_Active
            {
                ChangeCommandOffset = offset,
                ColumnValue         = cmd.ColumnValue,
                WaitingVerify       = otherList,
                WaitingAck          = otherList,
                Data         = amalgamatedData,
                ToDelete     = uniqueColumnValuesToDelete,
                ToSet        = otherUniqueColumns.Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name])).ToDictionary(a => a.Key, a => a.Value),
                VerifyFailed = new List <NameAndValue>()
            });

            // finally, send an enter command to the relevant key/values
            // that need locking.
            foreach (var cs in inProgressState_Active[cmd.Correlation].WaitingVerify)
            {
                var enterCommand = new Command_Enter
                {
                    Correlation       = cmd.Correlation,
                    IsAddCommand      = isAddCommand,
                    ColumnValue       = cs.Value,
                    SourceColumnName  = this.columnName,
                    SourceColumnValue = cmd.ColumnValue
                };

                var tp = new TopicPartition(
                    tableSpecification.CommandTopicName(cs.Name),
                    Table.Partitioner(cs.Value, numPartitions)
                    );

                // TODO: verify producer settings ensure in order, gapless produce.
                //       with a long retry.
                cmdProducer.ProduceAsync(tp, new Message <Null, string> {
                    Value = JsonConvert.SerializeObject(enterCommand, Formatting.Indented, jsonSettings)
                })
                .FailIfFaulted("CHANGE", $"A fatal problem occured writing a lock command.");
            }
        }