Пример #1
0
        public static void Log(Command c, string columnName, int partition)
        {
            var colId = String.Format("{0,15}", $"[{columnName}|{partition}]");

            var summary = c switch
            {
                Command_Change cmd => $"{colId} CHANGE  {cmd.ColumnValue} | {cmd.ChangeType}",
                Command_Enter cmd => $"{colId} ENTER   {cmd.ColumnValue}",
                Command_Verify cmd => $"{colId} VERIFY  {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
                Command_Exit cmd => $"{colId} EXIT    {cmd.ColumnValue}",
                Command_Ack cmd => $"{colId} ACK     {cmd.SourceColumnName} | {cmd.SourceColumnValue}",
                _ => $"unknown command"
            };

            var logLine = String.Format("{0,-60} .." + c.Correlation.Substring(c.Correlation.Length - 8), summary);

            Console.WriteLine(logLine);
        }
        private void HandleEnter(Command_Enter cmd, Offset offset)
        {
            bool canChange = true;

            // deal with case where value is locked already.
            if (locked.ContainsKey(cmd.ColumnValue))
            {
                if (locked[cmd.ColumnValue] != cmd.Correlation) // dedupe.
                {
                    Logger.Log("ENTER", $"Duplicate lock command received, ignoring [{cmd.Correlation}]");
                    return;
                }
                else
                {
                    Logger.Log("ENTER", $"Attempting to lock key that is already locked: blocking the change operation [{cmd.Correlation}]");
                    canChange = false;
                }
            }

            // deal with case where value is already materialized.
            if (materialized.ContainsKey(cmd.ColumnValue))
            {
                if (cmd.IsAddCommand)
                {
                    Logger.Log("ENTER", $"Attempting to add a new row with unique column '{this.columnName}' value '{cmd.ColumnValue}' that already exists. [{cmd.Correlation}]");
                    canChange = false;
                }

                if (!cmd.IsAddCommand)
                {
                    // in the case of an update, it is fine for the key to exist, if the row corresponds to the one being updated.
                    if (materialized[cmd.ColumnValue][cmd.SourceColumnName] != cmd.SourceColumnValue)
                    {
                        Logger.Log("ENTER", $"Attempting to update a row with unique column '{this.columnName}' value '{cmd.ColumnValue}' that already exists for some other row. [{cmd.Correlation}]");
                        canChange = false;
                    }
                }
            }

            var verifyCommand = new Command_Verify
            {
                Correlation       = cmd.Correlation,
                Verified          = canChange,
                SourceColumnName  = this.columnName,
                SourceColumnValue = cmd.ColumnValue
            };

            var tp = new TopicPartition(
                tableSpecification.CommandTopicName(cmd.SourceColumnName),
                Table.Partitioner(cmd.SourceColumnValue, numPartitions));

            cmdProducer.ProduceAsync(tp, new Message <Null, string> {
                Value = JsonConvert.SerializeObject(verifyCommand, Formatting.Indented, jsonSettings)
            })
            .FailIfFaulted("ENTER", $"A fatal problem occured writing a verify command.");

            // locking is only required if the value may possibly be changing.
            if (canChange)
            {
                // lock the value until the command is complete [this may not be necessary].
                locked.Add(cmd.ColumnValue, cmd.Correlation);

                // don't allow commit until corresponding exit command
                blockedForCommit.Add(offset, cmd.Correlation);

                // remember the offset, to remove from blockedForCommit on exit.
                inProgressState_Secondary.Add(cmd.Correlation, new InProgressState_Secondary {
                    EnterCommandOffset = offset
                });
            }
        }
        /// <summary>
        ///     Handle a brand new add or update command.
        /// </summary>
        private void HandleChange(Command_Change cmd, Offset offset)
        {
            if (inProgressState_Active.ContainsKey(cmd.Correlation))
            {
                // if the command_change message is received more than once, this is a duplicate
                // message in the log and can simply be ignored.
                Logger.Log("CHANGE", $"ignoring duplicate change command [{cmd.Correlation}]");
                return;
            }

            // if the column value is locked, then fail the command immediately.
            if (locked.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} locked, can't change [{cmd.Correlation}]"));
                return;
            }

            // also abort if this is an add operation and the value exists already.
            if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Add &&
                this.materialized.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} exists, can't add [{cmd.Correlation}]"));
                return;
            }

            // also abort if this is an update operation and the value doesn't exist already.
            if (cmd.ChangeType == Howlett.Kafka.Extensions.Experiment.ChangeType.Update &&
                !this.materialized.ContainsKey(cmd.ColumnValue))
            {
                CompleteChangeRequest(cmd.Correlation, new Exception($"key {cmd.ColumnValue} doesn't exists, can't update [{cmd.Correlation}]"));
                return;
            }

            // AddOrUpdate can be dis-ambiguated at this point, and is not considered further
            // in the workflow.
            var isAddCommand = !materialized.ContainsKey(cmd.ColumnValue) || cmd.ChangeType == Experiment.ChangeType.Add;

            var amalgamatedData            = new Dictionary <string, string>(cmd.Data);
            var uniqueColumnValuesToDelete = new Dictionary <string, string>();

            // if this is an update:
            if (!isAddCommand)
            {
                // 1. amalgamate with existing data.
                var existing = materialized[cmd.ColumnValue];
                foreach (var e in existing)
                {
                    if (!amalgamatedData.ContainsKey(e.Key))
                    {
                        amalgamatedData.Add(e.Key, e.Value);
                    }
                }

                // 2. work out unique values (other than this) that have changed -
                //    we need the old values to be removed.
                //
                //    notes:
                //     1. the current partition column value can't have changed (obviously)
                //     2. even if a unique column value hasn't changed, it
                //        needs to get locked (be included in the operation workflow)
                //        because it's data is changing.
                foreach (var other in this.otherUniqueColumns)
                {
                    if (cmd.Data.ContainsKey(other.Name))
                    {
                        if (cmd.Data[other.Name] != materialized[cmd.ColumnValue][other.Name])
                        {
                            uniqueColumnValuesToDelete.Add(other.Name, this.materialized[cmd.ColumnValue][other.Name]);
                        }
                    }
                }
            }

            // check that all unique columns have a value.
            foreach (var other in this.otherUniqueColumns)
            {
                if (!amalgamatedData.ContainsKey(other.Name))
                {
                    CompleteChangeRequest(cmd.Correlation, new Exception("change request does not contain value for key {other.Name} [{cmd.Correlation}]"));
                    return;
                }
            }

            // lock this column's value - at this point, we're going to try and apply the change.
            locked.Add(cmd.ColumnValue, cmd.Correlation);

            // prevent a commit of this offset until the final write in the workflow is done.
            blockedForCommit.Add(offset, cmd.Correlation);

            // keep track of info related to this command including the columns we'll be waiting for a verify from.
            var otherList = otherUniqueColumns
                            .Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name]))
                            .Concat(uniqueColumnValuesToDelete.ToList())
                            .Select(a => new NameAndValue {
                Name = a.Key, Value = a.Value
            })
                            .ToList();

            inProgressState_Active.Add(cmd.Correlation, new InProgressState_Active
            {
                ChangeCommandOffset = offset,
                ColumnValue         = cmd.ColumnValue,
                WaitingVerify       = otherList,
                WaitingAck          = otherList,
                Data         = amalgamatedData,
                ToDelete     = uniqueColumnValuesToDelete,
                ToSet        = otherUniqueColumns.Select(a => new KeyValuePair <string, string>(a.Name, amalgamatedData[a.Name])).ToDictionary(a => a.Key, a => a.Value),
                VerifyFailed = new List <NameAndValue>()
            });

            // finally, send an enter command to the relevant key/values
            // that need locking.
            foreach (var cs in inProgressState_Active[cmd.Correlation].WaitingVerify)
            {
                var enterCommand = new Command_Enter
                {
                    Correlation       = cmd.Correlation,
                    IsAddCommand      = isAddCommand,
                    ColumnValue       = cs.Value,
                    SourceColumnName  = this.columnName,
                    SourceColumnValue = cmd.ColumnValue
                };

                var tp = new TopicPartition(
                    tableSpecification.CommandTopicName(cs.Name),
                    Table.Partitioner(cs.Value, numPartitions)
                    );

                // TODO: verify producer settings ensure in order, gapless produce.
                //       with a long retry.
                cmdProducer.ProduceAsync(tp, new Message <Null, string> {
                    Value = JsonConvert.SerializeObject(enterCommand, Formatting.Indented, jsonSettings)
                })
                .FailIfFaulted("CHANGE", $"A fatal problem occured writing a lock command.");
            }
        }