public void CheckpointErrorHandling()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction(null)
{
Error = "oh noes"
},
new ShardEndedAction()
);
bool handlerCodeCalled = false;
CheckpointErrorHandler handler = (sequenceNumber, errorMsg, checkpointer) =>
{
handlerCodeCalled = true;
};
TestRecordProcessor recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last(), handler)
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
Assert.IsTrue(handlerCodeCalled);
}
public void HappyCase()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction("456"),
new ShardEndedAction(),
new CheckpointAction("456")
);
TestRecordProcessor recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last()),
ShardEndedFunc = (input) => input.Checkpointer.Checkpoint()
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
List <Action> outputActions = ParseActionsFromOutput();
dynamic a = outputActions[0];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == InitializeAction.ACTION);
a = outputActions[1];
Assert.IsTrue(a is CheckpointAction);
Assert.IsTrue(a.SequenceNumber == "456" && a.Error == null);
a = outputActions[2];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == ProcessRecordsAction.ACTION);
a = outputActions[3];
Assert.IsTrue(a is CheckpointAction);
Assert.IsTrue(a.SequenceNumber == null && a.Error == null);
a = outputActions[4];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == ShardEndedAction.ACTION);
}
public void CheckpointErrorHandling()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction(null) { Error = "oh noes" },
new ShutdownAction("TERMINATE")
);
var handlerCodeCalled = false;
CheckpointErrorHandler handler = (sequenceNumber, errorMsg, checkpointer) =>
{
handlerCodeCalled = true;
};
var recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last(), handler)
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
Assert.IsTrue(handlerCodeCalled);
}
public void RetryHandler()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" },
new ShutdownAction(ShutdownReason.TERMINATE.ToString()),
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" },
new CheckpointAction(null) { Error = "oh noes" }
);
const int numRetries = 3;
var recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last(),
RetryingCheckpointErrorHandler.Create(numRetries, TimeSpan.Zero)),
ShutdownFunc = (input) => input.Checkpointer.Checkpoint(
RetryingCheckpointErrorHandler.Create(numRetries, TimeSpan.Zero))
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
var outputActions = ParseActionsFromOutput();
Console.Error.WriteLine(string.Join("\n", outputActions.Select(x => x.ToString()).ToList()));
var i = 0;
dynamic a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual("initialize", a.ResponseFor);
for (int j = 0; j <= numRetries; j++)
{
a = outputActions[i++];
Assert.IsTrue(a is CheckpointAction, "Action " + (i - 1) + " should be CheckpointAction");
Assert.AreEqual("456", a.SequenceNumber);
Assert.IsNull(a.Error);
}
a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual("processRecords", a.ResponseFor);
for (int j = 0; j <= numRetries; j++)
{
a = outputActions[i++];
Assert.IsTrue(a is CheckpointAction, "Action " + (i - 1) + " should be CheckpointAction");
Assert.IsNull(a.SequenceNumber);
Assert.IsNull(a.Error);
}
a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual("shutdown", a.ResponseFor);
}
public void HappyCase()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction("456"),
new ShutdownAction("TERMINATE"),
new CheckpointAction("456")
);
var recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last()),
ShutdownFunc = (input) => input.Checkpointer.Checkpoint()
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
var outputActions = ParseActionsFromOutput();
dynamic a = outputActions[0];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == "initialize");
a = outputActions[1];
Assert.IsTrue(a is CheckpointAction);
Assert.IsTrue(a.SequenceNumber == "456" && a.Error == null);
a = outputActions[2];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == "processRecords");
a = outputActions[3];
Assert.IsTrue(a is CheckpointAction);
Assert.IsTrue(a.SequenceNumber == null && a.Error == null);
a = outputActions[4];
Assert.IsTrue(a is StatusAction);
Assert.IsTrue(a.ResponseFor == "shutdown");
}
public void RetryHandler()
{
WriteActions(
new InitializeAction("0"),
new ProcessRecordsAction(new DefaultRecord("456", "cat", "bWVvdw==")),
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
},
new ShardEndedAction(),
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
},
new CheckpointAction(null)
{
Error = "oh noes"
}
);
const int numRetries = 3;
TestRecordProcessor recordProcessor = new TestRecordProcessor
{
ProcessFunc = (input) => input.Checkpointer.Checkpoint(input.Records.Last(),
RetryingCheckpointErrorHandler.Create(numRetries, TimeSpan.Zero)),
ShardEndedFunc = (input) => input.Checkpointer.Checkpoint(
RetryingCheckpointErrorHandler.Create(numRetries, TimeSpan.Zero))
};
KclProcess.Create(recordProcessor, _ioHandler).Run();
List <Action> outputActions = ParseActionsFromOutput();
Console.Error.WriteLine(String.Join("\n", outputActions.Select(x => x.ToJson()).ToList()));
int i = 0;
dynamic a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual(InitializeAction.ACTION, a.ResponseFor);
for (int j = 0; j <= numRetries; j++)
{
a = outputActions[i++];
Assert.IsTrue(a is CheckpointAction, "Action " + (i - 1) + " should be CheckpointAction");
Assert.AreEqual("456", a.SequenceNumber);
Assert.IsNull(a.Error);
}
a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual(ProcessRecordsAction.ACTION, a.ResponseFor);
for (int j = 0; j <= numRetries; j++)
{
a = outputActions[i++];
Assert.IsTrue(a is CheckpointAction, "Action " + (i - 1) + " should be CheckpointAction");
Assert.IsNull(a.SequenceNumber);
Assert.IsNull(a.Error);
}
a = outputActions[i++];
Assert.IsTrue(a is StatusAction, "Action " + (i - 1) + " should be StatusAction");
Assert.AreEqual(ShardEndedAction.ACTION, a.ResponseFor);
}
