public static void waitUntilState(LayerManager db, ReplHandler srvr, ReplState state)
{
for (int x = 0; x < 20; x++) {
if (srvr.State == state) {
break;
}
Console.WriteLine("waiting for ({0}) to become {1}.. (currently: {2})",
srvr.ToString(),state, srvr.State);
Thread.Sleep(1000);
}
if (srvr.State != state) {
db.debugDump();
Console.WriteLine("server({0}) failed to become {1}, aborting test",
srvr.ToString(), state);
Environment.Exit(1);
}
Console.WriteLine("Server ({0}) is now {1}!", srvr, state);
}
public int?GetLineNumber(ITextViewLine viewLine, ITextSnapshotLine snapshotLine, ref object state)
{
if (!viewLine.IsFirstTextViewLineForSnapshotLine)
{
return(null);
}
ReplState replState;
if (state == null)
{
state = replState = new ReplState();
}
else
{
replState = (ReplState)state;
}
if (replState.BufferInfo.Buffer == null || viewLine.Start.Position < replState.BufferInfo.Buffer.Span.Start || viewLine.Start.Position >= replState.BufferInfo.Buffer.Span.End)
{
var subBufferInfo = replEditor.FindBuffer(viewLine.Start.Position);
var snapshot = viewLine.Snapshot;
Debug.Assert(subBufferInfo.Buffer.Span.Start <= snapshot.Length);
if (subBufferInfo.Buffer.Span.Start > snapshot.Length)
{
return(null);
}
replState.BufferInfo = subBufferInfo;
replState.BufferStartLine = snapshot.GetLineFromPosition(subBufferInfo.Buffer.Span.Start);
}
int lineNumber = snapshotLine.LineNumber - replState.BufferStartLine.LineNumber;
Debug.Assert(lineNumber >= 0);
if (lineNumber < 0)
{
return(null);
}
return(lineNumber + 1);
}
private void worker_logResume()
{
bool can_log_replay = true;
bool need_resolve = false;
IReplConnection srvr;
List<string> rebuild_reasons = new List<string>();
try {
srvr = pusher.getRandomSeed();
} catch (ReplPusher.NoServersAvailableException) {
// TODO: How will the first resume decide he is current enough to go active if there is
// no Seed?
Console.WriteLine("Repl({0}): no servers available for log resume...",
ctx.server_guid);
return;
}
// (0) check that our data-instance-ids match
if (srvr.getDataInstanceId().CompareTo(this.data_instance_id) != 0) {
this.state = ReplState.error;
return;
}
// (1) see if we can resume from our commit_head pointers
var our_log_status_dict = new Dictionary<string, LogStatus>();
List<LogStatus> client_log_status = this.getStatusForLogs().ToList();
Console.WriteLine("worker_logResume({0}) - ourlogs: {1}",
ctx.server_guid,String.Join(",",client_log_status));
foreach (var ls in client_log_status) {
our_log_status_dict[ls.server_guid] = ls;
}
List<LogStatus> srvr_log_status = srvr.getStatusForLogs().ToList();
Console.WriteLine("worker_logResume({0}) - serverlogs({1}): {2}",
ctx.server_guid, srvr.getServerGuid(),
String.Join(",",srvr_log_status));
foreach (var ls in srvr_log_status) {
if (!our_log_status_dict.ContainsKey(ls.server_guid)) {
// we are missing an entire log...
if (ls.oldest_entry_pointer.GetLong().Equals(0)) {
// it's the magic start of log pointer, so we can resume from it
} else {
// otherwise, we need a full rebuild!
rebuild_reasons.Add(String.Format("we are entirely missing log data: {0}", ls));
can_log_replay = false;
}
} else {
// if our log_head is before their oldest_entry, we need a full rebuild!
var our_ls = our_log_status_dict[ls.server_guid];
if (our_ls.log_commit_head.CompareTo(ls.oldest_entry_pointer) < 0) {
rebuild_reasons.Add(String.Format("log:{0}, our log_head:{1} < their oldest:{2}",
ls.server_guid,our_ls.log_commit_head,ls.oldest_entry_pointer));
can_log_replay = false;
}
if (our_ls.log_commit_head.CompareTo(ls.log_commit_head) > 0) {
// we have newer log entries than they do for at least one log!!
rebuild_reasons.Add(String.Format("log:{0}, our log_head:{1} > their head:{2} need resolve",
ls.server_guid, our_ls.log_commit_head, ls.log_commit_head));
need_resolve = true;
}
}
}
if (!can_log_replay) {
if (!need_resolve) {
// stop all the fetchers!
this._stopFetchers();
// schedule a full rebuild
Console.WriteLine("Repl({0}) logs don't match, we need a full rebuild. Reasons: {1}",
ctx.server_guid, String.Join(",",rebuild_reasons));
this.state = ReplState.rebuild;
return;
} else {
// TODO: do we really need to do anything?
Console.WriteLine("Repl({0}) our log has newer changes than somebody, we expect he'll resolve with us. Reasons: {1}",
ctx.server_guid, String.Join(",",rebuild_reasons));
// this.state = ReplState.resolve;
// return;
}
}
bool all_caught_up = true;
// (3) make sure we have a fetcher for every log_server_guid
// (4) check the to see if we're caught up on all logs
foreach (var ls in srvr_log_status) {
if (ls.server_guid.Equals(this.getServerGuid())) {
// don't try to fetch entries for our own log.
// TODO: check for agreement about our log entries.
continue;
}
// make sure we have a fetcher...
lock (this.fetcher_for_logserverguid) {
if (!this.fetcher_for_logserverguid.ContainsKey(ls.server_guid)) {
this.fetcher_for_logserverguid[ls.server_guid] = new ReplLogFetcher(this, ls.server_guid);
}
if (!this.fetcher_for_logserverguid[ls.server_guid].IsCaughtUp) {
all_caught_up = false;
}
}
}
// if we're all caught up, and we're currently not active, make us active!!
if (all_caught_up && (this.state != ReplState.active)) {
Console.WriteLine("** Server {0} becoming ACTIVE!!", ctx.server_guid);
state = ReplState.active; // we are up to date and online!!
}
// TODO: if we're NOT all caught up, we should go back to inactive!
}
private void worker_fullRebuild()
{
IReplConnection srvr;
// TODO: make sure servers are only listed as seeds when they are "active".
try {
srvr = pusher.getRandomSeed();
// double check that we didn't get ourself, because that won't work
if (srvr.getServerGuid().CompareTo(this.getServerGuid()) == 0) {
Console.WriteLine("******************* ERROR: getRandomSeed() returned US when we're trying to full rebuild");
this.state = ReplState.error;
return;
}
} catch (ReplPusher.NoServersAvailableException) {
Console.WriteLine("Repl({0}): fullRebuild - no servers available... return to init",
ctx.server_guid);
this.state = ReplState.init;
return;
}
// (1) clear our keyspace
// TODO: How do we trigger a reinit on a new prefix, so we don't
// have to actually delete everything?? Maybe we can make SubsetStage
// capable of clearing by dropping the old prefix-id? Or maybe we will
// use built-in support for a range-deletion-tombstone?
// TODO: how do we verify that this isn't going to lose important information?
// TODO: probably should just delete/copy _data and _logs
// so we don't lose our seeds and config info
Console.WriteLine("Rebuild({0}): deleting our keys", ctx.server_guid);
foreach (var row in this.next_stage.scanForward(ScanRange<RecordKey>.All())) {
Console.WriteLine(" Rebuild({0}): deleting {1}", ctx.server_guid, row);
this.next_stage.setValue(row.Key, RecordUpdate.DeletionTombstone());
}
// (2) re-record our data-instance id, so we don't get confused
next_stage.setValue(new RecordKey()
.appendKeyPart("_config")
.appendKeyPart("DATA-INSTANCE-ID"),
RecordUpdate.WithPayload(this.data_instance_id));
// (3) ask for a snapshot
IStepsKVDB snapshot = srvr.getSnapshot();
// (4) then copy the snapshot
// TODO: make sure we get data keys before logs, or that we do something to
// be sure to know whether this copy completes successfully or not.
// TODO: probably want to record our copy-progress, so we can continue copy after
// restart without having to do it from scratch!!
// TODO: need to be able to see tombstones in this scan!!
foreach (var row in snapshot.scanForward(ScanRange<RecordKey>.All())) {
Console.WriteLine(" + Rebuild({0}) setValue: {1}", ctx.server_guid, row);
this.next_stage.setValue(row.Key,RecordUpdate.WithPayload(row.Value.data));
}
// (5) make sure to record our server-id correctly
next_stage.setValue(new RecordKey()
.appendKeyPart("_config")
.appendKeyPart("MY-SERVER-ID"),
RecordUpdate.WithPayload(ctx.server_guid));
// (6) if our log is empty, write our log-start
LogStatus status = this.getStatusForLog(ctx.server_guid);
if (status.log_commit_head.GetLong().CompareTo(0) == 0) {
next_stage.setValue(new RecordKey()
.appendKeyPart("_logs")
.appendKeyPart(ctx.server_guid)
.appendKeyPart(new RecordKeyType_Long(0)),
RecordUpdate.WithPayload(new byte[0]));
}
// (7) make sure there is a seed/log entry for ourselves
next_stage.setValue(new RecordKey()
.appendKeyPart("_config").appendKeyPart("seeds").appendKeyPart(ctx.server_guid),
RecordUpdate.WithPayload(""));
Console.WriteLine("Rebuild({0}): finished, sending to init", ctx.server_guid);
this.state = ReplState.init; // now we should be able to log resume!!
}
private void workerThread()
{
ReplState last_state = this.state;
int error_count = 0;
while (true) {
try {
workerFunc();
} catch (Exception e) {
error_count++;
Console.WriteLine("Server ({0}) exception {1}:\n{2}",
ctx.server_guid, error_count, e.ToString());
if (error_count > 5) {
Console.WriteLine("too many exceptions, shutting down");
this.ctx.connector.unregisterServer(ctx.server_guid);
this.state = ReplState.shutdown;
return;
}
}
if (this.state != last_state) {
Console.WriteLine("++ Server ({0}) changed state {1} -> {2}", ctx.server_guid, last_state, this.state);
last_state = this.state;
}
if (state == ReplState.shutdown) {
Console.WriteLine("worker ending..");
return;
}
Thread.Sleep(1000);
}
}
private void workerFunc()
{
// Make sure we try to stay connnected to all seeds so we can push writes to them
// as fast as possible.
pusher.scanSeeds();
if (this.should_shutdown) {
this.state = ReplState.do_shutdown;
}
switch (this.state) {
case ReplState.init:
// we are initializing.... check our log state and see if we need a full rebuild
Console.WriteLine("Repl({0}): log resume", ctx.server_guid);
worker_logResume();
break;
case ReplState.rebuild:
// we need a FULL rebuild
Console.WriteLine("Repl({0}): full rebuild started", ctx.server_guid);
worker_fullRebuild();
Console.WriteLine("Repl({0}): full rebuild finished", ctx.server_guid);
break;
case ReplState.resolve:
Console.WriteLine("Repl({0}): resolve needed!!", ctx.server_guid);
break;
case ReplState.active:
// we are just running!!
ctx.connector.registerServer(ctx.server_guid, this.my_repl_interface);
// pop back to init to check log tails
worker_logResume();
break;
case ReplState.error:
Console.WriteLine("Repl({0}): error, stalled", ctx.server_guid);
break;
case ReplState.do_shutdown:
// do whatever cleanup we need
_stopFetchers();
this.state = ReplState.shutdown;
break;
case ReplState.shutdown:
break;
default:
// UNKNOWN ReplState
throw new Exception("unknown ReplState: " + this.state.ToString());
}
}
public int? GetLineNumber(ITextViewLine viewLine, ITextSnapshotLine snapshotLine, ref object state) {
if (!viewLine.IsFirstTextViewLineForSnapshotLine)
return null;
ReplState replState;
if (state == null)
state = replState = new ReplState();
else
replState = (ReplState)state;
if (replState.BufferInfo.Buffer == null || viewLine.Start.Position < replState.BufferInfo.Buffer.Span.Start || viewLine.Start.Position >= replState.BufferInfo.Buffer.Span.End) {
var subBufferInfo = replEditor.FindBuffer(viewLine.Start.Position);
var snapshot = viewLine.Snapshot;
Debug.Assert(subBufferInfo.Buffer.Span.Start <= snapshot.Length);
if (subBufferInfo.Buffer.Span.Start > snapshot.Length)
return null;
replState.BufferInfo = subBufferInfo;
replState.BufferStartLine = snapshot.GetLineFromPosition(subBufferInfo.Buffer.Span.Start);
}
int lineNumber = snapshotLine.LineNumber - replState.BufferStartLine.LineNumber;
Debug.Assert(lineNumber >= 0);
if (lineNumber < 0)
return null;
return lineNumber + 1;
}
public Repl(ReplState replState) => ReplState = replState;