private void SendSynchronously(IRingMasterBackendRequest req)
{
Stopwatch sw = Stopwatch.StartNew();
try
{
this.backend.ProcessMessage(
req,
this.session,
(r, e) =>
{
RequestResponse resp = r;
req.NotifyComplete(resp.ResultCode, resp.Content, resp.Stat, resp.ResponsePath);
this.OnComplete(req, resp.ResultCode, sw.ElapsedMilliseconds);
sw.Stop();
});
}
catch (TimeoutException)
{
this.OnComplete(req, (int)RingMasterException.Code.Operationtimeout, sw.ElapsedMilliseconds);
sw.Stop();
}
catch (Exception)
{
this.OnComplete(req, (int)RingMasterException.Code.Systemerror, sw.ElapsedMilliseconds);
sw.Stop();
}
}
/// <summary>
/// gets or creates a locklist
/// </summary>
/// <param name="req">the request this operation is for</param>
/// <param name="createChangeList">Function to create change list</param>
/// <param name="lockDownPaths">if not null, this is a list of paths that are in RW lockdown mode.</param>
/// <param name="txId">Transaction ID unique in the lifetime of service partition</param>
/// <returns>the locklist for this thread</returns>
internal ILockListTransaction GetOrCreateLockList(IRingMasterBackendRequest req, Func <Persistence.IChangeList> createChangeList, LockDownSet lockDownPaths, long txId)
{
IOperationOverrides over = req.Overrides;
ILockListTransaction ll;
ISessionAuth auth = req.Auth;
if (auth == null)
{
auth = this.authenticationInfo;
}
if (this.ROInterfaceRequiresLocks || this.WritesAllowed || !useROLocks)
{
ll = new LockListForRW(createChangeList, lockDownPaths, this.OnlyEphemeralChangesAllowed, txId);
}
else
{
ll = new LockListForRO();
}
try
{
ll.Initialize(auth, over);
}
catch (Exception)
{
ll.Dispose();
throw;
}
return(ll);
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
BackendRequestWithContext<TRequest, TReturn> other = obj as BackendRequestWithContext<TRequest, TReturn>;
// note we don't need to validate the request type because the previous check covers us on that
if (this.Uid != other?.Uid)
{
return false;
}
if (this.ExecutionQueueTimeoutMillis != other.ExecutionQueueTimeoutMillis)
{
return false;
}
if (Guid.Equals(this.ExecutionQueueId, other.ExecutionQueueId))
{
return false;
}
if (!string.Equals(this.Path, other.Path))
{
return false;
}
return EqualityHelper.Equals(this.context, other.context);
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
RequestInit other = obj as RequestInit;
// note we don't need to validate the request type because the previous check covers us on that
if (this.SessionId != other?.SessionId)
{
return(false);
}
return(string.Equals(this.SessionPwd, other.SessionPwd));
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
RequestSetAuth other = obj as RequestSetAuth;
if (other == null)
{
return(false);
}
// note we don't need to validate the request type because the previous check covers us on that
return(string.Equals(this.ClientId, other.ClientId));
}
/// <summary>
/// commands are encoded in a create operation for a path.
/// This routines tells if the given request is a single command
/// </summary>
/// <param name="req">the request potentially containing the command</param>
/// <returns>true if the request is a command</returns>
public virtual bool IsCommand(IRingMasterBackendRequest req)
{
if (req == null)
{
return(false);
}
if (req.RequestType != RingMasterRequestType.Create)
{
return(false);
}
return(this.IsCommand(req.Path));
}
public override void Send(IRingMasterBackendRequest req)
{
if (req == null)
{
return;
}
List <Action> actions = new List <Action>();
RequestResponse resp;
if (req.RequestType == RingMasterRequestType.Multi && ((Backend.RequestMulti)req).ScheduledName != null)
{
string scheduledName = ((Backend.RequestMulti)req).ScheduledName;
((Backend.RequestMulti)req).ScheduledName = null;
Requests.RequestCreate crReq = new Requests.RequestCreate("/$metadata/scheduler/commands/" + scheduledName, ScheduledCommand.GetBytes(req, this.marshaller), null, CreateMode.Persistent);
RequestResponse aux = this.ProcessT(crReq, actions);
this.ev.PushEvent(this.ToString(crReq));
resp = new RequestResponse()
{
CallId = 0,
Content = new List <OpResult>()
{
OpResult.GetOpResult(RingMasterRequestType.Create, aux)
}.AsReadOnly(),
ResponsePath = string.Empty,
Stat = null,
ResultCode = aux.ResultCode
};
}
else
{
resp = this.Process(req.WrappedRequest, actions);
this.ev.PushEvent(this.ToString(req.WrappedRequest));
}
foreach (Action action in actions)
{
action();
}
ThreadPool.QueueUserWorkItem(_ =>
{
req.NotifyComplete(resp.ResultCode, resp.Content, resp.Stat, resp.ResponsePath);
});
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
RequestGetData other = obj as RequestGetData;
if (other == null)
{
return(false);
}
if (!base.DataEquals(other))
{
return(false);
}
return(true);
}
/// <summary>
/// Process the given request.
/// </summary>
/// <param name="request">Request that must be processed</param>
/// <param name="session">Client session associtated with the request</param>
/// <param name="onCompletion">Action that must be invoked when the request is completed</param>
public void ProcessMessage(
IRingMasterBackendRequest request,
ClientSession session,
Action <RequestResponse, Exception> onCompletion)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (session == null)
{
throw new ArgumentNullException(nameof(session));
}
PendingRequest pendingRequest = new PendingRequest
{
SequenceNumber = Interlocked.Increment(ref this.lastAssignedSequenceNumber),
Lifetime = Stopwatch.StartNew(),
Request = request,
Session = session,
OnCompletion = onCompletion ?? NoAction,
};
if (this.requestQueue.TryAdd(pendingRequest))
{
RingMasterEventSource.Log.Executor_RequestQueued(pendingRequest.SequenceNumber, session.SessionId, request.Uid);
this.instrumentation?.OnExecutionScheduled(this.requestQueue.Count, this.requestQueue.BoundedCapacity);
}
else
{
RingMasterEventSource.Log.Executor_RequestQueueOverflow(pendingRequest.SequenceNumber, session.SessionId, request.Uid);
pendingRequest.OnCompletion(
new RequestResponse
{
ResponsePath = request.Path,
ResultCode = (int)RingMasterException.Code.ServerOperationTimeout,
},
null);
this.instrumentation?.OnQueueOverflow(this.requestQueue.Count, this.requestQueue.BoundedCapacity);
}
}
/// <summary>
/// returns the bytes corresponding to the request given
/// </summary>
/// <param name="req">request to serialize</param>
/// <param name="marshaller">marshaller to use</param>
/// <returns>serialization bytes</returns>
public static byte[] GetBytes(IRingMasterBackendRequest req, IByteArrayMarshaller marshaller)
{
if (req == null)
{
throw new ArgumentNullException("req");
}
if (marshaller == null)
{
throw new ArgumentNullException("marshaller");
}
RequestCall call = new RequestCall()
{
CallId = 0,
Request = req,
};
return(marshaller.SerializeRequestAsBytes(call));
}
private RequestResponse Process(IRingMasterRequest req, List <Action> actions)
{
IRingMasterBackendRequest breq = req as IRingMasterBackendRequest;
if (breq != null)
{
req = breq.WrappedRequest;
}
switch (req.RequestType)
{
case RingMasterRequestType.Check:
return(this.ProcessT((Requests.RequestCheck)req, actions));
case RingMasterRequestType.Create:
return(this.ProcessT((Requests.RequestCreate)req, actions));
case RingMasterRequestType.GetChildren:
return(this.ProcessT((Requests.RequestGetChildren)req, actions));
case RingMasterRequestType.GetData:
return(this.ProcessT((Requests.RequestGetData)req, actions));
case RingMasterRequestType.Delete:
return(this.ProcessT((Requests.RequestDelete)req, actions));
case RingMasterRequestType.Exists:
return(this.ProcessT((Requests.RequestExists)req, actions));
case RingMasterRequestType.Multi:
return(this.ProcessT((Requests.RequestMulti)req, actions));
case RingMasterRequestType.Move:
return(this.ProcessT((Requests.RequestMove)req, actions));
}
return(new RequestResponse()
{
ResultCode = (int)RingMasterException.Code.Unimplemented
});
}
/// <inheritdoc />
public override void Send(IRingMasterBackendRequest req)
{
if (req == null)
{
throw new ArgumentNullException("req");
}
if (this.client == null)
{
SecureTransport transport = null;
try
{
var endpoints = SecureTransport.ParseConnectionString(this.ConnectString);
transport = new SecureTransport(this.transportConfiguration);
this.client = new RingMasterClient(this.protocol, transport);
this.secureTransport = transport;
// The lifetime of transport is now owned by RingMasterClient
transport = null;
this.secureTransport.StartClient(endpoints);
}
finally
{
transport?.Dispose();
}
}
this.client.Request(req.WrappedRequest).ContinueWith(responseTask =>
{
try
{
RequestResponse response = responseTask.Result;
req.NotifyComplete(response.ResultCode, response.Content, response.Stat, response.ResponsePath);
}
catch (System.Exception)
{
req.NotifyComplete((int)RingMasterException.Code.Systemerror, null, null, null);
}
});
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
RequestSetAcl other = obj as RequestSetAcl;
if (other == null)
{
return(false);
}
if (!base.DataEquals(other))
{
return(false);
}
if (this.Version != other.Version)
{
return(false);
}
return(EqualityHelper.Equals(this.Acl, other.Acl));
}
/// <inheritdoc />
public override bool DataEquals(IRingMasterBackendRequest obj)
{
RequestSetData other = obj as RequestSetData;
if (this.Version != other?.Version)
{
return(false);
}
if (this.IsDataCommand != other.IsDataCommand)
{
return(false);
}
if (!base.DataEquals(other))
{
return(false);
}
return(EqualityHelper.Equals(this.Data, other.Data));
}
private string ToString(IRingMasterRequest req)
{
IRingMasterBackendRequest breq = req as IRingMasterBackendRequest;
if (breq != null)
{
req = breq.WrappedRequest;
}
if (req.RequestType == RingMasterRequestType.Multi)
{
return(string.Format("Request[Type:{0}, Requests:{1}]", req.RequestType, string.Join(";", ((Requests.RequestMulti)req).Requests.Select(c => this.ToString(c)).ToArray())));
}
if (req.RequestType == RingMasterRequestType.Move)
{
return(string.Format("Request[Type:{0}, Path:{1}, PathDst:{2}]", req.RequestType, req.Path, ((Requests.RequestMove)req).PathDst));
}
return(string.Format("Request[Type:{0}, Path:{1}]", req.RequestType, req.Path));
}
/// <summary>
/// returns the bytes corresponding to the request given
/// </summary>
/// <param name="req">request to serialize</param>
/// <returns>serialization bytes</returns>
public byte[] GetBytes(IRingMasterBackendRequest req)
{
return(GetBytes(req, this.marshaller));
}
/// <inheritdoc />
public override void Send(IRingMasterBackendRequest req)
{
this.executionQueue.Enqueue <IRingMasterBackendRequest>(this.SendSynchronously, req);
}
protected override void OnComplete(IRingMasterBackendRequest req, int resultcode, double timeInMillis)
{
}
/// <inheritdoc />
protected override void OnComplete(IRingMasterBackendRequest req, int resultcode, double timeInMillis)
{
this.del?.Invoke(req, resultcode, timeInMillis);
}
/// <summary>
/// Processes a request in the backend
/// </summary>
/// <param name="backend">Backend for the operation</param>
/// <param name="session">Session for the operation</param>
/// <param name="request">Request to execute</param>
/// <param name="expectedResponseCode">Expected response code for the request</param>
private static void ProcessRequest(RingMasterBackendCore backend, ClientSession session, IRingMasterBackendRequest request, Code expectedResponseCode)
{
var evt = ManualResetEventPool.InstancePool.GetOne();
backend.ProcessMessage(
request,
session,
(response, ex) =>
{
response.ResultCode.Should().Be((int)expectedResponseCode);
evt.Set();
});
ManualResetEventPool.InstancePool.WaitOneAndReturn(ref evt);
}
/// <summary>
/// Executes a single command, with the given name. If there is an uncollected "inflight" item, this method will move it back to commands tree, and throw an exception,
/// hence making the whole scheduler to restart but this time including those commands that were in the inflight bucket.
/// </summary>
/// <param name="scheduledName">the name of the command to execute</param>
private void ExecuteCommand(string scheduledName)
{
byte[] resultbytes = null;
string scheduledCommandPath = GetCommandPath(scheduledName);
string faultPath = GetFailurePath(scheduledName);
RingMasterEventSource.Log.ExecuteScheduledCommandStarted(scheduledName, scheduledCommandPath, faultPath);
Stopwatch sw = Stopwatch.StartNew();
bool rethrow = false;
try
{
IStat stat = null;
IRingMasterBackendRequest req = this.GetRequestOnCommandNode(scheduledCommandPath, ref stat);
try
{
resultbytes = this.ExecuteCommandsAndGetResultsOnError(scheduledCommandPath, stat, req);
}
catch (InflightExistException inflight)
{
Trace.TraceInformation("Inflight command found: " + inflight);
string[] inflightNames = this.MoveInflightPathIntoCommandPath();
rethrow = true;
if (inflightNames != null)
{
throw new InvalidOperationException($"There were objects in the inflight bucket: [{string.Join(",", inflightNames)}]. We moved them back to the commands tree and we will restart processing");
}
throw new InvalidOperationException($"There were objects in the inflight bucket but no useful command. We will restart processing");
}
}
catch (Exception ex)
{
RingMasterEventSource.Log.ExecuteScheduledCommandException(scheduledName, ex.ToString());
if (rethrow)
{
throw;
}
resultbytes = this.GetBytes(ex);
}
if (resultbytes == null)
{
// resultbytes is null, means success, and we are done.
RingMasterServerInstrumentation.Instance.OnScheduledCommandFinished(true, sw.ElapsedMilliseconds);
RingMasterEventSource.Log.ExecuteScheduledCommandCompleted(scheduledName, sw.ElapsedMilliseconds);
return;
}
// delete the command and report the result.
// an exception here will mean we suspend the scheduler.
this.DeleteScheduledCommandAndWriteFailureNode(scheduledCommandPath, faultPath, resultbytes);
RingMasterServerInstrumentation.Instance.OnScheduledCommandFinished(false, sw.ElapsedMilliseconds);
RingMasterEventSource.Log.ExecuteScheduledCommandFailed(scheduledName, sw.ElapsedMilliseconds);
}
/// <summary> /// Compares this <see cref="IRingMasterBackendRequest"/> with the given request. /// </summary> /// <param name="ringMasterRequest">The request to compare with</param> /// <returns><c>true</c> if objects are equal (including data), <c>false</c> otherwise.</returns> public abstract bool DataEquals(IRingMasterBackendRequest ringMasterRequest);
/// <summary>
/// Executes the given command and obtains the result bytes on error.
/// </summary>
/// <param name="scheduledCommandPath">the path to the command</param>
/// <param name="stat">the expected stat of the command node</param>
/// <param name="req">the request to execute</param>
/// <returns>null on success, or the bytes corresponsing to the execution error</returns>
private byte[] ExecuteCommandsAndGetResultsOnError(string scheduledCommandPath, IStat stat, IRingMasterBackendRequest req)
{
if (req == null)
{
return(this.GetBytes(Code.Badarguments, null));
}
IStat readStat = this.self.Exists(scheduledCommandPath, false, false);
if (readStat == null)
{
throw new InvalidOperationException($"Scheduled command stat of [{scheduledCommandPath}] is null");
}
if (readStat.Version != stat.Version || readStat.Cversion != stat.Cversion)
{
throw new InvalidOperationException(
$"Scheduled command stat of [{scheduledCommandPath}] is different from expected command stat; " +
$"Expected [{stat.Version} {stat.Cversion}], " +
$"Actual [{readStat.Version} {readStat.Cversion}]");
}
ManualResetEvent e;
Code resultCode = Code.Unknown;
IReadOnlyList <OpResult> results = null;
// setup the command into the inflight tree
string inflightPath = this.MoveCommandToInflight(scheduledCommandPath, stat);
// run the command, and atomically remove the inflight marks on success
IRingMasterBackendRequest[] ops =
{
req,
new RequestDelete(inflightPath, null, stat.Version, null, DeleteMode.SuccessEvenIfNodeDoesntExist),
new RequestDelete(PathInflightToken,null, -1, null, DeleteMode.SuccessEvenIfNodeDoesntExist),
};
e = ManualResetEventPool.InstancePool.GetOne();
resultCode = Code.Unknown;
results = null;
try
{
this.self.Multi(
ops,
(rc, p, c) =>
{
resultCode = (Code)rc;
if (resultCode == Code.Ok)
{
results = p;
}
e.Set();
},
null,
true);
}
finally
{
ManualResetEventPool.InstancePool.WaitOneAndReturn(ref e);
}
bool allok = this.AllResultsOk(resultCode, results);
if (allok)
{
// if all went well, we are done since the command was deleted from the inflight tree atomically with the operation.
return(null);
}
// otherwise, we need to undo the setup now
this.MoveInflightPathIntoCommandPath();
return(this.GetBytes(resultCode, results));
}
/// <summary>
/// Deletes a command from the commands tree and writes a corresponding node in the failed commands tree.
/// </summary>
/// <param name="scheduledCommandPath">the path to the scheduled command</param>
/// <param name="faultPath">the path to the failure node</param>
/// <param name="resultbytes">the result bytes (describing the failure) to write in the failure node</param>
private void DeleteScheduledCommandAndWriteFailureNode(string scheduledCommandPath, string faultPath, byte[] resultbytes)
{
IRingMasterBackendRequest[] ops = new IRingMasterBackendRequest[]
{
new RequestCreate(faultPath, null, null, null, CreateMode.Persistent, null),
new RequestMove(scheduledCommandPath, null, -1, faultPath, null),
new RequestCreate(faultPath + "/ResultData", null, resultbytes, null, CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag, null),
};
ManualResetEvent e = ManualResetEventPool.InstancePool.GetOne();
IReadOnlyList <OpResult> results = null;
int resultCode = -1;
try
{
this.self.Multi(
ops,
(rc, p, c) =>
{
resultCode = rc;
if (rc == (int)Code.Ok)
{
results = p;
}
e.Set();
},
null,
true);
}
finally
{
ManualResetEventPool.InstancePool.WaitOneAndReturn(ref e);
}
if (resultCode != (int)Code.Ok)
{
throw new AggregateException("error code was not success: " + resultCode);
}
if (results == null)
{
throw new AggregateException("result list came null");
}
if (results.Count == 0)
{
throw new AggregateException("result list came empty");
}
if (results[0].ErrCode != Code.Ok)
{
throw new AggregateException("scheduleCommand failure node could not be created: " + results[0].ErrCode);
}
if (results[1].ErrCode != Code.Ok)
{
throw new AggregateException("scheduleCommand node could not be moved into scheduledFailure tree: " + results[1].ErrCode);
}
if (results[2].ErrCode != Code.Ok)
{
throw new AggregateException("scheduleFailure result node could not be created: " + results[2].ErrCode);
}
}