private void SendSynchronously(IRingMasterBackendRequest req) { Stopwatch sw = Stopwatch.StartNew(); try { this.backend.ProcessMessage( req, this.session, (r, e) => { RequestResponse resp = r; req.NotifyComplete(resp.ResultCode, resp.Content, resp.Stat, resp.ResponsePath); this.OnComplete(req, resp.ResultCode, sw.ElapsedMilliseconds); sw.Stop(); }); } catch (TimeoutException) { this.OnComplete(req, (int)RingMasterException.Code.Operationtimeout, sw.ElapsedMilliseconds); sw.Stop(); } catch (Exception) { this.OnComplete(req, (int)RingMasterException.Code.Systemerror, sw.ElapsedMilliseconds); sw.Stop(); } }
/// <summary> /// gets or creates a locklist /// </summary> /// <param name="req">the request this operation is for</param> /// <param name="createChangeList">Function to create change list</param> /// <param name="lockDownPaths">if not null, this is a list of paths that are in RW lockdown mode.</param> /// <param name="txId">Transaction ID unique in the lifetime of service partition</param> /// <returns>the locklist for this thread</returns> internal ILockListTransaction GetOrCreateLockList(IRingMasterBackendRequest req, Func <Persistence.IChangeList> createChangeList, LockDownSet lockDownPaths, long txId) { IOperationOverrides over = req.Overrides; ILockListTransaction ll; ISessionAuth auth = req.Auth; if (auth == null) { auth = this.authenticationInfo; } if (this.ROInterfaceRequiresLocks || this.WritesAllowed || !useROLocks) { ll = new LockListForRW(createChangeList, lockDownPaths, this.OnlyEphemeralChangesAllowed, txId); } else { ll = new LockListForRO(); } try { ll.Initialize(auth, over); } catch (Exception) { ll.Dispose(); throw; } return(ll); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { BackendRequestWithContext<TRequest, TReturn> other = obj as BackendRequestWithContext<TRequest, TReturn>; // note we don't need to validate the request type because the previous check covers us on that if (this.Uid != other?.Uid) { return false; } if (this.ExecutionQueueTimeoutMillis != other.ExecutionQueueTimeoutMillis) { return false; } if (Guid.Equals(this.ExecutionQueueId, other.ExecutionQueueId)) { return false; } if (!string.Equals(this.Path, other.Path)) { return false; } return EqualityHelper.Equals(this.context, other.context); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { RequestInit other = obj as RequestInit; // note we don't need to validate the request type because the previous check covers us on that if (this.SessionId != other?.SessionId) { return(false); } return(string.Equals(this.SessionPwd, other.SessionPwd)); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { RequestSetAuth other = obj as RequestSetAuth; if (other == null) { return(false); } // note we don't need to validate the request type because the previous check covers us on that return(string.Equals(this.ClientId, other.ClientId)); }
/// <summary> /// commands are encoded in a create operation for a path. /// This routines tells if the given request is a single command /// </summary> /// <param name="req">the request potentially containing the command</param> /// <returns>true if the request is a command</returns> public virtual bool IsCommand(IRingMasterBackendRequest req) { if (req == null) { return(false); } if (req.RequestType != RingMasterRequestType.Create) { return(false); } return(this.IsCommand(req.Path)); }
public override void Send(IRingMasterBackendRequest req) { if (req == null) { return; } List <Action> actions = new List <Action>(); RequestResponse resp; if (req.RequestType == RingMasterRequestType.Multi && ((Backend.RequestMulti)req).ScheduledName != null) { string scheduledName = ((Backend.RequestMulti)req).ScheduledName; ((Backend.RequestMulti)req).ScheduledName = null; Requests.RequestCreate crReq = new Requests.RequestCreate("/$metadata/scheduler/commands/" + scheduledName, ScheduledCommand.GetBytes(req, this.marshaller), null, CreateMode.Persistent); RequestResponse aux = this.ProcessT(crReq, actions); this.ev.PushEvent(this.ToString(crReq)); resp = new RequestResponse() { CallId = 0, Content = new List <OpResult>() { OpResult.GetOpResult(RingMasterRequestType.Create, aux) }.AsReadOnly(), ResponsePath = string.Empty, Stat = null, ResultCode = aux.ResultCode }; } else { resp = this.Process(req.WrappedRequest, actions); this.ev.PushEvent(this.ToString(req.WrappedRequest)); } foreach (Action action in actions) { action(); } ThreadPool.QueueUserWorkItem(_ => { req.NotifyComplete(resp.ResultCode, resp.Content, resp.Stat, resp.ResponsePath); }); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { RequestGetData other = obj as RequestGetData; if (other == null) { return(false); } if (!base.DataEquals(other)) { return(false); } return(true); }
/// <summary> /// Process the given request. /// </summary> /// <param name="request">Request that must be processed</param> /// <param name="session">Client session associtated with the request</param> /// <param name="onCompletion">Action that must be invoked when the request is completed</param> public void ProcessMessage( IRingMasterBackendRequest request, ClientSession session, Action <RequestResponse, Exception> onCompletion) { if (request == null) { throw new ArgumentNullException(nameof(request)); } if (session == null) { throw new ArgumentNullException(nameof(session)); } PendingRequest pendingRequest = new PendingRequest { SequenceNumber = Interlocked.Increment(ref this.lastAssignedSequenceNumber), Lifetime = Stopwatch.StartNew(), Request = request, Session = session, OnCompletion = onCompletion ?? NoAction, }; if (this.requestQueue.TryAdd(pendingRequest)) { RingMasterEventSource.Log.Executor_RequestQueued(pendingRequest.SequenceNumber, session.SessionId, request.Uid); this.instrumentation?.OnExecutionScheduled(this.requestQueue.Count, this.requestQueue.BoundedCapacity); } else { RingMasterEventSource.Log.Executor_RequestQueueOverflow(pendingRequest.SequenceNumber, session.SessionId, request.Uid); pendingRequest.OnCompletion( new RequestResponse { ResponsePath = request.Path, ResultCode = (int)RingMasterException.Code.ServerOperationTimeout, }, null); this.instrumentation?.OnQueueOverflow(this.requestQueue.Count, this.requestQueue.BoundedCapacity); } }
/// <summary> /// returns the bytes corresponding to the request given /// </summary> /// <param name="req">request to serialize</param> /// <param name="marshaller">marshaller to use</param> /// <returns>serialization bytes</returns> public static byte[] GetBytes(IRingMasterBackendRequest req, IByteArrayMarshaller marshaller) { if (req == null) { throw new ArgumentNullException("req"); } if (marshaller == null) { throw new ArgumentNullException("marshaller"); } RequestCall call = new RequestCall() { CallId = 0, Request = req, }; return(marshaller.SerializeRequestAsBytes(call)); }
private RequestResponse Process(IRingMasterRequest req, List <Action> actions) { IRingMasterBackendRequest breq = req as IRingMasterBackendRequest; if (breq != null) { req = breq.WrappedRequest; } switch (req.RequestType) { case RingMasterRequestType.Check: return(this.ProcessT((Requests.RequestCheck)req, actions)); case RingMasterRequestType.Create: return(this.ProcessT((Requests.RequestCreate)req, actions)); case RingMasterRequestType.GetChildren: return(this.ProcessT((Requests.RequestGetChildren)req, actions)); case RingMasterRequestType.GetData: return(this.ProcessT((Requests.RequestGetData)req, actions)); case RingMasterRequestType.Delete: return(this.ProcessT((Requests.RequestDelete)req, actions)); case RingMasterRequestType.Exists: return(this.ProcessT((Requests.RequestExists)req, actions)); case RingMasterRequestType.Multi: return(this.ProcessT((Requests.RequestMulti)req, actions)); case RingMasterRequestType.Move: return(this.ProcessT((Requests.RequestMove)req, actions)); } return(new RequestResponse() { ResultCode = (int)RingMasterException.Code.Unimplemented }); }
/// <inheritdoc /> public override void Send(IRingMasterBackendRequest req) { if (req == null) { throw new ArgumentNullException("req"); } if (this.client == null) { SecureTransport transport = null; try { var endpoints = SecureTransport.ParseConnectionString(this.ConnectString); transport = new SecureTransport(this.transportConfiguration); this.client = new RingMasterClient(this.protocol, transport); this.secureTransport = transport; // The lifetime of transport is now owned by RingMasterClient transport = null; this.secureTransport.StartClient(endpoints); } finally { transport?.Dispose(); } } this.client.Request(req.WrappedRequest).ContinueWith(responseTask => { try { RequestResponse response = responseTask.Result; req.NotifyComplete(response.ResultCode, response.Content, response.Stat, response.ResponsePath); } catch (System.Exception) { req.NotifyComplete((int)RingMasterException.Code.Systemerror, null, null, null); } }); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { RequestSetAcl other = obj as RequestSetAcl; if (other == null) { return(false); } if (!base.DataEquals(other)) { return(false); } if (this.Version != other.Version) { return(false); } return(EqualityHelper.Equals(this.Acl, other.Acl)); }
/// <inheritdoc /> public override bool DataEquals(IRingMasterBackendRequest obj) { RequestSetData other = obj as RequestSetData; if (this.Version != other?.Version) { return(false); } if (this.IsDataCommand != other.IsDataCommand) { return(false); } if (!base.DataEquals(other)) { return(false); } return(EqualityHelper.Equals(this.Data, other.Data)); }
private string ToString(IRingMasterRequest req) { IRingMasterBackendRequest breq = req as IRingMasterBackendRequest; if (breq != null) { req = breq.WrappedRequest; } if (req.RequestType == RingMasterRequestType.Multi) { return(string.Format("Request[Type:{0}, Requests:{1}]", req.RequestType, string.Join(";", ((Requests.RequestMulti)req).Requests.Select(c => this.ToString(c)).ToArray()))); } if (req.RequestType == RingMasterRequestType.Move) { return(string.Format("Request[Type:{0}, Path:{1}, PathDst:{2}]", req.RequestType, req.Path, ((Requests.RequestMove)req).PathDst)); } return(string.Format("Request[Type:{0}, Path:{1}]", req.RequestType, req.Path)); }
/// <summary> /// returns the bytes corresponding to the request given /// </summary> /// <param name="req">request to serialize</param> /// <returns>serialization bytes</returns> public byte[] GetBytes(IRingMasterBackendRequest req) { return(GetBytes(req, this.marshaller)); }
/// <inheritdoc /> public override void Send(IRingMasterBackendRequest req) { this.executionQueue.Enqueue <IRingMasterBackendRequest>(this.SendSynchronously, req); }
protected override void OnComplete(IRingMasterBackendRequest req, int resultcode, double timeInMillis) { }
/// <inheritdoc /> protected override void OnComplete(IRingMasterBackendRequest req, int resultcode, double timeInMillis) { this.del?.Invoke(req, resultcode, timeInMillis); }
/// <summary> /// Processes a request in the backend /// </summary> /// <param name="backend">Backend for the operation</param> /// <param name="session">Session for the operation</param> /// <param name="request">Request to execute</param> /// <param name="expectedResponseCode">Expected response code for the request</param> private static void ProcessRequest(RingMasterBackendCore backend, ClientSession session, IRingMasterBackendRequest request, Code expectedResponseCode) { var evt = ManualResetEventPool.InstancePool.GetOne(); backend.ProcessMessage( request, session, (response, ex) => { response.ResultCode.Should().Be((int)expectedResponseCode); evt.Set(); }); ManualResetEventPool.InstancePool.WaitOneAndReturn(ref evt); }
/// <summary> /// Executes a single command, with the given name. If there is an uncollected "inflight" item, this method will move it back to commands tree, and throw an exception, /// hence making the whole scheduler to restart but this time including those commands that were in the inflight bucket. /// </summary> /// <param name="scheduledName">the name of the command to execute</param> private void ExecuteCommand(string scheduledName) { byte[] resultbytes = null; string scheduledCommandPath = GetCommandPath(scheduledName); string faultPath = GetFailurePath(scheduledName); RingMasterEventSource.Log.ExecuteScheduledCommandStarted(scheduledName, scheduledCommandPath, faultPath); Stopwatch sw = Stopwatch.StartNew(); bool rethrow = false; try { IStat stat = null; IRingMasterBackendRequest req = this.GetRequestOnCommandNode(scheduledCommandPath, ref stat); try { resultbytes = this.ExecuteCommandsAndGetResultsOnError(scheduledCommandPath, stat, req); } catch (InflightExistException inflight) { Trace.TraceInformation("Inflight command found: " + inflight); string[] inflightNames = this.MoveInflightPathIntoCommandPath(); rethrow = true; if (inflightNames != null) { throw new InvalidOperationException($"There were objects in the inflight bucket: [{string.Join(",", inflightNames)}]. We moved them back to the commands tree and we will restart processing"); } throw new InvalidOperationException($"There were objects in the inflight bucket but no useful command. We will restart processing"); } } catch (Exception ex) { RingMasterEventSource.Log.ExecuteScheduledCommandException(scheduledName, ex.ToString()); if (rethrow) { throw; } resultbytes = this.GetBytes(ex); } if (resultbytes == null) { // resultbytes is null, means success, and we are done. RingMasterServerInstrumentation.Instance.OnScheduledCommandFinished(true, sw.ElapsedMilliseconds); RingMasterEventSource.Log.ExecuteScheduledCommandCompleted(scheduledName, sw.ElapsedMilliseconds); return; } // delete the command and report the result. // an exception here will mean we suspend the scheduler. this.DeleteScheduledCommandAndWriteFailureNode(scheduledCommandPath, faultPath, resultbytes); RingMasterServerInstrumentation.Instance.OnScheduledCommandFinished(false, sw.ElapsedMilliseconds); RingMasterEventSource.Log.ExecuteScheduledCommandFailed(scheduledName, sw.ElapsedMilliseconds); }
/// <summary> /// Compares this <see cref="IRingMasterBackendRequest"/> with the given request. /// </summary> /// <param name="ringMasterRequest">The request to compare with</param> /// <returns><c>true</c> if objects are equal (including data), <c>false</c> otherwise.</returns> public abstract bool DataEquals(IRingMasterBackendRequest ringMasterRequest);
/// <summary> /// Executes the given command and obtains the result bytes on error. /// </summary> /// <param name="scheduledCommandPath">the path to the command</param> /// <param name="stat">the expected stat of the command node</param> /// <param name="req">the request to execute</param> /// <returns>null on success, or the bytes corresponsing to the execution error</returns> private byte[] ExecuteCommandsAndGetResultsOnError(string scheduledCommandPath, IStat stat, IRingMasterBackendRequest req) { if (req == null) { return(this.GetBytes(Code.Badarguments, null)); } IStat readStat = this.self.Exists(scheduledCommandPath, false, false); if (readStat == null) { throw new InvalidOperationException($"Scheduled command stat of [{scheduledCommandPath}] is null"); } if (readStat.Version != stat.Version || readStat.Cversion != stat.Cversion) { throw new InvalidOperationException( $"Scheduled command stat of [{scheduledCommandPath}] is different from expected command stat; " + $"Expected [{stat.Version} {stat.Cversion}], " + $"Actual [{readStat.Version} {readStat.Cversion}]"); } ManualResetEvent e; Code resultCode = Code.Unknown; IReadOnlyList <OpResult> results = null; // setup the command into the inflight tree string inflightPath = this.MoveCommandToInflight(scheduledCommandPath, stat); // run the command, and atomically remove the inflight marks on success IRingMasterBackendRequest[] ops = { req, new RequestDelete(inflightPath, null, stat.Version, null, DeleteMode.SuccessEvenIfNodeDoesntExist), new RequestDelete(PathInflightToken,null, -1, null, DeleteMode.SuccessEvenIfNodeDoesntExist), }; e = ManualResetEventPool.InstancePool.GetOne(); resultCode = Code.Unknown; results = null; try { this.self.Multi( ops, (rc, p, c) => { resultCode = (Code)rc; if (resultCode == Code.Ok) { results = p; } e.Set(); }, null, true); } finally { ManualResetEventPool.InstancePool.WaitOneAndReturn(ref e); } bool allok = this.AllResultsOk(resultCode, results); if (allok) { // if all went well, we are done since the command was deleted from the inflight tree atomically with the operation. return(null); } // otherwise, we need to undo the setup now this.MoveInflightPathIntoCommandPath(); return(this.GetBytes(resultCode, results)); }
/// <summary> /// Deletes a command from the commands tree and writes a corresponding node in the failed commands tree. /// </summary> /// <param name="scheduledCommandPath">the path to the scheduled command</param> /// <param name="faultPath">the path to the failure node</param> /// <param name="resultbytes">the result bytes (describing the failure) to write in the failure node</param> private void DeleteScheduledCommandAndWriteFailureNode(string scheduledCommandPath, string faultPath, byte[] resultbytes) { IRingMasterBackendRequest[] ops = new IRingMasterBackendRequest[] { new RequestCreate(faultPath, null, null, null, CreateMode.Persistent, null), new RequestMove(scheduledCommandPath, null, -1, faultPath, null), new RequestCreate(faultPath + "/ResultData", null, resultbytes, null, CreateMode.PersistentAllowPathCreation | CreateMode.SuccessEvenIfNodeExistsFlag, null), }; ManualResetEvent e = ManualResetEventPool.InstancePool.GetOne(); IReadOnlyList <OpResult> results = null; int resultCode = -1; try { this.self.Multi( ops, (rc, p, c) => { resultCode = rc; if (rc == (int)Code.Ok) { results = p; } e.Set(); }, null, true); } finally { ManualResetEventPool.InstancePool.WaitOneAndReturn(ref e); } if (resultCode != (int)Code.Ok) { throw new AggregateException("error code was not success: " + resultCode); } if (results == null) { throw new AggregateException("result list came null"); } if (results.Count == 0) { throw new AggregateException("result list came empty"); } if (results[0].ErrCode != Code.Ok) { throw new AggregateException("scheduleCommand failure node could not be created: " + results[0].ErrCode); } if (results[1].ErrCode != Code.Ok) { throw new AggregateException("scheduleCommand node could not be moved into scheduledFailure tree: " + results[1].ErrCode); } if (results[2].ErrCode != Code.Ok) { throw new AggregateException("scheduleFailure result node could not be created: " + results[2].ErrCode); } }