public IObservable <T> ObserveState <T>()
        {
            var subject = new Subject <T>();

            Cluster.SubscribeToChannel <T>(ActorInboxCommon.ClusterStatePubSubKey(ProcessId), msg => subject.OnNext(msg));
            return(subject.AsObservable());
        }
        public Unit Startup(IActor process, ActorItem parent, Option <ICluster> cluster, int maxMailboxSize)
        {
            if (cluster.IsNone)
            {
                throw new Exception("Remote inboxes not supported when there's no cluster");
            }
            this.tokenSource    = new CancellationTokenSource();
            this.actor          = (Actor <S, T>)process;
            this.cluster        = cluster.LiftUnsafe();
            this.maxMailboxSize = maxMailboxSize < 0
                ? ActorConfig.Default.MaxMailboxSize
                : maxMailboxSize;
            this.parent = parent;

            actorPath = actor.Id.ToString();

            userNotify = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ClusterUserInboxKey, true));
            sysNotify  = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ClusterSystemInboxKey, false));

            SubscribeToSysInboxChannel();
            SubscribeToUserInboxChannel();

            this.cluster.SetValue(ActorInboxCommon.ClusterMetaDataKey(actor.Id), new ProcessMetaData(
                                      new[] { typeof(T).FullName },
                                      typeof(S).FullName
                                      ));

            return(unit);
        }
Example #3
0
        InboxDirective StatefulUserInbox(Actor <S, T> actor, IActorInbox inbox, UserControlMessage msg, ActorItem parent)
        {
            if (IsPaused)
            {
                userQueue = userQueue.Enqueue(msg);
            }
            else
            {
                while (userQueue.Count > 0)
                {
                    // Don't process messages if we've been paused
                    if (IsPaused)
                    {
                        return(InboxDirective.Pause);
                    }

                    var qmsg = userQueue.Peek();
                    userQueue = userQueue.Dequeue();
                    ProcessInboxDirective(ActorInboxCommon.UserMessageInbox(actor, inbox, qmsg, parent), qmsg);
                }

                if (IsPaused)
                {
                    // Don't process the message if we've been paused
                    userQueue = userQueue.Enqueue(msg);
                    return(InboxDirective.Pause);
                }

                return(ProcessInboxDirective(ActorInboxCommon.UserMessageInbox(actor, inbox, msg, parent), msg));
            }
            return(InboxDirective.Default);
        }
 void SubscribeToSysInboxChannel()
 {
     // System inbox is just listening to the notifications, that means that system
     // messages don't persist.
     cluster.UnsubscribeChannel(ActorInboxCommon.ClusterSystemInboxNotifyKey(actor.Id));
     cluster.SubscribeToChannel <RemoteMessageDTO>(ActorInboxCommon.ClusterSystemInboxNotifyKey(actor.Id)).Subscribe(SysInbox);
 }
        public Unit Startup(IActor process, ActorItem parent, Option <ICluster> cluster, int maxMailboxSize)
        {
            if (cluster.IsNone)
            {
                throw new Exception("Remote inboxes not supported when there's no cluster");
            }
            this.tokenSource = new CancellationTokenSource();
            this.actor       = (Actor <S, T>)process;
            this.cluster     = cluster.LiftUnsafe();

            this.maxMailboxSize = maxMailboxSize;
            this.parent         = parent;

            userNotify = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ActorInboxCommon.ClusterUserInboxKey(actor.Id), true));
            sysNotify  = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ActorInboxCommon.ClusterSystemInboxKey(actor.Id), false));

            SubscribeToSysInboxChannel();
            SubscribeToUserInboxChannel();

            this.cluster.SetValue(ActorInboxCommon.ClusterMetaDataKey(actor.Id), new ProcessMetaData(
                                      new[] { typeof(T).AssemblyQualifiedName },
                                      typeof(S).AssemblyQualifiedName,
                                      typeof(S).GetTypeInfo().ImplementedInterfaces.Map(x => x.AssemblyQualifiedName).ToArray()
                                      ));

            return(unit);
        }
        InboxDirective StatefulUserInbox(Actor <S, T> actor, IActorInbox inbox, UserControlMessage msg, ActorItem parent)
        {
            if (IsPaused)
            {
                userQueue = userQueue.Enqueue(msg);
            }
            else
            {
                while (userQueue.Count > 0)
                {
                    var qmsg = userQueue.Peek();
                    userQueue = userQueue.Dequeue();
                    ActorInboxCommon.UserMessageInbox(actor, inbox, qmsg, parent);
                }
                var directive = ActorInboxCommon.UserMessageInbox(actor, inbox, msg, parent);

                if (directive == InboxDirective.PushToFrontOfQueue)
                {
                    var newQueue = Que <UserControlMessage> .Empty;

                    while (userQueue.Count > 0)
                    {
                        newQueue = newQueue.Enqueue(userQueue.Peek());
                        userQueue.Dequeue();
                    }

                    userQueue = newQueue;
                }
            }
            return(InboxDirective.Default);
        }
Example #7
0
        private void SetupClusterStatePersist(Option <ICluster> cluster, ProcessFlags flags)
        {
            cluster.IfSome(c =>
            {
                if ((flags & ProcessFlags.PersistState) == ProcessFlags.PersistState)
                {
                    try
                    {
                        stateSubject.Subscribe(state => c.SetValue(StateKey, state));
                    }
                    catch (Exception e)
                    {
                        logSysErr(e);
                    }
                }

                if ((flags & ProcessFlags.RemoteStatePublish) == ProcessFlags.RemoteStatePublish)
                {
                    try
                    {
                        stateSubject.Subscribe(state => c.PublishToChannel(ActorInboxCommon.ClusterPubSubKey(Id), state));
                    }
                    catch (Exception e)
                    {
                        logSysErr(e);
                    }
                }
            });
        }
Example #8
0
        /// <summary>
        /// Shutdown everything from this node down
        /// </summary>
        public Unit Shutdown(bool maintainState)
        {
            lock (sync)
            {
                if (maintainState == false)
                {
                    cluster.IfSome(c =>
                    {
                        // TODO: Make this transactional
                        // {
                        c.Delete(StateKey);
                        c.Delete(ActorInboxCommon.ClusterUserInboxKey(Id));
                        c.Delete(ActorInboxCommon.ClusterSystemInboxKey(Id));
                        c.Delete(ActorInboxCommon.ClusterMetaDataKey(Id));
                        ActorContext.DeregisterById(Id);
                        // }
                    });
                }

                RemoveAllSubscriptions();
                publishSubject.OnCompleted();
                stateSubject.OnCompleted();
                remoteSubsAcquired = false;
                strategyState      = StrategyState.Empty;
                DisposeState();

                ActorContext.DispatchTerminate(Id);

                return(unit);
            }
        }
Example #9
0
        public void CheckRemoteInbox(string key, ICluster cluster, ProcessId self, FSharpMailboxProcessor <SystemMessage> sysInbox, FSharpMailboxProcessor <UserControlMessage> userInbox, bool pausable)
        {
            try
            {
                int count = cluster.QueueLength(key);

                while (count > 0 && (!pausable || !IsPaused))
                {
                    Option <Tuple <RemoteMessageDTO, Message> > pair;
                    lock (sync)
                    {
                        pair = ActorInboxCommon.GetNextMessage(cluster, self, key);
                        pair.IfSome(x => cluster.Dequeue <RemoteMessageDTO>(key));
                    }

                    pair.IfSome(x => iter(x, (dto, msg) =>
                    {
                        switch (msg.MessageType)
                        {
                        case Message.Type.System: sysInbox.Post((SystemMessage)msg); break;

                        case Message.Type.User: userInbox.Post((UserControlMessage)msg); break;

                        case Message.Type.UserControl: userInbox.Post((UserControlMessage)msg); break;
                        }
                    }));
                    count--;
                }
            }
            catch (Exception e)
            {
                logSysErr($"CheckRemoteInbox failed for {self}", e);
            }
        }
        public bool CanAccept <T>()
        {
            Try <bool> valid = () =>
            {
                if (Cluster.Exists(ActorInboxCommon.ClusterMetaDataKey(ProcessId)))
                {
                    if (typeof(T) == typeof(TerminatedMessage) || typeof(T) == typeof(UserControlMessage) || typeof(T) == typeof(SystemMessage))
                    {
                        return(true);
                    }
                    var meta = Cluster.GetValue <ProcessMetaData>(ActorInboxCommon.ClusterMetaDataKey(ProcessId));
                    return(meta == null || meta.MsgTypeNames == null
                        ? true
                        : meta.MsgTypeNames.Fold(false, (value, typ) =>
                    {
                        var lhsType = Type.GetType(typ)?.GetTypeInfo();
                        var rhsType = typeof(T).GetTypeInfo();

                        return value
                                ? true
                                : lhsType.IsAssignableFrom(rhsType);
                    }));
                }
                else
                {
                    return(true);
                }
            };

            return(valid.IfFail(true));
        }
        void ValidateMessageType(object message, ProcessId sender)
        {
            if (message == null)
            {
                throw new ProcessException($"Invalid message.  Null is not allowed for Process ({ProcessId}).", ProcessId.Path, sender.Path, null);
            }
            if (message is TerminatedMessage || message is UserControlMessage || message is SystemMessage)
            {
                return;
            }
            if (Cluster.Exists(ActorInboxCommon.ClusterMetaDataKey(ProcessId)))
            {
                Try <bool> valid = () =>
                {
                    var meta = Cluster.GetValue <ProcessMetaData>(ActorInboxCommon.ClusterMetaDataKey(ProcessId));
                    return(meta == null || meta.MsgTypeNames == null
                        ? true
                        : meta.MsgTypeNames.Fold(false, (value, typ) =>
                    {
                        var lhsType = Type.GetType(typ)?.GetTypeInfo();
                        var rhsType = message.GetType().GetTypeInfo();

                        return value
                                    ? true
                                    : lhsType.IsAssignableFrom(rhsType);
                    }));
                };

                if (!valid.IfFail(true))
                {
                    throw new ProcessException($"Invalid message-type ({message.GetType().Name}) for Process ({ProcessId}).", ProcessId.Path, sender.Path, null);
                }
            }
        }
Example #12
0
        private void CheckRemoteInbox(string key)
        {
            var inbox = this;
            var count = cluster.QueueLength(key);

            while (count > 0)
            {
                ActorInboxCommon.GetNextMessage(cluster, actor.Id, key).IfSome(
                    x => iter(x, (dto, msg) =>
                {
                    try
                    {
                        switch (msg.MessageType)
                        {
                        case Message.Type.System: ActorInboxCommon.SystemMessageInbox(actor, inbox, (SystemMessage)msg, parent); break;

                        case Message.Type.User: ActorInboxCommon.UserMessageInbox(actor, inbox, (UserControlMessage)msg, parent); break;

                        case Message.Type.UserControl: ActorInboxCommon.UserMessageInbox(actor, inbox, (UserControlMessage)msg, parent); break;
                        }
                    }
                    catch (Exception e)
                    {
                        ActorContext.WithContext(new ActorItem(actor, inbox, actor.Flags), parent, dto.Sender, msg as ActorRequest, msg, () => replyErrorIfAsked(e));
                        tell(ActorContext.DeadLetters, DeadLetter.create(dto.Sender, actor.Id, e, "Remote message inbox.", msg));
                        logSysErr(e);
                    }
                    finally
                    {
                        cluster.Dequeue <RemoteMessageDTO>(key);
                    }
                }));
                count--;
            }
        }
        public Unit TellSystem(SystemMessage message, ProcessId sender)
        {
            var dto            = RemoteMessageDTO.Create(message, ProcessId, sender, Message.Type.System, message.Tag);
            var clientsReached = Cluster.PublishToChannel(ActorInboxCommon.ClusterSystemInboxNotifyKey(ProcessId), dto);

            return(unit);
        }
Example #14
0
        public Unit Startup(IActor process, ActorItem parent, Option <ICluster> cluster, int version = 0)
        {
            if (cluster.IsNone)
            {
                throw new Exception("Remote inboxes not supported when there's no cluster");
            }
            this.tokenSource = new CancellationTokenSource();
            this.actor       = (Actor <S, T>)process;
            this.cluster     = cluster.LiftUnsafe();
            this.version     = version;
            this.parent      = parent;

            // Registered process remote address hack
            actorPath = actor.Id.Path.StartsWith(Registered.Path)
                ? actor.Id.Skip(1).ToString()
                : actor.Id.ToString();

            // Preparing for message versioning support
            //actorPath += "-" + version;

            userNotify = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ClusterUserInboxKey));
            sysNotify  = ActorInboxCommon.StartNotifyMailbox(tokenSource.Token, msgId => CheckRemoteInbox(ClusterSystemInboxKey));

            this.cluster.SubscribeToChannel <string>(ClusterUserInboxNotifyKey, msg => userNotify.Post(msg));
            this.cluster.SubscribeToChannel <string>(ClusterSystemInboxNotifyKey, msg => sysNotify.Post(msg));

            // We want the check done asyncronously, in case the setup function creates child processes that
            // won't exist if we invoke directly.
            this.cluster.PublishToChannel(ClusterUserInboxNotifyKey, Guid.NewGuid().ToString());
            this.cluster.PublishToChannel(ClusterSystemInboxNotifyKey, Guid.NewGuid().ToString());

            return(unit);
        }
 void SubscribeToUserInboxChannel()
 {
     cluster.UnsubscribeChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id));
     cluster.SubscribeToChannel <string>(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id)).Subscribe(msg => userNotify.Post(msg));
     // We want the check done asyncronously, in case the setup function creates child processes that
     // won't exist if we invoke directly.
     cluster.PublishToChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id), Guid.NewGuid().ToString());
 }
Example #16
0
 public Unit Tell(object message, ProcessId sender)
 {
     if (userInbox != null)
     {
         ActorInboxCommon.PreProcessMessage <T>(sender, actor.Id, message).IfSome(msg => userInbox.Post(msg));
     }
     return(unit);
 }
Example #17
0
        public void Dispose()
        {
            tokenSource?.Cancel();
            tokenSource?.Dispose();
            tokenSource = null;

            cluster?.UnsubscribeChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id));
            cluster?.UnsubscribeChannel(ActorInboxCommon.ClusterSystemInboxNotifyKey(actor.Id));
            cluster = null;
        }
        public Unit Tell(object message, ProcessId sender, string inbox, Message.Type type, Message.TagSpec tag)
        {
            var dto            = RemoteMessageDTO.Create(message, ProcessId, sender, type, tag);
            var inboxKey       = ActorInboxCommon.ClusterInboxKey(ProcessId, inbox);
            var inboxNotifyKey = ActorInboxCommon.ClusterInboxNotifyKey(ProcessId, inbox);

            Cluster.Enqueue(inboxKey, dto);
            long clientsReached = Cluster.PublishToChannel(inboxNotifyKey, dto.MessageId);

            return(unit);
        }
Example #19
0
 public static Unit Publish(object message)
 {
     if (cluster.IsSome && (ProcessFlags & ProcessFlags.RemotePublish) == ProcessFlags.RemotePublish)
     {
         cluster.IfSome(c => c.PublishToChannel(ActorInboxCommon.ClusterPubSubKey(Self), message));
     }
     else
     {
         SelfProcess.Actor.Publish(message);
     }
     return(unit);
 }
Example #20
0
 public Unit Pause()
 {
     lock (sync)
     {
         if (!IsPaused)
         {
             IsPaused = true;
             cluster.UnsubscribeChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id));
         }
     }
     return(unit);
 }
        void CheckRemoteInbox(string key, bool pausable)
        {
            var inbox = this;
            var count = cluster?.QueueLength(key) ?? 0;

            while (count > 0 && (!pausable || !IsPaused))
            {
                var directive = InboxDirective.Default;

                ActorInboxCommon.GetNextMessage(cluster, actor.Id, key).IfSome(
                    x => iter(x, (dto, msg) =>
                {
                    try
                    {
                        switch (msg.MessageType)
                        {
                        case Message.Type.User:        directive = ActorInboxCommon.UserMessageInbox(actor, inbox, (UserControlMessage)msg, parent); break;

                        case Message.Type.UserControl: directive = ActorInboxCommon.UserMessageInbox(actor, inbox, (UserControlMessage)msg, parent); break;
                        }
                    }
                    catch (Exception e)
                    {
                        var session = msg.SessionId == null
                                ? None
                                : Some(new SessionId(msg.SessionId));

                        ActorContext.System(actor.Id).WithContext(new ActorItem(actor, inbox, actor.Flags), parent, dto.Sender, msg as ActorRequest, msg, session, () => replyErrorIfAsked(e));
                        tell(ActorContext.System(actor.Id).DeadLetters, DeadLetter.create(dto.Sender, actor.Id, e, "Remote message inbox.", msg));
                        logSysErr(e);
                    }
                    finally
                    {
                        if ((directive & InboxDirective.Pause) != 0)
                        {
                            IsPaused  = true;
                            directive = directive & (~InboxDirective.Pause);
                        }

                        if (directive == InboxDirective.Default)
                        {
                            cluster?.Dequeue <RemoteMessageDTO>(key);
                        }
                    }
                }));

                if (directive == InboxDirective.Default)
                {
                    count--;
                }
            }
        }
Example #22
0
 void SubscribeToUserInboxChannel()
 {
     cluster.UnsubscribeChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id));
     cluster.SubscribeToChannel <string>(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id)).Subscribe(
         msg =>
     {
         if (userInbox.CurrentQueueLength == 0)
         {
             CheckRemoteInbox(ActorInboxCommon.ClusterUserInboxKey(actor.Id), cluster, actor.Id, sysInbox, userInbox, true);
         }
     });
     cluster.PublishToChannel(ActorInboxCommon.ClusterUserInboxNotifyKey(actor.Id), Guid.NewGuid().ToString());
 }
Example #23
0
        public Unit Tell(object message, ProcessId sender)
        {
            if (Count >= MailboxSize)
            {
                throw new ProcessInboxFullException(actor.Id, MailboxSize, "user");
            }

            if (userInbox != null)
            {
                ActorInboxCommon.PreProcessMessage <T>(sender, actor.Id, message).IfSome(msg => userInbox.Post(msg));
            }
            return(unit);
        }
Example #24
0
        void SetupRemoteSubscriptions(Option <ICluster> cluster, ProcessFlags flags)
        {
            if (remoteSubsAcquired)
            {
                return;
            }

            cluster.IfSome(c =>
            {
                // Watches for local state-changes and persists them
                if ((flags & ProcessFlags.PersistState) == ProcessFlags.PersistState)
                {
                    try
                    {
                        stateSubject.Subscribe(state => c.SetValue(StateKey, state));
                    }
                    catch (Exception e)
                    {
                        logSysErr(e);
                    }
                }

                // Watches for local state-changes and publishes them remotely
                if ((flags & ProcessFlags.RemoteStatePublish) == ProcessFlags.RemoteStatePublish)
                {
                    try
                    {
                        stateSubject.Subscribe(state => c.PublishToChannel(ActorInboxCommon.ClusterStatePubSubKey(Id), state));
                    }
                    catch (Exception e)
                    {
                        logSysErr(e);
                    }
                }

                // Watches for publish events and remotely publishes them
                if ((flags & ProcessFlags.RemotePublish) == ProcessFlags.RemotePublish)
                {
                    try
                    {
                        publishSubject.Subscribe(msg => c.PublishToChannel(ActorInboxCommon.ClusterPubSubKey(Id), msg));
                    }
                    catch (Exception e)
                    {
                        logSysErr(e);
                    }
                }
            });

            remoteSubsAcquired = true;
        }
Example #25
0
 public Either <string, bool> CanAccept <T>()
 {
     if (Cluster.Exists(ActorInboxCommon.ClusterMetaDataKey(ProcessId)))
     {
         var meta = Cluster.GetValue <ProcessMetaData>(ActorInboxCommon.ClusterMetaDataKey(ProcessId));
         return(meta == null
             ? true
             : TypeHelper.IsMessageValidForProcess(typeof(T), meta.MsgTypeNames).Map(_ => true));
     }
     else
     {
         return(true);
     }
 }
Example #26
0
        void ValidateMessageType(object message, ProcessId sender)
        {
            if (Cluster.Exists(ActorInboxCommon.ClusterMetaDataKey(ProcessId)))
            {
                var meta = Cluster.GetValue <ProcessMetaData>(ActorInboxCommon.ClusterMetaDataKey(ProcessId));
                if (meta == null)
                {
                    return;
                }

                TypeHelper.IsMessageValidForProcess(message, meta.MsgTypeNames).IfLeft((string err) =>
                {
                    throw new ProcessException($"{err} for Process ({ProcessId}).", ProcessId.Path, sender.Path, null);
                });
            }
        }
Example #27
0
        public Either <string, bool> HasStateTypeOf <T>()
        {
            if (Cluster.Exists(ActorInboxCommon.ClusterMetaDataKey(ProcessId)))
            {
                var meta = Cluster.GetValue <ProcessMetaData>(ActorInboxCommon.ClusterMetaDataKey(ProcessId));
                if (meta == null)
                {
                    return(true);
                }

                return(TypeHelper.HasStateTypeOf(typeof(T), meta.StateTypeInterfaces));
            }
            else
            {
                return(true);
            }
        }
Example #28
0
 private FSharpMailboxProcessor <TMsg> StartMailbox <TMsg>(Actor <S, T> actor, string key, CancellationToken cancelToken, Action <Actor <S, T>, IActorInbox, TMsg, ActorItem> handler)
     where TMsg : Message =>
 ActorInboxCommon.Mailbox <TMsg>(cancelToken, msg =>
 {
     try
     {
         handler(actor, this, msg, parent);
     }
     catch (Exception e)
     {
         replyErrorIfAsked(e);
         tell(ActorContext.DeadLetters, DeadLetter.create(ActorContext.Sender, actor.Id, e, "Remote message inbox.", msg));
         logSysErr(e);
     }
     finally
     {
         // Remove from queue, then see if there are any more to process.
         CheckRemoteInbox(key, cluster, actor.Id, sysInbox, userInbox);
     }
 });
Example #29
0
        public Unit Startup(IActor process, ActorItem parent, Option <ICluster> cluster, int maxMailboxSize)
        {
            if (cluster.IsNone)
            {
                throw new Exception("Remote inboxes not supported when there's no cluster");
            }
            this.tokenSource = new CancellationTokenSource();
            this.actor       = (Actor <S, T>)process;
            this.parent      = parent;
            this.cluster     = cluster.LiftUnsafe();

            this.maxMailboxSize = maxMailboxSize;

            userInbox = StartMailbox <UserControlMessage>(actor, ActorInboxCommon.ClusterUserInboxKey(actor.Id), tokenSource.Token, StatefulUserInbox, true);
            sysInbox  = StartMailbox <SystemMessage>(actor, ActorInboxCommon.ClusterSystemInboxKey(actor.Id), tokenSource.Token, ActorInboxCommon.SystemMessageInbox, false);

            SubscribeToSysInboxChannel();
            SubscribeToUserInboxChannel();

            return(unit);
        }
Example #30
0
 FSharpMailboxProcessor <TMsg> StartMailbox <TMsg>(Actor <S, T> actor, string key, CancellationToken cancelToken, Func <Actor <S, T>, IActorInbox, TMsg, ActorItem, InboxDirective> handler, bool pausable)
     where TMsg : Message =>
 ActorInboxCommon.Mailbox <TMsg>(cancelToken, msg =>
 {
     try
     {
         var directive = handler(actor, this, msg, parent);
         // TODO: Push msg to front of queue if directive requests it
     }
     catch (Exception e)
     {
         replyErrorIfAsked(e);
         tell(ActorContext.System(actor.Id).DeadLetters, DeadLetter.create(ActorContext.Request.Sender, actor.Id, e, "Remote message inbox.", msg));
         logSysErr(e);
     }
     finally
     {
         // Remove from queue, then see if there are any more to process.
         CheckRemoteInbox(key, cluster, actor.Id, sysInbox, userInbox, pausable);
     }
 });