private IEnumerable <SituationEdge <T> > DevelopRuleInputEdges(string RuleName) { ProcessingQueue <string, SituationEdge <T> > queue; queue = new ProcessingQueue <string, SituationEdge <T> >(); queue.Add(RuleName); queue.Process((q, ruleName) => { foreach (IRule <T> rule in GetRootNodes(ruleName).Select(item => item.Rule)) { foreach (SituationEdge <T> edge in GetRuleInputEdges(rule)) { q.AddResult(edge); if (!(edge.Predicate is INonTerminalPredicate <T> nonTerminal)) { continue; } q.Add(nonTerminal.Name); } } }); return(queue.Results); }
//--- Methods --- protected override IEnumerator <IYield> Start(XDoc config, Result result) { yield return(Coroutine.Invoke(base.Start, config, new Result())); _processingQueue = new ProcessingQueue <XDoc>(CallUpdate); _instances = new ExpiringDictionary <string, PackageUpdater>(TimerFactory, true); _apikey = config["apikey"].AsText; _instanceTtl = TimeSpan.FromSeconds(config["instance-ttl"].AsInt ?? 10 * 60); _packagePath = config["package-path"].AsText; if (string.IsNullOrEmpty(_packagePath)) { throw new ArgumentException("No value was provided for configuration key 'package-path'"); } try { _packagePath = PhpUtil.ConvertToFormatString(_packagePath); // Note (arnec): _packagePath may contain a {0} for injecting wikiid, so we want to make sure the // path string can be formatted without an exception string.Format(_packagePath, "dummy"); } catch { throw new ArgumentException(string.Format("The package path '{0}' contains an illegal formmating directive", _packagePath)); } // set up subscription for pubsub yield return(Coroutine.Invoke(SubscribeInstanceEvents, PubSub, new Result())); result.Return(); }
/// <summary> /// Constructor /// </summary> public LogViewModel(LogOutput parent) { Parent = parent; // Add handlers Parent.OutputViewer.SizeChanged += OutputViewerSizeChanged; Parent.Output.TextChanged += OnTextChanged; Parent.ClearButton.Click += OnClearButton; Parent.SaveButton.Click += OnSaveButton; // Update the internal state var document = new FlowDocument() { Background = new SolidColorBrush(Color.FromArgb(0xFF, 0x20, 0x20, 0x20)) }; _paragraph = new Paragraph(); document.Blocks.Add(_paragraph); Parent.Output.Document = document; // TODO: Can we dynamically add matchers *only* during dumping? _matchers = new List <Matcher?>(); AddAaruMatchers(); AddDiscImageCreatorMatchers(); logQueue = new ProcessingQueue <LogLine>(ProcessLogLine); }
public UnityPluginInstaller( Lifetime lifetime, ILogger logger, ISolution solution, IShellLocks shellLocks, UnityPluginDetector detector, RdNotificationsModel notifications, ISettingsStore settingsStore, ApplicationPackages applicationPackages, ApplicationPackagesLocallyInstalled applicationPackagesLocallyInstalled, IEnumerable <ApplicationPackageArtifact> packages, IDeployedPackagesExpandLocationResolver resolver) { myPluginInstallations = new JetHashSet <FileSystemPath>(); myLifetime = lifetime; myLogger = logger; mySolution = solution; myShellLocks = shellLocks; myDetector = detector; myNotifications = notifications; myApplicationPackages = applicationPackages; myApplicationPackagesLocallyInstalled = applicationPackagesLocallyInstalled; myPackages = packages; myResolver = resolver; myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext())); myQueue = new ProcessingQueue(myShellLocks, myLifetime); }
public MosquitoService(IWindsorContainer container) { _container = container; _queue = new ProcessingQueue(); _workers = new WorkersManager(_queue, new HandlersFactory(_container)); }
public UnityPluginInstaller( Lifetime lifetime, ILogger logger, ISolution solution, IShellLocks shellLocks, UnityPluginDetector detector, RdNotificationsModel notifications, ISettingsStore settingsStore, PluginPathsProvider pluginPathsProvider, UnityVersionDetector unityVersionDetector) { myPluginInstallations = new JetHashSet <FileSystemPath>(); myLifetime = lifetime; myLogger = logger; mySolution = solution; myShellLocks = shellLocks; myDetector = detector; myNotifications = notifications; myPluginPathsProvider = pluginPathsProvider; myUnityVersionDetector = unityVersionDetector; myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext())); myQueue = new ProcessingQueue(myShellLocks, myLifetime); }
public async Task AllTasksAreDequeuedOnce() { const int count = 5000; const int workerCount = 10; ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { }); foreach (int task in Enumerable.Range(0, count)) { queue.Enqueue(task); } var tasks = Enumerable.Range(0, workerCount) .Select(workerIndex => Task.Run(() => { var dequeuedItems = new List <int>(); while (queue.TryDequeue(out int item)) { dequeuedItems.Add(item); } return(dequeuedItems); })) .ToList(); await Task.WhenAll(tasks); var countsPerThread = tasks.Select(t => t.Result.Count).ToList(); // just verify it does not have an extremely peaked distribution var allDequeuedItems = tasks.Select(t => t.Result) .Concat() .Distinct() .ToList(); Assert.AreEqual(count, allDequeuedItems.Count); }
//--- Constructors --- /// <summary> /// Create a new dispatcher. /// </summary> /// <param name="config">Configuration instance injected from pub sub service.</param> public Dispatcher(DispatcherConfig config) { _owner = config.ServiceUri.AsServerUri(); _serviceKeySetCookie = config.ServiceAccessCookie; _combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie); _dispatchQueue = new ProcessingQueue <DispatcherEvent>(DispatchFromQueue, 10); }
protected BaseTemplate(IEnumerable <IToken> fTokens) : this( new ImmutableMap <TokenKey, IToken>(), ImmutableList.Empty <TokenReference>(), new ImmutableMap <InputKey, IInput>(), ProcessingQueue.Empty()) { foreach (var tok in fTokens) { if (tokenMap.ContainsKey(tok.Key)) { tokenSeq = tokenSeq.ConsLast(new TokenReference(tok.Key)); } else { foreach (var input in tok.Inputs) { if (!inputs.ContainsKey(input.Key)) { var newInput = input.Key.Type == InputType.Instance ? (IInput) new UnboundInput(input.Key) : new UnboundStaticInput(input.Key); inputs = inputs.Add(input.Key, newInput); } } tokenMap = tokenMap.Add(tok.Key, tok); tokenSeq = tokenSeq.ConsLast(new TokenReference(tok.Key)); } } }
public void Enqueue(string queueName, ProcessingAction act) { lock (_Queues) { retry: if (stopped) { return; } Cleanup(); bool created = false; if (!_Queues.TryGetValue(queueName, out var queue)) { queue = new ProcessingQueue(); _Queues.Add(queueName, queue); created = true; } if (!queue.Chan.Writer.TryWrite(act)) { goto retry; } if (created) { queue.ProcessTask = queue.Process(cts.Token); } } }
//--- Constructors --- public NotificationDelayQueue(TimeSpan delay, CoroutineHandler<NotificationUpdateRecord, Result> callback) { _delay = delay; _callback = callback; _queueTimer = new TaskTimer(CheckExpire, null); _queueTimer.Change(_delay, TaskEnv.None); _dispatchQueue = new ProcessingQueue<NotificationUpdateRecord>(Dispatch, 10); }
//--- Constructors --- public UpdateRecordDispatcher(CoroutineHandler <UpdateRecord, Result> callback, int maxParallelism, int maxRetry, TimeSpan retrySleep) { _dispatchQueue = new ProcessingQueue <QueueItem>(DispatchRecord, maxParallelism); _callback = callback; _maxRetry = maxRetry; _retrySleep = retrySleep; }
//--- Constructors --- public NotificationDelayQueue(TimeSpan delay, CoroutineHandler <NotificationUpdateRecord, Result> callback) { _delay = delay; _callback = callback; _queueTimer = new TaskTimer(CheckExpire, null); _queueTimer.Change(_delay, TaskEnv.None); _dispatchQueue = new ProcessingQueue <NotificationUpdateRecord>(Dispatch, 10); }
public FileWatcher(Action <Exception> onException = null) { this.onException = onException ?? throw new ArgumentNullException(nameof(onException)); this.watchers = new ConcurrentBag <System.IO.FileSystemWatcher>(); this.watchedDirectories = new Dictionary <Uri, ImmutableHashSet <string> >(); this.processing = new ProcessingQueue(this.onException, "error in file system watcher"); this.globPatterns = new ConcurrentDictionary <Uri, IEnumerable <string> >(); }
//--- Constructors --- public DekiChangeSink(string wikiid, XUri apiUri, Plug publishPlug) { _wikiid = wikiid; _apiUri = apiUri; _publishPlug = publishPlug; _channel = new XUri(string.Format("event://{0}/deki/", _wikiid)); _changeQueue = new ProcessingQueue <ChangeData>(Publish, 10); }
private async Task DoWork(object state) { TaskState taskState = (TaskState)state; Stopwatch stopwatch = new Stopwatch(); while (active) { taskState.Status = State.Running; if (!await CanProcess() && downloadersActive) { taskState.Status = State.Paused; await Task.Delay(250); continue; } if (!PageProcessingQueue.IsEmpty) { UnparsedPage page; if (!PageProcessingQueue.TryDequeue(out page)) { //another thread grabbed the page continue; } stopwatch.Time(() => { return(ProcessThreadPage(page.Thread, page.Html)); }, (long time) => TelemetryManager.Incriment(TelemetryType.processed_pages, taskState.Id, time) ); } if (!ProcessingQueue.IsEmpty) { UnparsedThread thread; if (!ProcessingQueue.TryDequeue(out thread)) { continue; } stopwatch.Time(() => { ProcessThread(thread.Id, thread.Html); }, (long time) => TelemetryManager.Incriment(TelemetryType.processed_threads, taskState.Id, time) ); } else if (!downloadersActive) //Queue is empty, no more downloads are being performed, set active to false after 1s delay { taskState.Status = State.Paused; await Task.Delay(1000); active = false; break; } } taskState.Status = State.Complete; }
protected BaseTemplate (ImmutableMap <TokenKey, IToken> tokenMap, ImmutableList <TokenReference> tokenSeq, ImmutableMap <InputKey, IInput> inputs, ProcessingQueue postProcessingQueue) { this.tokenSeq = tokenSeq; this.inputs = inputs; this.tokenMap = tokenMap; this.postProcessingQueue = postProcessingQueue; }
void OnSocketConnected(Socket socket) { var networkStream = new NetworkStream(socket); queue = new ProcessingQueue(networkStream, new Logger()); queue.OnReceive += messageReceived; queue.Start(); }
public void UpdateStats() { Statistics.RequestQueueLength = RequestManager.Count(t => t.Tileset == this); Statistics.ActiveDownloads = RequestManager.CountActiveDownloads(t => t.Tileset == this); Statistics.ProcessingQueueLength = ProcessingQueue.Count(t => t.Tileset == this); Statistics.DownloadedTiles = TileCache.Count(t => t.Tileset == this); Statistics.ReadyTiles = TileCache.Count(t => t.Tileset == this && t.ContentState == Unity3DTileContentState.READY); Unity3DTilesetStatistics.MaxLoadedTiles = TileCache.MaxSize; }
internal override void TryLog(LogData logData) { try { ProcessingQueue.TryAdd(logData); } catch (Exception ex) { Logger.Error($"Addition of logEntries to queue on processing is impossible. Likely processing thread doesn't working. {ex.GetType()}:{ex.Message}"); } }
private void UpdateProcessingQueue(WorkItem workItem) { if (ProcessingQueue.CapacitiesLeft && workItem != null) { CreateAndEnqueueInstuction(methodName: ProductionAgent.InstuctionsMethods.ProductionStarted.ToString(), objectToProcess: workItem, targetAgent: workItem.ProductionAgent); ProcessingQueue.Enqueue(workItem); Queue.Remove(workItem); } }
public App(IHostApplicationLifetime host, ILoggerFactory loggerFactory, IServiceProvider services, ProgressTracker progress, ProcessingQueue processingQueue, DeadLetterQueue deadLetterQueue) { _host = host; _services = services; _progress = progress; _logger = loggerFactory.CreateLogger(GetType().FullName); _processingQueue = processingQueue; _deadLetterQueue = deadLetterQueue; _workerTokenSource = new(); }
public void ShouldAddAndContains() { ProcessingQueue <string, string> queue; queue = new ProcessingQueue <string, string>( ); for (int t = 0; t < 10; t++) { Assert.IsFalse(queue.Contains(t.ToString())); queue.Add(t.ToString()); Assert.IsTrue(queue.Contains(t.ToString())); } }
public async Task DequeuedItemIsStillIncludedInCount() { const int item = 0; ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { }); queue.Enqueue(item); await DelayDelta(); bool dequeued = queue.TryDequeue(out int dequeuedItem); Assert.AreEqual(1, queue.Count); }
public async Task NeverDequeuedItemCannotBeRemoved() { const int item = 0; const int otherItem = 1; ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { }); queue.Enqueue(item); await DelayDelta(); // should throw because otherItem is not being processed yet queue.OnProcessed(otherItem); }
private async Task MessageReceivedAsync(QueueMessage <TMessage> message, CancellationToken cancellationToken) { if (!cancellationToken.IsCancellationRequested && !_tasks.Any(t => t.Value.IsFaulted)) { if (Behaviour == AcknowledgeBehaviour.BeforeProcess) { _queueConsumer.AcknowledgeMessage(message.DeliveryTag); } string processingSequenceIdentifier = GetProcessingSequenceIdentifier(message.RoutingKey); if (_processingQueues.ContainsKey(processingSequenceIdentifier)) { // Add a message to the processing queue, and signal the processing thread to alert it to the new message. var processingQueue = _processingQueues[processingSequenceIdentifier]; processingQueue.Queue.Enqueue(message); processingQueue.AutoResetEvent.Set(); } else { // create a new processing queue and kick off a task to process it. var processingQueue = new ProcessingQueue <TMessage>(); processingQueue.Queue.Enqueue(message); _processingQueues[processingSequenceIdentifier] = processingQueue; var t = Task.Run(RunSequentialProcessor(processingQueue, cancellationToken)); _tasks.TryAdd(processingSequenceIdentifier, t); } _logger.Info($"Received new message {message.DeliveryTag}, processor queue length {_processingQueues.First().Value.Queue.Count()}"); } // Remove completed queues var processingQueuesToRemove = new List <string>(); foreach (var processingQueue in _processingQueues) { if (!processingQueue.Value.Queue.Any()) { processingQueue.Value.AutoResetEvent.Set(); processingQueuesToRemove.Add(processingQueue.Key); } } foreach (var processingQueueToRemove in processingQueuesToRemove) { _processingQueues.TryRemove(processingQueueToRemove, out _); _tasks.TryRemove(processingQueueToRemove, out _); } // TODO: is it possible to remove executed tasks from the Task list and processingqueues from the processing queues _logger.Info($"Number of tasks {_tasks.Count()}, number of queues {_processingQueues.Count()}."); }
static void Main(string[] args) { var processingQueue = new ProcessingQueue(); Task.Factory.StartNew(processingQueue.Consume); while (true) { var input = Console.ReadLine(); processingQueue.Produce(input); } }
public UnityPluginInstaller( Lifetime lifetime, ILogger logger, ISolution solution, IShellLocks shellLocks, UnityPluginDetector detector, NotificationsModel notifications, ISettingsStore settingsStore, PluginPathsProvider pluginPathsProvider, UnityVersion unityVersion, UnityHost unityHost, UnitySolutionTracker unitySolutionTracker, UnityRefresher refresher) { myPluginInstallations = new JetHashSet <FileSystemPath>(); myLifetime = lifetime; myLogger = logger; mySolution = solution; myShellLocks = shellLocks; myDetector = detector; myNotifications = notifications; myPluginPathsProvider = pluginPathsProvider; myUnityVersion = unityVersion; myUnitySolutionTracker = unitySolutionTracker; myRefresher = refresher; myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext())); myQueue = new ProcessingQueue(myShellLocks, myLifetime); unityHost.PerformModelAction(rdUnityModel => { rdUnityModel.InstallEditorPlugin.AdviseNotNull(lifetime, x => { myShellLocks.ExecuteOrQueueReadLockEx(myLifetime, "UnityPluginInstaller.InstallEditorPlugin", () => { var installationInfo = myDetector.GetInstallationInfo(myCurrentVersion); QueueInstall(installationInfo, true); }); }); }); unitySolutionTracker.IsUnityProjectFolder.AdviseOnce(lifetime, args => { if (!args) { return; } myShellLocks.ExecuteOrQueueReadLockEx(myLifetime, "IsAbleToEstablishProtocolConnectionWithUnity", InstallPluginIfRequired); BindToInstallationSettingChange(); }); }
private async Task OpenChannel(int port, string hostId) { var cacheContextAccessor = new CacheContextAccessor(); var cache = new Cache(cacheContextAccessor); var namedDependencyProvider = new NamedCacheDependencyProvider(); var contexts = new Dictionary<int, ApplicationContext>(); var services = new ServiceProvider(_services); var protocolManager = new ProtocolManager(maxVersion: 2); // This fixes the mono incompatibility but ties it to ipv4 connections var listenSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); listenSocket.Bind(new IPEndPoint(IPAddress.Loopback, port)); listenSocket.Listen(10); Console.WriteLine("Listening on port {0}", port); for (; ;) { var acceptSocket = await AcceptAsync(listenSocket); Console.WriteLine("Client accepted {0}", acceptSocket.LocalEndPoint); var stream = new NetworkStream(acceptSocket); var queue = new ProcessingQueue(stream); var connection = new ConnectionContext( contexts, services, cache, cacheContextAccessor, namedDependencyProvider, queue, protocolManager, hostId); queue.OnReceive += message => { // Enumerates all project contexts and return them to the // sender if (message.MessageType == "EnumerateProjectContexts") { WriteProjectContexts(message, queue, contexts); } else { // Otherwise it's a context specific message connection.OnReceive(message); } }; queue.Start(); } }
public async Task DequeuedItemIsInProcessingList() { const int item = 0; ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { }); queue.Enqueue(item); await DelayDelta(); queue.TryDequeue(out int dequeuedItem); queue.OnProcessed(dequeuedItem); Assert.IsTrue(queue.CurrentlyProcessingTasks.Contains(dequeuedItem)); }
public async Task AddedItemCanBeDequeued() { const int item = 0; ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { }); queue.Enqueue(item); await DelayDelta(); bool dequeued = queue.TryDequeue(out int dequeuedItem); Assert.IsTrue(dequeued); Assert.AreEqual(item, dequeuedItem); }
public void RunBatchProcessing() { Task task = Task.Factory.StartNew(async() => { try { await Log(); } catch (Exception ex) { Logger.Error($"Processing thread was dropped. {ex.GetType()}:{ex.Message}"); } ProcessingQueue.CompleteAdding(); }, TaskCreationOptions.LongRunning); }
public ConnectionContext(IDictionary<int, ApplicationContext> contexts, IServiceProvider services, ICache cache, ICacheContextAccessor cacheContextAccessor, INamedCacheDependencyProvider namedDependencyProvider, ProcessingQueue queue, string hostId) { _contexts = contexts; _services = services; _cache = cache; _cacheContextAccessor = cacheContextAccessor; _namedDependencyProvider = namedDependencyProvider; _queue = queue; _hostId = hostId; }
//--- Constructor --- public SubscriptionManager(XUri destination, List<Tuplet<string, List<XDoc>>> subscriptions) { _destination = destination; _recordChangeQueue = new ProcessingQueue<UserInfo>(RecordsChange_Helper, 1); _subscriptionChangeQueue = new ProcessingQueue<Empty>(UpdateSubscriptions_Helper, 1); if(subscriptions == null) { return; } foreach(Tuplet<string, List<XDoc>> subscription in subscriptions) { string wikiId = subscription.Item1; SiteInfo siteInfo = new SiteInfo(wikiId); _subscriptions.Add(wikiId, siteInfo); foreach(XDoc userDoc in subscription.Item2) { UserInfo userInfo = UserInfo.FromXDoc(wikiId, userDoc); if(userInfo == null) { continue; } lock(siteInfo) { siteInfo.Users.Add(userInfo.Id, userInfo); } userInfo.ResourcesChanged += OnSubscriptionChange; userInfo.DataChanged += OnRecordsChange; } } }
//--- Methods --- protected override IEnumerator<IYield> Start(XDoc config, Result result) { yield return Coroutine.Invoke(base.Start, config, new Result()); _processingQueue = new ProcessingQueue<XDoc>(CallUpdate); _instances = new ExpiringDictionary<string, PackageUpdater>(TimerFactory, true); _apikey = config["apikey"].AsText; _instanceTtl = TimeSpan.FromSeconds(config["instance-ttl"].AsInt ?? 10 * 60); _packagePath = config["package-path"].AsText; if(string.IsNullOrEmpty(_packagePath)) { throw new ArgumentException("No value was provided for configuration key 'package-path'"); } try { _packagePath = PhpUtil.ConvertToFormatString(_packagePath); // Note (arnec): _packagePath may contain a {0} for injecting wikiid, so we want to make sure the // path string can be formatted without an exception string.Format(_packagePath, "dummy"); } catch { throw new ArgumentException(string.Format("The package path '{0}' contains an illegal formmating directive", _packagePath)); } // set up subscription for pubsub yield return Coroutine.Invoke(SubscribeInstanceEvents, PubSub, new Result()); result.Return(); }
private static void WriteProjectContexts(Message message, ProcessingQueue queue, IDictionary<int, ApplicationContext> contexts) { try { var projectContexts = contexts.Values.Select(p => new { Id = p.Id, ProjectPath = p.ApplicationPath }) .ToList(); var versionToken = message.Payload.HasValues ? message.Payload?["Version"] : null; var version = versionToken != null ? versionToken.Value<int>() : 0; queue.Send(writer => { if (version == 0) { writer.Write("ProjectContexts"); writer.Write(projectContexts.Count); for (int i = 0; i < projectContexts.Count; i++) { writer.Write(projectContexts[i].ProjectPath); writer.Write(projectContexts[i].Id); } } else { var obj = new JObject(); obj["MessageType"] = "ProjectContexts"; var projects = new JObject(); obj["Projects"] = projects; foreach (var pair in projectContexts) { projects[pair.ProjectPath] = pair.Id; } writer.Write(obj.ToString(Formatting.None)); } }); } catch (Exception ex) { var error = new JObject(); error["Message"] = ex.Message; queue.Send(new Message { MessageType = "Error", Payload = error }); throw; } }
private static void WriteProjectContexts(ProcessingQueue queue, IDictionary<int, ApplicationContext> contexts) { var projects = contexts.Values.Select(p => new { Id = p.Id, ProjectPath = p.ApplicationPath }) .ToList(); queue.Send(writer => { writer.Write("ProjectContexts"); writer.Write(projects.Count); for (int i = 0; i < projects.Count; i++) { writer.Write(projects[i].ProjectPath); writer.Write(projects[i].Id); } }); }
//--- Constructors --- /// <summary> /// Create a new dispatcher. /// </summary> /// <param name="config">Configuration instance injected from pub sub service.</param> /// <param name="queueRepository">Factory for dispatch queues used by persisted (i.e. expiring) subscriptions</param> public Dispatcher(DispatcherConfig config, IPubSubDispatchQueueRepository queueRepository) { _queueRepository = queueRepository; _owner = config.ServiceUri.AsServerUri(); _serviceKeySetCookie = config.ServiceAccessCookie; _combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie); _dispatchQueue = new ProcessingQueue<DispatcherEvent>(DispatchFromQueue, 10); _defaultQueue = new ImmediatePubSubDispatchQueue(TryDispatchItem); var pubSubSubscriptionSets = queueRepository.GetUninitializedSets(); // Note (arnec): only invoking lock here, so that RegisterSet and Update don't do it over and over lock(_subscriptionsByOwner) { foreach(var set in pubSubSubscriptionSets) { RegisterSet(set, true); } Update(); } queueRepository.InitializeRepository(TryDispatchItem); }
//--- Constructors --- /// <summary> /// Create a new storage manager. /// </summary> /// <param name="catalog">Database catalog to use.</param> /// <param name="config">Collection and index configuration.</param> public MysqlDocStoreManager(IDataCatalog catalog, XDoc config) { _catalog = catalog; _config = config; _name = "docstore_" + _config["name"].AsText; _indexLookupTable = _name + "_indicies"; if(string.IsNullOrEmpty(_name)) { throw new ArgumentException("Missing name for store table"); } if(_catalog == null) { throw new ArgumentException("Missing DataCatalog"); } _namespaceMap.Add(new KeyValuePair<string, string>("docstore", "mindtouch.dream.docstore")); foreach(XDoc doc in _config["namespaces/namespace"]) { _namespaceMap.Add(new KeyValuePair<string, string>(doc["@prefix"].AsText, doc["@urn"].AsText)); } _processingQueue = new ProcessingQueue<WorkItem>(Update, 5); // create storage & index lookup tables if required _catalog.NewQuery(string.Format(@" CREATE TABLE IF NOT EXISTS {0} ( id int primary key auto_increment not null, revision int not null default 1, doc_id varchar(255) unique not null, doc text not null )", _name)) .Execute(); _catalog.NewQuery(string.Format(@" CREATE TABLE IF NOT EXISTS {0} ( idx_name varchar(255) primary key not null, idx_xpath text not null )", _indexLookupTable)) .Execute(); RefreshIndicies(); }
//--- Constructors --- /// <summary> /// Create a new dispatcher. /// </summary> /// <param name="config">Configuration instance injected from pub sub service.</param> public Dispatcher(DispatcherConfig config) { _owner = config.ServiceUri.AsServerUri(); _serviceKeySetCookie = config.ServiceAccessCookie; _combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie); _dispatchQueue = new ProcessingQueue<DispatcherEvent>(DispatchFromQueue, 10); }
protected override Yield Stop(Result result) { if(!IsRunning) { result.Return(); yield break; } try { // BUG #810: announce to all root-level services that we're shutting down // dismiss all pending requests _requestQueue = null; // shutdown all services, except host and sub-services (the latter should be cleaned-up by their owners) _log.Debug("Stopping stand-alone services"); Dictionary<string, ServiceEntry> services; lock(_services) { services = new Dictionary<string, ServiceEntry>(_services); } foreach(KeyValuePair<string, ServiceEntry> entry in services) { if((entry.Value.Owner == null) && !(ReferenceEquals(this, entry.Value.Service))) { StopService(entry.Value.Service.Self); } } // now destroy support services _log.Debug("Stopping host"); } catch(Exception ex) { _log.ErrorExceptionMethodCall(ex, "Stop: host failed to deinitialize"); } // stop storage service if(_storage != null) { yield return _storage.Delete(new Result<DreamMessage>(TimeSpan.MaxValue)).CatchAndLog(_log); _storage = null; } // check if any inner services failed to stop foreach(KeyValuePair<string, ServiceEntry> entry in _services) { _log.WarnMethodCall("Stop: service did not shutdown", entry.Key); } // invoke base.Stop yield return Coroutine.Invoke(base.Stop, new Result()).CatchAndLog(_log); // deinitialize fields _blueprints = null; _registeredTypes.Clear(); _infos.Clear(); _activities.Clear(); _features = new DreamFeatureDirectory(); _services.Clear(); _aliases.Clear(); // mark host as not running Plug.RemoveEndpoint(this); _running = false; _shutdown.Set(); result.Return(); }
//--- Constructors --- public UpdateRecordDispatcher(CoroutineHandler<UpdateRecord, Result> callback) { _callback = callback; _dispatchQueue = new ProcessingQueue<UpdateRecord>(DispatchQueued, 10); }
//--- Constructors --- public UpdateRecordDispatcher(CoroutineHandler<UpdateRecord, Result> callback, int maxParallelism, int maxRetry, TimeSpan retrySleep) { _dispatchQueue = new ProcessingQueue<QueueItem>(DispatchRecord, maxParallelism); _callback = callback; _maxRetry = maxRetry; _retrySleep = retrySleep; }
public void Initialize(XDoc config) { if(_running) { _log.WarnMethodCall("Initialize: host already initailized"); throw new InvalidOperationException("already initialized"); } try { // initialize container var containerConfig = config["components"]; if(!containerConfig.IsEmpty) { _log.Debug("registering host level module"); var builder = new ContainerBuilder(); builder.RegisterModule(new XDocAutofacContainerConfigurator(containerConfig, DreamContainerScope.Host)); builder.Update(_container); } // make sure we have an IServiceActivator if(!_container.IsRegistered<IServiceActivator>()) { var builder = new ContainerBuilder(); builder.RegisterType<DefaultServiceActivator>().As<IServiceActivator>(); builder.Update(_container); } _serviceActivator = _container.Resolve<IServiceActivator>(); _running = true; _shutdown = new ManualResetEvent(false); _rootRedirect = config["root-redirect"].AsText; _debugMode = config["debug"].AsText.IfNullOrEmpty("false").ToLowerInvariant(); _memorizeAliases = config["memorize-aliases"].AsBool ?? true; // add default prologues/epilogues _defaultPrologues = new[] { new DreamFeatureStage("dream.in.*", PrologueDreamIn, DreamAccess.Public) }; _defaultEpilogues = new[] { new DreamFeatureStage("dream.out.*", EpilogueDreamOut, DreamAccess.Public) }; // initialize identity _id = !config["guid"].IsEmpty ? new Guid(config["guid"].AsText) : Guid.NewGuid(); _localMachineUri = new XUri(string.Format("local://{0}", _id.ToString("N"))); _aliases[_localMachineUri] = _localMachineUri; // initialize environment string path = config["host-path"].AsText ?? "host"; _publicUri = config["uri.public"].AsUri ?? new XUri("http://*****:*****@type"].AsText ?? "local"; if("s3".EqualsInvariant(_storageType)) { _storagePath = config["storage/root"].AsText ?? ""; _storageConfig = config["storage"].Clone(); } else { _storagePath = config["storage/path"].AsText ?? config["storage-dir"].AsText ?? config["service-dir"].AsText; if(!Path.IsPathRooted(_storagePath)) { throw new ArgumentException("missing or invalid storage-dir"); } } // log initialization settings _log.DebugMethodCall("Initialize: guid", _id); _log.DebugMethodCall("Initialize: apikey", config["apikey"].AsText ?? "(auto)"); _log.DebugMethodCall("Initialize: uri.public", _publicUri); _log.DebugMethodCall("Initialize: storage-type", _storageType); _log.DebugMethodCall("Initialize: storage-dir", _storagePath); _log.DebugMethodCall("Initialize: host-path", path); _log.DebugMethodCall("Initialize: connect-limit", _connectionLimit); // add path & type information config = config.Root; config.Elem("path", path); config.Elem("class", GetType().FullName); config.Elem("sid", "sid://mindtouch.com/2007/03/dream/host"); // set root-uri Plug.AddEndpoint(this); // check if we need to fill in the TYPE information using the type XDoc blueprint = CreateServiceBlueprint(GetType()); // start service if(_connectionLimit > 0) { _requestQueue = new ProcessingQueue<Action<Action>>(RequestQueueCallback, _connectionLimit); } Coroutine.Invoke(StartService, this, blueprint, path, config, new Result<XDoc>()).Wait(); } catch { _running = false; _shutdown.Set(); throw; } }
//--- Constructors --- public DekiChangeSink(string wikiid, XUri apiUri, Plug publishPlug) { _wikiid = wikiid; _apiUri = apiUri; _publishPlug = publishPlug; _channel = new XUri(string.Format("event://{0}/deki/", _wikiid)); _changeQueue = new ProcessingQueue<ChangeData>(Publish, 10); }