private IEnumerable <SituationEdge <T> > DevelopRuleInputEdges(string RuleName)
{
ProcessingQueue <string, SituationEdge <T> > queue;
queue = new ProcessingQueue <string, SituationEdge <T> >();
queue.Add(RuleName);
queue.Process((q, ruleName) =>
{
foreach (IRule <T> rule in GetRootNodes(ruleName).Select(item => item.Rule))
{
foreach (SituationEdge <T> edge in GetRuleInputEdges(rule))
{
q.AddResult(edge);
if (!(edge.Predicate is INonTerminalPredicate <T> nonTerminal))
{
continue;
}
q.Add(nonTerminal.Name);
}
}
});
return(queue.Results);
}
//--- Methods ---
protected override IEnumerator <IYield> Start(XDoc config, Result result)
{
yield return(Coroutine.Invoke(base.Start, config, new Result()));
_processingQueue = new ProcessingQueue <XDoc>(CallUpdate);
_instances = new ExpiringDictionary <string, PackageUpdater>(TimerFactory, true);
_apikey = config["apikey"].AsText;
_instanceTtl = TimeSpan.FromSeconds(config["instance-ttl"].AsInt ?? 10 * 60);
_packagePath = config["package-path"].AsText;
if (string.IsNullOrEmpty(_packagePath))
{
throw new ArgumentException("No value was provided for configuration key 'package-path'");
}
try {
_packagePath = PhpUtil.ConvertToFormatString(_packagePath);
// Note (arnec): _packagePath may contain a {0} for injecting wikiid, so we want to make sure the
// path string can be formatted without an exception
string.Format(_packagePath, "dummy");
} catch {
throw new ArgumentException(string.Format("The package path '{0}' contains an illegal formmating directive", _packagePath));
}
// set up subscription for pubsub
yield return(Coroutine.Invoke(SubscribeInstanceEvents, PubSub, new Result()));
result.Return();
}
/// <summary>
/// Constructor
/// </summary>
public LogViewModel(LogOutput parent)
{
Parent = parent;
// Add handlers
Parent.OutputViewer.SizeChanged += OutputViewerSizeChanged;
Parent.Output.TextChanged += OnTextChanged;
Parent.ClearButton.Click += OnClearButton;
Parent.SaveButton.Click += OnSaveButton;
// Update the internal state
var document = new FlowDocument()
{
Background = new SolidColorBrush(Color.FromArgb(0xFF, 0x20, 0x20, 0x20))
};
_paragraph = new Paragraph();
document.Blocks.Add(_paragraph);
Parent.Output.Document = document;
// TODO: Can we dynamically add matchers *only* during dumping?
_matchers = new List <Matcher?>();
AddAaruMatchers();
AddDiscImageCreatorMatchers();
logQueue = new ProcessingQueue <LogLine>(ProcessLogLine);
}
public UnityPluginInstaller(
Lifetime lifetime,
ILogger logger,
ISolution solution,
IShellLocks shellLocks,
UnityPluginDetector detector,
RdNotificationsModel notifications,
ISettingsStore settingsStore,
ApplicationPackages applicationPackages,
ApplicationPackagesLocallyInstalled applicationPackagesLocallyInstalled,
IEnumerable <ApplicationPackageArtifact> packages, IDeployedPackagesExpandLocationResolver resolver)
{
myPluginInstallations = new JetHashSet <FileSystemPath>();
myLifetime = lifetime;
myLogger = logger;
mySolution = solution;
myShellLocks = shellLocks;
myDetector = detector;
myNotifications = notifications;
myApplicationPackages = applicationPackages;
myApplicationPackagesLocallyInstalled = applicationPackagesLocallyInstalled;
myPackages = packages;
myResolver = resolver;
myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext()));
myQueue = new ProcessingQueue(myShellLocks, myLifetime);
}
public MosquitoService(IWindsorContainer container)
{
_container = container;
_queue = new ProcessingQueue();
_workers = new WorkersManager(_queue, new HandlersFactory(_container));
}
public UnityPluginInstaller(
Lifetime lifetime,
ILogger logger,
ISolution solution,
IShellLocks shellLocks,
UnityPluginDetector detector,
RdNotificationsModel notifications,
ISettingsStore settingsStore,
PluginPathsProvider pluginPathsProvider,
UnityVersionDetector unityVersionDetector)
{
myPluginInstallations = new JetHashSet <FileSystemPath>();
myLifetime = lifetime;
myLogger = logger;
mySolution = solution;
myShellLocks = shellLocks;
myDetector = detector;
myNotifications = notifications;
myPluginPathsProvider = pluginPathsProvider;
myUnityVersionDetector = unityVersionDetector;
myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext()));
myQueue = new ProcessingQueue(myShellLocks, myLifetime);
}
public async Task AllTasksAreDequeuedOnce()
{
const int count = 5000;
const int workerCount = 10;
ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { });
foreach (int task in Enumerable.Range(0, count))
{
queue.Enqueue(task);
}
var tasks = Enumerable.Range(0, workerCount)
.Select(workerIndex => Task.Run(() =>
{
var dequeuedItems = new List <int>();
while (queue.TryDequeue(out int item))
{
dequeuedItems.Add(item);
}
return(dequeuedItems);
}))
.ToList();
await Task.WhenAll(tasks);
var countsPerThread = tasks.Select(t => t.Result.Count).ToList(); // just verify it does not have an extremely peaked distribution
var allDequeuedItems = tasks.Select(t => t.Result)
.Concat()
.Distinct()
.ToList();
Assert.AreEqual(count, allDequeuedItems.Count);
}
//--- Constructors ---
/// <summary>
/// Create a new dispatcher.
/// </summary>
/// <param name="config">Configuration instance injected from pub sub service.</param>
public Dispatcher(DispatcherConfig config)
{
_owner = config.ServiceUri.AsServerUri();
_serviceKeySetCookie = config.ServiceAccessCookie;
_combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie);
_dispatchQueue = new ProcessingQueue <DispatcherEvent>(DispatchFromQueue, 10);
}
protected BaseTemplate(IEnumerable <IToken> fTokens)
: this(
new ImmutableMap <TokenKey, IToken>(), ImmutableList.Empty <TokenReference>(), new ImmutableMap <InputKey, IInput>(),
ProcessingQueue.Empty())
{
foreach (var tok in fTokens)
{
if (tokenMap.ContainsKey(tok.Key))
{
tokenSeq = tokenSeq.ConsLast(new TokenReference(tok.Key));
}
else
{
foreach (var input in tok.Inputs)
{
if (!inputs.ContainsKey(input.Key))
{
var newInput = input.Key.Type == InputType.Instance
? (IInput) new UnboundInput(input.Key) : new UnboundStaticInput(input.Key);
inputs = inputs.Add(input.Key, newInput);
}
}
tokenMap = tokenMap.Add(tok.Key, tok);
tokenSeq = tokenSeq.ConsLast(new TokenReference(tok.Key));
}
}
}
public void Enqueue(string queueName, ProcessingAction act)
{
lock (_Queues)
{
retry:
if (stopped)
{
return;
}
Cleanup();
bool created = false;
if (!_Queues.TryGetValue(queueName, out var queue))
{
queue = new ProcessingQueue();
_Queues.Add(queueName, queue);
created = true;
}
if (!queue.Chan.Writer.TryWrite(act))
{
goto retry;
}
if (created)
{
queue.ProcessTask = queue.Process(cts.Token);
}
}
}
//--- Constructors ---
public NotificationDelayQueue(TimeSpan delay, CoroutineHandler<NotificationUpdateRecord, Result> callback) {
_delay = delay;
_callback = callback;
_queueTimer = new TaskTimer(CheckExpire, null);
_queueTimer.Change(_delay, TaskEnv.None);
_dispatchQueue = new ProcessingQueue<NotificationUpdateRecord>(Dispatch, 10);
}
//--- Constructors ---
public UpdateRecordDispatcher(CoroutineHandler <UpdateRecord, Result> callback, int maxParallelism, int maxRetry, TimeSpan retrySleep)
{
_dispatchQueue = new ProcessingQueue <QueueItem>(DispatchRecord, maxParallelism);
_callback = callback;
_maxRetry = maxRetry;
_retrySleep = retrySleep;
}
//--- Constructors ---
public NotificationDelayQueue(TimeSpan delay, CoroutineHandler <NotificationUpdateRecord, Result> callback)
{
_delay = delay;
_callback = callback;
_queueTimer = new TaskTimer(CheckExpire, null);
_queueTimer.Change(_delay, TaskEnv.None);
_dispatchQueue = new ProcessingQueue <NotificationUpdateRecord>(Dispatch, 10);
}
public FileWatcher(Action <Exception> onException = null)
{
this.onException = onException ?? throw new ArgumentNullException(nameof(onException));
this.watchers = new ConcurrentBag <System.IO.FileSystemWatcher>();
this.watchedDirectories = new Dictionary <Uri, ImmutableHashSet <string> >();
this.processing = new ProcessingQueue(this.onException, "error in file system watcher");
this.globPatterns = new ConcurrentDictionary <Uri, IEnumerable <string> >();
}
//--- Constructors ---
public DekiChangeSink(string wikiid, XUri apiUri, Plug publishPlug)
{
_wikiid = wikiid;
_apiUri = apiUri;
_publishPlug = publishPlug;
_channel = new XUri(string.Format("event://{0}/deki/", _wikiid));
_changeQueue = new ProcessingQueue <ChangeData>(Publish, 10);
}
private async Task DoWork(object state)
{
TaskState taskState = (TaskState)state;
Stopwatch stopwatch = new Stopwatch();
while (active)
{
taskState.Status = State.Running;
if (!await CanProcess() && downloadersActive)
{
taskState.Status = State.Paused;
await Task.Delay(250);
continue;
}
if (!PageProcessingQueue.IsEmpty)
{
UnparsedPage page;
if (!PageProcessingQueue.TryDequeue(out page))
{
//another thread grabbed the page
continue;
}
stopwatch.Time(() =>
{
return(ProcessThreadPage(page.Thread, page.Html));
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_pages, taskState.Id, time)
);
}
if (!ProcessingQueue.IsEmpty)
{
UnparsedThread thread;
if (!ProcessingQueue.TryDequeue(out thread))
{
continue;
}
stopwatch.Time(() =>
{
ProcessThread(thread.Id, thread.Html);
},
(long time) => TelemetryManager.Incriment(TelemetryType.processed_threads, taskState.Id, time)
);
}
else if (!downloadersActive) //Queue is empty, no more downloads are being performed, set active to false after 1s delay
{
taskState.Status = State.Paused;
await Task.Delay(1000);
active = false;
break;
}
}
taskState.Status = State.Complete;
}
protected BaseTemplate
(ImmutableMap <TokenKey, IToken> tokenMap, ImmutableList <TokenReference> tokenSeq,
ImmutableMap <InputKey, IInput> inputs, ProcessingQueue postProcessingQueue)
{
this.tokenSeq = tokenSeq;
this.inputs = inputs;
this.tokenMap = tokenMap;
this.postProcessingQueue = postProcessingQueue;
}
void OnSocketConnected(Socket socket)
{
var networkStream = new NetworkStream(socket);
queue = new ProcessingQueue(networkStream, new Logger());
queue.OnReceive += messageReceived;
queue.Start();
}
public void UpdateStats()
{
Statistics.RequestQueueLength = RequestManager.Count(t => t.Tileset == this);
Statistics.ActiveDownloads = RequestManager.CountActiveDownloads(t => t.Tileset == this);
Statistics.ProcessingQueueLength = ProcessingQueue.Count(t => t.Tileset == this);
Statistics.DownloadedTiles = TileCache.Count(t => t.Tileset == this);
Statistics.ReadyTiles =
TileCache.Count(t => t.Tileset == this && t.ContentState == Unity3DTileContentState.READY);
Unity3DTilesetStatistics.MaxLoadedTiles = TileCache.MaxSize;
}
internal override void TryLog(LogData logData)
{
try
{
ProcessingQueue.TryAdd(logData);
}
catch (Exception ex)
{
Logger.Error($"Addition of logEntries to queue on processing is impossible. Likely processing thread doesn't working. {ex.GetType()}:{ex.Message}");
}
}
private void UpdateProcessingQueue(WorkItem workItem)
{
if (ProcessingQueue.CapacitiesLeft && workItem != null)
{
CreateAndEnqueueInstuction(methodName: ProductionAgent.InstuctionsMethods.ProductionStarted.ToString(),
objectToProcess: workItem,
targetAgent: workItem.ProductionAgent);
ProcessingQueue.Enqueue(workItem);
Queue.Remove(workItem);
}
}
public App(IHostApplicationLifetime host, ILoggerFactory loggerFactory, IServiceProvider services,
ProgressTracker progress, ProcessingQueue processingQueue, DeadLetterQueue deadLetterQueue)
{
_host = host;
_services = services;
_progress = progress;
_logger = loggerFactory.CreateLogger(GetType().FullName);
_processingQueue = processingQueue;
_deadLetterQueue = deadLetterQueue;
_workerTokenSource = new();
}
public void ShouldAddAndContains()
{
ProcessingQueue <string, string> queue;
queue = new ProcessingQueue <string, string>( );
for (int t = 0; t < 10; t++)
{
Assert.IsFalse(queue.Contains(t.ToString()));
queue.Add(t.ToString());
Assert.IsTrue(queue.Contains(t.ToString()));
}
}
public async Task DequeuedItemIsStillIncludedInCount()
{
const int item = 0;
ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { });
queue.Enqueue(item);
await DelayDelta();
bool dequeued = queue.TryDequeue(out int dequeuedItem);
Assert.AreEqual(1, queue.Count);
}
public async Task NeverDequeuedItemCannotBeRemoved()
{
const int item = 0;
const int otherItem = 1;
ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { });
queue.Enqueue(item);
await DelayDelta();
// should throw because otherItem is not being processed yet
queue.OnProcessed(otherItem);
}
private async Task MessageReceivedAsync(QueueMessage <TMessage> message, CancellationToken cancellationToken)
{
if (!cancellationToken.IsCancellationRequested && !_tasks.Any(t => t.Value.IsFaulted))
{
if (Behaviour == AcknowledgeBehaviour.BeforeProcess)
{
_queueConsumer.AcknowledgeMessage(message.DeliveryTag);
}
string processingSequenceIdentifier = GetProcessingSequenceIdentifier(message.RoutingKey);
if (_processingQueues.ContainsKey(processingSequenceIdentifier))
{
// Add a message to the processing queue, and signal the processing thread to alert it to the new message.
var processingQueue = _processingQueues[processingSequenceIdentifier];
processingQueue.Queue.Enqueue(message);
processingQueue.AutoResetEvent.Set();
}
else
{
// create a new processing queue and kick off a task to process it.
var processingQueue = new ProcessingQueue <TMessage>();
processingQueue.Queue.Enqueue(message);
_processingQueues[processingSequenceIdentifier] = processingQueue;
var t = Task.Run(RunSequentialProcessor(processingQueue, cancellationToken));
_tasks.TryAdd(processingSequenceIdentifier, t);
}
_logger.Info($"Received new message {message.DeliveryTag}, processor queue length {_processingQueues.First().Value.Queue.Count()}");
}
// Remove completed queues
var processingQueuesToRemove = new List <string>();
foreach (var processingQueue in _processingQueues)
{
if (!processingQueue.Value.Queue.Any())
{
processingQueue.Value.AutoResetEvent.Set();
processingQueuesToRemove.Add(processingQueue.Key);
}
}
foreach (var processingQueueToRemove in processingQueuesToRemove)
{
_processingQueues.TryRemove(processingQueueToRemove, out _);
_tasks.TryRemove(processingQueueToRemove, out _);
}
// TODO: is it possible to remove executed tasks from the Task list and processingqueues from the processing queues
_logger.Info($"Number of tasks {_tasks.Count()}, number of queues {_processingQueues.Count()}.");
}
static void Main(string[] args)
{
var processingQueue = new ProcessingQueue();
Task.Factory.StartNew(processingQueue.Consume);
while (true)
{
var input = Console.ReadLine();
processingQueue.Produce(input);
}
}
public UnityPluginInstaller(
Lifetime lifetime,
ILogger logger,
ISolution solution,
IShellLocks shellLocks,
UnityPluginDetector detector,
NotificationsModel notifications,
ISettingsStore settingsStore,
PluginPathsProvider pluginPathsProvider,
UnityVersion unityVersion,
UnityHost unityHost,
UnitySolutionTracker unitySolutionTracker,
UnityRefresher refresher)
{
myPluginInstallations = new JetHashSet <FileSystemPath>();
myLifetime = lifetime;
myLogger = logger;
mySolution = solution;
myShellLocks = shellLocks;
myDetector = detector;
myNotifications = notifications;
myPluginPathsProvider = pluginPathsProvider;
myUnityVersion = unityVersion;
myUnitySolutionTracker = unitySolutionTracker;
myRefresher = refresher;
myBoundSettingsStore = settingsStore.BindToContextLive(myLifetime, ContextRange.Smart(solution.ToDataContext()));
myQueue = new ProcessingQueue(myShellLocks, myLifetime);
unityHost.PerformModelAction(rdUnityModel =>
{
rdUnityModel.InstallEditorPlugin.AdviseNotNull(lifetime, x =>
{
myShellLocks.ExecuteOrQueueReadLockEx(myLifetime, "UnityPluginInstaller.InstallEditorPlugin", () =>
{
var installationInfo = myDetector.GetInstallationInfo(myCurrentVersion);
QueueInstall(installationInfo, true);
});
});
});
unitySolutionTracker.IsUnityProjectFolder.AdviseOnce(lifetime, args =>
{
if (!args)
{
return;
}
myShellLocks.ExecuteOrQueueReadLockEx(myLifetime, "IsAbleToEstablishProtocolConnectionWithUnity", InstallPluginIfRequired);
BindToInstallationSettingChange();
});
}
private async Task OpenChannel(int port, string hostId)
{
var cacheContextAccessor = new CacheContextAccessor();
var cache = new Cache(cacheContextAccessor);
var namedDependencyProvider = new NamedCacheDependencyProvider();
var contexts = new Dictionary<int, ApplicationContext>();
var services = new ServiceProvider(_services);
var protocolManager = new ProtocolManager(maxVersion: 2);
// This fixes the mono incompatibility but ties it to ipv4 connections
var listenSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listenSocket.Bind(new IPEndPoint(IPAddress.Loopback, port));
listenSocket.Listen(10);
Console.WriteLine("Listening on port {0}", port);
for (; ;)
{
var acceptSocket = await AcceptAsync(listenSocket);
Console.WriteLine("Client accepted {0}", acceptSocket.LocalEndPoint);
var stream = new NetworkStream(acceptSocket);
var queue = new ProcessingQueue(stream);
var connection = new ConnectionContext(
contexts,
services,
cache,
cacheContextAccessor,
namedDependencyProvider,
queue,
protocolManager,
hostId);
queue.OnReceive += message =>
{
// Enumerates all project contexts and return them to the
// sender
if (message.MessageType == "EnumerateProjectContexts")
{
WriteProjectContexts(message, queue, contexts);
}
else
{
// Otherwise it's a context specific message
connection.OnReceive(message);
}
};
queue.Start();
}
}
public async Task DequeuedItemIsInProcessingList()
{
const int item = 0;
ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { });
queue.Enqueue(item);
await DelayDelta();
queue.TryDequeue(out int dequeuedItem);
queue.OnProcessed(dequeuedItem);
Assert.IsTrue(queue.CurrentlyProcessingTasks.Contains(dequeuedItem));
}
public async Task AddedItemCanBeDequeued()
{
const int item = 0;
ProcessingQueue <int> queue = new ProcessingQueue <int>(() => { });
queue.Enqueue(item);
await DelayDelta();
bool dequeued = queue.TryDequeue(out int dequeuedItem);
Assert.IsTrue(dequeued);
Assert.AreEqual(item, dequeuedItem);
}
public void RunBatchProcessing()
{
Task task = Task.Factory.StartNew(async() =>
{
try
{
await Log();
}
catch (Exception ex)
{
Logger.Error($"Processing thread was dropped. {ex.GetType()}:{ex.Message}");
}
ProcessingQueue.CompleteAdding();
}, TaskCreationOptions.LongRunning);
}
public ConnectionContext(IDictionary<int, ApplicationContext> contexts,
IServiceProvider services,
ICache cache,
ICacheContextAccessor cacheContextAccessor,
INamedCacheDependencyProvider namedDependencyProvider,
ProcessingQueue queue,
string hostId)
{
_contexts = contexts;
_services = services;
_cache = cache;
_cacheContextAccessor = cacheContextAccessor;
_namedDependencyProvider = namedDependencyProvider;
_queue = queue;
_hostId = hostId;
}
//--- Constructor ---
public SubscriptionManager(XUri destination, List<Tuplet<string, List<XDoc>>> subscriptions) {
_destination = destination;
_recordChangeQueue = new ProcessingQueue<UserInfo>(RecordsChange_Helper, 1);
_subscriptionChangeQueue = new ProcessingQueue<Empty>(UpdateSubscriptions_Helper, 1);
if(subscriptions == null) {
return;
}
foreach(Tuplet<string, List<XDoc>> subscription in subscriptions) {
string wikiId = subscription.Item1;
SiteInfo siteInfo = new SiteInfo(wikiId);
_subscriptions.Add(wikiId, siteInfo);
foreach(XDoc userDoc in subscription.Item2) {
UserInfo userInfo = UserInfo.FromXDoc(wikiId, userDoc);
if(userInfo == null) {
continue;
}
lock(siteInfo) {
siteInfo.Users.Add(userInfo.Id, userInfo);
}
userInfo.ResourcesChanged += OnSubscriptionChange;
userInfo.DataChanged += OnRecordsChange;
}
}
}
//--- Methods ---
protected override IEnumerator<IYield> Start(XDoc config, Result result) {
yield return Coroutine.Invoke(base.Start, config, new Result());
_processingQueue = new ProcessingQueue<XDoc>(CallUpdate);
_instances = new ExpiringDictionary<string, PackageUpdater>(TimerFactory, true);
_apikey = config["apikey"].AsText;
_instanceTtl = TimeSpan.FromSeconds(config["instance-ttl"].AsInt ?? 10 * 60);
_packagePath = config["package-path"].AsText;
if(string.IsNullOrEmpty(_packagePath)) {
throw new ArgumentException("No value was provided for configuration key 'package-path'");
}
try {
_packagePath = PhpUtil.ConvertToFormatString(_packagePath);
// Note (arnec): _packagePath may contain a {0} for injecting wikiid, so we want to make sure the
// path string can be formatted without an exception
string.Format(_packagePath, "dummy");
} catch {
throw new ArgumentException(string.Format("The package path '{0}' contains an illegal formmating directive", _packagePath));
}
// set up subscription for pubsub
yield return Coroutine.Invoke(SubscribeInstanceEvents, PubSub, new Result());
result.Return();
}
private static void WriteProjectContexts(Message message, ProcessingQueue queue, IDictionary<int, ApplicationContext> contexts)
{
try
{
var projectContexts = contexts.Values.Select(p => new
{
Id = p.Id,
ProjectPath = p.ApplicationPath
})
.ToList();
var versionToken = message.Payload.HasValues ? message.Payload?["Version"] : null;
var version = versionToken != null ? versionToken.Value<int>() : 0;
queue.Send(writer =>
{
if (version == 0)
{
writer.Write("ProjectContexts");
writer.Write(projectContexts.Count);
for (int i = 0; i < projectContexts.Count; i++)
{
writer.Write(projectContexts[i].ProjectPath);
writer.Write(projectContexts[i].Id);
}
}
else
{
var obj = new JObject();
obj["MessageType"] = "ProjectContexts";
var projects = new JObject();
obj["Projects"] = projects;
foreach (var pair in projectContexts)
{
projects[pair.ProjectPath] = pair.Id;
}
writer.Write(obj.ToString(Formatting.None));
}
});
}
catch (Exception ex)
{
var error = new JObject();
error["Message"] = ex.Message;
queue.Send(new Message
{
MessageType = "Error",
Payload = error
});
throw;
}
}
private static void WriteProjectContexts(ProcessingQueue queue, IDictionary<int, ApplicationContext> contexts)
{
var projects = contexts.Values.Select(p => new
{
Id = p.Id,
ProjectPath = p.ApplicationPath
})
.ToList();
queue.Send(writer =>
{
writer.Write("ProjectContexts");
writer.Write(projects.Count);
for (int i = 0; i < projects.Count; i++)
{
writer.Write(projects[i].ProjectPath);
writer.Write(projects[i].Id);
}
});
}
//--- Constructors ---
/// <summary>
/// Create a new dispatcher.
/// </summary>
/// <param name="config">Configuration instance injected from pub sub service.</param>
/// <param name="queueRepository">Factory for dispatch queues used by persisted (i.e. expiring) subscriptions</param>
public Dispatcher(DispatcherConfig config, IPubSubDispatchQueueRepository queueRepository)
{
_queueRepository = queueRepository;
_owner = config.ServiceUri.AsServerUri();
_serviceKeySetCookie = config.ServiceAccessCookie;
_combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie);
_dispatchQueue = new ProcessingQueue<DispatcherEvent>(DispatchFromQueue, 10);
_defaultQueue = new ImmediatePubSubDispatchQueue(TryDispatchItem);
var pubSubSubscriptionSets = queueRepository.GetUninitializedSets();
// Note (arnec): only invoking lock here, so that RegisterSet and Update don't do it over and over
lock(_subscriptionsByOwner) {
foreach(var set in pubSubSubscriptionSets) {
RegisterSet(set, true);
}
Update();
}
queueRepository.InitializeRepository(TryDispatchItem);
}
//--- Constructors ---
/// <summary>
/// Create a new storage manager.
/// </summary>
/// <param name="catalog">Database catalog to use.</param>
/// <param name="config">Collection and index configuration.</param>
public MysqlDocStoreManager(IDataCatalog catalog, XDoc config)
{
_catalog = catalog;
_config = config;
_name = "docstore_" + _config["name"].AsText;
_indexLookupTable = _name + "_indicies";
if(string.IsNullOrEmpty(_name)) {
throw new ArgumentException("Missing name for store table");
}
if(_catalog == null) {
throw new ArgumentException("Missing DataCatalog");
}
_namespaceMap.Add(new KeyValuePair<string, string>("docstore", "mindtouch.dream.docstore"));
foreach(XDoc doc in _config["namespaces/namespace"]) {
_namespaceMap.Add(new KeyValuePair<string, string>(doc["@prefix"].AsText, doc["@urn"].AsText));
}
_processingQueue = new ProcessingQueue<WorkItem>(Update, 5);
// create storage & index lookup tables if required
_catalog.NewQuery(string.Format(@"
CREATE TABLE IF NOT EXISTS {0} (
id int primary key auto_increment not null,
revision int not null default 1,
doc_id varchar(255) unique not null,
doc text not null )", _name))
.Execute();
_catalog.NewQuery(string.Format(@"
CREATE TABLE IF NOT EXISTS {0} (
idx_name varchar(255) primary key not null,
idx_xpath text not null )", _indexLookupTable))
.Execute();
RefreshIndicies();
}
//--- Constructors ---
/// <summary>
/// Create a new dispatcher.
/// </summary>
/// <param name="config">Configuration instance injected from pub sub service.</param>
public Dispatcher(DispatcherConfig config)
{
_owner = config.ServiceUri.AsServerUri();
_serviceKeySetCookie = config.ServiceAccessCookie;
_combinedSet = new PubSubSubscriptionSet(_owner, 0, _serviceKeySetCookie);
_dispatchQueue = new ProcessingQueue<DispatcherEvent>(DispatchFromQueue, 10);
}
protected override Yield Stop(Result result)
{
if(!IsRunning) {
result.Return();
yield break;
}
try {
// BUG #810: announce to all root-level services that we're shutting down
// dismiss all pending requests
_requestQueue = null;
// shutdown all services, except host and sub-services (the latter should be cleaned-up by their owners)
_log.Debug("Stopping stand-alone services");
Dictionary<string, ServiceEntry> services;
lock(_services) {
services = new Dictionary<string, ServiceEntry>(_services);
}
foreach(KeyValuePair<string, ServiceEntry> entry in services) {
if((entry.Value.Owner == null) && !(ReferenceEquals(this, entry.Value.Service))) {
StopService(entry.Value.Service.Self);
}
}
// now destroy support services
_log.Debug("Stopping host");
} catch(Exception ex) {
_log.ErrorExceptionMethodCall(ex, "Stop: host failed to deinitialize");
}
// stop storage service
if(_storage != null) {
yield return _storage.Delete(new Result<DreamMessage>(TimeSpan.MaxValue)).CatchAndLog(_log);
_storage = null;
}
// check if any inner services failed to stop
foreach(KeyValuePair<string, ServiceEntry> entry in _services) {
_log.WarnMethodCall("Stop: service did not shutdown", entry.Key);
}
// invoke base.Stop
yield return Coroutine.Invoke(base.Stop, new Result()).CatchAndLog(_log);
// deinitialize fields
_blueprints = null;
_registeredTypes.Clear();
_infos.Clear();
_activities.Clear();
_features = new DreamFeatureDirectory();
_services.Clear();
_aliases.Clear();
// mark host as not running
Plug.RemoveEndpoint(this);
_running = false;
_shutdown.Set();
result.Return();
}
//--- Constructors ---
public UpdateRecordDispatcher(CoroutineHandler<UpdateRecord, Result> callback) {
_callback = callback;
_dispatchQueue = new ProcessingQueue<UpdateRecord>(DispatchQueued, 10);
}
//--- Constructors ---
public UpdateRecordDispatcher(CoroutineHandler<UpdateRecord, Result> callback, int maxParallelism, int maxRetry, TimeSpan retrySleep) {
_dispatchQueue = new ProcessingQueue<QueueItem>(DispatchRecord, maxParallelism);
_callback = callback;
_maxRetry = maxRetry;
_retrySleep = retrySleep;
}
public void Initialize(XDoc config)
{
if(_running) {
_log.WarnMethodCall("Initialize: host already initailized");
throw new InvalidOperationException("already initialized");
}
try {
// initialize container
var containerConfig = config["components"];
if(!containerConfig.IsEmpty) {
_log.Debug("registering host level module");
var builder = new ContainerBuilder();
builder.RegisterModule(new XDocAutofacContainerConfigurator(containerConfig, DreamContainerScope.Host));
builder.Update(_container);
}
// make sure we have an IServiceActivator
if(!_container.IsRegistered<IServiceActivator>()) {
var builder = new ContainerBuilder();
builder.RegisterType<DefaultServiceActivator>().As<IServiceActivator>();
builder.Update(_container);
}
_serviceActivator = _container.Resolve<IServiceActivator>();
_running = true;
_shutdown = new ManualResetEvent(false);
_rootRedirect = config["root-redirect"].AsText;
_debugMode = config["debug"].AsText.IfNullOrEmpty("false").ToLowerInvariant();
_memorizeAliases = config["memorize-aliases"].AsBool ?? true;
// add default prologues/epilogues
_defaultPrologues = new[] {
new DreamFeatureStage("dream.in.*", PrologueDreamIn, DreamAccess.Public)
};
_defaultEpilogues = new[] {
new DreamFeatureStage("dream.out.*", EpilogueDreamOut, DreamAccess.Public)
};
// initialize identity
_id = !config["guid"].IsEmpty ? new Guid(config["guid"].AsText) : Guid.NewGuid();
_localMachineUri = new XUri(string.Format("local://{0}", _id.ToString("N")));
_aliases[_localMachineUri] = _localMachineUri;
// initialize environment
string path = config["host-path"].AsText ?? "host";
_publicUri = config["uri.public"].AsUri ?? new XUri("http://*****:*****@type"].AsText ?? "local";
if("s3".EqualsInvariant(_storageType)) {
_storagePath = config["storage/root"].AsText ?? "";
_storageConfig = config["storage"].Clone();
} else {
_storagePath = config["storage/path"].AsText ?? config["storage-dir"].AsText ?? config["service-dir"].AsText;
if(!Path.IsPathRooted(_storagePath)) {
throw new ArgumentException("missing or invalid storage-dir");
}
}
// log initialization settings
_log.DebugMethodCall("Initialize: guid", _id);
_log.DebugMethodCall("Initialize: apikey", config["apikey"].AsText ?? "(auto)");
_log.DebugMethodCall("Initialize: uri.public", _publicUri);
_log.DebugMethodCall("Initialize: storage-type", _storageType);
_log.DebugMethodCall("Initialize: storage-dir", _storagePath);
_log.DebugMethodCall("Initialize: host-path", path);
_log.DebugMethodCall("Initialize: connect-limit", _connectionLimit);
// add path & type information
config = config.Root;
config.Elem("path", path);
config.Elem("class", GetType().FullName);
config.Elem("sid", "sid://mindtouch.com/2007/03/dream/host");
// set root-uri
Plug.AddEndpoint(this);
// check if we need to fill in the TYPE information using the type
XDoc blueprint = CreateServiceBlueprint(GetType());
// start service
if(_connectionLimit > 0) {
_requestQueue = new ProcessingQueue<Action<Action>>(RequestQueueCallback, _connectionLimit);
}
Coroutine.Invoke(StartService, this, blueprint, path, config, new Result<XDoc>()).Wait();
} catch {
_running = false;
_shutdown.Set();
throw;
}
}
//--- Constructors ---
public DekiChangeSink(string wikiid, XUri apiUri, Plug publishPlug) {
_wikiid = wikiid;
_apiUri = apiUri;
_publishPlug = publishPlug;
_channel = new XUri(string.Format("event://{0}/deki/", _wikiid));
_changeQueue = new ProcessingQueue<ChangeData>(Publish, 10);
}
