private UpdateRecord(UpdateRecord current, XDoc meta) { Id = current.Id; ActionStack = current.ActionStack; Meta = meta; WikiId = current.WikiId; QueueIds.AddRange(current.QueueIds); }
//--- Methods --- public Result Dispatch(UpdateRecord updateRecord, Result result) { _log.DebugFormat("moving update record '{0}' to dispatch queue ({1})", updateRecord.Id, _dispatchQueue.Count); if(!_dispatchQueue.TryEnqueue(new QueueItem(updateRecord, result))) { throw new InvalidOperationException(string.Format("Enqueue of '{0}' failed.", updateRecord.Id)); } return result; }
public Result Dispatch(UpdateRecord updateRecord, Result result) { _log.DebugFormat("got dispatch"); Dispatches.Add(new Tuplet<DateTime, UpdateRecord>(DateTime.Now, updateRecord)); result.Return(); ResetEvent.Set(); return result; }
//--- Methods --- public Result Dispatch(UpdateRecord updateRecord, Result result) { _log.DebugFormat("moving update record '{0}' to dispatch queue ({1})", updateRecord.Id, _dispatchQueue.Count); if (!_dispatchQueue.TryEnqueue(new QueueItem(updateRecord, result))) { throw new InvalidOperationException(string.Format("Enqueue of '{0}' failed.", updateRecord.Id)); } return(result); }
public void Failed_task_sleep_does_not_block_next_task() { var handler = new DispatchHandler(); var dispatcher = new UpdateRecordDispatcher(handler.Dispatch, 1, 1, 2.Seconds()); var r1 = new UpdateRecord(new XUri("mock://foo"), new XDoc("meta"), "default"); var r2 = new UpdateRecord(new XUri("mock://foo"), new XDoc("meta"), "default"); var r1firstAttempt = handler.AddCallback((d, r) => r.Throw(new Exception()), new Result<UpdateRecord>(1.Seconds())); var r2firstAttempt = handler.AddCallback((d, r) => { }, new Result<UpdateRecord>(1.Seconds())); var r1SecondAttempt = handler.AddCallback((d, r) => { }, new Result<UpdateRecord>(5.Seconds())); dispatcher.Dispatch(r1, new Result()); dispatcher.Dispatch(r2, new Result()); Assert.IsFalse(r1firstAttempt.Block().HasException, "r1 first attempt wasn't called"); Assert.IsFalse(r1firstAttempt.Block().HasException, "r2 first attempt wasn't called"); Assert.IsFalse(r1SecondAttempt.Block().HasException, "r2 second attempt wasn't called"); }
private void Poll(TaskTimer timer) { if (!_poll) { timer.Change(TimeSpan.FromSeconds(1), TaskEnv.Current); return; } _poll = false; while (true) { // pull item from queue to store in out accumulation queue and hold on to it var item = _persistentQueue.Dequeue(TimeSpan.MaxValue); if (item == null) { // didn't find an item, drop out of loop and set timer to check again later timer.Change(TimeSpan.FromSeconds(1), TaskEnv.Current); return; } var doc = item.Value; var wikiid = doc["@wikiid"].AsText; var id = new XUri("http://" + wikiid + "/" + doc["path"].AsText); lock (_data) { UpdateRecord data; XUri channel = doc["channel"].AsUri; string action = channel.Segments[2]; if (!_data.TryGetValue(id, out data)) { _log.DebugFormat("queueing '{0}' for '{1}'", action, id); _queue.Enqueue(new Tuplet <DateTime, XUri>(DateTime.UtcNow.Add(_delay), id)); data = new UpdateRecord(id, doc, wikiid); } else { _log.DebugFormat("appending existing queue record '{0}' for '{1}'", action, id); data = data.With(doc); } if (action != "create" && action != "move") { data.ActionStack.PushDelete(); } if (action != "delete") { data.ActionStack.PushAdd(); } data.QueueIds.Add(item.Id); _data[id] = data; } } }
private void CheckExpire(TaskTimer timer) { while (true) { // get the next scheduled item UpdateRecord data = null; lock (_data) { if (_queue.Count == 0) { _queueTimer.Change(_delay, TaskEnv.None); return; } Tuplet <DateTime, XUri> key = _queue.Peek(); if (key.Item1 > DateTime.UtcNow) { _queueTimer.Change(key.Item1, TaskEnv.None); return; } data = _data[key.Item2]; _queue.Dequeue(); _data.Remove(key.Item2); } Interlocked.Increment(ref _pendingCount); _dispatcher.Dispatch(data, new Result(TimeSpan.MaxValue)).WhenDone(r => { // cleanup items from the queue var poll = false; foreach (var itemId in data.QueueIds) { if (!_persistentQueue.CommitDequeue(itemId)) { // if we couldn't take an item, it must have gone back to the queue, so we better poll again poll = true; } } if (poll) { _poll = true; } Interlocked.Decrement(ref _pendingCount); if (r.HasException) { _log.Error(string.Format("dispatch of '{0}' encountered an error", data.Id), r.Exception); } }); } }
private void Poll(TaskTimer timer) { if(!_poll) { timer.Change(TimeSpan.FromSeconds(1), TaskEnv.Current); return; } _poll = false; while(true) { // pull item from queue to store in out accumulation queue and hold on to it var item = _persistentQueue.Dequeue(TimeSpan.MaxValue); if(item == null) { // didn't find an item, drop out of loop and set timer to check again later timer.Change(TimeSpan.FromSeconds(1), TaskEnv.Current); return; } var doc = item.Value; var wikiid = doc["@wikiid"].AsText; var id = new XUri("http://" + wikiid + "/" + doc["path"].AsText); lock(_data) { UpdateRecord data; XUri channel = doc["channel"].AsUri; string action = channel.Segments[2]; if(!_data.TryGetValue(id, out data)) { _log.DebugFormat("queueing '{0}' for '{1}'", action, id); _queue.Enqueue(new Tuplet<DateTime, XUri>(DateTime.UtcNow.Add(_delay), id)); data = new UpdateRecord(id, doc, wikiid); } else { _log.DebugFormat("appending existing queue record '{0}' for '{1}'", action, id); data = data.With(doc); } if(action != "create" && action != "move") { data.ActionStack.PushDelete(); } if(action != "delete") { data.ActionStack.PushAdd(); } data.QueueIds.Add(item.Id); _data[id] = data; } } }
public Yield Dispatch(UpdateRecord data, Result result) { if(_callbacks.Any()) { var tuple = _callbacks.Dequeue(); tuple.Item1(data, result); tuple.Item2.Return(data); } if(!result.HasFinished) { result.Return(); } yield break; }
//--- Constructors --- public QueueItem(UpdateRecord record, Result result) { Record = record; Result = result; }
private Yield OnQueueExpire(UpdateRecord data, Result result) { _log.DebugFormat("indexing '{0}'", data.Id); XUri docId = data.Id.WithHost("localhost").WithPort(80); string wikiid = data.Id.Host; if(string.IsNullOrEmpty(wikiid)) { wikiid = "default"; } XDoc revision = null; XUri revisionUri = null; XUri channel = data.Meta["channel"].AsUri; string type = channel.Segments[1]; string action = channel.Segments[2]; string contentUri = string.Empty; _log.DebugFormat("processing action '{0}' for resource type '{1}' and id '{2}'", action, type, data.Id); Term deleteTerm; // if this is an Add we need to validate the data before we get to a possible delete string oldDocUri = docId.ToString().ToLowerInvariant(); switch(type) { case "pages": if(oldDocUri.Contains("@api/deki/archive/")) { oldDocUri = oldDocUri.Replace("@api/deki/archive/", "@api/deki/"); } deleteTerm = new Term("uri", oldDocUri); break; case "users": var userId = data.Meta["userid"].AsText; deleteTerm = new Term("id.user", userId); break; default: deleteTerm = new Term("uri", oldDocUri); break; } if(data.ActionStack.IsAdd) { if(data.Meta.IsEmpty) { throw new DreamBadRequestException("document is empty"); } switch(type) { case "files": revisionUri = data.Meta["revision.uri"].AsUri; contentUri = data.Meta["content.uri"].AsText; if(string.IsNullOrEmpty(contentUri)) { throw new DreamBadRequestException(string.Format("missing content uri for '{0}'", data.Id)); } break; case "pages": revisionUri = data.Meta["revision.uri"].AsUri; contentUri = data.Meta["content.uri[@type='application/xml']"].AsText; if(string.IsNullOrEmpty(contentUri)) { throw new DreamBadRequestException(string.Format("missing xml content uri for '{0}'", data.Id)); } break; case "comments": revisionUri = data.Meta["uri"].AsUri; break; case "users": revisionUri = data.Meta["uri"].AsUri; break; } if(revisionUri == null) { throw new DreamBadRequestException(string.Format("missing revision uri for '{0}'", data.Id)); } Result<DreamMessage> revisionResult; _log.DebugFormat("fetching revision for {1} from {0}", data.Id, revisionUri); yield return revisionResult = Plug.New(revisionUri).With("apikey", _apikey).GetAsync(); if(!revisionResult.Value.IsSuccessful) { throw BadRequestException(revisionResult.Value, "unable to fetch revision info from '{0}' (status: {1})", data.Meta["revision.uri"].AsText, revisionResult.Value.Status); } revision = revisionResult.Value.ToDocument(); } _log.DebugFormat("deleting '{0}' from index using uri {1}", data.Id, oldDocUri); GetInstance(wikiid).DeleteDocuments(deleteTerm); // build new document string text = string.Empty; if(data.ActionStack.IsAdd) { _log.DebugFormat("adding '{0}' to index", data.Id); var d = new Document(); d.Add(new Field("uri", docId.ToString().ToLowerInvariant(), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("mime", revision["contents/@type"].AsText ?? "", Field.Store.YES, Field.Index.TOKENIZED)); DateTime editDate; string editDateStringFromDoc = (type == "files") ? revision["date.created"].AsText : revision["date.edited"].AsText; DateTime.TryParse(editDateStringFromDoc, out editDate); if(type == "comments" && editDate == DateTime.MinValue) { // if editDate is still min, we didn't find an edit date and need to use post date DateTime.TryParse(revision["date.posted"].AsText, out editDate); } if(editDate != DateTime.MinValue) { var editDateString = editDate.ToUniversalTime().ToString("yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture.DateTimeFormat); d.Add(new Field("date.edited", editDateString, Field.Store.YES, Field.Index.UN_TOKENIZED)); } string language = null; switch(type) { case "pages": { // filter what we actually index var ns = revision["namespace"].AsText; if(Array.IndexOf(_indexNamespaceWhitelist, ns) < 0) { _log.DebugFormat("not indexing '{0}', namespace '{1}' is not in whitelist", data.Id, ns); result.Return(); yield break; } string path = revision["path"].AsText ?? string.Empty; d.Add(new Field("path", path, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("id.page", revision["@id"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("title", revision["title"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("title.sort", revision["title"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("namespace", ns ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("type", "wiki", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("author", revision["user.author/username"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("author.sort", revision["user.author/username"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); // store the original page title in case display title was set int index = path.LastIndexOf('/'); if(index > 0) { path = path.Substring(index + 1); } d.Add(new Field("path.title", path, Field.Store.YES, Field.Index.TOKENIZED)); var pageUri = data.Meta["uri"].AsUri; _log.DebugFormat("fetching page info: {0}", pageUri); Result<DreamMessage> pageResult; yield return pageResult = Plug.New(pageUri).With("apikey", _apikey).GetAsync(); DreamMessage page = pageResult.Value; if(!page.IsSuccessful) { throw BadRequestException(page, "unable to fetch page data from '{0}' for '{1}'", contentUri, data.Id); } XDoc pageDoc = page.ToDocument(); var score = pageDoc["rating/@score"].AsText; if(!string.IsNullOrEmpty(score)) { d.Add(new Field("rating.score", score, Field.Store.YES, Field.Index.UN_TOKENIZED)); } d.Add(new Field("creator", pageDoc["user.createdby/username"].AsText ?? string.Empty, Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("date.created", DateTimeToString(pageDoc["date.created"].AsDate), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("rating.count", pageDoc["rating/@count"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("title.parent", pageDoc["page.parent/title"].AsText ?? "", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("path.parent", pageDoc["page.parent/path"].AsText ?? "", Field.Store.YES, Field.Index.UN_TOKENIZED)); foreach(var ancestor in pageDoc["//page.parent/path"]) { var ancestorPath = ancestor.AsText; if(string.IsNullOrEmpty(ancestorPath)) { continue; } d.Add(new Field("path.ancestor", ancestorPath, Field.Store.YES, Field.Index.UN_TOKENIZED)); } var parentId = pageDoc["page.parent/@id"].AsUInt; if(parentId.HasValue) { d.Add(new Field("id.parent", parentId.Value.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED)); } // check if this is a redirect if(!pageDoc["page.redirectedto"].IsEmpty) { // redirect if(!(Config["index-redirects"].AsBool ?? false)) { _log.DebugFormat("indexing of redirects is disabled, not indexing '{0}'", data.Id); result.Return(); yield break; } _log.DebugFormat("indexing redirect, leave content empty"); d.Add(new Field("size", "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); } else { language = pageDoc["language"].AsText; // fetch the page _log.DebugFormat("fetching page content: {0}", contentUri); DreamMessage content = null; yield return Plug.New(contentUri).With("apikey", _apikey).WithTimeout(TimeSpan.FromMinutes(10)) .Get(new Result<DreamMessage>()) .Set(x => content = x); if(!content.IsSuccessful) { throw BadRequestException(content, "unable to fetch content from '{0}' for '{1}'", contentUri, data.Id); } text = _htmlConverter.Convert(content.ToDocument()); d.Add(new Field("size", content.ContentLength.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED)); } // process tags, if they exist if(!data.Meta["tags.uri"].IsEmpty) { Result<DreamMessage> tagsResult; yield return tagsResult = Plug.New(data.Meta["tags.uri"].AsUri).With("apikey", _apikey).GetAsync(); if(!tagsResult.Value.IsSuccessful) { throw BadRequestException(tagsResult.Value, "unable to fetch tags from '{0}' for '{1}'", data.Meta["tags.uri"].AsText, data.Id); } XDoc tags = tagsResult.Value.ToDocument(); StringBuilder sb = new StringBuilder(); foreach(XDoc v in tags["tag/@value"]) { sb.AppendFormat("{0}\n", v.AsText); } d.Add(new Field("tag", sb.ToString(), Field.Store.YES, Field.Index.TOKENIZED)); } //Save page properties yield return Coroutine.Invoke(AddPropertiesToDocument, d, pageDoc["properties"], new Result()); // set docuemnt boost based on namespace d.SetBoost(GetNamespaceBoost(revision["namespace"].AsText)); break; } case "files": { var ns = revision["page.parent/namespace"].AsText; if(Array.IndexOf(_indexNamespaceWhitelist, ns) < 0) { _log.DebugFormat("not indexing '{0}', namespace '{1}' is not in whitelist", data.Id, ns); result.Return(); yield break; } d.Add(new Field("namespace", ns ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); var filename = revision["filename"].AsText; string extension = Path.GetExtension(filename); d.Add(new Field("path", revision["page.parent/path"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("title.page", revision["page.parent/title"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("id.page", revision["page.parent/@id"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("id.file", revision["@id"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("extension", extension ?? string.Empty, Field.Store.NO, Field.Index.TOKENIZED)); d.Add(new Field("filename", filename ?? string.Empty, Field.Store.NO, Field.Index.TOKENIZED)); d.Add(new Field("title", filename ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("title.sort", filename ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("author", revision["user.createdby/username"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("author.sort", revision["user.createdby/username"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("description", revision["description"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("type", GetDocumentType(extension), Field.Store.YES, Field.Index.UN_TOKENIZED)); // convert binary types to text Result<Tuplet<string, int>> contentResult; yield return contentResult = Coroutine.Invoke(ConvertToText, extension, new XUri(contentUri), new Result<Tuplet<string, int>>()); Tuplet<string, int> content = contentResult.Value; text = content.Item1; var size = content.Item2; if(size == 0) { // since ConvertToText only gets the byte size if there is a converter for the filetype, // we fall back to the size in the document if it comes back as zero size = revision["contents/@size"].AsInt ?? 0; } d.Add(new Field("size", size.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED)); break; } case "comments": { var ns = revision["page.parent/namespace"].AsText; if(Array.IndexOf(_indexNamespaceWhitelist, ns) < 0) { _log.DebugFormat("not indexing '{0}', namespace '{1}' is not in whitelist", data.Id, ns); result.Return(); yield break; } d.Add(new Field("namespace", ns ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); text = revision["content"].AsText ?? string.Empty; d.Add(new Field("comments", text, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("type", "comment", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("path", revision["page.parent/path"].AsText ?? string.Empty, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("id.page", revision["page.parent/@id"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("title.page", revision["page.parent/title"].AsText ?? string.Empty, Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("id.comment", revision["@id"].AsText ?? "0", Field.Store.YES, Field.Index.UN_TOKENIZED)); string title = "Comment #" + revision["number"].AsInt; d.Add(new Field("title", title, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("title.sort", title, Field.Store.NO, Field.Index.UN_TOKENIZED)); string author = revision["user.editedby/username"].AsText ?? revision["user.createdby/username"].AsText ?? ""; d.Add(new Field("author", author, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("author.sort", author, Field.Store.NO, Field.Index.UN_TOKENIZED)); break; } case "users": { d.Add(new Field("type", "user", Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("id.user", revision["@id"].AsText, Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("username", revision["username"].AsText, Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("email", revision["email"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); var fullname = revision["fullname"].AsText ?? string.Empty; d.Add(new Field("fullname", fullname, Field.Store.YES, Field.Index.ANALYZED)); d.Add(new Field("fullname.sort", fullname, Field.Store.NO, Field.Index.NOT_ANALYZED)); d.Add(new Field("date.lastlogin", DateTimeToString(revision["date.lastlogin"].AsDate), Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("date.created", DateTimeToString(revision["date.created"].AsDate), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("language", revision["language"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("service.authentication.id", revision["service.authentication/@id"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); foreach(XDoc group in revision["groups/group"]) { d.Add(new Field("group.id", group["@id"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); d.Add(new Field("group", group["groupname"].AsText ?? string.Empty, Field.Store.NO, Field.Index.UN_TOKENIZED)); } // NOTE (MaxM): User properties are only automatically included for current user so they need to be retrieved. Result<DreamMessage> propertyResult; yield return propertyResult = Plug.New(revisionUri).At("properties").With("apikey", _apikey).GetAsync(); if(!propertyResult.Value.IsSuccessful) { throw BadRequestException(propertyResult.Value, "unable to fetch properties for user id '{0}' for '{1}'", revision["@id"].AsText, data.Id); } XDoc propertiesDoc = propertyResult.Value.ToDocument(); // Save user properties yield return Coroutine.Invoke(AddPropertiesToDocument, d, propertiesDoc, new Result()); break; } }// switch(type) string preview = text; if(preview.Length > _previewLength) { preview = preview.Substring(0, _previewLength); } d.Add(new Field("content", text, Field.Store.NO, Field.Index.TOKENIZED)); d.Add(new Field("preview", preview, Field.Store.YES, Field.Index.TOKENIZED)); d.Add(new Field("wordcount", _wordcountRegex.Matches(text).Count.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED)); if(type == "files" || type == "comments") { // fetch parent page for language string parentUri = revision["page.parent/@href"].AsText; if(!string.IsNullOrEmpty(parentUri)) { Result<DreamMessage> parentResult; yield return parentResult = Plug.New(parentUri).With("apikey", _apikey).GetAsync(); if(!parentResult.Value.IsSuccessful) { throw new DreamBadRequestException(string.Format("unable to fetch parent from '{0}' for '{1}'", contentUri, data.Id)); } XDoc parent = parentResult.Value.ToDocument(); language = parent["language"].AsText; } } if(string.IsNullOrEmpty(language)) { language = "neutral"; } d.Add(new Field("language", language, Field.Store.YES, Field.Index.UN_TOKENIZED)); _log.DebugFormat("Adding document for '{0}' to index", data.Id); GetInstance(wikiid).AddDocument(d); } _log.DebugFormat("completed indexing '{0}'", data.Id); result.Return(); }