private ScoreDoc[] GetDocsUnderTree(string path, bool recurse) { var field = recurse ? "InTree" : "Path"; var lq = LucQuery.Parse(String.Format("{0}:'{1}'", path, path.ToLower())); using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var idxReader = readerFrame.IndexReader; var searcher = new IndexSearcher(idxReader); var numDocs = idxReader.NumDocs(); try { var collector = TopScoreDocCollector.Create(numDocs, false); searcher.Search(lq.Query, collector); var topDocs = collector.TopDocs(0, numDocs); return(topDocs.ScoreDocs); } finally { if (searcher != null) { searcher.Close(); } searcher = null; } } }
protected override bool ProtectedExecute() { // getting common versioning info var head = NodeHead.Get(this.NodeId); var versioningInfo = new VersioningInfo { Delete = EmptyIntArray, Reindex = EmptyIntArray, LastDraftVersionId = head.LastMinorVersionId, LastPublicVersionId = head.LastMajorVersionId }; // delete documents by NodeId LuceneManager.DeleteDocuments(new[] { LuceneManager.GetNodeIdTerm(this.NodeId) }, false, this.Id, false, versioningInfo); // add documents of all versions var documents = IndexDocumentInfo.GetDocuments(head.Versions.Select(v => v.VersionId)); foreach (var document in documents) { LuceneManager.AddDocument(document, this.Id, this.IsUnprocessedActivity, versioningInfo); } return(true); }
private void AddFakeDocument(int fromNodeId) { //minimal fakeobject: NodeId, VersionId, Path, Version, NodeTimestamp, VersionTimestamp var node = Node.LoadNode(fromNodeId); var doc = IndexDocumentInfo.CreateDocument(node); doc.RemoveField(LucObject.FieldName.NodeId); doc.RemoveField(LucObject.FieldName.VersionId); doc.RemoveField(LucObject.FieldName.Name); doc.RemoveField(LucObject.FieldName.Path); var nf = new NumericField(LucObject.FieldName.NodeId, LucField.Store.YES, true); nf.SetIntValue(99999); doc.Add(nf); nf = new NumericField(LucObject.FieldName.VersionId, LucField.Store.YES, true); nf.SetIntValue(99999); doc.Add(nf); doc.Add(new LucField(LucObject.FieldName.Name, "fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO)); doc.Add(new LucField(LucObject.FieldName.Path, "/root/fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO)); LuceneManager.AddCompleteDocument(doc, 0, false); LuceneManager.UnregisterActivity(0, false); }
private void DeleteDocument(int versionId) { var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId)); LuceneManager._writer.DeleteDocuments(term); LuceneManager.ApplyChanges(); }
static void Main(string[] args) { LoadUlkeler(); LoadSehirler(); LoadIlceler(); LoadSemtMahalle(); var manager = new LuceneManager(); if (!manager.ExistSchema("address")) { manager.AddSchema("address"); } Console.WriteLine("Ülkeler indeksleniyor"); manager.Index("address", TableUlke); Console.WriteLine("Şehirler indeksleniyor"); manager.Index("address", TableSehirler); Console.WriteLine("İlceler indeksleniyor"); manager.Index("address", TableIlceler); Console.WriteLine("Semt mahalle indeksleniyor"); manager.Index("address", TableSemtMahalle); }
/// <summary> /// Synchronous backup method for console applications. /// </summary> public static void SynchronousBackupIndex() { using (var optrace = new OperationTrace("Backup index immediatelly.")) { EnsureEmptyDirctory(_backupDirectoryPath); LuceneManager.PauseIndexing(); while (!LuceneManager.Paused) { Thread.Sleep(100); } Thread.Sleep(1000); try { CopyIndexToBackupDirectory(); } finally { LuceneManager.ContinueIndexing(); } OptimizeCompressAndStore(); optrace.IsSuccessful = true; } }
private void DeleteDocument(int versionId) { var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId)); LuceneManager._writer.DeleteDocuments(term); LuceneManager.UnregisterActivity(0, false); }
public IEnumerable <LucObject> Execute(LucQuery lucQuery, bool allVersions) { this.LucQuery = lucQuery; using (var traceOperation = Logger.TraceOperation("Query execution", "Query: " + this.LucQuery.QueryText)) { Query currentQuery; if (this.LucQuery.EnableAutofilters || this.LucQuery.EnableLifespanFilter) { var fullQuery = new BooleanQuery(); fullQuery.Add(new BooleanClause(this.LucQuery.Query, BooleanClause.Occur.MUST)); if (this.LucQuery.EnableAutofilters) { fullQuery.Add(new BooleanClause(this.LucQuery.AutoFilterQuery, BooleanClause.Occur.MUST)); } if (this.LucQuery.EnableLifespanFilter && this.LucQuery.LifespanQuery != null) { fullQuery.Add(new BooleanClause(this.LucQuery.LifespanQuery, BooleanClause.Occur.MUST)); } currentQuery = fullQuery; } else { currentQuery = this.LucQuery.Query; } //var idxReader = LuceneManager.IndexReader; SearchResult r = null; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var idxReader = readerFrame.IndexReader; BeginFullExecutingTime(); try { r = DoExecute(currentQuery, allVersions, idxReader, timer); } finally { FinishFullExecutingTime(); } } TotalCount = r.totalCount; var searchtimer = r.searchTimer; var trace = lucQuery.TraceInfo; trace.KernelTime = searchtimer.KernelTime; trace.CollectingTime = searchtimer.CollectingTime; trace.PagingTime = searchtimer.PagingTime; trace.FullExecutingTime = FullExecutingTime; trace.Searches = r.searches; traceOperation.AdditionalObject = trace; traceOperation.IsSuccessful = true; return(r.result); } }
/// <summary> /// Determines if the given tag is blacklisted or not. /// </summary> /// <param name="tag">Tag to search on blacklist.</param> /// <param name="blacklistPaths">The blacklist paths.</param> /// <returns>True if blacklisted, false if not.</returns> public static bool IsBlacklisted(string tag, List <string> blacklistPaths) { LucQuery query; // escaping if (!string.IsNullOrEmpty(tag)) { tag = tag.Replace("\"", "\\\""); } using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var reader = readerFrame.IndexReader; var terms = reader.Terms(new Term("BlackListPath", "*")); string queryString; if (blacklistPaths != null && blacklistPaths.Count > 0) { var usedPaths = new List <string>(); do { if (terms.Term().Field() == "BlackListPath") { if (blacklistPaths.Any(path => path.ToLower().StartsWith(terms.Term().Text()))) { usedPaths.Add(terms.Term().Text()); } } } while (terms.Next()); var pathString = usedPaths.Aggregate(string.Empty, (current, usedPath) => String.Concat(current, usedPath, " ")); pathString = pathString.TrimEnd(' '); queryString = String.Format("+Type:tag +DisplayName:\"{0}\"", tag.ToLower()); queryString = String.Concat(queryString, String.IsNullOrEmpty(pathString) ? " +BlackListPath:/root" : String.Format(" +BlackListPath:({0})", pathString)); } else { queryString = String.Concat("+Type:tag +DisplayName:\"", tag.ToLower(), "\" +BlackListPath:/root"); } try { query = LucQuery.Parse(queryString); } catch (ParserException) { // wrong query: return no Logger.WriteWarning(1, "TagAdmin query parse error: " + queryString); return(false); } var result = query.Execute(); return(result.Count() > 0); } }
[Description("An activity execution with update activity after delete activity not throws any exception.")] // ?? public void Indexing_WritingGapAndGettingUnprocessedActivitiesWithGap() { var content = Content.CreateNew("Car", TestRoot, "Indexing_WritingGapAndGettingUnprocessedActivitiesWithGap"); var handler = (GenericContent)content.ContentHandler; content.Save(); var id = content.Id; for (int i = 0; i < 10; i++) { handler.Index++; handler.Save(); } var maxActivityIdSave = MissingActivityHandler.MaxActivityId; var savedGap = MissingActivityHandler.GetGap(); try { MissingActivityHandler.MaxActivityId -= 2; MissingActivityHandler.SetGap(new List <int>(new[] { maxActivityIdSave - 4, maxActivityIdSave - 5, maxActivityIdSave - 7 })); EnsureWriterHasChanges(); //LuceneManager.ApplyChanges(); var luceneManagerAcc = new PrivateType(typeof(LuceneManager)); luceneManagerAcc.InvokeStatic("Commit", BindingFlags.Static | BindingFlags.NonPublic, new object[] { true }); IDictionary <string, string> cud; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { cud = readerFrame.IndexReader.GetCommitUserData(); } var lastIdStr = cud[IndexManager.LastActivityIdKey]; var missingStr = cud[IndexManager.MissingActivitiesKey]; // [0]: {[LastActivityId, 10]} // [1]: {[MissingActivities, 8,7,5]} Assert.IsTrue(cud[IndexManager.LastActivityIdKey] == MissingActivityHandler.MaxActivityId.ToString(), "#1"); Assert.IsTrue(cud[IndexManager.MissingActivitiesKey] == String.Join(",", new[] { maxActivityIdSave - 4, maxActivityIdSave - 5, maxActivityIdSave - 7 }), "#2"); var mid = MissingActivityHandler.MaxActivityId; var activities = IndexingActivityManager.GetUnprocessedActivities(new[] { mid - 2, mid - 3, mid - 5 }); var exp = String.Join(",", new[] { mid - 5, mid - 3, mid - 2 }); var cur = String.Join(",", activities.Select(t => t.IndexingActivityId)); Assert.AreEqual(exp, cur); var max = 0; activities = IndexingActivityManager.GetUnprocessedActivities(mid, out max); exp = String.Join(",", new[] { mid + 1, mid + 2 }); cur = String.Join(",", activities.Select(t => t.IndexingActivityId)); Assert.AreEqual(exp, cur); } finally { MissingActivityHandler.SetGap(savedGap); MissingActivityHandler.MaxActivityId = maxActivityIdSave; } }
private long GetLastTimestampFromIndex(int versionId) { var document = LuceneManager.GetDocumentByVersionId(versionId)[0]; var vt = document.Get(LucObject.FieldName.VersionTimestamp); var versionTimestamp = Convert.ToInt64(vt); return(versionTimestamp); }
protected override bool ProtectedExecute() { if (Document != null) { return(LuceneManager.UpdateDocument(Document, Id, IsUnprocessedActivity, Versioning)); } return(LuceneManager.UpdateDocument(Id, IsUnprocessedActivity, Versioning)); }
internal override void Execute() { using (var optrace = new OperationTrace("WriterRestartActivity Execute")) { LuceneManager.Restart(); optrace.IsSuccessful = true; } }
public void ProcessRequest(HttpContext context) { LuceneManager.Backup(); using (var writer = new StreamWriter(context.Response.OutputStream)) { writer.Write("ok"); writer.Flush(); } }
public override void Analyse(LineCollection lines) { if (lines == null) { return; } manager = new LuceneManager(); if (!manager.ExistSchema("address")) { manager.AddSchema("address"); } var analyzeIndexList = new List <AnalyzeIndex>(); foreach (var line in lines) { foreach (var sentence in line.SentenceList) { foreach (var word in sentence.WordList) { if (word.SpellWord != null) { if (word.SpellWord.Root != null) { var adresList = AddressControl(word); if (adresList != null) { if (adresList.Count > 0) { var analyze = new AnalyzeIndex(); analyze.AnalyzeIndexCollection.AddRange(adresList); analyze.LineNumber = lines.IndexOf(line); analyze.WordNumber = sentence.WordList.IndexOf(word); analyze.SentenceNumber = line.SentenceList.IndexOf(sentence); analyzeIndexList.Add(analyze); } } } } } } } ClearAddressIndex(ref analyzeIndexList, lines); }
public void Indexing_ExecuteUnprocessedIndexingActivities() { lock (LuceneManager._executingUnprocessedIndexingActivitiesLock) // make sure indexhealthmonitor will not overlap { var initInfo = InitCarsForUnprocessedTests(); var carlist = initInfo.Item1; var lastActivityId = initInfo.Item2; var expectedLastActivityId = initInfo.Item3; // generate a gap and delete corresponding documents from index var activities = GetCarActivities(lastActivityId); var activitiesToDelete = new IndexingActivity[] { activities[0], activities[2], activities[5], activities[7], activities[8], activities[9] }; MissingActivityHandler.SetGap(activitiesToDelete.Take(3).Select(a => a.IndexingActivityId).ToList()); // 0,2,5 will be missing MissingActivityHandler.MaxActivityId = activities[6].IndexingActivityId; // 7,8,9 will be missing // commit gap and maxactivityid to the index EnsureWriterHasChanges(); LuceneManager.Commit(true); foreach (var activity in activitiesToDelete) { DeleteVersionFromIndex(activity.VersionId); DeleteVersionIdFromIndexingHistory(activity.VersionId); } // check: cars deleted can NOT be found in the index for (var i = 0; i < carlist.Count; i++) { var id = carlist[i].Id; if (activitiesToDelete.Select(a => a.NodeId).Contains(id)) { Assert.IsFalse(CheckCarInIndex(id), "Deleted car can still be found in the index."); // deleted car should not be in index } else { Assert.IsTrue(CheckCarInIndex(id), "Untouched car can not be found in the index."); // untouched car should still be in index } } // execute unprocessed indexing tasks LuceneManager.ExecuteUnprocessedIndexingActivities(null, true); // check: all cars can be found in the index again for (var i = 0; i < carlist.Count; i++) { Assert.IsTrue(CheckCarInIndex(carlist[i].Id), "ExecuteUnprocessedIndexingActivities did not repair lost document."); } Assert.AreEqual(expectedLastActivityId, MissingActivityHandler.MaxActivityId, "Maxtaskid was not updated correctly."); Assert.AreEqual(0, MissingActivityHandler.GetGap().Count, "Gap size is not 0."); } }
internal override void ExecuteIndexingActivity() { if (ProtectedExecute()) { _executed = true; } // ActivityFinished must be called after executing an activity, even if index is not changed LuceneManager.ActivityFinished(this.Id, this.IsUnprocessedActivity); }
internal override void Execute() { using (var optrace = new OperationTrace("RemoveDocumentActivity Execute")) { var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId)); LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename); base.Execute(); optrace.IsSuccessful = true; } }
internal override void Execute() { try { LuceneManager.AddTree(TreeRoot, this.ActivityId, this.FromExecutingUnprocessedActivities); } finally { base.Execute(); } }
internal override void Execute() { using (var optrace = new OperationTrace("RemoveTreeActivity Execute")) { var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) }; LuceneManager.DeleteDocuments(terms, MoveOrRename); base.Execute(); optrace.IsSuccessful = true; } }
protected override bool ProtectedExecute() { if (Document != null) { if (true == this.SingleVersion) { return(LuceneManager.AddCompleteDocument(Document, Id, IsUnprocessedActivity, Versioning)); } return(LuceneManager.AddDocument(Document, Id, IsUnprocessedActivity, Versioning)); } return(LuceneManager.AddDocument(Id, IsUnprocessedActivity, Versioning)); }
internal override void Execute() { try { var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) }; LuceneManager.DeleteDocuments(terms, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities); } finally { base.Execute(); } }
internal override void Execute() { try { var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId)); LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities); } finally { base.Execute(); } }
internal override void Execute() { using (var optrace = new OperationTrace("UpdateDocumentActivity Execute")) { if (Document != null) { LuceneManager.UpdateDocument(GetIdTerm(Document), Document); } base.Execute(); optrace.IsSuccessful = true; } }
[Description("An activity execution with update activity after delete activity not throws any exception.")] // ?? public void Indexing_ActivitesWithMissingVersion() { var content = Content.CreateNew("Car", TestRoot, "Car_Indexing_ActivitesWithMissingVersion"); var handler = (GenericContent)content.ContentHandler; //handler.VersioningMode = VersioningType.None; content.Save(); var id = content.Id; LuceneManager.UnregisterActivity(0, false); IndexingActivity[] act = new IndexingActivity[3]; act[0] = new IndexingActivity { ActivityType = IndexingActivityType.RemoveDocument, NodeId = _fakeId, VersionId = _fakeId }; act[1] = new IndexingActivity { ActivityType = IndexingActivityType.UpdateDocument, NodeId = _fakeId, VersionId = _fakeId }; act[2] = new IndexingActivity { ActivityType = IndexingActivityType.AddDocument, NodeId = _fakeId, VersionId = _fakeId }; try { using (var context = new IndexingDataContext()) { foreach (var a in act) { context.IndexingActivities.InsertOnSubmit(a); context.SubmitChanges(); } } var max = 0; var activities = IndexingActivityManager.GetUnprocessedActivities(act[2].IndexingActivityId - 1, out max); foreach (var a in activities) { IndexingActivityManager.ExecuteActivityDirect(a); } } finally { RemoveFakeTestActivity(); } }
private static void ContinueIndexingAndWriteResult(HttpContext context) { try { LuceneManager.ContinueIndexing(); context.Response.Write("OK"); } catch (Exception ex) { Logger.WriteException(ex); context.Response.Write(ex.Message); } }
internal override void Execute() { try { if (Document != null) { LuceneManager.UpdateDocument(GetIdTerm(Document), Document, this.ActivityId, this.FromExecutingUnprocessedActivities); } } finally { base.Execute(); } }
internal override void Execute() { try { var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) }; LuceneManager.DeleteDocuments(terms, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities); #if INDEX LuceneManager.LogVersionFlagConsistency(this.NodeId, "RemoveTreeActivity.DeleteDocuments", activityId: this.ActivityId); #endif } finally { base.Execute(); } }
/// <summary> /// Starts Lucene if it is not running. /// </summary> public void StartLucene() { if (LuceneManagerIsRunning) { ConsoleWrite("LuceneManager has already started."); return; } ConsoleWriteLine("Starting LuceneManager:"); //var x = Lucene.Net.Documents.Field.Index.NO; //var y = Lucene.Net.Documents.Field.Store.NO; //var z = Lucene.Net.Documents.Field.TermVector.NO; LuceneManager.Start(_settings.Console); ConsoleWriteLine("LuceneManager has started."); }
internal override void Execute() { try { var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId)); LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities); #if INDEX LuceneManager.LogVersionFlagConsistency(this.NodeId, "RemoveDocumentActivity.DeleteDocuments", activityId: this.ActivityId); #endif } finally { base.Execute(); } }