private ScoreDoc[] GetDocsUnderTree(string path, bool recurse)
{
var field = recurse ? "InTree" : "Path";
var lq = LucQuery.Parse(String.Format("{0}:'{1}'", path, path.ToLower()));
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var idxReader = readerFrame.IndexReader;
var searcher = new IndexSearcher(idxReader);
var numDocs = idxReader.NumDocs();
try
{
var collector = TopScoreDocCollector.Create(numDocs, false);
searcher.Search(lq.Query, collector);
var topDocs = collector.TopDocs(0, numDocs);
return(topDocs.ScoreDocs);
}
finally
{
if (searcher != null)
{
searcher.Close();
}
searcher = null;
}
}
}
protected override bool ProtectedExecute()
{
// getting common versioning info
var head = NodeHead.Get(this.NodeId);
var versioningInfo = new VersioningInfo
{
Delete = EmptyIntArray,
Reindex = EmptyIntArray,
LastDraftVersionId = head.LastMinorVersionId,
LastPublicVersionId = head.LastMajorVersionId
};
// delete documents by NodeId
LuceneManager.DeleteDocuments(new[] { LuceneManager.GetNodeIdTerm(this.NodeId) }, false, this.Id, false, versioningInfo);
// add documents of all versions
var documents = IndexDocumentInfo.GetDocuments(head.Versions.Select(v => v.VersionId));
foreach (var document in documents)
{
LuceneManager.AddDocument(document, this.Id, this.IsUnprocessedActivity, versioningInfo);
}
return(true);
}
private void AddFakeDocument(int fromNodeId)
{
//minimal fakeobject: NodeId, VersionId, Path, Version, NodeTimestamp, VersionTimestamp
var node = Node.LoadNode(fromNodeId);
var doc = IndexDocumentInfo.CreateDocument(node);
doc.RemoveField(LucObject.FieldName.NodeId);
doc.RemoveField(LucObject.FieldName.VersionId);
doc.RemoveField(LucObject.FieldName.Name);
doc.RemoveField(LucObject.FieldName.Path);
var nf = new NumericField(LucObject.FieldName.NodeId, LucField.Store.YES, true);
nf.SetIntValue(99999);
doc.Add(nf);
nf = new NumericField(LucObject.FieldName.VersionId, LucField.Store.YES, true);
nf.SetIntValue(99999);
doc.Add(nf);
doc.Add(new LucField(LucObject.FieldName.Name, "fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO));
doc.Add(new LucField(LucObject.FieldName.Path, "/root/fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO));
LuceneManager.AddCompleteDocument(doc, 0, false);
LuceneManager.UnregisterActivity(0, false);
}
private void DeleteDocument(int versionId)
{
var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId));
LuceneManager._writer.DeleteDocuments(term);
LuceneManager.ApplyChanges();
}
static void Main(string[] args)
{
LoadUlkeler();
LoadSehirler();
LoadIlceler();
LoadSemtMahalle();
var manager = new LuceneManager();
if (!manager.ExistSchema("address"))
{
manager.AddSchema("address");
}
Console.WriteLine("Ülkeler indeksleniyor");
manager.Index("address", TableUlke);
Console.WriteLine("Şehirler indeksleniyor");
manager.Index("address", TableSehirler);
Console.WriteLine("İlceler indeksleniyor");
manager.Index("address", TableIlceler);
Console.WriteLine("Semt mahalle indeksleniyor");
manager.Index("address", TableSemtMahalle);
}
/// <summary>
/// Synchronous backup method for console applications.
/// </summary>
public static void SynchronousBackupIndex()
{
using (var optrace = new OperationTrace("Backup index immediatelly."))
{
EnsureEmptyDirctory(_backupDirectoryPath);
LuceneManager.PauseIndexing();
while (!LuceneManager.Paused)
{
Thread.Sleep(100);
}
Thread.Sleep(1000);
try
{
CopyIndexToBackupDirectory();
}
finally
{
LuceneManager.ContinueIndexing();
}
OptimizeCompressAndStore();
optrace.IsSuccessful = true;
}
}
private void DeleteDocument(int versionId)
{
var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId));
LuceneManager._writer.DeleteDocuments(term);
LuceneManager.UnregisterActivity(0, false);
}
public IEnumerable <LucObject> Execute(LucQuery lucQuery, bool allVersions)
{
this.LucQuery = lucQuery;
using (var traceOperation = Logger.TraceOperation("Query execution", "Query: " + this.LucQuery.QueryText))
{
Query currentQuery;
if (this.LucQuery.EnableAutofilters || this.LucQuery.EnableLifespanFilter)
{
var fullQuery = new BooleanQuery();
fullQuery.Add(new BooleanClause(this.LucQuery.Query, BooleanClause.Occur.MUST));
if (this.LucQuery.EnableAutofilters)
{
fullQuery.Add(new BooleanClause(this.LucQuery.AutoFilterQuery, BooleanClause.Occur.MUST));
}
if (this.LucQuery.EnableLifespanFilter && this.LucQuery.LifespanQuery != null)
{
fullQuery.Add(new BooleanClause(this.LucQuery.LifespanQuery, BooleanClause.Occur.MUST));
}
currentQuery = fullQuery;
}
else
{
currentQuery = this.LucQuery.Query;
}
//var idxReader = LuceneManager.IndexReader;
SearchResult r = null;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var idxReader = readerFrame.IndexReader;
BeginFullExecutingTime();
try
{
r = DoExecute(currentQuery, allVersions, idxReader, timer);
}
finally
{
FinishFullExecutingTime();
}
}
TotalCount = r.totalCount;
var searchtimer = r.searchTimer;
var trace = lucQuery.TraceInfo;
trace.KernelTime = searchtimer.KernelTime;
trace.CollectingTime = searchtimer.CollectingTime;
trace.PagingTime = searchtimer.PagingTime;
trace.FullExecutingTime = FullExecutingTime;
trace.Searches = r.searches;
traceOperation.AdditionalObject = trace;
traceOperation.IsSuccessful = true;
return(r.result);
}
}
/// <summary>
/// Determines if the given tag is blacklisted or not.
/// </summary>
/// <param name="tag">Tag to search on blacklist.</param>
/// <param name="blacklistPaths">The blacklist paths.</param>
/// <returns>True if blacklisted, false if not.</returns>
public static bool IsBlacklisted(string tag, List <string> blacklistPaths)
{
LucQuery query;
// escaping
if (!string.IsNullOrEmpty(tag))
{
tag = tag.Replace("\"", "\\\"");
}
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var reader = readerFrame.IndexReader;
var terms = reader.Terms(new Term("BlackListPath", "*"));
string queryString;
if (blacklistPaths != null && blacklistPaths.Count > 0)
{
var usedPaths = new List <string>();
do
{
if (terms.Term().Field() == "BlackListPath")
{
if (blacklistPaths.Any(path => path.ToLower().StartsWith(terms.Term().Text())))
{
usedPaths.Add(terms.Term().Text());
}
}
} while (terms.Next());
var pathString = usedPaths.Aggregate(string.Empty, (current, usedPath) => String.Concat(current, usedPath, " "));
pathString = pathString.TrimEnd(' ');
queryString = String.Format("+Type:tag +DisplayName:\"{0}\"", tag.ToLower());
queryString = String.Concat(queryString, String.IsNullOrEmpty(pathString) ? " +BlackListPath:/root" : String.Format(" +BlackListPath:({0})", pathString));
}
else
{
queryString = String.Concat("+Type:tag +DisplayName:\"", tag.ToLower(), "\" +BlackListPath:/root");
}
try
{
query = LucQuery.Parse(queryString);
}
catch (ParserException)
{
// wrong query: return no
Logger.WriteWarning(1, "TagAdmin query parse error: " + queryString);
return(false);
}
var result = query.Execute();
return(result.Count() > 0);
}
}
[Description("An activity execution with update activity after delete activity not throws any exception.")] // ??
public void Indexing_WritingGapAndGettingUnprocessedActivitiesWithGap()
{
var content = Content.CreateNew("Car", TestRoot, "Indexing_WritingGapAndGettingUnprocessedActivitiesWithGap");
var handler = (GenericContent)content.ContentHandler;
content.Save();
var id = content.Id;
for (int i = 0; i < 10; i++)
{
handler.Index++;
handler.Save();
}
var maxActivityIdSave = MissingActivityHandler.MaxActivityId;
var savedGap = MissingActivityHandler.GetGap();
try
{
MissingActivityHandler.MaxActivityId -= 2;
MissingActivityHandler.SetGap(new List <int>(new[] { maxActivityIdSave - 4, maxActivityIdSave - 5, maxActivityIdSave - 7 }));
EnsureWriterHasChanges();
//LuceneManager.ApplyChanges();
var luceneManagerAcc = new PrivateType(typeof(LuceneManager));
luceneManagerAcc.InvokeStatic("Commit", BindingFlags.Static | BindingFlags.NonPublic, new object[] { true });
IDictionary <string, string> cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
cud = readerFrame.IndexReader.GetCommitUserData();
}
var lastIdStr = cud[IndexManager.LastActivityIdKey];
var missingStr = cud[IndexManager.MissingActivitiesKey];
// [0]: {[LastActivityId, 10]}
// [1]: {[MissingActivities, 8,7,5]}
Assert.IsTrue(cud[IndexManager.LastActivityIdKey] == MissingActivityHandler.MaxActivityId.ToString(), "#1");
Assert.IsTrue(cud[IndexManager.MissingActivitiesKey] == String.Join(",", new[] { maxActivityIdSave - 4, maxActivityIdSave - 5, maxActivityIdSave - 7 }), "#2");
var mid = MissingActivityHandler.MaxActivityId;
var activities = IndexingActivityManager.GetUnprocessedActivities(new[] { mid - 2, mid - 3, mid - 5 });
var exp = String.Join(",", new[] { mid - 5, mid - 3, mid - 2 });
var cur = String.Join(",", activities.Select(t => t.IndexingActivityId));
Assert.AreEqual(exp, cur);
var max = 0;
activities = IndexingActivityManager.GetUnprocessedActivities(mid, out max);
exp = String.Join(",", new[] { mid + 1, mid + 2 });
cur = String.Join(",", activities.Select(t => t.IndexingActivityId));
Assert.AreEqual(exp, cur);
}
finally
{
MissingActivityHandler.SetGap(savedGap);
MissingActivityHandler.MaxActivityId = maxActivityIdSave;
}
}
private long GetLastTimestampFromIndex(int versionId)
{
var document = LuceneManager.GetDocumentByVersionId(versionId)[0];
var vt = document.Get(LucObject.FieldName.VersionTimestamp);
var versionTimestamp = Convert.ToInt64(vt);
return(versionTimestamp);
}
protected override bool ProtectedExecute()
{
if (Document != null)
{
return(LuceneManager.UpdateDocument(Document, Id, IsUnprocessedActivity, Versioning));
}
return(LuceneManager.UpdateDocument(Id, IsUnprocessedActivity, Versioning));
}
internal override void Execute()
{
using (var optrace = new OperationTrace("WriterRestartActivity Execute"))
{
LuceneManager.Restart();
optrace.IsSuccessful = true;
}
}
public void ProcessRequest(HttpContext context)
{
LuceneManager.Backup();
using (var writer = new StreamWriter(context.Response.OutputStream))
{
writer.Write("ok");
writer.Flush();
}
}
public override void Analyse(LineCollection lines)
{
if (lines == null)
{
return;
}
manager = new LuceneManager();
if (!manager.ExistSchema("address"))
{
manager.AddSchema("address");
}
var analyzeIndexList = new List <AnalyzeIndex>();
foreach (var line in lines)
{
foreach (var sentence in line.SentenceList)
{
foreach (var word in sentence.WordList)
{
if (word.SpellWord != null)
{
if (word.SpellWord.Root != null)
{
var adresList = AddressControl(word);
if (adresList != null)
{
if (adresList.Count > 0)
{
var analyze = new AnalyzeIndex();
analyze.AnalyzeIndexCollection.AddRange(adresList);
analyze.LineNumber = lines.IndexOf(line);
analyze.WordNumber = sentence.WordList.IndexOf(word);
analyze.SentenceNumber = line.SentenceList.IndexOf(sentence);
analyzeIndexList.Add(analyze);
}
}
}
}
}
}
}
ClearAddressIndex(ref analyzeIndexList, lines);
}
public void Indexing_ExecuteUnprocessedIndexingActivities()
{
lock (LuceneManager._executingUnprocessedIndexingActivitiesLock) // make sure indexhealthmonitor will not overlap
{
var initInfo = InitCarsForUnprocessedTests();
var carlist = initInfo.Item1;
var lastActivityId = initInfo.Item2;
var expectedLastActivityId = initInfo.Item3;
// generate a gap and delete corresponding documents from index
var activities = GetCarActivities(lastActivityId);
var activitiesToDelete = new IndexingActivity[] { activities[0], activities[2], activities[5], activities[7], activities[8], activities[9] };
MissingActivityHandler.SetGap(activitiesToDelete.Take(3).Select(a => a.IndexingActivityId).ToList()); // 0,2,5 will be missing
MissingActivityHandler.MaxActivityId = activities[6].IndexingActivityId; // 7,8,9 will be missing
// commit gap and maxactivityid to the index
EnsureWriterHasChanges();
LuceneManager.Commit(true);
foreach (var activity in activitiesToDelete)
{
DeleteVersionFromIndex(activity.VersionId);
DeleteVersionIdFromIndexingHistory(activity.VersionId);
}
// check: cars deleted can NOT be found in the index
for (var i = 0; i < carlist.Count; i++)
{
var id = carlist[i].Id;
if (activitiesToDelete.Select(a => a.NodeId).Contains(id))
{
Assert.IsFalse(CheckCarInIndex(id), "Deleted car can still be found in the index."); // deleted car should not be in index
}
else
{
Assert.IsTrue(CheckCarInIndex(id), "Untouched car can not be found in the index."); // untouched car should still be in index
}
}
// execute unprocessed indexing tasks
LuceneManager.ExecuteUnprocessedIndexingActivities(null, true);
// check: all cars can be found in the index again
for (var i = 0; i < carlist.Count; i++)
{
Assert.IsTrue(CheckCarInIndex(carlist[i].Id), "ExecuteUnprocessedIndexingActivities did not repair lost document.");
}
Assert.AreEqual(expectedLastActivityId, MissingActivityHandler.MaxActivityId, "Maxtaskid was not updated correctly.");
Assert.AreEqual(0, MissingActivityHandler.GetGap().Count, "Gap size is not 0.");
}
}
internal override void ExecuteIndexingActivity()
{
if (ProtectedExecute())
{
_executed = true;
}
// ActivityFinished must be called after executing an activity, even if index is not changed
LuceneManager.ActivityFinished(this.Id, this.IsUnprocessedActivity);
}
internal override void Execute()
{
using (var optrace = new OperationTrace("RemoveDocumentActivity Execute"))
{
var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId));
LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename);
base.Execute();
optrace.IsSuccessful = true;
}
}
internal override void Execute()
{
try
{
LuceneManager.AddTree(TreeRoot, this.ActivityId, this.FromExecutingUnprocessedActivities);
}
finally
{
base.Execute();
}
}
internal override void Execute()
{
using (var optrace = new OperationTrace("RemoveTreeActivity Execute"))
{
var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) };
LuceneManager.DeleteDocuments(terms, MoveOrRename);
base.Execute();
optrace.IsSuccessful = true;
}
}
protected override bool ProtectedExecute()
{
if (Document != null)
{
if (true == this.SingleVersion)
{
return(LuceneManager.AddCompleteDocument(Document, Id, IsUnprocessedActivity, Versioning));
}
return(LuceneManager.AddDocument(Document, Id, IsUnprocessedActivity, Versioning));
}
return(LuceneManager.AddDocument(Id, IsUnprocessedActivity, Versioning));
}
internal override void Execute()
{
try
{
var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) };
LuceneManager.DeleteDocuments(terms, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities);
}
finally
{
base.Execute();
}
}
internal override void Execute()
{
try
{
var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId));
LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities);
}
finally
{
base.Execute();
}
}
internal override void Execute()
{
using (var optrace = new OperationTrace("UpdateDocumentActivity Execute"))
{
if (Document != null)
{
LuceneManager.UpdateDocument(GetIdTerm(Document), Document);
}
base.Execute();
optrace.IsSuccessful = true;
}
}
[Description("An activity execution with update activity after delete activity not throws any exception.")] // ??
public void Indexing_ActivitesWithMissingVersion()
{
var content = Content.CreateNew("Car", TestRoot, "Car_Indexing_ActivitesWithMissingVersion");
var handler = (GenericContent)content.ContentHandler;
//handler.VersioningMode = VersioningType.None;
content.Save();
var id = content.Id;
LuceneManager.UnregisterActivity(0, false);
IndexingActivity[] act = new IndexingActivity[3];
act[0] = new IndexingActivity
{
ActivityType = IndexingActivityType.RemoveDocument,
NodeId = _fakeId,
VersionId = _fakeId
};
act[1] = new IndexingActivity
{
ActivityType = IndexingActivityType.UpdateDocument,
NodeId = _fakeId,
VersionId = _fakeId
};
act[2] = new IndexingActivity
{
ActivityType = IndexingActivityType.AddDocument,
NodeId = _fakeId,
VersionId = _fakeId
};
try
{
using (var context = new IndexingDataContext())
{
foreach (var a in act)
{
context.IndexingActivities.InsertOnSubmit(a);
context.SubmitChanges();
}
}
var max = 0;
var activities = IndexingActivityManager.GetUnprocessedActivities(act[2].IndexingActivityId - 1, out max);
foreach (var a in activities)
{
IndexingActivityManager.ExecuteActivityDirect(a);
}
}
finally
{
RemoveFakeTestActivity();
}
}
private static void ContinueIndexingAndWriteResult(HttpContext context)
{
try
{
LuceneManager.ContinueIndexing();
context.Response.Write("OK");
}
catch (Exception ex)
{
Logger.WriteException(ex);
context.Response.Write(ex.Message);
}
}
internal override void Execute()
{
try
{
if (Document != null)
{
LuceneManager.UpdateDocument(GetIdTerm(Document), Document, this.ActivityId, this.FromExecutingUnprocessedActivities);
}
}
finally
{
base.Execute();
}
}
internal override void Execute()
{
try
{
var terms = new[] { new Term("InTree", TreeRoot), new Term("Path", TreeRoot) };
LuceneManager.DeleteDocuments(terms, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities);
#if INDEX
LuceneManager.LogVersionFlagConsistency(this.NodeId, "RemoveTreeActivity.DeleteDocuments", activityId: this.ActivityId);
#endif
}
finally
{
base.Execute();
}
}
/// <summary>
/// Starts Lucene if it is not running.
/// </summary>
public void StartLucene()
{
if (LuceneManagerIsRunning)
{
ConsoleWrite("LuceneManager has already started.");
return;
}
ConsoleWriteLine("Starting LuceneManager:");
//var x = Lucene.Net.Documents.Field.Index.NO;
//var y = Lucene.Net.Documents.Field.Store.NO;
//var z = Lucene.Net.Documents.Field.TermVector.NO;
LuceneManager.Start(_settings.Console);
ConsoleWriteLine("LuceneManager has started.");
}
internal override void Execute()
{
try
{
var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(this.VersionId));
LuceneManager.DeleteDocuments(new[] { delTerm }, MoveOrRename, this.ActivityId, this.FromExecutingUnprocessedActivities);
#if INDEX
LuceneManager.LogVersionFlagConsistency(this.NodeId, "RemoveDocumentActivity.DeleteDocuments", activityId: this.ActivityId);
#endif
}
finally
{
base.Execute();
}
}
