public virtual Document CreateDocument()
{
Document doc;
using (var op = SnTrace.Index.StartOperation("LM: LuceneDocumentActivity.CreateDocument (VersionId:{0})", VersionId))
{
if (IndexDocumentData != null)
{
// create document from indexdocumentdata if it has been supplied (eg via MSMQ if it was small enough to send it over)
var docInfo = IndexDocumentData.IndexDocumentInfo as IndexDocumentInfo;
doc = IndexDocumentInfo.CreateDocument(docInfo, IndexDocumentData);
if (doc == null)
{
SnTrace.Index.Write("LM: LuceneDocumentActivity.CreateDocument (VersionId:{0}): Document is NULL from QUEUE", VersionId);
}
}
else
{
// create document via loading it from db (eg when indexdocumentdata was too large to send over MSMQ)
doc = IndexDocumentInfo.GetDocument(this.VersionId);
if (doc == null)
{
SnTrace.Index.Write("LM: LuceneDocumentActivity.CreateDocument (VersionId:{0}): Document is NULL from DB.", VersionId);
}
}
op.Successful = true;
return(doc);
}
}
protected override bool ProtectedExecute()
{
// getting common versioning info
var head = NodeHead.Get(this.NodeId);
var versioningInfo = new VersioningInfo
{
Delete = EmptyIntArray,
Reindex = EmptyIntArray,
LastDraftVersionId = head.LastMinorVersionId,
LastPublicVersionId = head.LastMajorVersionId
};
// delete documents by NodeId
LuceneManager.DeleteDocuments(new[] { LuceneManager.GetNodeIdTerm(this.NodeId) }, false, this.Id, false, versioningInfo);
// add documents of all versions
var documents = IndexDocumentInfo.GetDocuments(head.Versions.Select(v => v.VersionId));
foreach (var document in documents)
{
LuceneManager.AddDocument(document, this.Id, this.IsUnprocessedActivity, versioningInfo);
}
return(true);
}
private void AddFakeDocument(int fromNodeId)
{
//minimal fakeobject: NodeId, VersionId, Path, Version, NodeTimestamp, VersionTimestamp
var node = Node.LoadNode(fromNodeId);
var doc = IndexDocumentInfo.CreateDocument(node);
doc.RemoveField(LucObject.FieldName.NodeId);
doc.RemoveField(LucObject.FieldName.VersionId);
doc.RemoveField(LucObject.FieldName.Name);
doc.RemoveField(LucObject.FieldName.Path);
var nf = new NumericField(LucObject.FieldName.NodeId, LucField.Store.YES, true);
nf.SetIntValue(99999);
doc.Add(nf);
nf = new NumericField(LucObject.FieldName.VersionId, LucField.Store.YES, true);
nf.SetIntValue(99999);
doc.Add(nf);
doc.Add(new LucField(LucObject.FieldName.Name, "fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO));
doc.Add(new LucField(LucObject.FieldName.Path, "/root/fakedocument", LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO));
LuceneManager.AddCompleteDocument(doc, 0, false);
LuceneManager.UnregisterActivity(0, false);
}
public void Indexing_SavingRepairsDuplicatedNodes()
{
var content = Content.CreateNew("Car", TestRoot, "CarBadIndex");
var handler = (GenericContent)content.ContentHandler;
handler.VersioningMode = VersioningType.MajorAndMinor;
content.Save();
var id = content.Id;
var versionIdList = new List <int>();
versionIdList.Add(content.ContentHandler.VersionId);
for (int i = 0; i < 4; i++)
{
content.ContentHandler.Index++;
content.Save();
versionIdList.Add(content.ContentHandler.VersionId);
}
var term = new Term(LucObject.FieldName.NodeId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(id));
LuceneManager._writer.DeleteDocuments(term);
LuceneManager._writer.Commit();
LuceneManager._reader = LuceneManager._writer.GetReader();
foreach (var versionId in versionIdList)
{
var node = Node.LoadNodeByVersionId(versionId);
var doc = IndexDocumentInfo.CreateDocument(node);
doc.RemoveField(LucObject.FieldName.IsLastDraft);
doc.Add(new LucField(LucObject.FieldName.IsLastDraft, BooleanIndexHandler.YES, LucField.Store.YES, LucField.Index.NOT_ANALYZED, LucField.TermVector.NO));
LuceneManager._writer.AddDocument(doc);
// LuceneManager._isDirtyReader = true; //??/
}
LuceneManager._writer.Commit();
LuceneManager._reader = LuceneManager._writer.GetReader();
var children1 = ((IFolder)TestRoot).Children.ToArray();
var result1 = ContentQuery.Query("Name:CarBadIndex", new QuerySettings {
EnableAutofilters = FilterStatus.Disabled, EnableLifespanFilter = FilterStatus.Disabled
});
content.ContentHandler.Index++;
content.Save();
var children2 = ((IFolder)TestRoot).Children.ToArray();
var result2 = ContentQuery.Query("Name:CarBadIndex", new QuerySettings {
EnableAutofilters = FilterStatus.Disabled, EnableLifespanFilter = FilterStatus.Disabled
});
if (result1.Identifiers.Count() <= result2.Identifiers.Count())
{
Assert.Inconclusive("Identifier counts are equal. We did nothing.");
}
Assert.IsTrue(result2.Identifiers.Count() == 1, String.Concat("Identifiers.Count is ", result2.Identifiers.Count(), ". Expected: 1"));
}
internal override void Execute()
{
using (var optrace = new OperationTrace("AddTreeActivity Execute"))
{
var count = 0;
foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(TreeRoot))
{
var doc = IndexDocumentInfo.GetDocument(docData);
LuceneManager.AddCompleteDocument(doc);
count++;
}
Logger.WriteInformation(String.Concat("AddTreeActivity: ", count, " item added"));
base.Execute();
optrace.IsSuccessful = true;
}
}
public virtual Document CreateDocument()
{
using (var optrace = new OperationTrace("CreateDocument"))
{
if (_indexDocumentData != null)
{
//Trace.WriteLine("###I> create document from indexdocumentdata");
// create document from indexdocumentdata if it has been supplied (eg via MSMQ if it was small enough to send it over)
var docInfo = _indexDocumentData.IndexDocumentInfo as IndexDocumentInfo;
var doc = IndexDocumentInfo.CreateDocument(docInfo, _indexDocumentData);
optrace.IsSuccessful = true;
return(doc);
}
else
{
//Trace.WriteLine("###I> get document from db");
// create document via loading it from db (eg when indexdocumentdata was too large to send over MSMQ)
var doc = IndexDocumentInfo.GetDocument(this.VersionId);
optrace.IsSuccessful = true;
return(doc);
}
}
}
public void IndexingHistory_FixUpdateDeleteOverlap()
{
// update overlaps with delete
var fieldvalue1 = "IndexingHistoryFixUpdateDeleteOverlap";
var history = LuceneManager._history;
var car = new GenericContent(TestRoot, "Car");
car.Name = Guid.NewGuid().ToString();
car.Save();
var id = car.Id;
// init 1
var node1 = Node.LoadNode(id);
node1["Description"] = fieldvalue1;
node1.Save();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node1.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node1.VersionId);
var docInfo1 = IndexDocumentInfo.Create(node1, false);
var docData1 = DataBackingStore.CreateIndexDocumentData(node1, docInfo1, null, null);
var document1 = IndexDocumentInfo.CreateDocument(docInfo1, docData1);
// init 2
var node2 = Node.LoadNode(id);
node2.ForceDelete();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node2.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node2.VersionId);
// 1 check indexing history
var versionId1 = history.GetVersionId(document1);
var timestamp1 = history.GetTimestamp(document1);
var historyOk = LuceneManager._history.CheckForUpdate(versionId1, timestamp1);
// timestamp in indexing history should be the newest
var actTimestamp1 = history.Get(versionId1);
Assert.AreEqual(timestamp1, actTimestamp1, "Timestamp in indexing history did not change.");
Assert.IsTrue(historyOk, "History indicates indexing should not be executed, but this is not true.");
// 2 check indexing history
var versionId2 = node2.VersionId;
var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId2));
LuceneManager._history.ProcessDelete(new Term[] { term });
// timestamp in indexing history should change
var actTimestamp2 = history.Get(versionId2);
Assert.AreEqual(Timestamps.MaxValue, actTimestamp2, "Timestamp in indexing history did not change.");
// 2 index
LuceneManager.SetFlagsForDelete(term);
LuceneManager._writer.DeleteDocuments(term);
var firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
// check indexing occured correctly
// thread 2 deletes from index
Assert.AreEqual(0, firstfound);
// 1 index
var document = document1;
var updateTerm = UpdateDocumentActivity.GetIdTerm(document);
LuceneManager.SetFlagsForUpdate(document);
LuceneManager._writer.UpdateDocument(updateTerm, document);
using (new SenseNet.ContentRepository.Storage.Security.SystemAccount())
firstfound = ContentQuery.Query(SafeQueries.Description, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}, fieldvalue1).Count;
// check indexing occured correctly
// thread 1 updates (adds) values to index, ghost document is created
Assert.AreEqual(1, firstfound);
// 1 detects problem
var detectChange1 = history.CheckHistoryChange(versionId1, timestamp1);
Assert.IsTrue(detectChange1, "Thread 1 did not detect indexing overlapping although it should have.");
// 1 fixes index
LuceneManager.RefreshDocument(versionId1, false);
firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
Assert.AreEqual(0, firstfound);
}
public void IndexingHistory_FixAddUpdateOverlap()
{
// add overlaps with update
var fieldvalue1 = "IndexingHistoryFixAddUpdateOverlapFirst";
var fieldvalue2 = "IndexingHistoryFixAddUpdateOverlapSecond";
var history = LuceneManager._history;
var car = new GenericContent(TestRoot, "Car");
car.Name = Guid.NewGuid().ToString();
car.Save();
var id = car.Id;
// init 1
var node1 = Node.LoadNode(id);
node1["Description"] = fieldvalue1;
node1.Save();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node1.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node1.VersionId);
var docInfo1 = IndexDocumentInfo.Create(node1, false);
var docData1 = DataBackingStore.CreateIndexDocumentData(node1, docInfo1, null, null);
var document1 = IndexDocumentInfo.CreateDocument(docInfo1, docData1);
// init 2
var node2 = Node.LoadNode(id);
node2["Description"] = fieldvalue2;
node2.Save();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node2.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node2.VersionId);
var docInfo2 = IndexDocumentInfo.Create(node2, false);
var docData2 = DataBackingStore.CreateIndexDocumentData(node2, docInfo2, null, null);
var document2 = IndexDocumentInfo.CreateDocument(docInfo2, docData2);
// 1 check indexing history
var versionId1 = history.GetVersionId(document1);
var timestamp1 = history.GetTimestamp(document1);
var historyOk = LuceneManager._history.CheckForAdd(versionId1, timestamp1);
// timestamp in indexing history should be the newest
var actTimestamp1 = history.Get(versionId1);
Assert.AreEqual(timestamp1, actTimestamp1, "Timestamp in indexing history did not change.");
Assert.IsTrue(historyOk, "History indicates indexing should not be executed, but this is not true.");
// 2 check indexing history
var versionId2 = history.GetVersionId(document2);
var timestamp2 = history.GetTimestamp(document2);
historyOk = LuceneManager._history.CheckForUpdate(versionId2, timestamp2);
// timestamp in indexing history should change
var actTimestamp2 = history.Get(versionId2);
Assert.AreEqual(timestamp2, actTimestamp2, "Timestamp in indexing history did not change.");
Assert.IsTrue(historyOk, "History indicates indexing should not be executed, but this is not true.");
// 2 index
var document = document2;
var updateTerm = UpdateDocumentActivity.GetIdTerm(document);
LuceneManager.SetFlagsForUpdate(document);
LuceneManager._writer.UpdateDocument(updateTerm, document);
var firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
var secondfound = ContentQuery.Query("Description:" + fieldvalue2, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
// check indexing occured correctly
// thread 2 writes values in index
Assert.AreEqual(0, firstfound);
Assert.AreEqual(1, secondfound);
// 1 index
document = document1;
LuceneManager._writer.AddDocument(document);
firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
secondfound = ContentQuery.Query("Description:" + fieldvalue2, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
// check indexing occured correctly
// thread 1 adds values to index, duplication occurs
Assert.AreEqual(1, firstfound);
Assert.AreEqual(1, secondfound);
// 1 detects problem
var detectChange1 = history.CheckHistoryChange(versionId1, timestamp1);
Assert.IsTrue(detectChange1, "Thread 1 did not detect indexing overlapping although it should have.");
// 2 detects no problem
var detectChange2 = history.CheckHistoryChange(versionId2, timestamp2);
Assert.IsFalse(detectChange2, "Thread 2 detected indexing overlapping although it should not have.");
// 1 fixes index
LuceneManager.RefreshDocument(versionId1, false);
firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
//secondfound = ContentQuery.Query("Description:" + fieldvalue2, new QuerySettings { EnableAutofilters = FilterStatus.Disabled }).Count;
var q = LucQuery.Parse("Description:" + fieldvalue2);
q.EnableAutofilters = FilterStatus.Disabled;
secondfound = q.Execute(true).Count();
Assert.AreEqual(0, firstfound);
Assert.AreEqual(1, secondfound);
var node = Node.LoadNode(id);
node.ForceDelete();
}
public void IndexingHistory_FixDeleteUpdateOverlap()
{
// delete overlaps with update
var fieldvalue1 = "IndexingHistoryFixDeleteAddOverlap";
var history = LuceneManager._history;
var car = new GenericContent(TestRoot, "Car");
car.Name = Guid.NewGuid().ToString();
car.Save();
var id = car.Id;
// init 2
var node2 = Node.LoadNode(id);
node2["Description"] = fieldvalue1;
node2.Save();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node2.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node2.VersionId);
var docInfo2 = IndexDocumentInfo.Create(node2, false);
var docData2 = DataBackingStore.CreateIndexDocumentData(node2, docInfo2, null, null);
var document2 = IndexDocumentInfo.CreateDocument(docInfo2, docData2);
// init 1
var node1 = Node.LoadNode(id);
node1.ForceDelete();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node1.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node1.VersionId);
// 1 check indexing history
var versionId1 = node2.VersionId;
var term = new Term(LucObject.FieldName.VersionId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(versionId1));
LuceneManager._history.ProcessDelete(new Term[] { term });
// timestamp in indexing history should change
var actTimestamp1 = history.Get(versionId1);
Assert.AreEqual(Timestamps.MaxValue, actTimestamp1, "Timestamp in indexing history did not change.");
// 2 check indexing history
var versionId2 = history.GetVersionId(document2);
var timestamp2 = history.GetTimestamp(document2);
var historyOk = LuceneManager._history.CheckForUpdate(versionId2, timestamp2);
// timestamp in indexing history should not change
var actTimestamp2 = history.Get(versionId2);
Assert.AreNotEqual(timestamp2, actTimestamp2, "Timestamp in indexing history changed, although it should not have.");
Assert.IsFalse(historyOk, "History indicates indexing can be executed, but this is not true.");
// 2 does not continue, returns
// 1 deletes index
LuceneManager.SetFlagsForDelete(term);
LuceneManager._writer.DeleteDocuments(term);
var firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
// check indexing occured correctly
// thread 1 deletes from index
Assert.AreEqual(0, firstfound);
}
public void IndexingHistory_FixAddAddOverlap()
{
// add overlaps with add
var fieldvalue1 = "IndexingHistoryFixAddAddOverlap";
var history = LuceneManager._history;
var car = new GenericContent(TestRoot, "Car");
car.Name = Guid.NewGuid().ToString();
car.Save();
var id = car.Id;
// init 1 & 2
var node1 = Node.LoadNode(id);
node1["Description"] = fieldvalue1;
node1.Save();
// delete changes from index
DataRowTimestampTest.DeleteVersionFromIndex(node1.VersionId);
DataRowTimestampTest.DeleteVersionIdFromIndexingHistory(node1.VersionId);
var docInfo1 = IndexDocumentInfo.Create(node1, false);
var docData1 = DataBackingStore.CreateIndexDocumentData(node1, docInfo1, null, null);
var document1 = IndexDocumentInfo.CreateDocument(docInfo1, docData1);
// 1 check indexing history
var versionId1 = history.GetVersionId(document1);
var timestamp1 = history.GetTimestamp(document1);
var historyOk = LuceneManager._history.CheckForAdd(versionId1, timestamp1);
// timestamp in indexing history should be the newest
var actTimestamp1 = history.Get(versionId1);
Assert.AreEqual(timestamp1, actTimestamp1, "Timestamp in indexing history did not change.");
Assert.IsTrue(historyOk, "History indicates indexing should not be executed, but this is not true.");
// 2 check indexing history
var versionId2 = history.GetVersionId(document1);
var timestamp2 = history.GetTimestamp(document1);
historyOk = LuceneManager._history.CheckForAdd(versionId2, timestamp2);
// timestamp in indexing history should not change
var actTimestamp2 = history.Get(versionId2);
Assert.AreEqual(timestamp2, actTimestamp2, "Timestamp in indexing history changed, although it should not have.");
Assert.IsFalse(historyOk, "History indicates indexing can be executed, but this is not true.");
// 2 does not continue, returns
// 1 index
var document = document1;
LuceneManager._writer.AddDocument(document);
var firstfound = ContentQuery.Query("Description:" + fieldvalue1, new QuerySettings {
EnableAutofilters = FilterStatus.Disabled
}).Count;
// check indexing occured correctly
// thread 1 adds values to index
Assert.AreEqual(1, firstfound);
// 1 detects no problem
var detectChange1 = history.CheckHistoryChange(versionId1, timestamp1);
Assert.IsFalse(detectChange1, "Thread 1 detected indexing overlapping although it should not have.");
var node = Node.LoadNode(id);
node.ForceDelete();
}
