| public UpdateDocument ( Term term, Lucene.Net.Documents.Document doc ) : void | ||
| term | Term | the term to identify the document(s) to be /// deleted /// |
| doc | Lucene.Net.Documents.Document | the document to be added /// |
| return | void | |
public void TestRollbackIntegrityWithBufferFlush()
{
Directory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
for (int i = 0; i < 5; i++)
{
Document doc = new Document();
doc.Add(new Field("pk", i.ToString(), Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
w.AddDocument(doc);
}
w.Close();
// If buffer size is small enough to cause a flush, errors ensue...
w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
w.SetMaxBufferedDocs(2);
Term pkTerm = new Term("pk", "");
for (int i = 0; i < 3; i++)
{
Document doc = new Document();
String value = i.ToString();
doc.Add(new Field("pk", value, Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
doc.Add(new Field("text", "foo", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
w.UpdateDocument(pkTerm.CreateTerm(value), doc);
}
w.Rollback();
IndexReader r = IndexReader.Open(dir, true);
Assert.AreEqual(5, r.NumDocs(), "index should contain same number of docs post rollback");
r.Close();
dir.Close();
}
public virtual void TestRollbackIntegrityWithBufferFlush()
{
Directory dir = NewDirectory();
RandomIndexWriter rw = new RandomIndexWriter(Random(), dir);
for (int i = 0; i < 5; i++)
{
Document doc = new Document();
doc.Add(NewStringField("pk", Convert.ToString(i), Field.Store.YES));
rw.AddDocument(doc);
}
rw.Dispose();
// If buffer size is small enough to cause a flush, errors ensue...
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND));
for (int i = 0; i < 3; i++)
{
Document doc = new Document();
string value = Convert.ToString(i);
doc.Add(NewStringField("pk", value, Field.Store.YES));
doc.Add(NewStringField("text", "foo", Field.Store.YES));
w.UpdateDocument(new Term("pk", value), doc);
}
w.Rollback();
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(5, r.NumDocs, "index should contain same number of docs post rollback");
r.Dispose();
dir.Dispose();
}
// This method indexes the given text.
private static void AddToIndex(int id, string text, IndexWriter writer)
{
Term term = new Term("id", id.ToString());
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
doc.Add(new Field("id", id.ToString(), Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("mainText", text, Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
writer.UpdateDocument(term, doc);
}
public int FindChangedAndIndex()
{
var lastDateTimeFile = Path.Combine(path, "last.time");
var lastDateTime = DateTime.MinValue;
try
{
if (File.Exists(lastDateTimeFile))
{
lastDateTime = DateTime.Parse(File.ReadAllText(lastDateTimeFile)).ToUniversalTime();
}
}
catch (FormatException) { }
catch (ArgumentNullException) { }
var copyLastDateTime = lastDateTime;
lastDateTime = DateTime.UtcNow;
var texts = SelectTextsForIndex(copyLastDateTime, true);
if (0 < texts.Count)
{
var directory = GetOrCreateDirectory(path);
var analyzer = new AnalyzersProvider().GetAnalyzer(tenant.GetCulture().TwoLetterISOLanguageName);
var create = directory.ListAll().Length == 0;
var index = new IndexWriter(directory, analyzer, create, IndexWriter.MaxFieldLength.UNLIMITED);
try
{
foreach (var t in texts)
{
var term = new Term("Id", t.Item1);
if (string.IsNullOrEmpty(t.Item2))
{
index.DeleteDocuments(term);
}
else
{
var doc = new Document();
doc.Add(new Field("Id", t.Item1, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES));
doc.Add(new Field("Text", t.Item2, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
index.UpdateDocument(term, doc);
}
}
}
finally
{
index.Optimize();
index.Commit();
index.Close();
}
File.WriteAllText(lastDateTimeFile, lastDateTime.ToString("o"));
}
return texts.Count;
}
/// <summary>
/// Indexes all the urls in the supplied tweet
/// </summary>
public void IndexUrlsInTweet(Tweet tweet, IList<string> indexes)
{
//find urls to index in the url index
foreach (Uri uri in tweet.GetUrlsFromTweet())
{
Console.WriteLine("URL" + uri);
//setup index writer
FSDirectory luceneDirectory = FSDirectory.Open(new DirectoryInfo(Settings.URL_INDEX_DIR));
IndexWriter writer = new IndexWriter(luceneDirectory, _analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
//need to check if its not already indexed
//if it is already indexed, then just add a user to the index field in lucene
Document existingDoc = _searchengine.GetDocumentForUrl(uri.ToString());
if (existingDoc != null)
{
//document already exists, add a user to it.
Console.WriteLine("Already Exists");
bool wasUpdated = false;
wasUpdated |= UpdateIndexes(existingDoc, indexes);
wasUpdated |= UpdateTweets(existingDoc, tweet.TweetId);
//only update document if it was changed.
if (wasUpdated)
writer.UpdateDocument(new Term(Settings.FIELD_URL_ID, existingDoc.GetField(Settings.FIELD_URL_ID).StringValue()), existingDoc);
writer.Close();
continue;
}
Document luceneDocument = IndexUrl(uri, indexes, tweet.TweetId);
if (luceneDocument != null)
writer.AddDocument(luceneDocument);
writer.Optimize();
writer.Close();
}
//update the date indexed on the tweet
_tweetIndexer.UpdateDateIndexed(tweet);
}
public void Execute(IndexWriter writer)
{
Document document = IndexDefinition.Convert(Entity);
Term index = IndexDefinition.GetIndex(Entity);
if (document == null || index == null)
return;
switch (Operation)
{
case LuceneOperation.Insert:
writer.AddDocument(document);
break;
case LuceneOperation.Update:
writer.UpdateDocument(index, document);
break;
case LuceneOperation.Delete:
writer.DeleteDocuments(index);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public virtual void TestWithPendingDeletes3()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
SetUpDirs(dir, aux);
IndexWriter writer = NewWriter(dir, false);
// Adds 10 docs, then replaces them with another 10
// docs, so 10 pending deletes:
for (int i = 0; i < 20; i++)
{
Document doc = new Document();
doc.Add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.UpdateDocument(new Term("id", "" + (i % 10)), doc);
}
// Deletes one of the 10 added docs, leaving 9:
PhraseQuery q = new PhraseQuery();
q.Add(new Term("content", "bbb"));
q.Add(new Term("content", "14"));
writer.DeleteDocuments(q);
writer.AddIndexesNoOptimize(new Directory[] { aux });
writer.Optimize();
writer.Commit();
VerifyNumDocs(dir, 1039);
VerifyTermDocs(dir, new Term("content", "aaa"), 1030);
VerifyTermDocs(dir, new Term("content", "bbb"), 9);
writer.Close();
dir.Close();
aux.Close();
}
public void CreateCollection(string collectionPath)
{
using (StreamReader r = new StreamReader(collectionPath))
{
string json = r.ReadToEnd();
dynamic array = JsonConvert.DeserializeObject(json);
foreach (var item in array)
{
foreach (var passage in item.passages)
{
Lucene.Net.Documents.Document document = new Document();
string title = GetWebTitleFromUrl(passage.url.ToString());
document.Add(new TextField("title", title, Field.Store.YES));
document.Add(new StringField("url", passage.url.ToString(), Field.Store.YES));
document.Add(new TextField("passage_text", passage.passage_text.ToString(), Field.Store.YES));
document.Add(new StringField("passage_id", passage.passage_ID.ToString(), Field.Store.YES));
writer.UpdateDocument(new Term("url", passage.url.ToString()), document);
//writer.AddDocument(document);
}
}
System.Console.WriteLine("All documents added.");
}
}
public virtual void TestRollbackIntegrityWithBufferFlush()
{
Directory dir = NewDirectory();
RandomIndexWriter rw = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
for (int i = 0; i < 5; i++)
{
Document doc = new Document();
doc.Add(NewStringField("pk", Convert.ToString(i), Field.Store.YES));
rw.AddDocument(doc);
}
rw.Dispose();
// If buffer size is small enough to cause a flush, errors ensue...
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetOpenMode(OpenMode.APPEND));
for (int i = 0; i < 3; i++)
{
Document doc = new Document();
string value = Convert.ToString(i);
doc.Add(NewStringField("pk", value, Field.Store.YES));
doc.Add(NewStringField("text", "foo", Field.Store.YES));
w.UpdateDocument(new Term("pk", value), doc);
}
w.Rollback();
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(5, r.NumDocs, "index should contain same number of docs post rollback");
r.Dispose();
dir.Dispose();
}
private void UpdateDoc(IndexWriter modifier, int id, int value_Renamed)
{
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
doc.Add(new Field("id", System.Convert.ToString(id), Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("value", System.Convert.ToString(value_Renamed), Field.Store.NO, Field.Index.NOT_ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(id)), doc);
}
public virtual void IndexDoc()
{
Document d = new Document();
System.Collections.ArrayList fields = new System.Collections.ArrayList();
System.String idString = GetIdString();
Field idField = new Field(Lucene.Net.Index.TestStressIndexing2.idTerm.Field(), idString, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
fields.Add(idField);
int nFields = NextInt(Lucene.Net.Index.TestStressIndexing2.maxFields);
for (int i = 0; i < nFields; i++)
{
Field.TermVector tvVal = Field.TermVector.NO;
switch (NextInt(4))
{
case 0:
tvVal = Field.TermVector.NO;
break;
case 1:
tvVal = Field.TermVector.YES;
break;
case 2:
tvVal = Field.TermVector.WITH_POSITIONS;
break;
case 3:
tvVal = Field.TermVector.WITH_POSITIONS_OFFSETS;
break;
}
switch (NextInt(4))
{
case 0:
fields.Add(new Field("f" + NextInt(100), GetString(1), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, tvVal));
break;
case 1:
fields.Add(new Field("f" + NextInt(100), GetString(0), Field.Store.NO, Field.Index.ANALYZED, tvVal));
break;
case 2:
fields.Add(new Field("f" + NextInt(100), GetString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
break;
case 3:
fields.Add(new Field("f" + NextInt(100), GetString(Lucene.Net.Index.TestStressIndexing2.bigFieldSize), Field.Store.YES, Field.Index.ANALYZED, tvVal));
break;
}
}
if (Lucene.Net.Index.TestStressIndexing2.sameFieldOrder)
{
SupportClass.CollectionsHelper.Sort(fields, Lucene.Net.Index.TestStressIndexing2.fieldNameComparator);
}
else
{
// random placement of id field also
int index = NextInt(fields.Count);
fields[0] = fields[index];
fields[index] = idField;
}
for (int i = 0; i < fields.Count; i++)
{
d.Add((Fieldable)fields[i]);
}
w.UpdateDocument(Lucene.Net.Index.TestStressIndexing2.idTerm.CreateTerm(idString), d);
// System.out.println("indexing "+d);
docs[idString] = d;
}
static void UpdateArticleThreadProc(Object obj)
{
Article article = obj as Article;
lock (myLock)
{
Term term = new Term("ArticleId", article.ArticleId.ToString());
Document doc = ArticleToDocument(article);
IndexWriter writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UpdateDocument(term, doc);
writer.Optimize();
writer.Commit();
writer.Close();
}
}
public override void Run()
{
DirectoryReader currentReader = null;
Random random = LuceneTestCase.Random();
try
{
Document doc = new Document();
doc.Add(new TextField("id", "1", Field.Store.NO));
Writer.AddDocument(doc);
Holder.Reader = currentReader = Writer.GetReader(true);
Term term = new Term("id");
for (int i = 0; i < NumOps && !Holder.Stop; i++)
{
float nextOp = (float)random.NextDouble();
if (nextOp < 0.3)
{
term.Set("id", new BytesRef("1"));
Writer.UpdateDocument(term, doc);
}
else if (nextOp < 0.5)
{
Writer.AddDocument(doc);
}
else
{
term.Set("id", new BytesRef("1"));
Writer.DeleteDocuments(term);
}
if (Holder.Reader != currentReader)
{
Holder.Reader = currentReader;
if (Countdown)
{
Countdown = false;
Latch.Signal();
}
}
if (random.NextBoolean())
{
Writer.Commit();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader);
if (newReader != null)
{
currentReader.DecRef();
currentReader = newReader;
}
if (currentReader.NumDocs == 0)
{
Writer.AddDocument(doc);
}
}
}
}
catch (Exception e)
{
Failed = e;
}
finally
{
Holder.Reader = null;
if (Countdown)
{
Latch.Signal();
}
if (currentReader != null)
{
try
{
currentReader.DecRef();
}
catch (IOException e)
{
}
}
}
if (VERBOSE)
{
Console.WriteLine("writer stopped - forced by reader: " + Holder.Stop);
}
}
protected virtual void UpdateDocument(Term term, IEnumerable <IIndexableField> doc)
{
m_writer.UpdateDocument(term, doc);
}
private void WriteToIndex(ContentItem item, IndexWriter iw)
{
var doc = CreateDocument(item);
if (doc != null)
{
iw.UpdateDocument(new Term(Properties.ID, item.ID.ToString()), doc);
iw.Commit();
}
}
public virtual void IndexDoc()
{
Document d = new Document();
FieldType customType1 = new FieldType(TextField.TYPE_STORED);
customType1.IsTokenized = false;
customType1.OmitNorms = true;
List <Field> fields = new List <Field>();
string idString = IdString;
Field idField = NewField("id", idString, customType1);
fields.Add(idField);
int nFields = NextInt(maxFields);
for (int i = 0; i < nFields; i++)
{
FieldType customType = new FieldType();
switch (NextInt(4))
{
case 0:
break;
case 1:
customType.StoreTermVectors = true;
break;
case 2:
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
break;
case 3:
customType.StoreTermVectors = true;
customType.StoreTermVectorOffsets = true;
break;
}
switch (NextInt(4))
{
case 0:
customType.IsStored = true;
customType.OmitNorms = true;
customType.IsIndexed = true;
fields.Add(NewField("f" + NextInt(100), GetString(1), customType));
break;
case 1:
customType.IsIndexed = true;
customType.IsTokenized = true;
fields.Add(NewField("f" + NextInt(100), GetString(0), customType));
break;
case 2:
customType.IsStored = true;
customType.StoreTermVectors = false;
customType.StoreTermVectorOffsets = false;
customType.StoreTermVectorPositions = false;
fields.Add(NewField("f" + NextInt(100), GetString(0), customType));
break;
case 3:
customType.IsStored = true;
customType.IsIndexed = true;
customType.IsTokenized = true;
fields.Add(NewField("f" + NextInt(100), GetString(bigFieldSize), customType));
break;
}
}
if (sameFieldOrder)
{
fields.Sort(fieldNameComparer);
}
else
{
// random placement of id field also
fields.Swap(NextInt(fields.Count), 0);
}
for (int i = 0; i < fields.Count; i++)
{
d.Add(fields[i]);
}
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": indexing id:" + idString);
}
w.UpdateDocument(new Term("id", idString), d);
//System.out.println(Thread.currentThread().getName() + ": indexing "+d);
docs[idString] = d;
}
public virtual void TestFlushPushedDeletesByCount()
{
Directory dir = NewDirectory();
// Cannot use RandomIndexWriter because we don't want to
// ever call commit() for this test:
int flushAtDelCount = AtLeast(1020);
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDeleteTerms(flushAtDelCount).SetMaxBufferedDocs(1000).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false));
int count = 0;
while (true)
{
Document doc = new Document();
doc.Add(new StringField("id", count + "", Field.Store.NO));
Term delTerm;
if (count == 1010)
{
// this is the only delete that applies
delTerm = new Term("id", "" + 0);
}
else
{
// These get buffered, taking up RAM, but delete
// nothing when applied:
delTerm = new Term("id", "x" + count);
}
w.UpdateDocument(delTerm, doc);
// Eventually segment 0 should get a del docs:
// TODO: fix this test
if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv"))
{
break;
}
count++;
if (count > flushAtDelCount)
{
Assert.Fail("delete's were not applied at count=" + flushAtDelCount);
}
}
w.Dispose();
dir.Dispose();
}
/// <summary>
/// Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void DoTestOperationsOnDiskFull(bool updates)
{
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
// First build up a starting index:
MockDirectoryWrapper startDir = NewMockDirectory();
// TODO: find the resource leak that only occurs sometimes here.
startDir.NoDeleteOpenFile = false;
IndexWriter writer = new IndexWriter(startDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)));
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(NewStringField("id", Convert.ToString(i), Field.Store.YES));
d.Add(NewTextField("content", "aaa " + i, Field.Store.NO));
if (DefaultCodecSupportsDocValues())
{
d.Add(new NumericDocValuesField("dv", i));
}
writer.AddDocument(d);
}
writer.Dispose();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
if (VERBOSE)
{
Console.WriteLine("TEST: cycle");
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(Random(), new RAMDirectory(startDir, NewIOContext(Random())));
dir.PreventDoubleWrite = false;
dir.AllowRandomFileNotFoundException = false;
IndexWriter modifier = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)).SetMaxBufferedDocs(1000).SetMaxBufferedDeleteTerms(1000).SetMergeScheduler(new ConcurrentMergeScheduler()));
((ConcurrentMergeScheduler)modifier.Config.MergeScheduler).SetSuppressExceptions();
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: x=" + x);
}
double rate = 0.1;
double diskRatio = ((double)diskFree) / diskUsage;
long thisDiskFree;
string testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (VERBOSE)
{
Console.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.Dispose() @ " + thisDiskFree + " bytes";
dir.RandomIOExceptionRateOnOpen = Random().NextDouble() * 0.01;
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (VERBOSE)
{
Console.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
dir.RandomIOExceptionRateOnOpen = 0.0;
}
dir.MaxSizeInBytes = thisDiskFree;
dir.RandomIOExceptionRate = rate;
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(NewStringField("id", Convert.ToString(i), Field.Store.YES));
d.Add(NewTextField("content", "bbb " + i, Field.Store.NO));
if (DefaultCodecSupportsDocValues())
{
d.Add(new NumericDocValuesField("dv", i));
}
modifier.UpdateDocument(new Term("id", Convert.ToString(docId)), d);
} // deletes
else
{
modifier.DeleteDocuments(new Term("id", Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Dispose();
success = true;
if (0 == x)
{
done = true;
}
}
catch (IOException e)
{
if (VERBOSE)
{
Console.WriteLine(" hit IOException: " + e);
Console.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// prevent throwing a random exception here!!
double randomIOExceptionRate = dir.RandomIOExceptionRate;
long maxSizeInBytes = dir.MaxSizeInBytes;
dir.RandomIOExceptionRate = 0.0;
dir.RandomIOExceptionRateOnOpen = 0.0;
dir.MaxSizeInBytes = 0;
if (!success)
{
// Must force the close else the writer can have
// open files which cause exc in MockRAMDir.close
if (VERBOSE)
{
Console.WriteLine("TEST: now rollback");
}
modifier.Rollback();
}
// If the close() succeeded, make sure there are
// no unreferenced files.
if (success)
{
TestUtil.CheckIndex(dir);
TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.close");
}
dir.RandomIOExceptionRate = randomIOExceptionRate;
dir.MaxSizeInBytes = maxSizeInBytes;
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = DirectoryReader.Open(dir);
}
catch (IOException e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = NewSearcher(newReader);
ScoreDoc[] hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
}
catch (IOException e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but hits.Length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName + ": method did not throw exception but hits.Length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
Console.WriteLine(err.ToString());
Console.Write(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.Length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
newReader.Dispose();
if (result2 == END_COUNT)
{
break;
}
}
dir.Dispose();
modifier.Dispose();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
startDir.Dispose();
}
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void TestOperationsOnDiskFull(bool updates)
{
bool debug = false;
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
// First build up a starting index:
MockRAMDirectory startDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(startDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED, null);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d, null);
}
writer.Close();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
System.IO.IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
dir.SetPreventDoubleWrite(false);
IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED, null);
modifier.SetMaxBufferedDocs(1000); // use flush or close
modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
double rate = 0.1;
double diskRatio = ((double)diskFree) / diskUsage;
long thisDiskFree;
System.String testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
{
System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.close() @ " + thisDiskFree + " bytes";
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
{
System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
}
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES,
Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d, null);
}
else
{
// deletes
modifier.DeleteDocuments(null, new Term("id", System.Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Close();
success = true;
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
if (debug)
{
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// If the close() succeeded, make sure there are
// no unreferenced files.
if (success)
{
Lucene.Net.Util._TestUtil.CheckIndex(dir);
TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.close");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = IndexReader.Open((Directory)dir, true, null);
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
ScoreDoc[] hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, 1000, null).ScoreDocs;
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName +
": method did not throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName +
": method did not throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
System.Console.Error.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher.Close();
newReader.Close();
if (result2 == END_COUNT)
{
break;
}
}
dir.Close();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
}
public virtual void TestRollingUpdates_Mem()
{
Random random = new Random(Random().Next());
BaseDirectoryWrapper dir = NewDirectory();
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues());
//provider.register(new MemoryCodec());
if ((!"Lucene3x".Equals(Codec.Default.Name)) && Random().NextBoolean())
{
Codec.Default =
TestUtil.AlwaysPostingsFormat(new MemoryPostingsFormat(Random().nextBoolean(), random.NextFloat()));
}
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
int SIZE = AtLeast(20);
int id = 0;
IndexReader r = null;
IndexSearcher s = null;
int numUpdates = (int)(SIZE * (2 + (TEST_NIGHTLY ? 200 * Random().NextDouble() : 5 * Random().NextDouble())));
if (VERBOSE)
{
Console.WriteLine("TEST: numUpdates=" + numUpdates);
}
int updateCount = 0;
// TODO: sometimes update ids not in order...
for (int docIter = 0; docIter < numUpdates; docIter++)
{
Documents.Document doc = docs.NextDoc();
string myID = "" + id;
if (id == SIZE - 1)
{
id = 0;
}
else
{
id++;
}
if (VERBOSE)
{
Console.WriteLine(" docIter=" + docIter + " id=" + id);
}
((Field)doc.GetField("docid")).StringValue = myID;
Term idTerm = new Term("docid", myID);
bool doUpdate;
if (s != null && updateCount < SIZE)
{
TopDocs hits = s.Search(new TermQuery(idTerm), 1);
Assert.AreEqual(1, hits.TotalHits);
doUpdate = !w.TryDeleteDocument(r, hits.ScoreDocs[0].Doc);
if (VERBOSE)
{
if (doUpdate)
{
Console.WriteLine(" tryDeleteDocument failed");
}
else
{
Console.WriteLine(" tryDeleteDocument succeeded");
}
}
}
else
{
doUpdate = true;
if (VERBOSE)
{
Console.WriteLine(" no searcher: doUpdate=true");
}
}
updateCount++;
if (doUpdate)
{
w.UpdateDocument(idTerm, doc);
}
else
{
w.AddDocument(doc);
}
if (docIter >= SIZE && Random().Next(50) == 17)
{
if (r != null)
{
r.Dispose();
}
bool applyDeletions = Random().NextBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: reopen applyDeletions=" + applyDeletions);
}
r = w.GetReader(applyDeletions);
if (applyDeletions)
{
s = NewSearcher(r);
}
else
{
s = null;
}
Assert.IsTrue(!applyDeletions || r.NumDocs == SIZE, "applyDeletions=" + applyDeletions + " r.NumDocs=" + r.NumDocs + " vs SIZE=" + SIZE);
updateCount = 0;
}
}
if (r != null)
{
r.Dispose();
}
w.Commit();
Assert.AreEqual(SIZE, w.NumDocs());
w.Dispose();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "leftover files after rolling updates");
docs.Dispose();
// LUCENE-4455:
SegmentInfos infos = new SegmentInfos();
infos.Read(dir);
long totalBytes = 0;
foreach (SegmentCommitInfo sipc in infos.Segments)
{
totalBytes += sipc.SizeInBytes();
}
long totalBytes2 = 0;
foreach (string fileName in dir.ListAll())
{
if (!fileName.StartsWith(IndexFileNames.SEGMENTS))
{
totalBytes2 += dir.FileLength(fileName);
}
}
Assert.AreEqual(totalBytes2, totalBytes);
dir.Dispose();
}
//更新索引
/**
调用UpdateDocument() 方法，传给它一个新的doc来更新数据：
Term term = new Term(“id”, “1″);
先去索引文件里查找id为1 的 Document，如果有就更新它（如果有多条，最后更新后只有一条)。如果没有就新增 Document 到索引中。
数据库更新的时候，我们可以只针对某个列来更新，而lucene只能针对一行数据更新。
*/
public void UpdateIndex()
{
Lucene.Net.Store.Directory indexPath =
FSDirectory.Open(new DirectoryInfo(HttpContext.Current.Server.MapPath("~/Search/")));
IndexWriter writer = new IndexWriter(indexPath, new PanGuAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
Term term = new Term("id", "1");
Document doc = new Document();
doc.Add(new Field("id", "100", Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.UpdateDocument(term, doc, new PanGuAnalyzer(true));
writer.Optimize();
writer.Close();
}
/// <summary>
/// Calls
/// <see cref="IndexWriter.UpdateDocument(Term, IEnumerable{IIndexableField}, Analyzer)"/>
/// and returns the generation that reflects this change.
/// </summary>
public virtual long UpdateDocument(Term t, IEnumerable <IIndexableField> d, Analyzer a)
{
writer.UpdateDocument(t, d, a);
// Return gen as of when indexing finished:
return(indexingGen.Get());
}
public virtual void TestAddCloseOpen()
{
// Can't use assertNoDeletes: this test pulls a non-NRT
// reader in the end:
Directory dir1 = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir1, iwc);
for (int i = 0; i < 97; i++)
{
DirectoryReader reader = writer.Reader;
if (i == 0)
{
writer.AddDocument(DocHelper.CreateDocument(i, "x", 1 + Random().Next(5)));
}
else
{
int previous = Random().Next(i);
// a check if the reader is current here could fail since there might be
// merges going on.
switch (Random().Next(5))
{
case 0:
case 1:
case 2:
writer.AddDocument(DocHelper.CreateDocument(i, "x", 1 + Random().Next(5)));
break;
case 3:
writer.UpdateDocument(new Term("id", "" + previous), DocHelper.CreateDocument(previous, "x", 1 + Random().Next(5)));
break;
case 4:
writer.DeleteDocuments(new Term("id", "" + previous));
break;
}
}
Assert.IsFalse(reader.Current);
reader.Dispose();
}
writer.ForceMerge(1); // make sure all merging is done etc.
DirectoryReader dirReader = writer.Reader;
writer.Commit(); // no changes that are not visible to the reader
Assert.IsTrue(dirReader.Current);
writer.Dispose();
Assert.IsTrue(dirReader.Current); // all changes are visible to the reader
iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
writer = new IndexWriter(dir1, iwc);
Assert.IsTrue(dirReader.Current);
writer.AddDocument(DocHelper.CreateDocument(1, "x", 1 + Random().Next(5)));
Assert.IsTrue(dirReader.Current); // segments in ram but IW is different to the readers one
writer.Dispose();
Assert.IsFalse(dirReader.Current); // segments written
dirReader.Dispose();
dir1.Dispose();
}
public virtual void TestUpdateDocument()
{
bool doFullMerge = true;
Directory dir1 = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if (iwc.MaxBufferedDocs < 20)
{
iwc.SetMaxBufferedDocs(20);
}
// no merging
if (Random().NextBoolean())
{
iwc.SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
}
else
{
iwc.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
}
if (VERBOSE)
{
Console.WriteLine("TEST: make index");
}
IndexWriter writer = new IndexWriter(dir1, iwc);
// create the index
CreateIndexNoClose(!doFullMerge, "index1", writer);
// writer.Flush(false, true, true);
// get a reader
DirectoryReader r1 = writer.Reader;
Assert.IsTrue(r1.Current);
string id10 = r1.Document(10).GetField("id").StringValue;
Document newDoc = r1.Document(10);
newDoc.RemoveField("id");
newDoc.Add(NewStringField("id", Convert.ToString(8000), Field.Store.YES));
writer.UpdateDocument(new Term("id", id10), newDoc);
Assert.IsFalse(r1.Current);
DirectoryReader r2 = writer.Reader;
Assert.IsTrue(r2.Current);
Assert.AreEqual(0, Count(new Term("id", id10), r2));
if (VERBOSE)
{
Console.WriteLine("TEST: verify id");
}
Assert.AreEqual(1, Count(new Term("id", Convert.ToString(8000)), r2));
r1.Dispose();
Assert.IsTrue(r2.Current);
writer.Dispose();
Assert.IsTrue(r2.Current);
DirectoryReader r3 = DirectoryReader.Open(dir1);
Assert.IsTrue(r3.Current);
Assert.IsTrue(r2.Current);
Assert.AreEqual(0, Count(new Term("id", id10), r3));
Assert.AreEqual(1, Count(new Term("id", Convert.ToString(8000)), r3));
writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
doc.Add(NewTextField("field", "a b c", Field.Store.NO));
writer.AddDocument(doc);
Assert.IsTrue(r2.Current);
Assert.IsTrue(r3.Current);
writer.Dispose();
Assert.IsFalse(r2.Current);
Assert.IsTrue(!r3.Current);
r2.Dispose();
r3.Dispose();
dir1.Dispose();
}
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void TestOperationsOnDiskFull(bool updates)
{
bool debug = false;
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
// First build up a starting index:
RAMDirectory startDir = new RAMDirectory();
IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.TOKENIZED));
writer.AddDocument(d);
}
writer.Close();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
System.IO.IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
modifier.SetMaxBufferedDocs(1000); // use flush or close
modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
double rate = 0.1;
//UPGRADE_WARNING: Data types in Visual C# might be different. Verify the accuracy of narrowing conversions. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1042'"
double diskRatio = ((double) diskFree) / diskUsage;
long thisDiskFree;
System.String testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
{
System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.close() @ " + thisDiskFree + " bytes";
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
{
System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
}
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.TOKENIZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d);
}
else
{
// deletes
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Close();
success = true;
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
if (debug)
{
//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// Whether we succeeded or failed, check that all
// un-referenced files were in fact deleted (ie,
// we did not create garbage). Just create a
// new IndexFileDeleter, have it delete
// unreferenced files, then verify that in fact
// no files were deleted:
System.String[] startFiles = dir.List();
SegmentInfos infos = new SegmentInfos();
infos.Read(dir);
new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
System.String[] endFiles = dir.List();
//UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
System.Array.Sort(startFiles);
//UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
System.Array.Sort(endFiles);
// for(int i=0;i<startFiles.length;i++) {
// System.out.println(" startFiles: " + i + ": " + startFiles[i]);
// }
if (!SupportClass.Compare.CompareStringArrays(startFiles, endFiles))
{
System.String successStr;
if (success)
{
successStr = "success";
}
else
{
successStr = "IOException";
System.Console.Error.WriteLine(err.StackTrace);
}
Assert.Fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles));
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = IndexReader.Open(dir);
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
Hits hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm));
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length();
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
System.Console.Error.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher.Close();
newReader.Close();
if (result2 == END_COUNT)
{
break;
}
}
dir.Close();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
}
}
public virtual void TestFlushPushedDeletesByRAM()
{
Directory dir = NewDirectory();
// Cannot use RandomIndexWriter because we don't want to
// ever call commit() for this test:
// note: tiny rambuffer used, as with a 1MB buffer the test is too slow (flush @ 128,999)
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.1f).SetMaxBufferedDocs(1000).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false));
int count = 0;
while (true)
{
Document doc = new Document();
doc.Add(new StringField("id", count + "", Field.Store.NO));
Term delTerm;
if (count == 1010)
{
// this is the only delete that applies
delTerm = new Term("id", "" + 0);
}
else
{
// These get buffered, taking up RAM, but delete
// nothing when applied:
delTerm = new Term("id", "x" + count);
}
w.UpdateDocument(delTerm, doc);
// Eventually segment 0 should get a del docs:
// TODO: fix this test
if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv"))
{
if (VERBOSE)
{
Console.WriteLine("TEST: deletes created @ count=" + count);
}
break;
}
count++;
// Today we applyDeletes @ count=21553; even if we make
// sizable improvements to RAM efficiency of buffered
// del term we're unlikely to go over 100K:
if (count > 100000)
{
Assert.Fail("delete's were not applied");
}
}
w.Dispose();
dir.Dispose();
}
public override void Run()
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
doc.Add(new NumericDocValuesField("dv", 5));
int idUpto = 0;
int fullCount = 0;
long stopTime = DateTime.Now.Millisecond + 200;
do
{
try
{
Writer.UpdateDocument(new Term("id", "" + (idUpto++)), doc);
AddCount++;
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: expected exc:");
Console.WriteLine(ioe.StackTrace);
}
//System.out.println(Thread.currentThread().getName() + ": hit exc");
//ioConsole.WriteLine(e.StackTrace);
if (ioe.Message.StartsWith("fake disk full at") || ioe.Message.Equals("now failing on purpose"))
{
DiskFull = true;
try
{
Thread.Sleep(1);
}
catch (ThreadInterruptedException ie)
{
throw new ThreadInterruptedException(ie);
}
if (fullCount++ >= 5)
{
break;
}
}
else
{
if (NoErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected IOException:");
Console.WriteLine(ioe.StackTrace);
Error = ioe;
}
break;
}
}
catch (Exception t)
{
//Console.WriteLine(t.StackTrace);
if (NoErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected Throwable:");
Console.WriteLine(t.StackTrace);
Error = t;
}
break;
}
} while (DateTime.Now.Millisecond < stopTime);
}
private void UpdateDoc(IndexWriter modifier, int id, int value)
{
Document doc = new Document();
doc.Add(NewTextField("content", "aaa", Field.Store.NO));
doc.Add(NewStringField("id", Convert.ToString(id), Field.Store.YES));
doc.Add(NewStringField("value", Convert.ToString(value), Field.Store.NO));
if (DefaultCodecSupportsDocValues())
{
doc.Add(new NumericDocValuesField("dv", value));
}
modifier.UpdateDocument(new Term("id", Convert.ToString(id)), doc);
}
public virtual void TestRollingUpdates_Mem()
{
Random random = new Random(Random().Next());
BaseDirectoryWrapper dir = NewDirectory();
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues());
//provider.register(new MemoryCodec());
if ((!"Lucene3x".Equals(Codec.Default.Name, StringComparison.Ordinal)) && Random().NextBoolean())
{
Codec.Default =
TestUtil.AlwaysPostingsFormat(new MemoryPostingsFormat(Random().nextBoolean(), random.NextFloat()));
}
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
int SIZE = AtLeast(20);
int id = 0;
IndexReader r = null;
IndexSearcher s = null;
int numUpdates = (int)(SIZE * (2 + (TEST_NIGHTLY ? 200 * Random().NextDouble() : 5 * Random().NextDouble())));
if (VERBOSE)
{
Console.WriteLine("TEST: numUpdates=" + numUpdates);
}
int updateCount = 0;
// TODO: sometimes update ids not in order...
for (int docIter = 0; docIter < numUpdates; docIter++)
{
Documents.Document doc = docs.NextDoc();
string myID = "" + id;
if (id == SIZE - 1)
{
id = 0;
}
else
{
id++;
}
if (VERBOSE)
{
Console.WriteLine(" docIter=" + docIter + " id=" + id);
}
((Field)doc.GetField("docid")).SetStringValue(myID);
Term idTerm = new Term("docid", myID);
bool doUpdate;
if (s != null && updateCount < SIZE)
{
TopDocs hits = s.Search(new TermQuery(idTerm), 1);
Assert.AreEqual(1, hits.TotalHits);
doUpdate = !w.TryDeleteDocument(r, hits.ScoreDocs[0].Doc);
if (VERBOSE)
{
if (doUpdate)
{
Console.WriteLine(" tryDeleteDocument failed");
}
else
{
Console.WriteLine(" tryDeleteDocument succeeded");
}
}
}
else
{
doUpdate = true;
if (VERBOSE)
{
Console.WriteLine(" no searcher: doUpdate=true");
}
}
updateCount++;
if (doUpdate)
{
w.UpdateDocument(idTerm, doc);
}
else
{
w.AddDocument(doc);
}
if (docIter >= SIZE && Random().Next(50) == 17)
{
if (r != null)
{
r.Dispose();
}
bool applyDeletions = Random().NextBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: reopen applyDeletions=" + applyDeletions);
}
r = w.GetReader(applyDeletions);
if (applyDeletions)
{
s = NewSearcher(r);
}
else
{
s = null;
}
Assert.IsTrue(!applyDeletions || r.NumDocs == SIZE, "applyDeletions=" + applyDeletions + " r.NumDocs=" + r.NumDocs + " vs SIZE=" + SIZE);
updateCount = 0;
}
}
if (r != null)
{
r.Dispose();
}
w.Commit();
Assert.AreEqual(SIZE, w.NumDocs);
w.Dispose();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "leftover files after rolling updates");
docs.Dispose();
// LUCENE-4455:
SegmentInfos infos = new SegmentInfos();
infos.Read(dir);
long totalBytes = 0;
foreach (SegmentCommitInfo sipc in infos.Segments)
{
totalBytes += sipc.GetSizeInBytes();
}
long totalBytes2 = 0;
foreach (string fileName in dir.ListAll())
{
if (!fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal))
{
totalBytes2 += dir.FileLength(fileName);
}
}
Assert.AreEqual(totalBytes2, totalBytes);
dir.Dispose();
}
/// <summary>
/// Updates the date indexed to be now.
/// </summary>
public void UpdateDateIndexed(Tweet tweetToUpdate)
{
FSDirectory tweetDirectory = FSDirectory.Open(new DirectoryInfo(Settings.TWEET_INDEX_DIR));
IndexWriter tweetWriter = new IndexWriter(tweetDirectory, _analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
//update the date indexed on the tweet
Document existingTweet = _searchengine.GetDocumentForTweetId(tweetToUpdate.TweetId);
//update the field when it was updated
Field dateUpdated = existingTweet.GetField(Settings.FIELD_TWEET_DATE_INDEXED);
dateUpdated.SetValue(DateTime.Now.ToString());
tweetWriter.UpdateDocument(new Term(Settings.FIELD_TWEET_ID, existingTweet.GetField(Settings.FIELD_TWEET_ID).StringValue()), existingTweet);
tweetWriter.Close();
}
protected internal virtual void UpdateDocument(Term term, IEnumerable <IndexableField> doc)
{
Writer.UpdateDocument(term, doc);
}
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void TestOperationsOnDiskFull(bool updates)
{
bool debug = false;
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
// First build up a starting index:
MockRAMDirectory startDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d);
}
writer.Close();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
System.IO.IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
dir.SetPreventDoubleWrite(false);
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
modifier.SetMaxBufferedDocs(1000); // use flush or close
modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
double rate = 0.1;
double diskRatio = ((double) diskFree) / diskUsage;
long thisDiskFree;
System.String testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
{
System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.close() @ " + thisDiskFree + " bytes";
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
{
System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
}
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d);
}
else
{
// deletes
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Close();
success = true;
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
if (debug)
{
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// If the close() succeeded, make sure there are
// no unreferenced files.
if (success)
{
Lucene.Net.Util._TestUtil.CheckIndex(dir);
TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.close");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = IndexReader.Open(dir);
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
ScoreDoc[] hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
System.Console.Error.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher.Close();
newReader.Close();
if (result2 == END_COUNT)
{
break;
}
}
dir.Close();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
}
}
public virtual void TestUpdateDocument()
{
bool optimize = true;
Directory dir1 = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
// create the index
CreateIndexNoClose(!optimize, "index1", writer);
// writer.flush(false, true, true);
// get a reader
IndexReader r1 = writer.GetReader();
Assert.IsTrue(r1.IsCurrent());
System.String id10 = r1.Document(10).GetField("id").StringValue;
Document newDoc = r1.Document(10);
newDoc.RemoveField("id");
newDoc.Add(new Field("id", System.Convert.ToString(8000), Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.UpdateDocument(new Term("id", id10), newDoc);
Assert.IsFalse(r1.IsCurrent());
IndexReader r2 = writer.GetReader();
Assert.IsTrue(r2.IsCurrent());
Assert.AreEqual(0, Count(new Term("id", id10), r2));
Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r2));
r1.Close();
writer.Close();
Assert.IsTrue(r2.IsCurrent());
IndexReader r3 = IndexReader.Open(dir1, true);
Assert.IsTrue(r3.IsCurrent());
Assert.IsTrue(r2.IsCurrent());
Assert.AreEqual(0, Count(new Term("id", id10), r3));
Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r3));
writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.Add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
Assert.IsTrue(r2.IsCurrent());
Assert.IsTrue(r3.IsCurrent());
writer.Close();
Assert.IsFalse(r2.IsCurrent());
Assert.IsTrue(!r3.IsCurrent());
r2.Close();
r3.Close();
dir1.Close();
}
public override void Run()
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
doc.Add(new NumericDocValuesField("dv", 5));
int idUpto = 0;
int fullCount = 0;
// LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
long stopTime = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) + timeToRunInMilliseconds; // LUCENENET specific: added the ability to change how much time to alot
do
{
try
{
writer.UpdateDocument(new Term("id", "" + (idUpto++)), doc);
addCount++;
}
catch (Exception ioe) when(ioe.IsIOException())
{
if (Verbose)
{
Console.WriteLine("TEST: expected exc:");
Console.WriteLine(ioe.StackTrace);
}
//System.out.println(Thread.currentThread().getName() + ": hit exc");
//ioConsole.WriteLine(e.StackTrace);
if (ioe.Message.StartsWith("fake disk full at", StringComparison.Ordinal) || ioe.Message.Equals("now failing on purpose", StringComparison.Ordinal))
{
diskFull = true;
try
{
Thread.Sleep(1);
}
catch (Exception ie) when(ie.IsInterruptedException())
{
throw new Util.ThreadInterruptedException(ie);
}
if (fullCount++ >= 5)
{
break;
}
}
else
{
if (noErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected IOException:");
Console.WriteLine(ioe.StackTrace);
error = ioe;
}
break;
}
}
catch (Exception t) when(t.IsThrowable())
{
//Console.WriteLine(t.StackTrace);
if (noErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected Throwable:");
Console.WriteLine(t.StackTrace);
error = t;
}
break;
}
} while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTime); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
private void btnUpdateDoc_Click(object sender, EventArgs e)
{
try
{
string path = System.Configuration.ConfigurationSettings.AppSettings["Data"].ToString();
Analyzer analyzer = new StockFooAnalyzer(path);
FSDirectory dy = FSDirectory.Open(new DirectoryInfo(System.Configuration.ConfigurationSettings.AppSettings["IndexDirectory"].ToString()));
IndexWriter indexWriter = new IndexWriter(dy, analyzer, false);
IOpen oldopen = CurrentIndex.GetCurrentOpendIndex();
Document olddoc = oldopen.Reader.Document(Convert.ToInt32(textBox1.Text));
string oldurl = olddoc.Get("url").ToString();
bool oldisnew = false;
if (oldurl.Trim().Equals(txturl.Text.Trim()))
oldisnew = true;
Document document = new Document();
Field ftitle = new Field("title", txttitle.Text, Field.Store.YES, Field.Index.ANALYZED);//存储，索引
document.Add(ftitle);
Field furl = new Field("url", txturl.Text, Field.Store.YES, Field.Index.NOT_ANALYZED);//存储，不索引
document.Add(furl);
Field fsite = new Field("site", txtsite.Text, Field.Store.YES, Field.Index.NOT_ANALYZED);//存储，不索引
document.Add(fsite);
Field fbody = new Field("body", txtbody.Text, Field.Store.YES, Field.Index.ANALYZED);//存储，索引
document.Add(fbody);
Field fpublishtime = new Field("publish_time", txtpublishtime.Text, Field.Store.YES, Field.Index.NOT_ANALYZED);//存储，不索引
document.Add(fpublishtime);
Term term = new Term("url", txturl.Text);
indexWriter.UpdateDocument(term, document, analyzer);
indexWriter.Optimize();
indexWriter.Close();
IndexOpen open = new IndexOpen(System.Configuration.ConfigurationSettings.AppSettings["IndexDirectory"].ToString(), false);
bool isOpend = open.Open();
if (isOpend)
{
CurrentIndex.SetCurrentOpendIndex(open);
}
if (!oldisnew)
{
FindDocuemnt(0);
label9.Text = "当前为文档1";
textBox1.Text = "0";
docNum = docNum + 1;
label1.Text = string.Format("共有文档{0}条", docNum);
}
else
{
docNum = open.Reader.MaxDoc();
FindDocuemnt(docNum - 1);
label9.Text = "当前为文档" + (docNum).ToString();
textBox1.Text = (docNum - 1).ToString();
}
MessageBox.Show("更新成功！");
}
catch (Exception ex)
{
MessageBox.Show("更新失败！\\n"+ex.Message);
}
}
public override void Run()
{
DirectoryReader currentReader = null;
Random random = LuceneTestCase.Random;
try
{
Document doc = new Document();
doc.Add(new TextField("id", "1", Field.Store.NO));
writer.AddDocument(doc);
holder.reader = currentReader = writer.GetReader(true);
Term term = new Term("id");
for (int i = 0; i < numOps && !holder.stop; i++)
{
float nextOp = (float)random.NextDouble();
if (nextOp < 0.3)
{
term.Set("id", new BytesRef("1"));
writer.UpdateDocument(term, doc);
}
else if (nextOp < 0.5)
{
writer.AddDocument(doc);
}
else
{
term.Set("id", new BytesRef("1"));
writer.DeleteDocuments(term);
}
if (holder.reader != currentReader)
{
holder.reader = currentReader;
if (countdown)
{
countdown = false;
latch.Signal();
}
}
if (random.NextBoolean())
{
writer.Commit();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader);
if (newReader != null)
{
currentReader.DecRef();
currentReader = newReader;
}
if (currentReader.NumDocs == 0)
{
writer.AddDocument(doc);
}
}
}
}
catch (Exception e)
{
failed = e;
}
finally
{
holder.reader = null;
if (countdown)
{
latch.Signal();
}
if (currentReader != null)
{
try
{
currentReader.DecRef();
}
#pragma warning disable 168
catch (IOException e)
#pragma warning restore 168
{
}
}
}
if (Verbose)
{
Console.WriteLine("writer stopped - forced by reader: " + holder.stop);
}
}
public void End(bool shouldClose)
{
if (!_is_started)
{
return;
}
if (!shouldClose)
{
return;
}
//build 2del file list
if (!_job_status.Cancelled)
{
TermEnum term_enum = _index_reader.Terms();
Term path_term = new Term("path");
int nb_terms = 0;
while (term_enum.SkipTo(path_term)) //skip to new term equal or *ABOVE* "path:" !!!
{
Term term = term_enum.Term();
if (term.Field() != path_term.Field())
{
break;
}
if (!File.Exists(term.Text()))
{
_del_file_list.Add(term.Text());
}
if (_job_status.Cancelled)
{
break;
}
nb_terms++;
}
term_enum.Close();
Logger.Log.Info("update: deletion: {0} analyzed terms, found {1} vanished files.", nb_terms, _del_file_list.Count);
}
_index_searcher.Close();
_index_reader.Close();
//--- deleting deprecated
if ((_del_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
int num_file = 0;
int nb_files = _del_file_list.Count;
IndexWriter writer = new IndexWriter(_index_path, _default_analyzer, false);
foreach (string path in _del_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: removing (from index) file {0}/{1} - {2}", num_file, _del_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.DeleteDocuments(new Term("path", path));
}
writer.Commit();
writer.Close();
watch.Stop();
}
//adding new files
if ((_add_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
IndexWriter writer = null;
try
{
writer = new IndexWriter(_index_path, _default_analyzer, false, new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
int num_file = 0;
int nb_files = _add_file_list.Count;
foreach (BasicFileInfo fi in _add_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: indexing new file {0}/{1} - {2}", num_file, _add_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.AddDocument(_doc_factory.CreateFromPath(fi.FilePath, fi.LastModification));
if (num_file % 20 == 0)
{
writer.Commit();
}
}
writer.Commit();
}
catch (System.Exception ex)
{
Log.Error(ex);
}
finally
{
if (writer != null)
{
writer.Close();
writer = null;
}
}
watch.Stop();
}
//updating modified files
if ((_upd_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
int num_file = 0;
int nb_files = _upd_file_list.Count;
IndexWriter writer = null;
try
{
writer = new IndexWriter(_index_path, _default_analyzer, false,
new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
foreach (BasicFileInfo fi in _upd_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: modified file {0}/{1} - {2}", num_file, _upd_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.UpdateDocument(new Term("path", fi.FilePath),
_doc_factory.CreateFromPath(fi.FilePath, fi.LastModification));
}
writer.Commit();
//LittleBeagle.Properties.Settings.Default.NbIndexedFiles = num_file;
}
catch (System.Exception ex)
{
Log.Error(ex);
}
finally
{
if (writer != null)
{
writer.Close();
writer = null;
}
}
watch.Stop();
}
}
public override void Run()
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
doc.Add(new NumericDocValuesField("dv", 5));
int idUpto = 0;
int fullCount = 0;
long stopTime = Environment.TickCount + TimeToRunInMilliseconds; // LUCENENET specific: added the ability to change how much time to alot
do
{
try
{
Writer.UpdateDocument(new Term("id", "" + (idUpto++)), doc);
AddCount++;
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: expected exc:");
Console.WriteLine(ioe.StackTrace);
}
//System.out.println(Thread.currentThread().getName() + ": hit exc");
//ioConsole.WriteLine(e.StackTrace);
if (ioe.Message.StartsWith("fake disk full at", StringComparison.Ordinal) || ioe.Message.Equals("now failing on purpose", StringComparison.Ordinal))
{
DiskFull = true;
//#if !NETSTANDARD1_6
// try
// {
//#endif
Thread.Sleep(1);
//#if !NETSTANDARD1_6
// }
// catch (ThreadInterruptedException ie) // LUCENENET NOTE: Senseless to catch and rethrow the same exception type
// {
// throw new ThreadInterruptedException(ie.toString(), ie);
// }
//#endif
if (fullCount++ >= 5)
{
break;
}
}
else
{
if (NoErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected IOException:");
Console.WriteLine(ioe.StackTrace);
Error = ioe;
}
break;
}
}
catch (Exception t)
{
//Console.WriteLine(t.StackTrace);
if (NoErrors)
{
Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: unexpected Throwable:");
Console.WriteLine(t.StackTrace);
Error = t;
}
break;
}
} while (Environment.TickCount < stopTime);
}
override public void Run()
{
Document doc = new Document();
doc.Add(new Field("content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
doc.Add(new Field("content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
doc.Add(new Field("content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
doc.Add(new Field("content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.Add(new Field("content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
long stopTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 3000;
while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < stopTime)
{
System.Threading.Thread.SetData(Enclosing_Instance.doFail, this);
System.String id = "" + r.Next(50);
idField.SetValue(id);
Term idTerm = new Term("id", id);
try
{
writer.UpdateDocument(idTerm, doc);
}
catch (System.SystemException re)
{
if (Lucene.Net.Index.TestIndexWriterExceptions.DEBUG)
{
System.Console.Out.WriteLine("EXC: ");
System.Console.Out.WriteLine(re.StackTrace);
}
try
{
_TestUtil.CheckIndex(writer.GetDirectory());
}
catch (System.IO.IOException ioe)
{
System.Console.Out.WriteLine(SupportClass.ThreadClass.Current().Name + ": unexpected exception1");
System.Console.Out.WriteLine(ioe.StackTrace);
failure = ioe;
break;
}
}
catch (System.Exception t)
{
System.Console.Out.WriteLine(SupportClass.ThreadClass.Current().Name + ": unexpected exception2");
System.Console.Out.WriteLine(t.StackTrace);
failure = t;
break;
}
System.Threading.Thread.SetData(Enclosing_Instance.doFail, null);
// After a possible exception (above) I should be able
// to add a new document without hitting an
// exception:
try
{
writer.UpdateDocument(idTerm, doc);
}
catch (System.Exception t)
{
System.Console.Out.WriteLine(SupportClass.ThreadClass.Current().Name + ": unexpected exception3");
System.Console.Out.WriteLine(t.StackTrace);
failure = t;
break;
}
}
}
public virtual void TestUpdateDocument()
{
bool optimize = true;
Directory dir1 = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
// create the index
CreateIndexNoClose(!optimize, "index1", writer);
// writer.flush(false, true, true);
// get a reader
IndexReader r1 = writer.GetReader();
Assert.IsTrue(r1.IsCurrent());
System.String id10 = r1.Document(10).GetField("id").StringValue();
Document newDoc = r1.Document(10);
newDoc.RemoveField("id");
newDoc.Add(new Field("id", System.Convert.ToString(8000), Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.UpdateDocument(new Term("id", id10), newDoc);
Assert.IsFalse(r1.IsCurrent());
IndexReader r2 = writer.GetReader();
Assert.IsTrue(r2.IsCurrent());
Assert.AreEqual(0, Count(new Term("id", id10), r2));
Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r2));
r1.Close();
writer.Close();
Assert.IsTrue(r2.IsCurrent());
IndexReader r3 = IndexReader.Open(dir1);
Assert.IsTrue(r3.IsCurrent());
Assert.IsTrue(r2.IsCurrent());
Assert.AreEqual(0, Count(new Term("id", id10), r3));
Assert.AreEqual(1, Count(new Term("id", System.Convert.ToString(8000)), r3));
writer = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.Add(new Field("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
Assert.IsTrue(r2.IsCurrent());
Assert.IsTrue(r3.IsCurrent());
writer.Close();
Assert.IsFalse(r2.IsCurrent());
Assert.IsTrue(!r3.IsCurrent());
r2.Close();
r3.Close();
dir1.Close();
}
private static void UpdateFarmInIndex(Farm farm, IndexWriter indexWriter, ApplicationDbContext db)
{
var doc = CreateDocumentFromFarm(farm);
indexWriter.UpdateDocument(new Term("id", farm.FarmId.ToString()), doc);
db.Entry(farm).State = EntityState.Modified;
farm.IndexDateTime = DateTime.Now;
}
