public virtual void TestErrorInDocsWriterAdd() { MockRAMDirectory.Failure failure = new AnonymousClassFailure1(this); // create a couple of files System.String[] keywords = new System.String[]{"1", "2"}; System.String[] unindexed = new System.String[]{"Netherlands", "Italy"}; System.String[] unstored = new System.String[]{"Amsterdam has lots of bridges", "Venice has lots of canals"}; System.String[] text = new System.String[]{"Amsterdam", "Venice"}; for (int pass = 0; pass < 2; pass++) { bool autoCommit = (0 == pass); MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); dir.FailOn(failure.Reset()); for (int i = 0; i < keywords.Length; i++) { Document doc = new Document(); doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED)); doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO)); doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED)); doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED)); try { modifier.AddDocument(doc); } catch (System.IO.IOException) { break; } } System.String[] startFiles = dir.List(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null); System.String[] endFiles = dir.List(); if (!SupportClass.Compare.CompareStringArrays(startFiles, endFiles)) { Assert.Fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles)); } modifier.Close(); } }
public virtual void TestOptimizeTempSpaceUsage() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); for (int j = 0; j < 500; j++) { AddDocWithIndex(writer, j); } writer.Close(); long startDiskUsage = 0; System.String[] files = dir.List(); for (int i = 0; i < files.Length; i++) { startDiskUsage += dir.FileLength(files[i]); } dir.ResetMaxUsedSizeInBytes(); writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false); writer.Optimize(); writer.Close(); long maxDiskUsage = dir.GetMaxUsedSizeInBytes(); Assert.IsTrue( maxDiskUsage <= 2 * startDiskUsage, "optimized used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (2 * startDiskUsage) + " (= 2X starting usage)" ); dir.Close(); }
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier /// remains consistent and usable. Similar to TestIndexReader.testDiskFull(). /// </summary> private void TestOperationsOnDiskFull(bool updates) { bool debug = false; Term searchTerm = new Term("content", "aaa"); int START_COUNT = 157; int END_COUNT = 144; for (int pass = 0; pass < 2; pass++) { bool autoCommit = (0 == pass); // First build up a starting index: RAMDirectory startDir = new RAMDirectory(); IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true); for (int i = 0; i < 157; i++) { Document d = new Document(); d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.TOKENIZED)); writer.AddDocument(d); } writer.Close(); long diskUsage = startDir.SizeInBytes(); long diskFree = diskUsage + 10; System.IO.IOException err = null; bool done = false; // Iterate w/ ever increasing free disk space: while (!done) { MockRAMDirectory dir = new MockRAMDirectory(startDir); IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer()); modifier.SetMaxBufferedDocs(1000); // use flush or close modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close // For each disk size, first try to commit against // dir that will hit random IOExceptions & disk // full; after, give it infinite disk space & turn // off random IOExceptions & retry w/ same reader: bool success = false; for (int x = 0; x < 2; x++) { double rate = 0.1; //UPGRADE_WARNING: Data types in Visual C# might be different. Verify the accuracy of narrowing conversions. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1042'" double diskRatio = ((double) diskFree) / diskUsage; long thisDiskFree; System.String testName; if (0 == x) { thisDiskFree = diskFree; if (diskRatio >= 2.0) { rate /= 2; } if (diskRatio >= 4.0) { rate /= 2; } if (diskRatio >= 6.0) { rate = 0.0; } if (debug) { System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes"); } testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; } else { thisDiskFree = 0; rate = 0.0; if (debug) { System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space"); } testName = "reader re-use after disk full"; } dir.SetMaxSizeInBytes(thisDiskFree); dir.SetRandomIOExceptionRate(rate, diskFree); try { if (0 == x) { int docId = 12; for (int i = 0; i < 13; i++) { if (updates) { Document d = new Document(); d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.TOKENIZED)); modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d); } else { // deletes modifier.DeleteDocuments(new Term("id", System.Convert.ToString(docId))); // modifier.setNorm(docId, "contents", (float)2.0); } docId += 12; } } modifier.Close(); success = true; if (0 == x) { done = true; } } catch (System.IO.IOException e) { if (debug) { //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'" System.Console.Out.WriteLine(" hit IOException: " + e); System.Console.Out.WriteLine(e.StackTrace); } err = e; if (1 == x) { System.Console.Error.WriteLine(e.StackTrace); Assert.Fail(testName + " hit IOException after disk space was freed up"); } } // Whether we succeeded or failed, check that all // un-referenced files were in fact deleted (ie, // we did not create garbage). Just create a // new IndexFileDeleter, have it delete // unreferenced files, then verify that in fact // no files were deleted: System.String[] startFiles = dir.List(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null); System.String[] endFiles = dir.List(); //UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'" System.Array.Sort(startFiles); //UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'" System.Array.Sort(endFiles); // for(int i=0;i<startFiles.length;i++) { // System.out.println(" startFiles: " + i + ": " + startFiles[i]); // } if (!SupportClass.Compare.CompareStringArrays(startFiles, endFiles)) { System.String successStr; if (success) { successStr = "success"; } else { successStr = "IOException"; System.Console.Error.WriteLine(err.StackTrace); } Assert.Fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles)); } // Finally, verify index is not corrupt, and, if // we succeeded, we see all docs changed, and if // we failed, we see either all docs or no docs // changed (transactional semantics): IndexReader newReader = null; try { newReader = IndexReader.Open(dir); } catch (System.IO.IOException e) { System.Console.Error.WriteLine(e.StackTrace); //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'" Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e); } IndexSearcher searcher = new IndexSearcher(newReader); Hits hits = null; try { hits = searcher.Search(new TermQuery(searchTerm)); } catch (System.IO.IOException e) { System.Console.Error.WriteLine(e.StackTrace); //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'" Assert.Fail(testName + ": exception when searching: " + e); } int result2 = hits.Length(); if (success) { if (x == 0 && result2 != END_COUNT) { Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); } else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT) { // It's possible that the first exception was // "recoverable" wrt pending deletes, in which // case the pending deletes are retained and // then re-flushing (with plenty of disk // space) will succeed in flushing the // deletes: Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT); } } else { // On hitting exception we still may have added // all docs: if (result2 != START_COUNT && result2 != END_COUNT) { System.Console.Error.WriteLine(err.StackTrace); Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT); } } searcher.Close(); newReader.Close(); if (result2 == END_COUNT) { break; } } dir.Close(); // Try again with 10 more bytes of free space: diskFree += 10; } } }