public virtual void TestTwoFiles() { CreateSequenceFile(dir, "d1", (byte)0, 15); CreateSequenceFile(dir, "d2", (byte)0, 114); CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf"); csw.AddFile("d1"); csw.AddFile("d2"); csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, "d.csf"); IndexInput expected = dir.OpenInput("d1"); IndexInput actual = csr.OpenInput("d1"); AssertSameStreams("d1", expected, actual); AssertSameSeekBehavior("d1", expected, actual); expected.Close(); actual.Close(); expected = dir.OpenInput("d2"); actual = csr.OpenInput("d2"); AssertSameStreams("d2", expected, actual); AssertSameSeekBehavior("d2", expected, actual); expected.Close(); actual.Close(); csr.Close(); }
protected internal override void DoClose() { TermVectorsReader tvReader = (TermVectorsReader)System.Threading.Thread.GetData(termVectorsLocal); if (tvReader != null) { tvReader.Close(); System.Threading.Thread.SetData(termVectorsLocal, null); } fieldsReader.Close(); tis.Close(); if (freqStream != null) { freqStream.Close(); } if (proxStream != null) { proxStream.Close(); } CloseNorms(); if (termVectorsReaderOrig != null) { termVectorsReaderOrig.Close(); } if (cfsReader != null) { cfsReader.Close(); } }
protected internal override void DoClose() { fieldsReader.Close(); tis.Close(); if (freqStream != null) { freqStream.Close(); } if (proxStream != null) { proxStream.Close(); } CloseNorms(); if (termVectorsReaderOrig != null) { termVectorsReaderOrig.Close(); } if (cfsReader != null) { cfsReader.Close(); } }
public virtual void TestFileNotFound() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null); // Open two files Assert.Throws <System.IO.IOException>(() => cr.OpenInput("bogus", null), "File not found"); cr.Close(); }
public virtual void TestClonedStreamsClosing() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // basic clone IndexInput expected = dir.OpenInput("f11"); // this test only works for FSIndexInput Assert.IsTrue(_TestHelper.IsSimpleFSIndexInput(expected)); Assert.IsTrue(_TestHelper.IsSimpleFSIndexInputOpen(expected)); IndexInput one = cr.OpenInput("f11"); Assert.IsTrue(IsCSIndexInputOpen(one)); IndexInput two = (IndexInput)one.Clone(); Assert.IsTrue(IsCSIndexInputOpen(two)); AssertSameStreams("basic clone one", expected, one); expected.Seek(0); AssertSameStreams("basic clone two", expected, two); // Now close the first stream one.Close(); Assert.IsTrue(IsCSIndexInputOpen(one), "Only close when cr is closed"); // The following should really fail since we couldn't expect to // access a file once close has been called on it (regardless of // buffering and/or clone magic) expected.Seek(0); two.Seek(0); AssertSameStreams("basic clone two/2", expected, two); // Now close the compound reader cr.Close(); Assert.IsFalse(IsCSIndexInputOpen(one), "Now closed one"); Assert.IsFalse(IsCSIndexInputOpen(two), "Now closed two"); // The following may also fail since the compound stream is closed expected.Seek(0); two.Seek(0); //assertSameStreams("basic clone two/3", expected, two); // Now close the second clone two.Close(); expected.Seek(0); two.Seek(0); //assertSameStreams("basic clone two/4", expected, two); expected.Close(); }
public virtual void TestRandomFiles() { // Setup the test segment System.String segment = "test"; int chunk = 1024; // internal buffer size used by the stream CreateRandomFile(dir, segment + ".zero", 0); CreateRandomFile(dir, segment + ".one", 1); CreateRandomFile(dir, segment + ".ten", 10); CreateRandomFile(dir, segment + ".hundred", 100); CreateRandomFile(dir, segment + ".big1", chunk); CreateRandomFile(dir, segment + ".big2", chunk - 1); CreateRandomFile(dir, segment + ".big3", chunk + 1); CreateRandomFile(dir, segment + ".big4", 3 * chunk); CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1); CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1); CreateRandomFile(dir, segment + ".big7", 1000 * chunk); // Setup extraneous files CreateRandomFile(dir, "onetwothree", 100); CreateRandomFile(dir, segment + ".notIn", 50); CreateRandomFile(dir, segment + ".notIn2", 51); // Now test CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs"); System.String[] data = new System.String[] { ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7" }; for (int i = 0; i < data.Length; i++) { csw.AddFile(segment + data[i]); } csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs"); for (int i = 0; i < data.Length; i++) { IndexInput check = dir.OpenInput(segment + data[i]); IndexInput test = csr.OpenInput(segment + data[i]); AssertSameStreams(data[i], check, test); AssertSameSeekBehavior(data[i], check, test); test.Close(); check.Close(); } csr.Close(); }
public virtual void TestReadPastEOF() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null); IndexInput is_Renamed = cr.OpenInput("f2", null); is_Renamed.Seek(is_Renamed.Length(null) - 10, null); byte[] b = new byte[100]; is_Renamed.ReadBytes(b, 0, 10, null); Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadByte(null), "Single byte read past end of file"); is_Renamed.Seek(is_Renamed.Length(null) - 10, null); Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50, null), "Block read past end of file"); is_Renamed.Close(); cr.Close(); }
public virtual void TestFileNotFound() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // Open two files try { IndexInput e1 = cr.OpenInput("bogus"); Assert.Fail("File not found"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: File Not Found: " + e); } cr.Close(); }
public virtual void TestSingleFile() { int[] data = new int[] { 0, 1, 10, 100 }; for (int i = 0; i < data.Length; i++) { System.String name = "t" + data[i]; CreateSequenceFile(dir, name, (byte)0, data[i]); CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs"); csw.AddFile(name); csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs"); IndexInput expected = dir.OpenInput(name); IndexInput actual = csr.OpenInput(name); AssertSameStreams(name, expected, actual); AssertSameSeekBehavior(name, expected, actual); expected.Close(); actual.Close(); csr.Close(); } }
public virtual void TestReadPastEOF() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); IndexInput is_Renamed = cr.OpenInput("f2"); is_Renamed.Seek(is_Renamed.Length() - 10); byte[] b = new byte[100]; is_Renamed.ReadBytes(b, 0, 10); try { byte test = is_Renamed.ReadByte(); Assert.Fail("Single byte read past end of file"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: single byte read past end of file: " + e); } is_Renamed.Seek(is_Renamed.Length() - 10); try { is_Renamed.ReadBytes(b, 0, 50); Assert.Fail("Block read past end of file"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: block read past end of file: " + e); } is_Renamed.Close(); cr.Close(); }
public virtual void TestClonedStreamsClosing() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // basic clone IndexInput expected = dir.OpenInput("f11"); // this test only works for FSIndexInput Assert.IsTrue(_TestHelper.IsSimpleFSIndexInput(expected)); Assert.IsTrue(_TestHelper.IsSimpleFSIndexInputOpen(expected)); IndexInput one = cr.OpenInput("f11"); Assert.IsTrue(IsCSIndexInputOpen(one)); IndexInput two = (IndexInput) one.Clone(); Assert.IsTrue(IsCSIndexInputOpen(two)); AssertSameStreams("basic clone one", expected, one); expected.Seek(0); AssertSameStreams("basic clone two", expected, two); // Now close the first stream one.Close(); Assert.IsTrue(IsCSIndexInputOpen(one), "Only close when cr is closed"); // The following should really fail since we couldn't expect to // access a file once close has been called on it (regardless of // buffering and/or clone magic) expected.Seek(0); two.Seek(0); AssertSameStreams("basic clone two/2", expected, two); // Now close the compound reader cr.Close(); Assert.IsFalse(IsCSIndexInputOpen(one), "Now closed one"); Assert.IsFalse(IsCSIndexInputOpen(two), "Now closed two"); // The following may also fail since the compound stream is closed expected.Seek(0); two.Seek(0); //assertSameStreams("basic clone two/3", expected, two); // Now close the second clone two.Close(); expected.Seek(0); two.Seek(0); //assertSameStreams("basic clone two/4", expected, two); expected.Close(); }
public virtual void TestRandomFiles() { // Setup the test segment System.String segment = "test"; int chunk = 1024; // internal buffer size used by the stream CreateRandomFile(dir, segment + ".zero", 0); CreateRandomFile(dir, segment + ".one", 1); CreateRandomFile(dir, segment + ".ten", 10); CreateRandomFile(dir, segment + ".hundred", 100); CreateRandomFile(dir, segment + ".big1", chunk); CreateRandomFile(dir, segment + ".big2", chunk - 1); CreateRandomFile(dir, segment + ".big3", chunk + 1); CreateRandomFile(dir, segment + ".big4", 3 * chunk); CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1); CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1); CreateRandomFile(dir, segment + ".big7", 1000 * chunk); // Setup extraneous files CreateRandomFile(dir, "onetwothree", 100); CreateRandomFile(dir, segment + ".notIn", 50); CreateRandomFile(dir, segment + ".notIn2", 51); // Now test CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs"); System.String[] data = new System.String[]{".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7"}; for (int i = 0; i < data.Length; i++) { csw.AddFile(segment + data[i]); } csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs"); for (int i = 0; i < data.Length; i++) { IndexInput check = dir.OpenInput(segment + data[i]); IndexInput test = csr.OpenInput(segment + data[i]); AssertSameStreams(data[i], check, test); AssertSameSeekBehavior(data[i], check, test); test.Close(); check.Close(); } csr.Close(); }
public virtual void TestTwoFiles() { CreateSequenceFile(dir, "d1", (byte) 0, 15); CreateSequenceFile(dir, "d2", (byte) 0, 114); CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf"); csw.AddFile("d1"); csw.AddFile("d2"); csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, "d.csf"); IndexInput expected = dir.OpenInput("d1"); IndexInput actual = csr.OpenInput("d1"); AssertSameStreams("d1", expected, actual); AssertSameSeekBehavior("d1", expected, actual); expected.Close(); actual.Close(); expected = dir.OpenInput("d2"); actual = csr.OpenInput("d2"); AssertSameStreams("d2", expected, actual); AssertSameSeekBehavior("d2", expected, actual); expected.Close(); actual.Close(); csr.Close(); }
public virtual void TestSingleFile() { int[] data = new int[]{0, 1, 10, 100}; for (int i = 0; i < data.Length; i++) { System.String name = "t" + data[i]; CreateSequenceFile(dir, name, (byte) 0, data[i]); CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs"); csw.AddFile(name); csw.Close(); CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs"); IndexInput expected = dir.OpenInput(name); IndexInput actual = csr.OpenInput(name); AssertSameStreams(name, expected, actual); AssertSameSeekBehavior(name, expected, actual); expected.Close(); actual.Close(); csr.Close(); } }
public static void Main(System.String[] args) { System.String filename = null; bool extract = false; for (int i = 0; i < args.Length; ++i) { if (args[i].Equals("-extract")) { extract = true; } else if (filename == null) { filename = args[i]; } } if (filename == null) { System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>"); return; } Directory dir = null; CompoundFileReader cfr = null; try { System.IO.FileInfo file = new System.IO.FileInfo(filename); System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName; filename = file.Name; dir = FSDirectory.GetDirectory(dirname, false); cfr = new CompoundFileReader(dir, filename); System.String[] files = cfr.List(); System.Array.Sort(files); // sort the array of filename so that the output is more readable for (int i = 0; i < files.Length; ++i) { long len = cfr.FileLength(files[i]); if (extract) { System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory..."); IndexInput ii = cfr.OpenInput(files[i]); System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create); // read and write with a small buffer, which is more effectiv than reading byte by byte byte[] buffer = new byte[1024]; int chunk = buffer.Length; while (len > 0) { int bufLen = (int)System.Math.Min(chunk, len); ii.ReadBytes(buffer, 0, bufLen); byte[] byteArray = new byte[buffer.Length]; for (int index = 0; index < buffer.Length; index++) { byteArray[index] = (byte)buffer[index]; } f.Write(byteArray, 0, bufLen); len -= bufLen; } f.Close(); ii.Close(); } else { System.Console.Out.WriteLine(files[i] + ": " + len + " bytes"); } } } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } finally { try { if (dir != null) { dir.Close(); } if (cfr != null) { cfr.Close(); } } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } } }
public virtual void TestReadPastEOF() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); IndexInput is_Renamed = cr.OpenInput("f2"); is_Renamed.Seek(is_Renamed.Length() - 10); byte[] b = new byte[100]; is_Renamed.ReadBytes(b, 0, 10); Assert.Throws<System.IO.IOException>(() => is_Renamed.ReadByte(), "Single byte read past end of file"); is_Renamed.Seek(is_Renamed.Length() - 10); Assert.Throws<System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50), "Block read past end of file"); is_Renamed.Close(); cr.Close(); }
public virtual void TestReadPastEOF() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); IndexInput is_Renamed = cr.OpenInput("f2"); is_Renamed.Seek(is_Renamed.Length() - 10); byte[] b = new byte[100]; is_Renamed.ReadBytes(b, 0, 10); try { byte test = is_Renamed.ReadByte(); Assert.Fail("Single byte read past end of file"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: single byte read past end of file: " + e); } is_Renamed.Seek(is_Renamed.Length() - 10); try { is_Renamed.ReadBytes(b, 0, 50); Assert.Fail("Block read past end of file"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: block read past end of file: " + e); } is_Renamed.Close(); cr.Close(); }
public virtual void TestFileNotFound() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // Open two files try { IndexInput e1 = cr.OpenInput("bogus"); Assert.Fail("File not found"); } catch (System.IO.IOException e) { /* success */ //System.out.println("SUCCESS: File Not Found: " + e); } cr.Close(); }
public virtual void TestExactFileNames() { for (int pass = 0; pass < 2; pass++) { System.String outputDir = "lucene.backwardscompat0.index"; RmDir(outputDir); try { Directory dir = FSDirectory.Open(new System.IO.FileInfo(FullDir(outputDir))); bool autoCommit = 0 == pass; IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); writer.SetRAMBufferSizeMB(16.0); for (int i = 0; i < 35; i++) { AddDoc(writer, i); } Assert.AreEqual(35, writer.DocCount(), "wrong doc count"); writer.Close(); // Delete one doc so we get a .del file: IndexReader reader = IndexReader.Open(dir); Term searchTerm = new Term("id", "7"); int delCount = reader.DeleteDocuments(searchTerm); Assert.AreEqual(1, delCount, "didn't delete the right number of documents"); // Set one norm so we get a .s0 file: reader.SetNorm(21, "content", (float) 1.5); reader.Close(); // The numbering of fields can vary depending on which // JRE is in use. On some JREs we see content bound to // field 0; on others, field 1. So, here we have to // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm"); int contentFieldIndex = - 1; for (int i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.name_ForNUnit.Equals("content")) { contentFieldIndex = i; break; } } cfsReader.Close(); Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment"); // Now verify file names: System.String[] expected; expected = new System.String[]{"_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_3", "segments.gen"}; System.String[] actual = dir.ListAll(); System.Array.Sort(expected); System.Array.Sort(actual); if (!SupportClass.CollectionsHelper.Equals(expected, actual)) { Assert.Fail("incorrect filenames in index: expected:\n " + AsString(expected) + "\n actual:\n " + AsString(actual)); } dir.Close(); } finally { RmDir(outputDir); } } }
public virtual void TestExactFileNames() { System.String outputDir = "lucene.backwardscompat0.index"; RmDir(outputDir); try { Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(FullDir(outputDir))); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetRAMBufferSizeMB(16.0); for (int i = 0; i < 35; i++) { AddDoc(writer, i); } Assert.AreEqual(35, writer.MaxDoc(), "wrong doc count"); writer.Close(); // Delete one doc so we get a .del file: IndexReader reader = IndexReader.Open(dir, false); Term searchTerm = new Term("id", "7"); int delCount = reader.DeleteDocuments(searchTerm); Assert.AreEqual(1, delCount, "didn't delete the right number of documents"); // Set one norm so we get a .s0 file: reader.SetNorm(21, "content", (float)1.5); reader.Close(); // The numbering of fields can vary depending on which // JRE is in use. On some JREs we see content bound to // field 0; on others, field 1. So, here we have to // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm"); int contentFieldIndex = -1; for (int i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.name_ForNUnit.Equals("content")) { contentFieldIndex = i; break; } } cfsReader.Close(); Assert.IsTrue(contentFieldIndex != -1, "could not locate the 'content' field number in the _2.cfs segment"); // Now verify file names: System.String[] expected; expected = new System.String[] { "_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_3", "segments.gen" }; System.String[] actual = dir.ListAll(); System.Array.Sort(expected); System.Array.Sort(actual); if (!CollectionsHelper.Equals(expected, actual)) { Assert.Fail("incorrect filenames in index: expected:\n " + AsString(expected) + "\n actual:\n " + AsString(actual)); } dir.Close(); } finally { RmDir(outputDir); } }
public virtual void TestDeleteLeftoverFiles() { Directory dir = new RAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(10); int i; for (i = 0; i < 35; i++) { AddDoc(writer, i); } writer.SetUseCompoundFile(false); for (; i < 45; i++) { AddDoc(writer, i); } writer.Close(); // Delete one doc so we get a .del file: IndexReader reader = IndexReader.Open(dir); Term searchTerm = new Term("id", "7"); int delCount = reader.DeleteDocuments(searchTerm); Assert.AreEqual(1, delCount, "didn't delete the right number of documents"); // Set one norm so we get a .s0 file: reader.SetNorm(21, "content", (float)1.5); reader.Close(); // Now, artificially create an extra .del file & extra // .s0 file: System.String[] files = dir.ListAll(); /* * for(int j=0;j<files.length;j++) { * System.out.println(j + ": " + files[j]); * } */ // The numbering of fields can vary depending on which // JRE is in use. On some JREs we see content bound to // field 0; on others, field 1. So, here we have to // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm"); int contentFieldIndex = -1; for (i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.name_ForNUnit.Equals("content")) { contentFieldIndex = i; break; } } cfsReader.Close(); Assert.IsTrue(contentFieldIndex != -1, "could not locate the 'content' field number in the _2.cfs segment"); System.String normSuffix = "s" + contentFieldIndex; // Create a bogus separate norms file for a // segment/field that actually has a separate norms file // already: CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix); // Create a bogus separate norms file for a // segment/field that actually has a separate norms file // already, using the "not compound file" extension: CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex); // Create a bogus separate norms file for a // segment/field that does not have a separate norms // file already: CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix); // Create a bogus separate norms file for a // segment/field that does not have a separate norms // file already using the "not compound file" extension: CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex); // Create a bogus separate del file for a // segment that already has a separate del file: CopyFile(dir, "_0_1.del", "_0_2.del"); // Create a bogus separate del file for a // segment that does not yet have a separate del file: CopyFile(dir, "_0_1.del", "_1_1.del"); // Create a bogus separate del file for a // non-existent segment: CopyFile(dir, "_0_1.del", "_188_1.del"); // Create a bogus segment file: CopyFile(dir, "_0.cfs", "_188.cfs"); // Create a bogus fnm file when the CFS already exists: CopyFile(dir, "_0.cfs", "_0.fnm"); // Create a deletable file: CopyFile(dir, "_0.cfs", "deletable"); // Create some old segments file: CopyFile(dir, "segments_3", "segments"); CopyFile(dir, "segments_3", "segments_2"); // Create a bogus cfs file shadowing a non-cfs segment: CopyFile(dir, "_2.cfs", "_3.cfs"); System.String[] filesPre = dir.ListAll(); // Open & close a writer: it should delete the above 4 // files and nothing more: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED); writer.Close(); System.String[] files2 = dir.ListAll(); dir.Close(); System.Array.Sort(files); System.Array.Sort(files2); System.Collections.Hashtable dif = DifFiles(files, files2); if (!SupportClass.CollectionsHelper.Equals(files, files2)) { Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2) + "\ndif: " + SupportClass.CollectionsHelper.CollectionToString(dif)); } }
protected internal override void DoClose() { bool hasReferencedReader = (referencedSegmentReader != null); termVectorsLocal.Close(); //termVectorsLocal.Clear(); if (hasReferencedReader) { referencedSegmentReader.DecRefReaderNotNorms(); referencedSegmentReader = null; } deletedDocs = null; // close the single norms stream if (singleNormStream != null) { // we can close this stream, even if the norms // are shared, because every reader has it's own // singleNormStream singleNormStream.Close(); singleNormStream = null; } // re-opened SegmentReaders have their own instance of FieldsReader if (fieldsReader != null) { fieldsReader.Close(); } if (!hasReferencedReader) { // close everything, nothing is shared anymore with other readers if (tis != null) { tis.Close(); } if (freqStream != null) { freqStream.Close(); } if (proxStream != null) { proxStream.Close(); } if (termVectorsReaderOrig != null) { termVectorsReaderOrig.Close(); } if (cfsReader != null) { cfsReader.Close(); } if (storeCFSReader != null) { storeCFSReader.Close(); } // maybe close directory base.DoClose(); } }
public virtual void TestRandomAccessClones() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // Open two files IndexInput e1 = cr.OpenInput("f11"); IndexInput e2 = cr.OpenInput("f3"); IndexInput a1 = (IndexInput) e1.Clone(); IndexInput a2 = (IndexInput) e2.Clone(); // Seek the first pair e1.Seek(100); a1.Seek(100); Assert.AreEqual(100, e1.FilePointer); Assert.AreEqual(100, a1.FilePointer); byte be1 = e1.ReadByte(); byte ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now seek the second pair e2.Seek(1027); a2.Seek(1027); Assert.AreEqual(1027, e2.FilePointer); Assert.AreEqual(1027, a2.FilePointer); byte be2 = e2.ReadByte(); byte ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Now make sure the first one didn't move Assert.AreEqual(101, e1.FilePointer); Assert.AreEqual(101, a1.FilePointer); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now more the first one again, past the buffer length e1.Seek(1910); a1.Seek(1910); Assert.AreEqual(1910, e1.FilePointer); Assert.AreEqual(1910, a1.FilePointer); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now make sure the second set didn't move Assert.AreEqual(1028, e2.FilePointer); Assert.AreEqual(1028, a2.FilePointer); be2 = e2.ReadByte(); ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Move the second set back, again cross the buffer size e2.Seek(17); a2.Seek(17); Assert.AreEqual(17, e2.FilePointer); Assert.AreEqual(17, a2.FilePointer); be2 = e2.ReadByte(); ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Finally, make sure the first set didn't move // Now make sure the first one didn't move Assert.AreEqual(1911, e1.FilePointer); Assert.AreEqual(1911, a1.FilePointer); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); e1.Close(); e2.Close(); a1.Close(); a2.Close(); cr.Close(); }
public virtual void TestFileNotFound() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // Open two files Assert.Throws<System.IO.IOException>(() => cr.OpenInput("bogus"), "File not found"); cr.Close(); }
public virtual void TestRandomAccessClones() { SetUp_2(); CompoundFileReader cr = new CompoundFileReader(dir, "f.comp"); // Open two files IndexInput e1 = cr.OpenInput("f11"); IndexInput e2 = cr.OpenInput("f3"); IndexInput a1 = (IndexInput)e1.Clone(); IndexInput a2 = (IndexInput)e2.Clone(); // Seek the first pair e1.Seek(100); a1.Seek(100); Assert.AreEqual(100, e1.GetFilePointer()); Assert.AreEqual(100, a1.GetFilePointer()); byte be1 = e1.ReadByte(); byte ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now seek the second pair e2.Seek(1027); a2.Seek(1027); Assert.AreEqual(1027, e2.GetFilePointer()); Assert.AreEqual(1027, a2.GetFilePointer()); byte be2 = e2.ReadByte(); byte ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Now make sure the first one didn't move Assert.AreEqual(101, e1.GetFilePointer()); Assert.AreEqual(101, a1.GetFilePointer()); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now more the first one again, past the buffer length e1.Seek(1910); a1.Seek(1910); Assert.AreEqual(1910, e1.GetFilePointer()); Assert.AreEqual(1910, a1.GetFilePointer()); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); // Now make sure the second set didn't move Assert.AreEqual(1028, e2.GetFilePointer()); Assert.AreEqual(1028, a2.GetFilePointer()); be2 = e2.ReadByte(); ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Move the second set back, again cross the buffer size e2.Seek(17); a2.Seek(17); Assert.AreEqual(17, e2.GetFilePointer()); Assert.AreEqual(17, a2.GetFilePointer()); be2 = e2.ReadByte(); ba2 = a2.ReadByte(); Assert.AreEqual(be2, ba2); // Finally, make sure the first set didn't move // Now make sure the first one didn't move Assert.AreEqual(1911, e1.GetFilePointer()); Assert.AreEqual(1911, a1.GetFilePointer()); be1 = e1.ReadByte(); ba1 = a1.ReadByte(); Assert.AreEqual(be1, ba1); e1.Close(); e2.Close(); a1.Close(); a2.Close(); cr.Close(); }
public static void Main(System.String[] args) { System.String filename = null; bool extract = false; for (int i = 0; i < args.Length; ++i) { if (args[i].Equals("-extract")) { extract = true; } else if (filename == null) { filename = args[i]; } } if (filename == null) { System.Console.Out.WriteLine("Usage: Lucene.Net.Index.IndexReader [-extract] <cfsfile>"); return ; } Directory dir = null; CompoundFileReader cfr = null; try { System.IO.FileInfo file = new System.IO.FileInfo(filename); System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName; filename = file.Name; dir = FSDirectory.Open(new System.IO.FileInfo(dirname)); cfr = new CompoundFileReader(dir, filename); System.String[] files = cfr.List(); System.Array.Sort(files); // sort the array of filename so that the output is more readable for (int i = 0; i < files.Length; ++i) { long len = cfr.FileLength(files[i]); if (extract) { System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory..."); IndexInput ii = cfr.OpenInput(files[i]); System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create); // read and write with a small buffer, which is more effectiv than reading byte by byte byte[] buffer = new byte[1024]; int chunk = buffer.Length; while (len > 0) { int bufLen = (int) System.Math.Min(chunk, len); ii.ReadBytes(buffer, 0, bufLen); f.Write(buffer, 0, bufLen); len -= bufLen; } f.Close(); ii.Close(); } else System.Console.Out.WriteLine(files[i] + ": " + len + " bytes"); } } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } finally { try { if (dir != null) dir.Close(); if (cfr != null) cfr.Close(); } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } } }
public virtual void TestDeleteLeftoverFiles() { Directory dir = new RAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); writer.SetMaxBufferedDocs(10); int i; for (i = 0; i < 35; i++) { AddDoc(writer, i); } writer.SetUseCompoundFile(false); for (; i < 45; i++) { AddDoc(writer, i); } writer.Close(); // Delete one doc so we get a .del file: IndexReader reader = IndexReader.Open(dir); Term searchTerm = new Term("id", "7"); int delCount = reader.DeleteDocuments(searchTerm); Assert.AreEqual(1, delCount, "didn't delete the right number of documents"); // Set one norm so we get a .s0 file: reader.SetNorm(21, "content", (float) 1.5); reader.Close(); // Now, artificially create an extra .del file & extra // .s0 file: System.String[] files = dir.List(); /* for(int i=0;i<files.length;i++) { System.out.println(i + ": " + files[i]); } */ // The numbering of fields can vary depending on which // JRE is in use. On some JREs we see content bound to // field 0; on others, field 1. So, here we have to // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm"); int contentFieldIndex = - 1; for (i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.Name_ForNUnitTest.Equals("content")) { contentFieldIndex = i; break; } } cfsReader.Close(); Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment"); System.String normSuffix = "s" + contentFieldIndex; // Create a bogus separate norms file for a // segment/field that actually has a separate norms file // already: CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix); // Create a bogus separate norms file for a // segment/field that actually has a separate norms file // already, using the "not compound file" extension: CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex); // Create a bogus separate norms file for a // segment/field that does not have a separate norms // file already: CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix); // Create a bogus separate norms file for a // segment/field that does not have a separate norms // file already using the "not compound file" extension: CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex); // Create a bogus separate del file for a // segment that already has a separate del file: CopyFile(dir, "_0_1.del", "_0_2.del"); // Create a bogus separate del file for a // segment that does not yet have a separate del file: CopyFile(dir, "_0_1.del", "_1_1.del"); // Create a bogus separate del file for a // non-existent segment: CopyFile(dir, "_0_1.del", "_188_1.del"); // Create a bogus segment file: CopyFile(dir, "_0.cfs", "_188.cfs"); // Create a bogus fnm file when the CFS already exists: CopyFile(dir, "_0.cfs", "_0.fnm"); // Create a deletable file: CopyFile(dir, "_0.cfs", "deletable"); // Create some old segments file: CopyFile(dir, "segments_a", "segments"); CopyFile(dir, "segments_a", "segments_2"); // Create a bogus cfs file shadowing a non-cfs segment: CopyFile(dir, "_2.cfs", "_3.cfs"); System.String[] filesPre = dir.List(); // Open & close a writer: it should delete the above 4 // files and nothing more: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false); writer.Close(); System.String[] files2 = dir.List(); dir.Close(); System.Array.Sort(files); System.Array.Sort(files2); if (!ArrayEquals(files, files2)) { Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2)); } }