public virtual void TestTwoFiles()
{
CreateSequenceFile(dir, "d1", (byte)0, 15);
CreateSequenceFile(dir, "d2", (byte)0, 114);
CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
csw.AddFile("d1");
csw.AddFile("d2");
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
IndexInput expected = dir.OpenInput("d1");
IndexInput actual = csr.OpenInput("d1");
AssertSameStreams("d1", expected, actual);
AssertSameSeekBehavior("d1", expected, actual);
expected.Close();
actual.Close();
expected = dir.OpenInput("d2");
actual = csr.OpenInput("d2");
AssertSameStreams("d2", expected, actual);
AssertSameSeekBehavior("d2", expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
/// <summary>Setup a larger compound file with a number of components, each of
/// which is a sequential file (so that we can easily tell that we are
/// reading in the right byte). The methods sets up 20 files - f0 to f19,
/// the size of each file is 1000 bytes.
/// </summary>
private void SetUp_2()
{
CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
for (int i = 0; i < 20; i++)
{
CreateSequenceFile(dir, "f" + i, (byte)0, 2000);
cw.AddFile("f" + i);
}
cw.Close();
}
public /*internal*/ System.Collections.IList CreateCompoundFile(System.String fileName)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
System.Collections.IList files = new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + 1);
// Basic files
for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
{
System.String ext = IndexFileNames.COMPOUND_EXTENSIONS[i];
if (ext.Equals(IndexFileNames.PROX_EXTENSION) && !HasProx())
{
continue;
}
if (mergeDocStores || (!ext.Equals(IndexFileNames.FIELDS_EXTENSION) && !ext.Equals(IndexFileNames.FIELDS_INDEX_EXTENSION)))
{
files.Add(segment + "." + ext);
}
}
// Fieldable norm files
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
files.Add(segment + "." + IndexFileNames.NORMS_EXTENSION);
break;
}
}
// Vector files
if (fieldInfos.HasVectors() && mergeDocStores)
{
for (int i = 0; i < IndexFileNames.VECTOR_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]);
}
}
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String)it.Current);
}
// Perform the merge
cfsWriter.Close();
return(files);
}
public /*internal*/ ICollection <string> CreateCompoundFile(System.String fileName)
{
ICollection <string> files = GetMergedFiles();
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
// Now merge all added files
foreach (var file in files)
{
cfsWriter.AddFile(file);
}
// Perform the merge
cfsWriter.Close();
return(files);
}
public /*internal*/ System.Collections.Generic.ICollection <string> CreateCompoundFile(System.String fileName)
{
System.Collections.Generic.ICollection <string> files = GetMergedFiles();
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String)it.Current);
}
// Perform the merge
cfsWriter.Close();
return(files);
}
public virtual void TestRandomFiles()
{
// Setup the test segment
System.String segment = "test";
int chunk = 1024; // internal buffer size used by the stream
CreateRandomFile(dir, segment + ".zero", 0);
CreateRandomFile(dir, segment + ".one", 1);
CreateRandomFile(dir, segment + ".ten", 10);
CreateRandomFile(dir, segment + ".hundred", 100);
CreateRandomFile(dir, segment + ".big1", chunk);
CreateRandomFile(dir, segment + ".big2", chunk - 1);
CreateRandomFile(dir, segment + ".big3", chunk + 1);
CreateRandomFile(dir, segment + ".big4", 3 * chunk);
CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1);
CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1);
CreateRandomFile(dir, segment + ".big7", 1000 * chunk);
// Setup extraneous files
CreateRandomFile(dir, "onetwothree", 100);
CreateRandomFile(dir, segment + ".notIn", 50);
CreateRandomFile(dir, segment + ".notIn2", 51);
// Now test
CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
System.String[] data = new System.String[] { ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7" };
for (int i = 0; i < data.Length; i++)
{
csw.AddFile(segment + data[i]);
}
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i = 0; i < data.Length; i++)
{
IndexInput check = dir.OpenInput(segment + data[i]);
IndexInput test = csr.OpenInput(segment + data[i]);
AssertSameStreams(data[i], check, test);
AssertSameSeekBehavior(data[i], check, test);
test.Close();
check.Close();
}
csr.Close();
}
public System.Collections.ArrayList CreateCompoundFile(System.String fileName)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName);
System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + 1));
// Basic files
for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.COMPOUND_EXTENSIONS[i]);
}
// Fieldable norm files
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
files.Add(segment + "." + IndexFileNames.NORMS_EXTENSION);
break;
}
}
// Vector files
if (fieldInfos.HasVectors())
{
for (int i = 0; i < IndexFileNames.VECTOR_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]);
}
}
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String)it.Current);
}
// Perform the merge
cfsWriter.Close();
return(files);
}
public virtual void TestSingleFile()
{
int[] data = new int[] { 0, 1, 10, 100 };
for (int i = 0; i < data.Length; i++)
{
System.String name = "t" + data[i];
CreateSequenceFile(dir, name, (byte)0, data[i]);
CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
csw.AddFile(name);
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
IndexInput expected = dir.OpenInput(name);
IndexInput actual = csr.OpenInput(name);
AssertSameStreams(name, expected, actual);
AssertSameSeekBehavior(name, expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
}
public virtual void TestTwoFiles()
{
CreateSequenceFile(dir, "d1", (byte) 0, 15);
CreateSequenceFile(dir, "d2", (byte) 0, 114);
CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
csw.AddFile("d1");
csw.AddFile("d2");
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
IndexInput expected = dir.OpenInput("d1");
IndexInput actual = csr.OpenInput("d1");
AssertSameStreams("d1", expected, actual);
AssertSameSeekBehavior("d1", expected, actual);
expected.Close();
actual.Close();
expected = dir.OpenInput("d2");
actual = csr.OpenInput("d2");
AssertSameStreams("d2", expected, actual);
AssertSameSeekBehavior("d2", expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
public virtual void TestSingleFile()
{
int[] data = new int[]{0, 1, 10, 100};
for (int i = 0; i < data.Length; i++)
{
System.String name = "t" + data[i];
CreateSequenceFile(dir, name, (byte) 0, data[i]);
CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
csw.AddFile(name);
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
IndexInput expected = dir.OpenInput(name);
IndexInput actual = csr.OpenInput(name);
AssertSameStreams(name, expected, actual);
AssertSameSeekBehavior(name, expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
}
public /*internal*/ System.Collections.IList CreateCompoundFile(System.String fileName)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
System.Collections.IList files = new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + 1);
// Basic files
for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
{
System.String ext = IndexFileNames.COMPOUND_EXTENSIONS[i];
if (ext.Equals(IndexFileNames.PROX_EXTENSION) && !HasProx())
continue;
if (mergeDocStores || (!ext.Equals(IndexFileNames.FIELDS_EXTENSION) && !ext.Equals(IndexFileNames.FIELDS_INDEX_EXTENSION)))
files.Add(segment + "." + ext);
}
// Fieldable norm files
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
files.Add(segment + "." + IndexFileNames.NORMS_EXTENSION);
break;
}
}
// Vector files
if (fieldInfos.HasVectors() && mergeDocStores)
{
for (int i = 0; i < IndexFileNames.VECTOR_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]);
}
}
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String) it.Current);
}
// Perform the merge
cfsWriter.Close();
return files;
}
public /*internal*/ ICollection<string> CreateCompoundFile(System.String fileName)
{
ICollection<string> files = GetMergedFiles();
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
// Now merge all added files
foreach(var file in files)
{
cfsWriter.AddFile(file);
}
// Perform the merge
cfsWriter.Close();
return files;
}
/// <summary>Build compound file for the segment we just flushed </summary>
internal void CreateCompoundFile(System.String segment)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
int size = newFiles.Count;
for (int i = 0; i < size; i++)
cfsWriter.AddFile((System.String) newFiles[i]);
// Perform the merge
cfsWriter.Close();
}
/// <summary>Tells the docWriter to close its currently open shared
/// doc stores (stored fields & vectors files).
/// Return value specifices whether new doc store files are compound or not.
/// </summary>
private bool FlushDocStores()
{
lock (this)
{
System.Collections.IList files = docWriter.Files();
bool useCompoundDocStore = false;
if (files.Count > 0)
{
System.String docStoreSegment;
bool success = false;
try
{
docStoreSegment = docWriter.CloseDocStore();
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
Message("hit exception closing doc store segment");
docWriter.Abort(null);
}
}
useCompoundDocStore = mergePolicy.UseCompoundDocStore(segmentInfos);
if (useCompoundDocStore && docStoreSegment != null)
{
// Now build compound doc store file
success = false;
int numSegments = segmentInfos.Count;
System.String compoundFileName = docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
try
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
int size = files.Count;
for (int i = 0; i < size; i++)
cfsWriter.AddFile((System.String) files[i]);
// Perform the merge
cfsWriter.Close();
for (int i = 0; i < numSegments; i++)
{
SegmentInfo si = segmentInfos.Info(i);
if (si.GetDocStoreOffset() != - 1 && si.GetDocStoreSegment().Equals(docStoreSegment))
si.SetDocStoreIsCompoundFile(true);
}
Checkpoint();
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
Message("hit exception building compound file doc store for segment " + docStoreSegment);
// Rollback to no compound file
for (int i = 0; i < numSegments; i++)
{
SegmentInfo si = segmentInfos.Info(i);
if (si.GetDocStoreOffset() != - 1 && si.GetDocStoreSegment().Equals(docStoreSegment))
si.SetDocStoreIsCompoundFile(false);
}
deleter.DeleteFile(compoundFileName);
DeletePartialSegmentsFile();
}
}
deleter.Checkpoint(segmentInfos, false);
}
}
return useCompoundDocStore;
}
}
public virtual void TestRandomFiles()
{
// Setup the test segment
System.String segment = "test";
int chunk = 1024; // internal buffer size used by the stream
CreateRandomFile(dir, segment + ".zero", 0);
CreateRandomFile(dir, segment + ".one", 1);
CreateRandomFile(dir, segment + ".ten", 10);
CreateRandomFile(dir, segment + ".hundred", 100);
CreateRandomFile(dir, segment + ".big1", chunk);
CreateRandomFile(dir, segment + ".big2", chunk - 1);
CreateRandomFile(dir, segment + ".big3", chunk + 1);
CreateRandomFile(dir, segment + ".big4", 3 * chunk);
CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1);
CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1);
CreateRandomFile(dir, segment + ".big7", 1000 * chunk);
// Setup extraneous files
CreateRandomFile(dir, "onetwothree", 100);
CreateRandomFile(dir, segment + ".notIn", 50);
CreateRandomFile(dir, segment + ".notIn2", 51);
// Now test
CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
System.String[] data = new System.String[]{".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7"};
for (int i = 0; i < data.Length; i++)
{
csw.AddFile(segment + data[i]);
}
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i = 0; i < data.Length; i++)
{
IndexInput check = dir.OpenInput(segment + data[i]);
IndexInput test = csr.OpenInput(segment + data[i]);
AssertSameStreams(data[i], check, test);
AssertSameSeekBehavior(data[i], check, test);
test.Close();
check.Close();
}
csr.Close();
}
/// <summary>Tells the docWriter to close its currently open shared
/// doc stores (stored fields & vectors files).
/// Return value specifices whether new doc store files are compound or not.
/// </summary>
private bool FlushDocStores()
{
lock (this)
{
if (infoStream != null)
{
Message("flushDocStores segment=" + docWriter.GetDocStoreSegment());
}
bool useCompoundDocStore = false;
if (infoStream != null)
{
Message("closeDocStores segment=" + docWriter.GetDocStoreSegment());
}
System.String docStoreSegment;
bool success = false;
try
{
docStoreSegment = docWriter.CloseDocStore();
success = true;
}
finally
{
if (!success && infoStream != null)
{
Message("hit exception closing doc store segment");
}
}
if (infoStream != null)
{
Message("flushDocStores files=" + docWriter.ClosedFiles());
}
useCompoundDocStore = mergePolicy.UseCompoundDocStore(segmentInfos);
if (useCompoundDocStore && docStoreSegment != null && docWriter.ClosedFiles().Count != 0)
{
// Now build compound doc store file
if (infoStream != null)
{
Message("create compound file " + docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION);
}
success = false;
int numSegments = segmentInfos.Count;
System.String compoundFileName = docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
try
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
System.Collections.IEnumerator it = docWriter.ClosedFiles().GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String) it.Current);
}
// Perform the merge
cfsWriter.Close();
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
Message("hit exception building compound file doc store for segment " + docStoreSegment);
deleter.DeleteFile(compoundFileName);
}
}
for (int i = 0; i < numSegments; i++)
{
SegmentInfo si = segmentInfos.Info(i);
if (si.GetDocStoreOffset() != - 1 && si.GetDocStoreSegment().Equals(docStoreSegment))
si.SetDocStoreIsCompoundFile(true);
}
Checkpoint();
// In case the files we just merged into a CFS were
// not previously checkpointed:
deleter.DeleteNewFiles(docWriter.ClosedFiles());
}
return useCompoundDocStore;
}
}
/** Build compound file for the segment we just flushed */
internal void CreateCompoundFile(string segment)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
IEnumerator<string> it = flushState.flushedFiles.Keys.GetEnumerator();
while (it.MoveNext())
cfsWriter.AddFile(it.Current);
// Perform the merge
cfsWriter.Close();
}
/// <summary>Build compound file for the segment we just flushed </summary>
internal void CreateCompoundFile(System.String segment)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
foreach(string s in flushState.flushedFiles)
{
cfsWriter.AddFile(s);
}
// Perform the merge
cfsWriter.Close();
}
public System.Collections.ArrayList CreateCompoundFile(System.String fileName)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName);
System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + fieldInfos.Size()));
// Basic files
for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.COMPOUND_EXTENSIONS[i]);
}
// Field norm files
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
files.Add(segment + ".f" + i);
}
}
// Vector files
if (fieldInfos.HasVectors())
{
for (int i = 0; i < IndexFileNames.VECTOR_EXTENSIONS.Length; i++)
{
files.Add(segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]);
}
}
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String) it.Current);
}
// Perform the merge
cfsWriter.Close();
return files;
}
public /*internal*/ System.Collections.Generic.ICollection<string> CreateCompoundFile(System.String fileName)
{
System.Collections.Generic.ICollection<string> files = GetMergedFiles();
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
// Now merge all added files
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String) it.Current);
}
// Perform the merge
cfsWriter.Close();
return files;
}
/// <summary>Setup a larger compound file with a number of components, each of
/// which is a sequential file (so that we can easily tell that we are
/// reading in the right byte). The methods sets up 20 files - f0 to f19,
/// the size of each file is 1000 bytes.
/// </summary>
private void SetUp_2()
{
CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
for (int i = 0; i < 20; i++)
{
CreateSequenceFile(dir, "f" + i, (byte) 0, 2000);
cw.AddFile("f" + i);
}
cw.Close();
}
/// <summary>Build compound file for the segment we just flushed </summary>
internal void CreateCompoundFile(System.String segment)
{
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
System.Collections.IEnumerator it = flushState.flushedFiles.GetEnumerator();
while (it.MoveNext())
{
cfsWriter.AddFile((System.String) ((System.Collections.DictionaryEntry) it.Current).Key);
}
// Perform the merge
cfsWriter.Close();
}
