/// <summary>
/// Compresses the specified byte range using the
/// specified compressionLevel (constants are defined in
/// java.util.zip.Deflater).
/// </summary>
public static byte[] Compress(byte[] value, int offset, int length, int compressionLevel)
{
/* Create an expandable byte array to hold the compressed data.
* You cannot use an array that's the same size as the orginal because
* there is no guarantee that the compressed data will be smaller than
* the uncompressed data. */
var bos = new ByteArrayOutputStream(length);
Deflater compressor = SharpZipLib.CreateDeflater();
try
{
compressor.SetLevel(compressionLevel);
compressor.SetInput(value, offset, length);
compressor.Finish();
// Compress the data
var buf = new byte[1024];
while (!compressor.IsFinished)
{
int count = compressor.Deflate(buf);
bos.Write(buf, 0, count);
}
}
finally
{
}
return bos.ToArray();
}
public virtual void TestInfoStreamGetsFieldName()
{
Directory dir = NewDirectory();
IndexWriter writer;
IndexWriterConfig c = new IndexWriterConfig(TEST_VERSION_CURRENT, new ThrowingAnalyzer());
ByteArrayOutputStream infoBytes = new ByteArrayOutputStream();
StreamWriter infoPrintStream = new StreamWriter(infoBytes, Encoding.UTF8);
PrintStreamInfoStream printStreamInfoStream = new PrintStreamInfoStream(infoPrintStream);
c.InfoStream = printStreamInfoStream;
writer = new IndexWriter(dir, c);
Document doc = new Document();
doc.Add(NewField("distinctiveFieldName", "aaa ", StoredTextType));
try
{
writer.AddDocument(doc);
Assert.Fail("Failed to fail.");
}
catch (BadNews badNews)
{
infoPrintStream.Flush();
string infoStream = Encoding.UTF8.GetString(infoBytes.GetBuffer());
Assert.IsTrue(infoStream.Contains("distinctiveFieldName"));
}
writer.Dispose();
dir.Dispose();
}
public void TestWriteUTF()
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
dos.WriteUTF("Hello, World!"); // 15
dos.Flush();
if (baos.Length != dos.Length)
fail("Miscounted bytes in DataOutputStream.");
}
public override Field[] CreateIndexableFields(IShape shape)
{
int bufSize = Math.Max(128, (int)(this.indexLastBufSize * 1.5));//50% headroom over last
ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize);
BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes
try
{
ctx.BinaryCodec.WriteShape(new BinaryWriter(byteStream), shape);
//this is a hack to avoid redundant byte array copying by byteStream.toByteArray()
byteStream.WriteTo(new OutputStreamAnonymousHelper(bytesRef));
}
catch (IOException e)
{
throw new ApplicationException(e.Message, e);
}
this.indexLastBufSize = bytesRef.Length;//cache heuristic
return new Field[] { new BinaryDocValuesField(FieldName, bytesRef) };
}
public void TestBoundsCheck()
{
byte[] data = { 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 };
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DummyFilterStream dfs = new DummyFilterStream(bos);
bool caughtException = false;
// -ve length
try
{
dfs.Write(data, 0, -5);
}
#pragma warning disable 168
catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
fail("Test failed");
}
// -ve offset
caughtException = false;
try
{
dfs.Write(data, -2, 5);
}
#pragma warning disable 168
catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
fail("Test failed");
}
// off + len > data.length
caughtException = false;
try
{
dfs.Write(data, 6, 5);
}
#pragma warning disable 168
catch (ArgumentException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
fail("Test failed");
}
// null data
caughtException = false;
try
{
dfs.Write(null, 0, 5);
}
#pragma warning disable 168
catch (ArgumentNullException re)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
fail("Test failed");
}
}
public virtual void TestDeletesCheckIndexOutput()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMaxBufferedDocs(2);
IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
Document doc = new Document();
doc.Add(NewField("field", "0", StringField.TYPE_NOT_STORED));
w.AddDocument(doc);
doc = new Document();
doc.Add(NewField("field", "1", StringField.TYPE_NOT_STORED));
w.AddDocument(doc);
w.Commit();
Assert.AreEqual(1, w.SegmentCount);
w.DeleteDocuments(new Term("field", "0"));
w.Commit();
Assert.AreEqual(1, w.SegmentCount);
w.Dispose();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
//MemoryStream bos = new MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
CheckIndex.Status indexStatus = checker.DoCheckIndex(null);
Assert.IsTrue(indexStatus.Clean);
checker.FlushInfoStream();
string s = bos.ToString();
// Segment should have deletions:
Assert.IsTrue(s.Contains("has deletions"), "string was: " + s);
w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
w.ForceMerge(1);
w.Dispose();
bos = new ByteArrayOutputStream(1024);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
indexStatus = checker.DoCheckIndex(null);
Assert.IsTrue(indexStatus.Clean);
checker.FlushInfoStream();
s = bos.ToString();
Assert.IsFalse(s.Contains("has deletions"));
dir.Dispose();
}
public virtual void TestDeletedDocs()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
for (int i = 0; i < 19; i++)
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa" + i, customType));
writer.AddDocument(doc);
}
writer.ForceMerge(1);
writer.Commit();
writer.DeleteDocuments(new Term("field", "aaa5"));
writer.Dispose();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
if (VERBOSE)
{
checker.InfoStream = Console.Out;
}
CheckIndex.Status indexStatus = checker.DoCheckIndex();
if (indexStatus.Clean == false)
{
Console.WriteLine("CheckIndex failed");
checker.FlushInfoStream();
Console.WriteLine(bos.ToString());
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
Assert.IsTrue(seg.OpenReaderPassed);
Assert.IsNotNull(seg.Diagnostics);
Assert.IsNotNull(seg.FieldNormStatus);
Assert.IsNull(seg.FieldNormStatus.Error);
Assert.AreEqual(1, seg.FieldNormStatus.TotFields);
Assert.IsNotNull(seg.TermIndexStatus);
Assert.IsNull(seg.TermIndexStatus.Error);
Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
Assert.AreEqual(18, seg.TermIndexStatus.TotPos);
Assert.IsNotNull(seg.StoredFieldStatus);
Assert.IsNull(seg.StoredFieldStatus.Error);
Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);
Assert.IsNotNull(seg.TermVectorStatus);
Assert.IsNull(seg.TermVectorStatus.Error);
Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);
Assert.IsTrue(seg.Diagnostics.Count > 0);
IList<string> onlySegments = new List<string>();
onlySegments.Add("_0");
Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
dir.Dispose();
}
public virtual void TestBasic()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
// Writes facet ords to a separate directory from the
// main index:
var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
FacetsConfig config = new FacetsConfig();
config.SetHierarchical("Publish Date", true);
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
Document doc = new Document();
doc.Add(new FacetField("Author", "Bob"));
doc.Add(new FacetField("Publish Date", "2010", "10", "15"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("Author", "Lisa"));
doc.Add(new FacetField("Publish Date", "2010", "10", "20"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("Author", "Lisa"));
doc.Add(new FacetField("Publish Date", "2012", "1", "1"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("Author", "Susan"));
doc.Add(new FacetField("Publish Date", "2012", "1", "7"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("Author", "Frank"));
doc.Add(new FacetField("Publish Date", "1999", "5", "5"));
writer.AddDocument(config.Build(taxoWriter, doc));
// NRT open
IndexSearcher searcher = NewSearcher(writer.Reader);
// NRT open
var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
// Aggregate the facet counts:
FacetsCollector c = new FacetsCollector();
// MatchAllDocsQuery is for "browsing" (counts facets
// for all non-deleted docs in the index); normally
// you'd use a "normal" query, and use MultiCollector to
// wrap collecting the "normal" hits and also facets:
searcher.Search(new MatchAllDocsQuery(), c);
Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
// Retrieve & verify results:
Assert.AreEqual("dim=Publish Date path=[] value=5 childCount=3\n 2010 (2)\n 2012 (2)\n 1999 (1)\n", facets.GetTopChildren(10, "Publish Date").ToString());
Assert.AreEqual("dim=Author path=[] value=5 childCount=4\n Lisa (2)\n Bob (1)\n Susan (1)\n Frank (1)\n", facets.GetTopChildren(10, "Author").ToString());
// Now user drills down on Publish Date/2010:
DrillDownQuery q2 = new DrillDownQuery(config);
q2.Add("Publish Date", "2010");
c = new FacetsCollector();
searcher.Search(q2, c);
facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
Assert.AreEqual("dim=Author path=[] value=2 childCount=2\n Bob (1)\n Lisa (1)\n", facets.GetTopChildren(10, "Author").ToString());
Assert.AreEqual(1, facets.GetSpecificValue("Author", "Lisa"));
Assert.Null(facets.GetTopChildren(10, "Non exitent dim"));
// Smoke test PrintTaxonomyStats:
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintTaxonomyStats.PrintStats(taxoReader, Console.Out, true);
string result = bos.ToString();
Assert.True(result.IndexOf("/Author: 4 immediate children; 5 total categories", StringComparison.Ordinal) != -1);
Assert.True(result.IndexOf("/Publish Date: 3 immediate children; 12 total categories", StringComparison.Ordinal) != -1);
// Make sure at least a few nodes of the tree came out:
Assert.True(result.IndexOf(" /1999", StringComparison.Ordinal) != -1);
Assert.True(result.IndexOf(" /2012", StringComparison.Ordinal) != -1);
Assert.True(result.IndexOf(" /20", StringComparison.Ordinal) != -1);
IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, taxoDir, dir);
}
/// <summary>
/// Decompress the byte array previously returned by
/// compress
/// </summary>
public static byte[] Decompress(byte[] value, int offset, int length)
{
// Create an expandable byte array to hold the decompressed data
var bos = new ByteArrayOutputStream(length);
Inflater decompressor = SharpZipLib.CreateInflater();
try
{
decompressor.SetInput(value);
// Decompress the data
var buf = new byte[1024];
while (!decompressor.IsFinished)
{
int count = decompressor.Inflate(buf);
bos.Write(buf, 0, count);
}
}
finally
{
}
return bos.ToArray();
}
public void TestBoundsCheck()
{
byte[] data = { 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 };
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DummyFilterStream dfs = new DummyFilterStream(bos);
bool caughtException = false;
// -ve length
try
{
dfs.Write(data, 0, -5);
}
#pragma warning disable 168
catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
{
fail("Test failed");
}
}
// -ve offset
caughtException = false;
try
{
dfs.Write(data, -2, 5);
}
#pragma warning disable 168
catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
{
fail("Test failed");
}
}
// off + len > data.length
caughtException = false;
try
{
dfs.Write(data, 6, 5);
}
#pragma warning disable 168
catch (ArgumentException ie)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
{
fail("Test failed");
}
}
// null data
caughtException = false;
try
{
dfs.Write(null, 0, 5);
}
#pragma warning disable 168
catch (ArgumentNullException re)
#pragma warning restore 168
{
caughtException = true;
}
finally
{
if (!caughtException)
{
fail("Test failed");
}
}
}
