public virtual void TestCompressionTools()
{
IIndexableField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.Compress(binaryValCompressed.GetBytes(Encoding.UTF8)));
IIndexableField stringFldCompressed = new StoredField("stringCompressed", CompressionTools.CompressString(binaryValCompressed));
var doc = new Documents.Document {
binaryFldCompressed, stringFldCompressed
};
using Directory dir = NewDirectory();
using RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
writer.AddDocument(doc);
using IndexReader reader = writer.GetReader();
Document docFromReader = reader.Document(0);
Assert.IsTrue(docFromReader != null);
string binaryFldCompressedTest =
Encoding.UTF8.GetString(
CompressionTools.Decompress(docFromReader.GetBinaryValue("binaryCompressed")));
//new string(CompressionTools.Decompress(docFromReader.GetBinaryValue("binaryCompressed")), IOUtils.CHARSET_UTF_8);
Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed, StringComparison.Ordinal));
Assert.IsTrue(
CompressionTools.DecompressString(docFromReader.GetBinaryValue("stringCompressed"))
.Equals(binaryValCompressed, StringComparison.Ordinal));
}
public virtual void TestCompressionTools()
{
Fieldable binaryFldCompressed = new Field("binaryCompressed", CompressionTools.Compress(System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed)), Field.Store.YES);
Fieldable stringFldCompressed = new Field("stringCompressed", CompressionTools.CompressString(binaryValCompressed), Field.Store.YES);
Document doc = new Document();
doc.Add(binaryFldCompressed);
doc.Add(stringFldCompressed);
/** add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.AddDocument(doc);
writer.Close();
/** open a reader and fetch the document */
IndexReader reader = IndexReader.Open(dir);
Document docFromReader = reader.Document(0);
Assert.IsTrue(docFromReader != null);
/** fetch the binary compressed field and compare it's content with the original one */
System.String binaryFldCompressedTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(CompressionTools.Decompress(docFromReader.GetBinaryValue("binaryCompressed"))));
Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed));
Assert.IsTrue(CompressionTools.DecompressString(docFromReader.GetBinaryValue("stringCompressed")).Equals(binaryValCompressed));
reader.Close();
dir.Close();
}
public virtual void TestBinaryField()
{
Document doc = new Document();
Fieldable stringFld = new Field("string", binaryVal, Field.Store.YES, Field.Index.NO);
Fieldable binaryFld = new Field("binary", System.Text.UTF8Encoding.UTF8.GetBytes(binaryVal), Field.Store.YES);
Fieldable binaryFld2 = new Field("binary", System.Text.UTF8Encoding.UTF8.GetBytes(binaryVal2), Field.Store.YES);
doc.Add(stringFld);
doc.Add(binaryFld);
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
Assert.IsTrue(binaryFld.IsBinary());
Assert.IsTrue(binaryFld.IsStored());
Assert.IsFalse(binaryFld.IsIndexed());
Assert.IsFalse(binaryFld.IsTokenized());
System.String binaryTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(doc.GetBinaryValue("binary")));
Assert.IsTrue(binaryTest.Equals(binaryVal));
System.String stringTest = doc.Get("string");
Assert.IsTrue(binaryTest.Equals(stringTest));
doc.Add(binaryFld2);
Assert.AreEqual(3, doc.fields_ForNUnit.Count);
byte[][] binaryTests = doc.GetBinaryValues("binary");
Assert.AreEqual(2, binaryTests.Length);
binaryTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(binaryTests[0]));
System.String binaryTest2 = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(binaryTests[1]));
Assert.IsFalse(binaryTest.Equals(binaryTest2));
Assert.IsTrue(binaryTest.Equals(binaryVal));
Assert.IsTrue(binaryTest2.Equals(binaryVal2));
doc.RemoveField("string");
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
doc.RemoveFields("binary");
Assert.AreEqual(0, doc.fields_ForNUnit.Count);
}
public virtual void TestBinaryFieldInIndex()
{
Fieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES);
Fieldable binaryFldCompressed = new Field("binaryCompressed", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed), Field.Store.COMPRESS);
Fieldable stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);
Fieldable stringFldCompressed = new Field("stringCompressed", binaryValCompressed, Field.Store.COMPRESS, Field.Index.NO, Field.TermVector.NO);
try
{
// binary fields with store off are not allowed
new Field("fail", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed), Field.Store.NO);
Assert.Fail();
}
catch (System.ArgumentException iae)
{
;
}
Document doc = new Document();
doc.Add(binaryFldStored);
doc.Add(binaryFldCompressed);
doc.Add(stringFldStored);
doc.Add(stringFldCompressed);
/** test for field count */
Assert.AreEqual(4, doc.fields_ForNUnit.Count);
/** add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.AddDocument(doc);
writer.Close();
/** open a reader and fetch the document */
IndexReader reader = IndexReader.Open(dir);
Document docFromReader = reader.Document(0);
Assert.IsTrue(docFromReader != null);
/** fetch the binary stored field and compare it's content with the original one */
System.String binaryFldStoredTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryStored")));
Assert.IsTrue(binaryFldStoredTest.Equals(binaryValStored));
/** fetch the binary compressed field and compare it's content with the original one */
System.String binaryFldCompressedTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryCompressed")));
Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed));
/** fetch the string field and compare it's content with the original one */
System.String stringFldStoredTest = docFromReader.Get("stringStored");
Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored));
/** fetch the compressed string field and compare it's content with the original one */
System.String stringFldCompressedTest = docFromReader.Get("stringCompressed");
Assert.IsTrue(stringFldCompressedTest.Equals(binaryValCompressed));
/** delete the document from index */
reader.DeleteDocument(0);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
dir.Close();
}
public virtual void TestBinaryField()
{
Document doc = new Document();
Fieldable stringFld = new Field("string", binaryVal, Field.Store.YES, Field.Index.NO);
Fieldable binaryFld = new Field("binary", System.Text.UTF8Encoding.UTF8.GetBytes(binaryVal), Field.Store.YES);
Fieldable binaryFld2 = new Field("binary", System.Text.UTF8Encoding.UTF8.GetBytes(binaryVal2), Field.Store.YES);
doc.Add(stringFld);
doc.Add(binaryFld);
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
Assert.IsTrue(binaryFld.IsBinary());
Assert.IsTrue(binaryFld.IsStored());
Assert.IsFalse(binaryFld.IsIndexed());
Assert.IsFalse(binaryFld.IsTokenized());
System.String binaryTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(doc.GetBinaryValue("binary")));
Assert.IsTrue(binaryTest.Equals(binaryVal));
System.String stringTest = doc.Get("string");
Assert.IsTrue(binaryTest.Equals(stringTest));
doc.Add(binaryFld2);
Assert.AreEqual(3, doc.fields_ForNUnit.Count);
byte[][] binaryTests = doc.GetBinaryValues("binary");
Assert.AreEqual(2, binaryTests.Length);
binaryTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(binaryTests[0]));
System.String binaryTest2 = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(binaryTests[1]));
Assert.IsFalse(binaryTest.Equals(binaryTest2));
Assert.IsTrue(binaryTest.Equals(binaryVal));
Assert.IsTrue(binaryTest2.Equals(binaryVal2));
doc.RemoveField("string");
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
doc.RemoveFields("binary");
Assert.AreEqual(0, doc.fields_ForNUnit.Count);
}
public virtual void TestBinaryFieldInIndex()
{
IFieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES);
IFieldable stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);
// binary fields with store off are not allowed
Assert.Throws <ArgumentException>(
() => new Field("fail", System.Text.Encoding.UTF8.GetBytes(binaryValStored), Field.Store.NO));
Document doc = new Document();
doc.Add(binaryFldStored);
doc.Add(stringFldStored);
/* test for field count */
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
/* add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.AddDocument(doc);
writer.Close();
/* open a reader and fetch the document */
IndexReader reader = IndexReader.Open(dir, false);
Document docFromReader = reader.Document(0);
Assert.IsTrue(docFromReader != null);
/* fetch the binary stored field and compare it's content with the original one */
System.String binaryFldStoredTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryStored")));
Assert.IsTrue(binaryFldStoredTest.Equals(binaryValStored));
/* fetch the string field and compare it's content with the original one */
System.String stringFldStoredTest = docFromReader.Get("stringStored");
Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored));
/* delete the document from index */
reader.DeleteDocument(0);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
dir.Close();
}
// [Test] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass
public virtual void TestWriteReadMerge()
{
// get another codec, other than the default: so we are merging segments across different codecs
Codec otherCodec;
/*if ("SimpleText".Equals(Codec.Default.Name))
{*/
otherCodec = new Lucene46Codec();
/*}
else
{
otherCodec = new SimpleTextCodec();
}*/
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, (IndexWriterConfig)iwConf.Clone());
int docCount = AtLeast(200);
var data = new byte[docCount][][];
for (int i = 0; i < docCount; ++i)
{
int fieldCount = Rarely() ? RandomInts.NextIntBetween(Random(), 1, 500) : RandomInts.NextIntBetween(Random(), 1, 5);
data[i] = new byte[fieldCount][];
for (int j = 0; j < fieldCount; ++j)
{
int length = Rarely() ? Random().Next(1000) : Random().Next(10);
int max = Rarely() ? 256 : 2;
data[i][j] = RandomByteArray(length, max);
}
}
FieldType type = new FieldType(StringField.TYPE_STORED);
type.Indexed = false;
type.Freeze();
IntField id = new IntField("id", 0, Field.Store.YES);
for (int i = 0; i < data.Length; ++i)
{
Document doc = new Document();
doc.Add(id);
id.IntValue = i;
for (int j = 0; j < data[i].Length; ++j)
{
Field f = new Field("bytes" + j, data[i][j], type);
doc.Add(f);
}
iw.w.AddDocument(doc);
if (Random().NextBoolean() && (i % (data.Length / 10) == 0))
{
iw.w.Dispose();
// test merging against a non-compressing codec
if (iwConf.Codec == otherCodec)
{
iwConf.SetCodec(Codec.Default);
}
else
{
iwConf.SetCodec(otherCodec);
}
iw = new RandomIndexWriter(Random(), dir, (IndexWriterConfig)iwConf.Clone());
}
}
for (int i = 0; i < 10; ++i)
{
int min = Random().Next(data.Length);
int max = min + Random().Next(20);
iw.DeleteDocuments(NumericRangeQuery.NewIntRange("id", min, max, true, false));
}
iw.ForceMerge(2); // force merges with deletions
iw.Commit();
DirectoryReader ir = DirectoryReader.Open(dir);
Assert.IsTrue(ir.NumDocs > 0);
int numDocs = 0;
for (int i = 0; i < ir.MaxDoc; ++i)
{
Document doc = ir.Document(i);
if (doc == null)
{
continue;
}
++numDocs;
int docId = (int)doc.GetField("id").NumericValue;
Assert.AreEqual(data[docId].Length + 1, doc.Fields.Count);
for (int j = 0; j < data[docId].Length; ++j)
{
var arr = data[docId][j];
BytesRef arr2Ref = doc.GetBinaryValue("bytes" + j);
var arr2 = Arrays.CopyOfRange(arr2Ref.Bytes, arr2Ref.Offset, arr2Ref.Offset + arr2Ref.Length);
Assert.AreEqual(arr, arr2);
}
}
Assert.IsTrue(ir.NumDocs <= numDocs);
ir.Dispose();
iw.DeleteAll();
iw.Commit();
iw.ForceMerge(1);
iw.Dispose();
dir.Dispose();
}
