예제 #1
0
        /// <summary>
        /// Compresses the specified byte range using the
        ///  specified compressionLevel (constants are defined in
        ///  java.util.zip.Deflater).
        /// </summary>
        public static byte[] Compress(byte[] value, int offset, int length, int compressionLevel)
        {
            /* Create an expandable byte array to hold the compressed data.
             * You cannot use an array that's the same size as the orginal because
             * there is no guarantee that the compressed data will be smaller than
             * the uncompressed data. */
            var bos = new ByteArrayOutputStream(length);

            Deflater compressor = SharpZipLib.CreateDeflater();

            try
            {
                compressor.SetLevel(compressionLevel);
                compressor.SetInput(value, offset, length);
                compressor.Finish();

                // Compress the data
                var buf = new byte[1024];
                while (!compressor.IsFinished)
                {
                    int count = compressor.Deflate(buf);
                    bos.Write(buf, 0, count);
                }
            }
            finally
            {
            }

            return bos.ToArray();
        }
        public virtual void TestInfoStreamGetsFieldName()
        {
            Directory dir = NewDirectory();
            IndexWriter writer;
            IndexWriterConfig c = new IndexWriterConfig(TEST_VERSION_CURRENT, new ThrowingAnalyzer());
            ByteArrayOutputStream infoBytes = new ByteArrayOutputStream();
            StreamWriter infoPrintStream = new StreamWriter(infoBytes, Encoding.UTF8);
            PrintStreamInfoStream printStreamInfoStream = new PrintStreamInfoStream(infoPrintStream);
            c.InfoStream = printStreamInfoStream;
            writer = new IndexWriter(dir, c);
            Document doc = new Document();
            doc.Add(NewField("distinctiveFieldName", "aaa ", StoredTextType));
            try
            {
                writer.AddDocument(doc);
                Assert.Fail("Failed to fail.");
            }
            catch (BadNews badNews)
            {
                infoPrintStream.Flush();
                string infoStream = Encoding.UTF8.GetString(infoBytes.GetBuffer());
                Assert.IsTrue(infoStream.Contains("distinctiveFieldName"));
            }

            writer.Dispose();
            dir.Dispose();
        }
예제 #3
0
 public void TestWriteUTF()
 {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream dos = new DataOutputStream(baos);
     dos.WriteUTF("Hello, World!");  // 15
     dos.Flush();
     if (baos.Length != dos.Length)
         fail("Miscounted bytes in DataOutputStream.");
 }
예제 #4
0
        public override Field[] CreateIndexableFields(IShape shape)
        {
            int bufSize = Math.Max(128, (int)(this.indexLastBufSize * 1.5));//50% headroom over last
            ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize);
            BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes
            try
            {
                ctx.BinaryCodec.WriteShape(new BinaryWriter(byteStream), shape);

                //this is a hack to avoid redundant byte array copying by byteStream.toByteArray()
                byteStream.WriteTo(new OutputStreamAnonymousHelper(bytesRef));
            }
            catch (IOException e)
            {
                throw new ApplicationException(e.Message, e);
            }
            this.indexLastBufSize = bytesRef.Length;//cache heuristic
            return new Field[] { new BinaryDocValuesField(FieldName, bytesRef) };
        }
예제 #5
0
        public void TestBoundsCheck()
        {
            byte[] data = { 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 };
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DummyFilterStream dfs = new DummyFilterStream(bos);
            bool caughtException = false;

            // -ve length
            try
            {
                dfs.Write(data, 0, -5);
            }
#pragma warning disable 168
            catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                    fail("Test failed");
            }

            // -ve offset
            caughtException = false;
            try
            {
                dfs.Write(data, -2, 5);
            }
#pragma warning disable 168
            catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                    fail("Test failed");
            }

            // off + len > data.length
            caughtException = false;
            try
            {
                dfs.Write(data, 6, 5);
            }
#pragma warning disable 168
            catch (ArgumentException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                    fail("Test failed");
            }

            // null data
            caughtException = false;
            try
            {
                dfs.Write(null, 0, 5);
            }
#pragma warning disable 168
            catch (ArgumentNullException re)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                    fail("Test failed");
            }
        }
예제 #6
0
        public virtual void TestDeletesCheckIndexOutput()
        {
            Directory dir = NewDirectory();
            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
            iwc.SetMaxBufferedDocs(2);
            IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
            Document doc = new Document();
            doc.Add(NewField("field", "0", StringField.TYPE_NOT_STORED));
            w.AddDocument(doc);

            doc = new Document();
            doc.Add(NewField("field", "1", StringField.TYPE_NOT_STORED));
            w.AddDocument(doc);
            w.Commit();
            Assert.AreEqual(1, w.SegmentCount);

            w.DeleteDocuments(new Term("field", "0"));
            w.Commit();
            Assert.AreEqual(1, w.SegmentCount);
            w.Dispose();

            ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
            //MemoryStream bos = new MemoryStream(1024);
            CheckIndex checker = new CheckIndex(dir);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            CheckIndex.Status indexStatus = checker.DoCheckIndex(null);
            Assert.IsTrue(indexStatus.Clean);
            checker.FlushInfoStream();
            string s = bos.ToString();

            // Segment should have deletions:
            Assert.IsTrue(s.Contains("has deletions"), "string was: " + s);
            w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
            w.ForceMerge(1);
            w.Dispose();

            bos = new ByteArrayOutputStream(1024);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            indexStatus = checker.DoCheckIndex(null);
            Assert.IsTrue(indexStatus.Clean);
            checker.FlushInfoStream();
            s = bos.ToString();
            Assert.IsFalse(s.Contains("has deletions"));
            dir.Dispose();
        }
예제 #7
0
        public virtual void TestDeletedDocs()
        {
            Directory dir = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
            for (int i = 0; i < 19; i++)
            {
                Document doc = new Document();
                FieldType customType = new FieldType(TextField.TYPE_STORED);
                customType.StoreTermVectors = true;
                customType.StoreTermVectorPositions = true;
                customType.StoreTermVectorOffsets = true;
                doc.Add(NewField("field", "aaa" + i, customType));
                writer.AddDocument(doc);
            }
            writer.ForceMerge(1);
            writer.Commit();
            writer.DeleteDocuments(new Term("field", "aaa5"));
            writer.Dispose();

            ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
            CheckIndex checker = new CheckIndex(dir);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            if (VERBOSE)
            {
                checker.InfoStream = Console.Out;
            }
            CheckIndex.Status indexStatus = checker.DoCheckIndex();
            if (indexStatus.Clean == false)
            {
                Console.WriteLine("CheckIndex failed");
                checker.FlushInfoStream();
                Console.WriteLine(bos.ToString());
                Assert.Fail();
            }

            CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
            Assert.IsTrue(seg.OpenReaderPassed);

            Assert.IsNotNull(seg.Diagnostics);

            Assert.IsNotNull(seg.FieldNormStatus);
            Assert.IsNull(seg.FieldNormStatus.Error);
            Assert.AreEqual(1, seg.FieldNormStatus.TotFields);

            Assert.IsNotNull(seg.TermIndexStatus);
            Assert.IsNull(seg.TermIndexStatus.Error);
            Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
            Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
            Assert.AreEqual(18, seg.TermIndexStatus.TotPos);

            Assert.IsNotNull(seg.StoredFieldStatus);
            Assert.IsNull(seg.StoredFieldStatus.Error);
            Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
            Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);

            Assert.IsNotNull(seg.TermVectorStatus);
            Assert.IsNull(seg.TermVectorStatus.Error);
            Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
            Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);

            Assert.IsTrue(seg.Diagnostics.Count > 0);
            IList<string> onlySegments = new List<string>();
            onlySegments.Add("_0");

            Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
            dir.Dispose();
        }
        public virtual void TestBasic()
        {
            Store.Directory dir = NewDirectory();
            Store.Directory taxoDir = NewDirectory();

            // Writes facet ords to a separate directory from the
            // main index:
            var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);

            FacetsConfig config = new FacetsConfig();
            config.SetHierarchical("Publish Date", true);

            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);

            Document doc = new Document();
            doc.Add(new FacetField("Author", "Bob"));
            doc.Add(new FacetField("Publish Date", "2010", "10", "15"));
            writer.AddDocument(config.Build(taxoWriter, doc));

            doc = new Document();
            doc.Add(new FacetField("Author", "Lisa"));
            doc.Add(new FacetField("Publish Date", "2010", "10", "20"));
            writer.AddDocument(config.Build(taxoWriter, doc));

            doc = new Document();
            doc.Add(new FacetField("Author", "Lisa"));
            doc.Add(new FacetField("Publish Date", "2012", "1", "1"));
            writer.AddDocument(config.Build(taxoWriter, doc));

            doc = new Document();
            doc.Add(new FacetField("Author", "Susan"));
            doc.Add(new FacetField("Publish Date", "2012", "1", "7"));
            writer.AddDocument(config.Build(taxoWriter, doc));

            doc = new Document();
            doc.Add(new FacetField("Author", "Frank"));
            doc.Add(new FacetField("Publish Date", "1999", "5", "5"));
            writer.AddDocument(config.Build(taxoWriter, doc));

            // NRT open
            IndexSearcher searcher = NewSearcher(writer.Reader);

            // NRT open
            var taxoReader = new DirectoryTaxonomyReader(taxoWriter);

            // Aggregate the facet counts:
            FacetsCollector c = new FacetsCollector();

            // MatchAllDocsQuery is for "browsing" (counts facets
            // for all non-deleted docs in the index); normally
            // you'd use a "normal" query, and use MultiCollector to
            // wrap collecting the "normal" hits and also facets:
            searcher.Search(new MatchAllDocsQuery(), c);

            Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, c);

            // Retrieve & verify results:
            Assert.AreEqual("dim=Publish Date path=[] value=5 childCount=3\n  2010 (2)\n  2012 (2)\n  1999 (1)\n", facets.GetTopChildren(10, "Publish Date").ToString());
            Assert.AreEqual("dim=Author path=[] value=5 childCount=4\n  Lisa (2)\n  Bob (1)\n  Susan (1)\n  Frank (1)\n", facets.GetTopChildren(10, "Author").ToString());

            // Now user drills down on Publish Date/2010:
            DrillDownQuery q2 = new DrillDownQuery(config);
            q2.Add("Publish Date", "2010");
            c = new FacetsCollector();
            searcher.Search(q2, c);
            facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
            Assert.AreEqual("dim=Author path=[] value=2 childCount=2\n  Bob (1)\n  Lisa (1)\n", facets.GetTopChildren(10, "Author").ToString());

            Assert.AreEqual(1, facets.GetSpecificValue("Author", "Lisa"));

            Assert.Null(facets.GetTopChildren(10, "Non exitent dim"));

            // Smoke test PrintTaxonomyStats:
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            PrintTaxonomyStats.PrintStats(taxoReader, Console.Out, true);
            string result = bos.ToString();
            Assert.True(result.IndexOf("/Author: 4 immediate children; 5 total categories", StringComparison.Ordinal) != -1);
            Assert.True(result.IndexOf("/Publish Date: 3 immediate children; 12 total categories", StringComparison.Ordinal) != -1);
            // Make sure at least a few nodes of the tree came out:
            Assert.True(result.IndexOf("  /1999", StringComparison.Ordinal) != -1);
            Assert.True(result.IndexOf("  /2012", StringComparison.Ordinal) != -1);
            Assert.True(result.IndexOf("      /20", StringComparison.Ordinal) != -1);

            IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, taxoDir, dir);
        }
예제 #9
0
        /// <summary>
        /// Decompress the byte array previously returned by
        ///  compress
        /// </summary>
        public static byte[] Decompress(byte[] value, int offset, int length)
        {
            // Create an expandable byte array to hold the decompressed data
            var bos = new ByteArrayOutputStream(length);

            Inflater decompressor = SharpZipLib.CreateInflater();

            try
            {
                decompressor.SetInput(value);

                // Decompress the data
                var buf = new byte[1024];
                while (!decompressor.IsFinished)
                {
                    int count = decompressor.Inflate(buf);
                    bos.Write(buf, 0, count);
                }
            }
            finally
            {
            }

            return bos.ToArray();
        }
        public void TestBoundsCheck()
        {
            byte[] data = { 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 };
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DummyFilterStream     dfs = new DummyFilterStream(bos);
            bool caughtException      = false;

            // -ve length
            try
            {
                dfs.Write(data, 0, -5);
            }
#pragma warning disable 168
            catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                {
                    fail("Test failed");
                }
            }

            // -ve offset
            caughtException = false;
            try
            {
                dfs.Write(data, -2, 5);
            }
#pragma warning disable 168
            catch (ArgumentOutOfRangeException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                {
                    fail("Test failed");
                }
            }

            // off + len > data.length
            caughtException = false;
            try
            {
                dfs.Write(data, 6, 5);
            }
#pragma warning disable 168
            catch (ArgumentException ie)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                {
                    fail("Test failed");
                }
            }

            // null data
            caughtException = false;
            try
            {
                dfs.Write(null, 0, 5);
            }
#pragma warning disable 168
            catch (ArgumentNullException re)
#pragma warning restore 168
            {
                caughtException = true;
            }
            finally
            {
                if (!caughtException)
                {
                    fail("Test failed");
                }
            }
        }