public virtual void TestTotalBytesSize()
        {
            Directory d = NewDirectory();
            if (d is MockDirectoryWrapper)
            {
                ((MockDirectoryWrapper)d).Throttling = Throttling.NEVER;
            }
            IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
            iwc.SetMaxBufferedDocs(5);
            iwc.SetMergeScheduler(new TrackingCMS());
            if (TestUtil.GetPostingsFormat("id").Equals("SimpleText", StringComparison.Ordinal))
            {
                // no
                iwc.SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()));
            }
            RandomIndexWriter w = new RandomIndexWriter(Random, d, iwc);
            for (int i = 0; i < 1000; i++)
            {
                Document doc = new Document();
                doc.Add(new StringField("id", "" + i, Field.Store.NO));
                w.AddDocument(doc);

                if (Random.NextBoolean())
                {
                    w.DeleteDocuments(new Term("id", "" + Random.Next(i + 1)));
                }
            }
            Assert.IsTrue(((TrackingCMS)w.IndexWriter.Config.MergeScheduler).TotMergedBytes != 0);
            w.Dispose();
            d.Dispose();
        }
        public virtual void TestMaxMergeCount()
        {
            Directory dir = NewDirectory();
            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));

            int maxMergeCount = TestUtil.NextInt32(Random, 1, 5);
            int maxMergeThreads = TestUtil.NextInt32(Random, 1, maxMergeCount);
            CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount);
            AtomicInt32 runningMergeCount = new AtomicInt32(0);
            AtomicBoolean failed = new AtomicBoolean();

            if (VERBOSE)
            {
                Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads);
            }

            ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed);
            cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
            iwc.SetMergeScheduler(cms);
            iwc.SetMaxBufferedDocs(2);

            TieredMergePolicy tmp = new TieredMergePolicy();
            iwc.SetMergePolicy(tmp);
            tmp.MaxMergeAtOnce = 2;
            tmp.SegmentsPerTier = 2;

            IndexWriter w = new IndexWriter(dir, iwc);
            Document doc = new Document();
            doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED));
            while (enoughMergesWaiting.CurrentCount != 0 && !failed)
            {
                for (int i = 0; i < 10; i++)
                {
                    w.AddDocument(doc);
                }
            }
            w.Dispose(false);
            dir.Dispose();
        }
Beispiel #3
0
 private void MostlyAscendingStrategy(Entry[] arr, int i)
 {
     arr[i] = i == 0
     ? new Entry(random.nextInt(6), 0)
     : new Entry(arr[i - 1].Value + TestUtil.NextInt32(random, -8, 10), i);
 }
Beispiel #4
0
        public static string RandomSubString(Random random, int wordLength, bool simple)
        {
            if (wordLength == 0)
            {
                return("");
            }

            int evilness = TestUtil.NextInt32(random, 0, 20);

            StringBuilder sb = new StringBuilder();

            while (sb.Length < wordLength)
            {
                ;
                if (simple)
                {
                    sb.Append(random.NextBoolean() ? TestUtil.RandomSimpleString(random, wordLength) : TestUtil.RandomHtmlishString(random, wordLength));
                }
                else
                {
                    if (evilness < 10)
                    {
                        sb.Append(TestUtil.RandomSimpleString(random, wordLength));
                    }
                    else if (evilness < 15)
                    {
                        Assert.AreEqual(0, sb.Length); // we should always get wordLength back!
                        sb.Append(TestUtil.RandomRealisticUnicodeString(random, wordLength, wordLength));
                    }
                    else if (evilness == 16)
                    {
                        sb.Append(TestUtil.RandomHtmlishString(random, wordLength));
                    }
                    else if (evilness == 17)
                    {
                        // gives a lot of punctuation
                        sb.Append(TestUtil.RandomRegexpishString(random, wordLength));
                    }
                    else
                    {
                        sb.Append(TestUtil.RandomUnicodeString(random, wordLength));
                    }
                }
            }
            if (sb.Length > wordLength)
            {
                sb.Length = wordLength;
                if (char.IsHighSurrogate(sb[wordLength - 1]))
                {
                    sb.Length = wordLength - 1;
                }
            }

            if (random.Next(17) == 0)
            {
                // mix up case
                string mixedUp = TestUtil.RandomlyRecaseString(random, sb.ToString());
                Assert.True(mixedUp.Length == sb.Length, "Lengths are not the same: mixedUp = " + mixedUp + ", length = " + mixedUp.Length + ", sb = " + sb + ", length = " + sb.Length);
                return(mixedUp);
            }
            else
            {
                return(sb.ToString());
            }
        }
Beispiel #5
0
 private void StrictlyDescendingStrategy(Entry[] arr, int i)
 {
     arr[i] = i == 0
     ? new Entry(random.nextInt(6), 0)
     : new Entry(arr[i - 1].Value - TestUtil.NextInt32(random, 1, 5), i);
 }
Beispiel #6
0
        internal virtual void DoRandomSets(int maxSize, int iter, int mode)
        {
            BitArray   a0 = null;
            LongBitSet b0 = null;

            for (int i = 0; i < iter; i++)
            {
                int        sz = TestUtil.NextInt(Random(), 2, maxSize);
                BitArray   a  = new BitArray(sz);
                LongBitSet b  = new LongBitSet(sz);

                // test the various ways of setting bits
                if (sz > 0)
                {
                    int nOper = Random().Next(sz);
                    for (int j = 0; j < nOper; j++)
                    {
                        int idx;

                        idx = Random().Next(sz);
                        a.SafeSet(idx, true);
                        b.Set(idx);

                        idx = Random().Next(sz);
                        a.SafeSet(idx, false);
                        b.Clear(idx);

                        idx = Random().Next(sz);
                        a.SafeSet(idx, !a.SafeGet(idx));
                        b.Flip(idx, idx + 1);

                        idx = Random().Next(sz);
                        a.SafeSet(idx, !a.SafeGet(idx));
                        b.Flip(idx, idx + 1);

                        bool val2 = b.Get(idx);
                        bool val  = b.GetAndSet(idx);
                        Assert.IsTrue(val2 == val);
                        Assert.IsTrue(b.Get(idx));

                        if (!val)
                        {
                            b.Clear(idx);
                        }
                        Assert.IsTrue(b.Get(idx) == val);
                    }
                }

                // test that the various ways of accessing the bits are equivalent
                DoGet(a, b);

                // test ranges, including possible extension
                int fromIndex, toIndex;
                fromIndex = Random().Next(sz / 2);
                toIndex   = fromIndex + Random().Next(sz - fromIndex);
                BitArray aa = (BitArray)a.Clone();
                aa.Flip(fromIndex, toIndex);
                LongBitSet bb = b.Clone();
                bb.Flip(fromIndex, toIndex);

                fromIndex = Random().Next(sz / 2);
                toIndex   = fromIndex + Random().Next(sz - fromIndex);
                aa        = (BitArray)a.Clone();
                aa.Clear(fromIndex, toIndex);
                bb = b.Clone();
                bb.Clear(fromIndex, toIndex);

                DoNextSetBit(aa, bb); // a problem here is from clear() or nextSetBit

                DoPrevSetBit(aa, bb);

                fromIndex = Random().Next(sz / 2);
                toIndex   = fromIndex + Random().Next(sz - fromIndex);
                aa        = (BitArray)a.Clone();
                aa.Set(fromIndex, toIndex);
                bb = b.Clone();
                bb.Set(fromIndex, toIndex);

                DoNextSetBit(aa, bb); // a problem here is from set() or nextSetBit

                DoPrevSetBit(aa, bb);

                if (b0 != null && b0.Length() <= b.Length())
                {
                    Assert.AreEqual(a.Cardinality(), b.Cardinality());

                    BitArray a_and = (BitArray)a.Clone();
                    a_and = a_and.And_UnequalLengths(a0);
                    BitArray a_or = (BitArray)a.Clone();
                    a_or = a_or.Or_UnequalLengths(a0);
                    BitArray a_xor = (BitArray)a.Clone();
                    a_xor = a_xor.Xor_UnequalLengths(a0);
                    BitArray a_andn = (BitArray)a.Clone();
                    a_andn.AndNot(a0);

                    LongBitSet b_and = b.Clone();
                    Assert.AreEqual(b, b_and);
                    b_and.And(b0);
                    LongBitSet b_or = b.Clone();
                    b_or.Or(b0);
                    LongBitSet b_xor = b.Clone();
                    b_xor.Xor(b0);
                    LongBitSet b_andn = b.Clone();
                    b_andn.AndNot(b0);

                    Assert.AreEqual(a0.Cardinality(), b0.Cardinality());
                    Assert.AreEqual(a_or.Cardinality(), b_or.Cardinality());

                    Assert.AreEqual(a_and.Cardinality(), b_and.Cardinality());
                    Assert.AreEqual(a_or.Cardinality(), b_or.Cardinality());
                    Assert.AreEqual(a_xor.Cardinality(), b_xor.Cardinality());
                    Assert.AreEqual(a_andn.Cardinality(), b_andn.Cardinality());
                }

                a0 = a;
                b0 = b;
            }
        }
Beispiel #7
0
 public override PForDeltaDocIdSet CopyOf(BitArray bs, int length)
 {
     PForDeltaDocIdSet.Builder builder = (new PForDeltaDocIdSet.Builder()).SetIndexInterval(TestUtil.NextInt(Random(), 1, 20));
     for (int doc = bs.NextSetBit(0); doc != -1; doc = bs.NextSetBit(doc + 1))
     {
         builder.Add(doc);
     }
     return(builder.Build());
 }
Beispiel #8
0
        public virtual void TestDataInputOutput()
        {
            Random random = Random();

            for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
            {
                BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("testOverflow"));
                if (dir is MockDirectoryWrapper)
                {
                    ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
                }
                int         blockBits = TestUtil.NextInt(random, 1, 20);
                int         blockSize = 1 << blockBits;
                PagedBytes  p         = new PagedBytes(blockBits);
                IndexOutput @out      = dir.CreateOutput("foo", IOContext.DEFAULT);
                int         numBytes  = TestUtil.NextInt(Random(), 2, 10000000);

                byte[] answer = new byte[numBytes];
                Random().NextBytes(answer);
                int written = 0;
                while (written < numBytes)
                {
                    if (Random().Next(10) == 7)
                    {
                        @out.WriteByte(answer[written++]);
                    }
                    else
                    {
                        int chunk = Math.Min(Random().Next(1000), numBytes - written);
                        @out.WriteBytes(answer, written, chunk);
                        written += chunk;
                    }
                }

                @out.Dispose();
                IndexInput input = dir.OpenInput("foo", IOContext.DEFAULT);
                DataInput  @in   = (DataInput)input.Clone();

                p.Copy(input, input.Length());
                PagedBytes.Reader reader = p.Freeze(random.NextBoolean());

                byte[] verify = new byte[numBytes];
                int    read   = 0;
                while (read < numBytes)
                {
                    if (Random().Next(10) == 7)
                    {
                        verify[read++] = @in.ReadByte();
                    }
                    else
                    {
                        int chunk = Math.Min(Random().Next(1000), numBytes - read);
                        @in.ReadBytes(verify, read, chunk);
                        read += chunk;
                    }
                }
                Assert.IsTrue(Arrays.Equals(answer, verify));

                BytesRef slice = new BytesRef();
                for (int iter2 = 0; iter2 < 100; iter2++)
                {
                    int pos = random.Next(numBytes - 1);
                    int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
                    reader.FillSlice(slice, pos, len);
                    for (int byteUpto = 0; byteUpto < len; byteUpto++)
                    {
                        Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
                    }
                }
                input.Dispose();
                dir.Dispose();
            }
        }
Beispiel #9
0
        public virtual void TestDataInputOutput2()
        {
            Random random = Random();

            for (int iter = 0; iter < 5 * RANDOM_MULTIPLIER; iter++)
            {
                int        blockBits = TestUtil.NextInt(random, 1, 20);
                int        blockSize = 1 << blockBits;
                PagedBytes p         = new PagedBytes(blockBits);
                DataOutput @out      = p.DataOutput;
                int        numBytes  = Random().Next(10000000);

                byte[] answer = new byte[numBytes];
                Random().NextBytes(answer);
                int written = 0;
                while (written < numBytes)
                {
                    if (Random().Next(10) == 7)
                    {
                        @out.WriteByte(answer[written++]);
                    }
                    else
                    {
                        int chunk = Math.Min(Random().Next(1000), numBytes - written);
                        @out.WriteBytes(answer, written, chunk);
                        written += chunk;
                    }
                }

                PagedBytes.Reader reader = p.Freeze(random.NextBoolean());

                DataInput @in = p.DataInput;

                byte[] verify = new byte[numBytes];
                int    read   = 0;
                while (read < numBytes)
                {
                    if (Random().Next(10) == 7)
                    {
                        verify[read++] = @in.ReadByte();
                    }
                    else
                    {
                        int chunk = Math.Min(Random().Next(1000), numBytes - read);
                        @in.ReadBytes(verify, read, chunk);
                        read += chunk;
                    }
                }
                Assert.IsTrue(Arrays.Equals(answer, verify));

                BytesRef slice = new BytesRef();
                for (int iter2 = 0; iter2 < 100; iter2++)
                {
                    int pos = random.Next(numBytes - 1);
                    int len = random.Next(Math.Min(blockSize + 1, numBytes - pos));
                    reader.FillSlice(slice, pos, len);
                    for (int byteUpto = 0; byteUpto < len; byteUpto++)
                    {
                        Assert.AreEqual(answer[pos + byteUpto], (byte)slice.Bytes[slice.Offset + byteUpto]);
                    }
                }
            }
        }