/// <summary>
 /// Creates a new iterator, buffering entries from the specified iterator </summary>
 public BufferedInputIterator(InputIterator source)
 {
     BytesRef spare;
     int freqIndex = 0;
     hasPayloads = source.HasPayloads;
     hasContexts_Renamed = source.HasContexts;
     while ((spare = source.Next()) != null)
     {
         entries.Append(spare);
         if (hasPayloads)
         {
             payloads.Append(source.Payload);
         }
         if (hasContexts_Renamed)
         {
             contextSets.Add(source.Contexts);
         }
         if (freqIndex >= freqs.Length)
         {
             freqs = ArrayUtil.Grow(freqs, freqs.Length + 1);
         }
         freqs[freqIndex++] = source.Weight;
     }
     comp = source.Comparator;
 }
예제 #2
0
        Mismatch <T>(InputIterator <T> begin, InputIterator <T> end, InputIterator <T> cmpBegin, IEqualityComparer <T> comparer)
        {
            for (begin = IteratorUtil.Clone(begin), cmpBegin = IteratorUtil.Clone(cmpBegin); !begin.Equals(end); begin.MoveNext(), cmpBegin.MoveNext())
            {
                T t1 = begin.Read();
                T t2 = cmpBegin.Read();
                if (!comparer.Equals(t1, t2))
                {
                    return(new CSTL.Utility.Pair <InputIterator <T>, InputIterator <T> >(IteratorUtil.Clone(begin), IteratorUtil.Clone(cmpBegin)));
                }
            }

            return(new CSTL.Utility.Pair <InputIterator <T>, InputIterator <T> >(null, null));//begin, cmpBegin);
        }
예제 #3
0
        public void IsPartitionedTest3()
        {
            int[] data = { 2, 7, 4, 6 };

            UnaryPredicate <int> isEven = delegate(int x)
            {
                return(x % 2 == 0);
            };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                Assert.IsFalse(Algorithm.IsPartitioned(inputIterator, isEven));
            }
        }
        public void EqualTest3()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 8 };

            int[] data2 = { 1, 2, 3, 4, 5, 7 };

            using (IInputIterator <int> inputIterator1 = new InputIterator <int>(data1))
            {
                using (IInputIterator <int> inputIterator2 = new InputIterator <int>(data2))
                {
                    Assert.IsFalse(Algorithm.Equal(inputIterator1, inputIterator2));
                }
            }
        }
        public void FindIfTest()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 9, 8 };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data1))
            {
                bool result = Algorithm.FindIf(inputIterator, delegate(int x)
                {
                    return(x == 9);
                }).IsEnd();

                Assert.IsFalse(result);
            }
        }
예제 #6
0
 public static void ReplaceCopy <T>(InputIterator <T> begin, InputIterator <T> end, OutputIterator <T> dest, T oldValue, T newValue, IEqualityComparer <T> comparer)
 {
     for (begin = IteratorUtil.Clone(begin), dest = IteratorUtil.Clone(dest); !begin.Equals(end); begin.MoveNext(), dest.MoveNext())
     {
         T srcValue = begin.Read();
         if (comparer.Equals(oldValue, srcValue))
         {
             dest.Write(newValue);
         }
         else
         {
             dest.Write(srcValue);
         }
     }
 }
        public void SearchNTest2()
        {
            int[] data = { 1, 2, 3, 4, 5 };

            int expectedIndex = -1;

            int actualIndex = 0;

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                actualIndex = Algorithm.SearchN(inputIterator, 2, 3);
            }

            Assert.IsTrue(expectedIndex == actualIndex);
        }
        public void AdjacentFindTest2()
        {
            char[] data = { 'x', 'r', 'q', 'w', 'e', 'v', 'y', 't' };

            char expected = 't';

            char actual = '?';

            using (IInputIterator <char> inputIterator = new InputIterator <char>(data))
            {
                actual = Algorithm.AdjacentFind(inputIterator);
            }

            Assert.IsTrue(expected == actual);
        }
        public void CountTest()
        {
            int expectedCount = 4;

            int actualCount = 0;

            int[] data = { 1, 2, 3, 1, 1, 4, 1 };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                actualCount = Algorithm.Count(inputIterator, 1);
            }

            Assert.IsTrue(expectedCount == actualCount);
        }
        public void SearchTest2()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 8 };

            int[] data2 = { 77, 78, 79, 80, 81, 45 };

            using (IInputIterator <int> inputIterator1 = new InputIterator <int>(data1))
            {
                using (IInputIterator <int> inputIterator2 = new InputIterator <int>(data2))
                {
                    bool result = Algorithm.Search(inputIterator1, inputIterator2).IsEnd();

                    Assert.IsTrue(result);
                }
            }
        }
        public void ForEachTest()
        {
            int sum = 0;

            int[] data = { 1, 2, 3 };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                Algorithm.ForEach(inputIterator, delegate(int x)
                {
                    sum += x;
                });
            }

            Assert.IsTrue(sum == 6);
        }
예제 #12
0
        public void IncludesTest4()
        {
            char[] data = { 'a', 'b', 'c', 'f', 'h', 'x' };

            char[] data2 = { 'a', 'c', 'g' };

            using (IInputIterator <char> inputIterator = new InputIterator <char>(data))
            {
                using (IInputIterator <char> inputIterator2 = new InputIterator <char>(data2))
                {
                    bool result = Algorithm.Includes(inputIterator, inputIterator2);

                    Assert.IsFalse(result);
                }
            }
        }
예제 #13
0
 /// <summary>
 /// Creates a new iterator, wrapping the specified iterator and
 /// returning elements in a random order.
 /// </summary>
 public UnsortedInputIterator(InputIterator source)
     : base(source)
 {
     ords = new int[entries.Size()];
     Random random = new Random();
     for (int i = 0; i < ords.Length; i++)
     {
         ords[i] = i;
     }
     for (int i = 0; i < ords.Length; i++)
     {
         int randomPosition = random.Next(ords.Length);
         int temp = ords[i];
         ords[i] = ords[randomPosition];
         ords[randomPosition] = temp;
     }
 }
예제 #14
0
        public static bool Equal <T>(InputIterator <T> begin, InputIterator <T> end, InputIterator <T> cmpBegin, IEqualityComparer <T> comparer)
        {
            begin    = IteratorUtil.Clone(begin);
            cmpBegin = IteratorUtil.Clone(cmpBegin);

            for (; !begin.Equals(end); begin.MoveNext(), cmpBegin.MoveNext())
            {
                T t1 = begin.Read();
                T t2 = cmpBegin.Read();
                if (!comparer.Equals(t1, t2))
                {
                    return(false);
                }
            }

            return(true);
        }
        public void FindIfNotTest()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 9, 8, -1, 9 };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data1))
            {
                Algorithm.FindIfNot(inputIterator, delegate(int x)
                {
                    return(x > 0);
                });

                ICursor cursor = (ICursor)inputIterator;

                int index = cursor.GetPosition();

                Assert.IsTrue(index == 8);
            }
        }
예제 #16
0
        /// <summary>
        /// Creates a new iterator, wrapping the specified iterator and
        /// returning elements in a random order.
        /// </summary>
        public UnsortedInputIterator(InputIterator source)
            : base(source)
        {
            ords = new int[entries.Size()];
            Random random = new Random();

            for (int i = 0; i < ords.Length; i++)
            {
                ords[i] = i;
            }
            for (int i = 0; i < ords.Length; i++)
            {
                int randomPosition = random.Next(ords.Length);
                int temp           = ords[i];
                ords[i] = ords[randomPosition];
                ords[randomPosition] = temp;
            }
        }
        public void MismatchTest()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 8 };

            int[] data2 = { 1, 2, 3, 4, 6, 7, 8 };

            KeyValuePair <int, int> value = new KeyValuePair <int, int>();

            using (IInputIterator <int> inputIterator1 = new InputIterator <int>(data1))
            {
                using (IInputIterator <int> inputIterator2 = new InputIterator <int>(data2))
                {
                    value = Algorithm.Mismatch(inputIterator1, inputIterator2);
                }
            }

            Assert.IsTrue(value.Key == 5 && value.Value == 6);
        }
예제 #18
0
        public static int Count <T>(InputIterator <T> begin, InputIterator <T> end, T t, IComparer <T> comparer)
        {
            return(Count(IteratorUtil.CreateEnumerator(begin, end), t, comparer));

#if NEVER
            int count = 0;
            while (!begin.Equals(end))
            {
                if (comparer.Compare(t, begin.Read()) == 0)
                {
                    ++count;
                }
                begin.MoveNext();
            }

            return(count);
#endif
        }
예제 #19
0
        public void UpperBoundTest()
        {
            int[] data = { 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6 };

            int expected = 5;

            int expectedIndex = 10;

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                int actual = Algorithm.UpperBound(inputIterator, 4).Read();

                int actualIndex = ((ICursor)inputIterator).GetPosition();

                Assert.IsTrue(expected == actual);
                Assert.IsTrue(expectedIndex == actualIndex);
            }
        }
예제 #20
0
        public void IncludesTest2()
        {
            char[] data = { 'a', 'b', 'c', 'f', 'h', 'x' };

            char[] data2 = { 'a', 'c' };

            bool expected = true;

            using (IInputIterator <char> inputIterator = new InputIterator <char>(data))
            {
                using (IInputIterator <char> inputIterator2 = new InputIterator <char>(data2))
                {
                    bool actual = Algorithm.Includes(inputIterator, inputIterator2);

                    Assert.IsTrue(expected == actual);
                }
            }
        }
예제 #21
0
        public static int CountIf <T>(InputIterator <T> begin, InputIterator <T> end, Functional.UnaryPredicate <T> func)
        {
            return(CountIf(IteratorUtil.CreateEnumerator(begin, end), func));

#if NEVER
            int count = 0;
            while (!begin.Equals(end))
            {
                if (func(begin.Read()))
                {
                    ++count;
                }
                begin.MoveNext();
            }

            return(count);
#endif
        }
        public void CountIfTest()
        {
            int expectedCount = 4;

            int actualCount = 0;

            int[] data = { 1, 2, 3, 1, 1, 4, 1 };

            using (IInputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                actualCount = Algorithm.CountIf(inputIterator, delegate(int x)
                {
                    return(x == 1);
                }
                                                );
            }

            Assert.IsTrue(expectedCount == actualCount);
        }
예제 #23
0
// Not sure why I coded these. The STL doesn't have an analgous version of these
        public static bool Equal <T>(InputIterator <T> begin, InputIterator <T> end, InputIterator <T> cmpBegin, InputIterator <T> cmpEnd)
            where T : IEquatable <T>
        {
            return(Equal(begin, end, cmpBegin, cmpEnd, new Functional.EqualComparer <T>()));

#if NEVER
            for (; !begin.Equals(end); begin.MoveNext(), cmpBegin.MoveNext())
            {
                T t1 = begin.Read();
                T t2 = cmpBegin.Read();
                if (!t1.Equals(t2))
                {
                    return(false);
                }
            }

            return(true);
#endif
        }
예제 #24
0
        public static int Count <T>(InputIterator <T> begin, InputIterator <T> end, T value)
            where T : IEquatable <T>
        {
            return(Count(IteratorUtil.CreateEnumerator(begin, end), value));

#if NEVER
            int count = 0;
            while (!begin.Equals(end))
            {
                if (value.Equals(begin.Read()))
                {
                    ++count;
                }
                begin.MoveNext();
            }

            return(count);
#endif
        }
예제 #25
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private static long importData(String title, int numRunners, InputIterable data, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, System.Func<EntityImporter> visitors, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, org.neo4j.unsafe.impl.batchimport.stats.StatsProvider memoryStatsProvider) throws java.io.IOException
        private static long ImportData(string title, int numRunners, InputIterable data, BatchingNeoStores stores, System.Func <EntityImporter> visitors, ExecutionMonitor executionMonitor, StatsProvider memoryStatsProvider)
        {
            LongAdder        roughEntityCountProgress = new LongAdder();
            ExecutorService  pool         = Executors.newFixedThreadPool(numRunners, new NamedThreadFactory(title + "Importer"));
            IoMonitor        writeMonitor = new IoMonitor(stores.IoTracer);
            ControllableStep step         = new ControllableStep(title, roughEntityCountProgress, Configuration.DEFAULT, writeMonitor, memoryStatsProvider);
            StageExecution   execution    = new StageExecution(title, null, Configuration.DEFAULT, Collections.singletonList(step), 0);
            long             startTime    = currentTimeMillis();

            using (InputIterator dataIterator = data.GetEnumerator())
            {
                executionMonitor.Start(execution);
                for (int i = 0; i < numRunners; i++)
                {
                    pool.submit(new ExhaustingEntityImporterRunnable(execution, dataIterator, visitors(), roughEntityCountProgress));
                }
                pool.shutdown();

                long nextWait = 0;
                try
                {
                    while (!pool.awaitTermination(nextWait, TimeUnit.MILLISECONDS))
                    {
                        executionMonitor.Check(execution);
                        nextWait = executionMonitor.NextCheckTime() - currentTimeMillis();
                    }
                }
                catch (InterruptedException e)
                {
                    Thread.CurrentThread.Interrupt();
                    throw new IOException(e);
                }
            }

            execution.AssertHealthy();
            step.MarkAsCompleted();
            writeMonitor.Stop();
            executionMonitor.End(execution, currentTimeMillis() - startTime);
            execution.AssertHealthy();

            return(roughEntityCountProgress.sum());
        }
        public void SearchTest()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 7, 8 };

            int[] data2 = { 4, 5, 79, 1, 23, 45 };

            int expected = 1;

            int actual = -1;

            using (IInputIterator <int> inputIterator1 = new InputIterator <int>(data1))
            {
                using (IInputIterator <int> inputIterator2 = new InputIterator <int>(data2))
                {
                    actual = Algorithm.Search(inputIterator1, inputIterator2).Read();
                }
            }

            Assert.IsTrue(expected == actual);
        }
예제 #27
0
        public void acceptInput(string input)
        {
            currentText = null;
            input       = input.Trim();

            ArrayList brokenInput = new ArrayList(input.Split());

            Command command = identifyCommand(brokenInput);

            iterator = new InputIterator(archivedInput, brokenInput);

            if (command == lib.options)
            {
                optionsCommand();
            }
            else if (command == lib.create)
            {
                createCommand();
            }
        }
예제 #28
0
        public static bool Equal <T>(InputIterator <T> begin, InputIterator <T> end, InputIterator <T> cmpBegin, InputIterator <T> cmpEnd, IComparer <T> comparer)
        {
            for (; !begin.Equals(end) && !cmpBegin.Equals(cmpEnd); begin.MoveNext(), cmpBegin.MoveNext())
            {
                T t1 = begin.Read();
                T t2 = cmpBegin.Read();
                if (comparer.Compare(t1, t2) != 0)
                {
                    return(false);
                }
            }

            if (begin.Equals(end) && cmpBegin.Equals(cmpEnd))
            {
                return(true);
            }
            else
            {
                return(false);
            }
        }
예제 #29
0
        public void RemoveTest()
        {
            int[] data = { 1, 2, 2, 3 };

            int expectedCount = 2;

            int actualCount = 0;

            using (InputIterator <int> inputIterator = new InputIterator <int>(data))
            {
                Algorithm.Remove(inputIterator, 2);

                actualCount = inputIterator.Count();

                Assert.IsTrue(expectedCount == actualCount);

                bool isCorrectData = (inputIterator[0] == 1 && inputIterator[1] == 3);

                Assert.IsTrue(isCorrectData);
            }
        }
예제 #30
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void consume(String name, org.neo4j.unsafe.impl.batchimport.InputIterator entities, org.neo4j.unsafe.impl.batchimport.input.csv.Header header, Deserializer deserializer) throws java.io.IOException
        private void Consume(string name, InputIterator entities, Header header, Deserializer deserializer)
        {
            using (PrintStream @out = File(name + "header.csv"))
            {
                Serialize(@out, header);
            }

            try
            {
                int             threads  = Runtime.Runtime.availableProcessors();
                ExecutorService executor = Executors.newFixedThreadPool(threads);
                for (int i = 0; i < threads; i++)
                {
                    int id = i;
                    executor.submit((Callable <Void>)() =>
                    {
                        StringDeserialization deserialization = new StringDeserialization(_config);
                        using (PrintStream @out = File(name + "-" + id + ".csv"), InputChunk chunk = entities.NewChunk())
                        {
                            InputEntity entity = new InputEntity();
                            while (entities.Next(chunk))
                            {
                                while (chunk.next(entity))
                                {
                                    @out.println(deserializer.Apply(entity, deserialization, header));
                                }
                            }
                        }
                        return(null);
                    });
                }
                executor.shutdown();
                executor.awaitTermination(10, TimeUnit.MINUTES);
            }
            catch (InterruptedException e)
            {
                Thread.CurrentThread.Interrupt();
                throw new IOException(e);
            }
        }
        public void FindFirstOfTest()
        {
            int[] data1 = { 1, 2, 3, 4, 5, 6 };

            int[] data2 = { 7, 8, 9, 10, 3, 11 };

            int expectedData = 3;

            int actualData = 0;

            bool result = false;

            using (IInputIterator <int> inputIterator1 = new InputIterator <int>(data1))
            {
                using (IInputIterator <int> inputIterator2 = new InputIterator <int>(data2))
                {
                    result = Algorithm.FindFirstOf(inputIterator1, inputIterator2, out actualData);
                }
            }

            Assert.IsTrue((result == true) && (expectedData == actualData));
        }
예제 #32
0
        public override void Build(InputIterator iterator)
        {
            if (iterator.HasPayloads)
            {
                throw new ArgumentException("this suggester doesn't support payloads");
            }
            if (iterator.HasContexts)
            {
                throw new ArgumentException("this suggester doesn't support contexts");
            }
            count = 0;
            var           scratch     = new BytesRef();
            InputIterator iter        = new WFSTInputIterator(this, iterator);
            var           scratchInts = new IntsRef();
            BytesRef      previous    = null;
            var           outputs     = PositiveIntOutputs.Singleton;
            var           builder     = new Builder <long?>(FST.INPUT_TYPE.BYTE1, outputs);

            while ((scratch = iter.Next()) != null)
            {
                long cost = iter.Weight;

                if (previous == null)
                {
                    previous = new BytesRef();
                }
                else if (scratch.Equals(previous))
                {
                    continue; // for duplicate suggestions, the best weight is actually
                    // added
                }
                Lucene.Net.Util.Fst.Util.ToIntsRef(scratch, scratchInts);
                builder.Add(scratchInts, cost);
                previous.CopyBytes(scratch);
                count++;
            }
            fst = builder.Finish();
        }
예제 #33
0
 public MathematicalFormula(InputIterator iterator)
     : base(iterator)
 {
 }
예제 #34
0
        public override void Build(InputIterator tfit)
        {
            if (tfit.HasPayloads)
            {
                throw new ArgumentException("this suggester doesn't support payloads");
            }
            if (tfit.Comparator != null)
            {
                // make sure it's unsorted
                // WTF - this could result in yet another sorted iteration....
                tfit = new UnsortedInputIterator(tfit);
            }
            if (tfit.HasContexts)
            {
                throw new System.ArgumentException("this suggester doesn't support contexts");
            }
            count = 0;
            trie = new JaspellTernarySearchTrie { MatchAlmostDiff = editDistance };
            BytesRef spare;

            var charsSpare = new CharsRef();

            while ((spare = tfit.Next()) != null)
            {

                long weight = tfit.Weight;
                if (spare.Length == 0)
                {
                    continue;
                }
                charsSpare.Grow(spare.Length);
                UnicodeUtil.UTF8toUTF16(spare.Bytes, spare.Offset, spare.Length, charsSpare);
                trie.Put(charsSpare.ToString(), Convert.ToInt64(weight));
            }
        }
예제 #35
0
        public override void Build(InputIterator iter)
        {
            if (searcherMgr != null)
            {
                searcherMgr.Dispose();
                searcherMgr = null;
            }

            if (writer != null)
            {
                writer.Dispose();
                writer = null;
            }

            AtomicReader r = null;
            bool success = false;
            try
            {
                // First pass: build a temporary normal Lucene index,
                // just indexing the suggestions as they iterate:
                writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GramAnalyzer, IndexWriterConfig.OpenMode.CREATE));
                //long t0 = System.nanoTime();

                // TODO: use threads?
                BytesRef text;
                while ((text = iter.Next()) != null)
                {
                    BytesRef payload;
                    if (iter.HasPayloads)
                    {
                        payload = iter.Payload;
                    }
                    else
                    {
                        payload = null;
                    }

                    Add(text, iter.Contexts, iter.Weight, payload);
                }

                //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");

                searcherMgr = new SearcherManager(writer, true, null);
                success = true;
            }
            finally
            {
                if (success)
                {
                    IOUtils.Close(r);
                }
                else
                {
                    IOUtils.CloseWhileHandlingException(writer, r);
                    writer = null;
                }
            }
        }
예제 #36
0
 public SimpleXml(InputIterator iterator)
     : base(iterator)
 {
 }
 public LimitingRepetitionInfiniteLoopTest(InputIterator iterator)
     : base(iterator)
 {
 }
예제 #38
0
        public override void Build(InputIterator tfit)
        {
            if (tfit.HasPayloads)
            {
                throw new System.ArgumentException("this suggester doesn't support payloads");
            }
            if (tfit.HasContexts)
            {
                throw new System.ArgumentException("this suggester doesn't support contexts");
            }
            root = new TernaryTreeNode();
            // buffer first
            if (tfit.Comparator != BytesRef.UTF8SortedAsUTF16Comparator)
            {
                // make sure it's sorted and the comparator uses UTF16 sort order
                tfit = new SortedInputIterator(tfit, BytesRef.UTF8SortedAsUTF16Comparator);
            }

            List<string> tokens = new List<string>();
            List<Number> vals = new List<Number>();
            BytesRef spare;
            CharsRef charsSpare = new CharsRef();
            while ((spare = tfit.Next()) != null)
            {
                charsSpare.Grow(spare.Length);
                UnicodeUtil.UTF8toUTF16(spare.Bytes, spare.Offset, spare.Length, charsSpare);
                tokens.Add(charsSpare.ToString());
                vals.Add(Convert.ToInt64(tfit.Weight));
            }
            autocomplete.BalancedTree(tokens.ToArray(), vals.ToArray(), 0, tokens.Count - 1, root);
        }
예제 #39
0
 public PhoneNumber(InputIterator iterator)
     : base(iterator)
 {
 }
예제 #40
0
        public override void Build(InputIterator iterator)
        {
            if (iterator.HasPayloads)
            {
                throw new System.ArgumentException("this suggester doesn't support payloads");
            }
            if (iterator.HasContexts)
            {
                throw new System.ArgumentException("this suggester doesn't support contexts");
            }
            File tempInput = File.CreateTempFile(typeof(FSTCompletionLookup).Name, ".input", OfflineSorter.defaultTempDir());
            File tempSorted = File.CreateTempFile(typeof(FSTCompletionLookup).Name, ".sorted", OfflineSorter.defaultTempDir());

            OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
            OfflineSorter.ByteSequencesReader reader = null;
            ExternalRefSorter sorter = null;

            // Push floats up front before sequences to sort them. For now, assume they are non-negative.
            // If negative floats are allowed some trickery needs to be done to find their byte order.
            bool success = false;
            count = 0;
            try
            {
                sbyte[] buffer = new sbyte[0];
                ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
                BytesRef spare;
                while ((spare = iterator.Next()) != null)
                {
                    if (spare.Length + 4 >= buffer.Length)
                    {
                        buffer = ArrayUtil.Grow(buffer, spare.Length + 4);
                    }

                    output.Reset(buffer);
                    output.WriteInt(EncodeWeight(iterator.Weight));
                    output.WriteBytes(spare.Bytes, spare.Offset, spare.Length);
                    writer.Write(buffer, 0, output.Position);
                }
                writer.Dispose();

                // We don't know the distribution of scores and we need to bucket them, so we'll sort
                // and divide into equal buckets.
                OfflineSorter.SortInfo info = (new OfflineSorter()).Sort(tempInput, tempSorted);
                tempInput.Delete();
                FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);

                int inputLines = info.Lines;
                reader = new OfflineSorter.ByteSequencesReader(tempSorted);
                long line = 0;
                int previousBucket = 0;
                int previousScore = 0;
                ByteArrayDataInput input = new ByteArrayDataInput();
                BytesRef tmp1 = new BytesRef();
                BytesRef tmp2 = new BytesRef();
                while (reader.Read(tmp1))
                {
                    input.Reset(tmp1.Bytes);
                    int currentScore = input.ReadInt();

                    int bucket;
                    if (line > 0 && currentScore == previousScore)
                    {
                        bucket = previousBucket;
                    }
                    else
                    {
                        bucket = (int)(line * buckets / inputLines);
                    }
                    previousScore = currentScore;
                    previousBucket = bucket;

                    // Only append the input, discard the weight.
                    tmp2.Bytes = tmp1.Bytes;
                    tmp2.Offset = input.Position;
                    tmp2.Length = tmp1.Length - input.Position;
                    builder.Add(tmp2, bucket);

                    line++;
                    count++;
                }

                // The two FSTCompletions share the same automaton.
                this.higherWeightsCompletion = builder.Build();
                this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst);

                success = true;
            }
            finally
            {
                if (success)
                {
                    IOUtils.Close(reader, writer, sorter);
                }
                else
                {
                    IOUtils.CloseWhileHandlingException(reader, writer, sorter);
                }

                tempInput.Delete();
                tempSorted.Delete();
            }
        }
 public ZeroOrMoreInfiniteLoopTest(InputIterator iterator)
     : base(iterator)
 {
 }
예제 #42
0
 public NpegNode(InputIterator iterator)
     : base(iterator)
 {
 }