public override void SetUp()
        {
            base.SetUp();

            FSTCompletionBuilder builder = new FSTCompletionBuilder();
            foreach (Input tf in EvalKeys())
            {
                builder.Add(tf.term, (int)tf.v);
            }
            completion = builder.Build();
            completionAlphabetical = new FSTCompletion(completion.FST, false, true);
        }
 public override bool Load(DataInput input)
 {
     lock (this)
     {
         count = input.ReadVLong();
         this.higherWeightsCompletion = new FSTCompletion(new FST<object>(input, NoOutputs.Singleton));
         this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst);
         return true;
     }
 }
        public override void Build(IInputIterator iterator)
        {
            if (iterator.HasPayloads)
            {
                throw new System.ArgumentException("this suggester doesn't support payloads");
            }
            if (iterator.HasContexts)
            {
                throw new System.ArgumentException("this suggester doesn't support contexts");
            }
            FileInfo tempInput = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".input", OfflineSorter.DefaultTempDir());
            FileInfo tempSorted = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".sorted", OfflineSorter.DefaultTempDir());

            OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
            OfflineSorter.ByteSequencesReader reader = null;
            ExternalRefSorter sorter = null;

            // Push floats up front before sequences to sort them. For now, assume they are non-negative.
            // If negative floats are allowed some trickery needs to be done to find their byte order.
            bool success = false;
            count = 0;
            try
            {
                byte[] buffer = new byte[0];
                ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
                BytesRef spare;
                while ((spare = iterator.Next()) != null)
                {
                    if (spare.Length + 4 >= buffer.Length)
                    {
                        buffer = ArrayUtil.Grow(buffer, spare.Length + 4);
                    }

                    output.Reset(buffer);
                    output.WriteInt(EncodeWeight(iterator.Weight));
                    output.WriteBytes(spare.Bytes, spare.Offset, spare.Length);
                    writer.Write(buffer, 0, output.Position);
                }
                writer.Dispose();

                // We don't know the distribution of scores and we need to bucket them, so we'll sort
                // and divide into equal buckets.
                OfflineSorter.SortInfo info = (new OfflineSorter()).Sort(tempInput, tempSorted);
                tempInput.Delete();
                FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);

                int inputLines = info.Lines;
                reader = new OfflineSorter.ByteSequencesReader(tempSorted);
                long line = 0;
                int previousBucket = 0;
                int previousScore = 0;
                ByteArrayDataInput input = new ByteArrayDataInput();
                BytesRef tmp1 = new BytesRef();
                BytesRef tmp2 = new BytesRef();
                while (reader.Read(tmp1))
                {
                    input.Reset(tmp1.Bytes);
                    int currentScore = input.ReadInt();

                    int bucket;
                    if (line > 0 && currentScore == previousScore)
                    {
                        bucket = previousBucket;
                    }
                    else
                    {
                        bucket = (int)(line * buckets / inputLines);
                    }
                    previousScore = currentScore;
                    previousBucket = bucket;

                    // Only append the input, discard the weight.
                    tmp2.Bytes = tmp1.Bytes;
                    tmp2.Offset = input.Position;
                    tmp2.Length = tmp1.Length - input.Position;
                    builder.Add(tmp2, bucket);

                    line++;
                    count++;
                }

                // The two FSTCompletions share the same automaton.
                this.higherWeightsCompletion = builder.Build();
                this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst);

                success = true;
            }
            finally
            {
                if (success)
                {
                    IOUtils.Close(reader, writer, sorter);
                }
                else
                {
                    IOUtils.CloseWhileHandlingException(reader, writer, sorter);
                }

                tempInput.Delete();
                tempSorted.Delete();
            }
        }
 /// <summary>
 /// This constructor takes a pre-built automaton.
 /// </summary>
 ///  <param name="completion"> 
 ///          An instance of <see cref="FSTCompletion"/>. </param>
 ///  <param name="exactMatchFirst">
 ///          If <code>true</code> exact matches are promoted to the top of the
 ///          suggestions list. Otherwise they appear in the order of
 ///          discretized weight and alphabetical within the bucket. </param>
 public FSTCompletionLookup(FSTCompletion completion, bool exactMatchFirst)
     : this(INVALID_BUCKETS_COUNT, exactMatchFirst)
 {
     this.normalCompletion = new FSTCompletion(completion.FST, false, exactMatchFirst);
     this.higherWeightsCompletion = new FSTCompletion(completion.FST, true, exactMatchFirst);
 }
 public void TestEmptyInput()
 {
     completion = new FSTCompletionBuilder().Build();
     AssertMatchEquals(completion.DoLookup(StringToCharSequence("").ToString(), 10));
 }
        public void TestRequestedCount()
        {
            // 'one' is promoted after collecting two higher ranking results.
            AssertMatchEquals(completion.DoLookup(StringToCharSequence("one").ToString(), 2),
                "one/0.0",
                "oneness/1.0");

            // 'four' is collected in a bucket and then again as an exact match. 
            AssertMatchEquals(completion.DoLookup(StringToCharSequence("four").ToString(), 2),
                "four/0.0",
                "fourblah/1.0");

            // Check reordering of exact matches. 
            AssertMatchEquals(completion.DoLookup(StringToCharSequence("four").ToString(), 4),
                "four/0.0",
                "fourblah/1.0",
                "fourteen/1.0",
                "fourier/0.0");

            // 'one' is at the top after collecting all alphabetical results.
            AssertMatchEquals(completionAlphabetical.DoLookup(StringToCharSequence("one").ToString(), 2),
                "one/0.0",
                "oneness/1.0");

            // 'one' is not promoted after collecting two higher ranking results.
            FSTCompletion noPromotion = new FSTCompletion(completion.FST, true, false);
            AssertMatchEquals(noPromotion.DoLookup(StringToCharSequence("one").ToString(), 2),
                "oneness/1.0",
                "onerous/1.0");

            // 'one' is at the top after collecting all alphabetical results. 
            AssertMatchEquals(completionAlphabetical.DoLookup(StringToCharSequence("one").ToString(), 2),
                "one/0.0",
                "oneness/1.0");
        }