// we have a segment, in NFD. Find all the strings that are canonically equivalent to it.
private string[] GetEquivalents(string segment)
{
ISet <string> result = new JCG.HashSet <string>();
ISet <string> basic = GetEquivalents2(segment);
ISet <string> permutations = new JCG.HashSet <string>();
// now get all the permutations
// add only the ones that are canonically equivalent
// TODO: optimize by not permuting any class zero.
using (IEnumerator <string> it = basic.GetEnumerator())
{
while (it.MoveNext())
{
string item = it.Current;
permutations.Clear();
#pragma warning disable 612, 618
Permute(item, SKIP_ZEROS, permutations);
#pragma warning restore 612, 618
using (IEnumerator <string> it2 = permutations.GetEnumerator())
{
while (it2.MoveNext())
{
string possible = it2.Current;
/*
* String attempt = Normalizer.normalize(possible, Normalizer.DECOMP, 0);
* if (attempt.equals(segment)) {
*/
if (Normalizer.Compare(possible, segment, 0) == 0)
{
if (PROGRESS)
{
Console.Out.WriteLine("Adding Permutation: " + Utility.Hex(possible));
}
result.Add(possible);
}
else
{
if (PROGRESS)
{
Console.Out.WriteLine("-Skipping Permutation: " + Utility.Hex(possible));
}
}
}
}
}
}
// convert into a String[] to clean up storage
string[] finalResult = new string[result.Count];
result.CopyTo(finalResult, 0);
return(finalResult);
}
/**
* @param me
* @param stayWithMe
*/
private void checkEquals(UnicodeMap <Integer> me, JCG.SortedDictionary <String, Integer> stayWithMe)
{
temp.Clear();
foreach (var e in me.EntrySet())
{
temp.Add(e);
}
ISet <KeyValuePair <String, Integer> > entrySet = new JCG.HashSet <KeyValuePair <string, Integer> >(stayWithMe);
if (!entrySet.SetEquals(temp))
{
Logln(me.EntrySet().ToString());
Logln(me.ToString());
assertEquals("are in parallel", entrySet, temp);
// we failed. Reset and start again
entrySet.Clear();
temp.Clear();
return;
}
// ICU4N: looping through pairs instead of doing explicity table lookups is much faster
foreach (var pair in stayWithMe)
{
assertEquals("containsKey", stayWithMe.ContainsKey(pair.Key), me.ContainsKey(pair.Key));
Integer value = pair.Value;
assertEquals("get", value, me.Get(pair.Key));
assertEquals("containsValue", stayWithMe.ContainsValue(value), me.ContainsValue(value));
int cp = UnicodeSet.GetSingleCodePoint(pair.Key);
if (cp != int.MaxValue)
{
assertEquals("get", value, me.Get(cp));
}
}
// ICU4N TODO: complete implementation
//ISet<String> nonCodePointStrings = stayWithMe.tailMap("").keySet();
//if (nonCodePointStrings.Count == 0) nonCodePointStrings = null; // for parallel api
//assertEquals("getNonRangeStrings", nonCodePointStrings, me.GetNonRangeStrings());
ISet <Integer> values = new JCG.SortedSet <Integer>(stayWithMe.Values);
ISet <Integer> myValues = new JCG.SortedSet <Integer>(me.Values());
assertEquals("values", myValues, values);
foreach (String key in stayWithMe.Keys)
{
assertEquals("containsKey", stayWithMe.ContainsKey(key), me.ContainsKey(key));
}
}
public static void Permute(string source, bool skipZeros, ISet <string> output)
{
// TODO: optimize
//if (PROGRESS) System.out.println("Permute: " + source);
// optimization:
// if zero or one character, just return a set with it
// we check for length < 2 to keep from counting code points all the time
if (source.Length <= 2 && UTF16.CountCodePoint(source) <= 1)
{
output.Add(source);
return;
}
// otherwise iterate through the string, and recursively permute all the other characters
ISet <string> subpermute = new JCG.HashSet <string>();
int cp;
for (int i = 0; i < source.Length; i += UTF16.GetCharCount(cp))
{
cp = UTF16.CharAt(source, i);
// optimization:
// if the character is canonical combining class zero,
// don't permute it
if (skipZeros && i != 0 && UChar.GetCombiningClass(cp) == 0)
{
//System.out.println("Skipping " + Utility.hex(UTF16.valueOf(source, i)));
continue;
}
// see what the permutations of the characters before and after this one are
subpermute.Clear();
Permute(source.Substring(0, i - 0) // ICU4N: Checked 2nd parameter
+ source.Substring(i + UTF16.GetCharCount(cp)), skipZeros, subpermute); // ICU4N: Substring only has 1 parameter
// prefix this character to all of them
string chStr = UTF16.ValueOf(source, i);
foreach (string s in subpermute)
{
string piece = chStr + s;
//if (PROGRESS) System.out.println(" Piece: " + piece);
output.Add(piece);
}
}
}
private void PruneDups(IList <OneSyn> syns)
{
ISet <string> seen = new JCG.HashSet <string>();
foreach (OneSyn syn in syns)
{
int idx = 0;
while (idx < [email protected])
{
string @out = syn.@out[idx];
if (!seen.Contains(@out))
{
seen.Add(@out);
idx++;
}
else
{
[email protected](idx);
}
}
seen.Clear();
}
}
public override void Build(IInputEnumerator enumerator)
{
if (enumerator.HasContexts)
{
throw new ArgumentException("this suggester doesn't support contexts");
}
string prefix = this.GetType().Name;
var directory = OfflineSorter.DefaultTempDir();
var tempInput = FileSupport.CreateTempFile(prefix, ".input", directory);
var tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory);
hasPayloads = enumerator.HasPayloads;
var writer = new OfflineSorter.ByteSequencesWriter(tempInput);
OfflineSorter.ByteSequencesReader reader = null;
var scratch = new BytesRef();
TokenStreamToAutomaton ts2a = GetTokenStreamToAutomaton();
bool success = false;
count = 0;
byte[] buffer = new byte[8];
try
{
var output = new ByteArrayDataOutput(buffer);
BytesRef surfaceForm;
while (enumerator.MoveNext())
{
surfaceForm = enumerator.Current;
ISet <Int32sRef> paths = ToFiniteStrings(surfaceForm, ts2a);
maxAnalyzedPathsForOneInput = Math.Max(maxAnalyzedPathsForOneInput, paths.Count);
foreach (Int32sRef path in paths)
{
Util.Fst.Util.ToBytesRef(path, scratch);
// length of the analyzed text (FST input)
if (scratch.Length > ushort.MaxValue - 2)
{
throw new ArgumentException("cannot handle analyzed forms > " + (ushort.MaxValue - 2) +
" in length (got " + scratch.Length + ")");
}
ushort analyzedLength = (ushort)scratch.Length;
// compute the required length:
// analyzed sequence + weight (4) + surface + analyzedLength (short)
int requiredLength = analyzedLength + 4 + surfaceForm.Length + 2;
BytesRef payload;
if (hasPayloads)
{
if (surfaceForm.Length > (ushort.MaxValue - 2))
{
throw new ArgumentException("cannot handle surface form > " + (ushort.MaxValue - 2) +
" in length (got " + surfaceForm.Length + ")");
}
payload = enumerator.Payload;
// payload + surfaceLength (short)
requiredLength += payload.Length + 2;
}
else
{
payload = null;
}
buffer = ArrayUtil.Grow(buffer, requiredLength);
output.Reset(buffer);
output.WriteInt16((short)analyzedLength);
output.WriteBytes(scratch.Bytes, scratch.Offset, scratch.Length);
output.WriteInt32(EncodeWeight(enumerator.Weight));
if (hasPayloads)
{
for (int i = 0; i < surfaceForm.Length; i++)
{
if (surfaceForm.Bytes[i] == PAYLOAD_SEP)
{
throw new ArgumentException(
"surface form cannot contain unit separator character U+001F; this character is reserved");
}
}
output.WriteInt16((short)surfaceForm.Length);
output.WriteBytes(surfaceForm.Bytes, surfaceForm.Offset, surfaceForm.Length);
output.WriteBytes(payload.Bytes, payload.Offset, payload.Length);
}
else
{
output.WriteBytes(surfaceForm.Bytes, surfaceForm.Offset, surfaceForm.Length);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(output.Position == requiredLength, () => output.Position + " vs " + requiredLength);
}
writer.Write(buffer, 0, output.Position);
}
count++;
}
writer.Dispose();
// Sort all input/output pairs (required by FST.Builder):
(new OfflineSorter(new AnalyzingComparer(hasPayloads))).Sort(tempInput, tempSorted);
// Free disk space:
tempInput.Delete();
reader = new OfflineSorter.ByteSequencesReader(tempSorted);
var outputs = new PairOutputs <long?, BytesRef>(PositiveInt32Outputs.Singleton,
ByteSequenceOutputs.Singleton);
var builder = new Builder <PairOutputs <long?, BytesRef> .Pair>(FST.INPUT_TYPE.BYTE1, outputs);
// Build FST:
BytesRef previousAnalyzed = null;
BytesRef analyzed = new BytesRef();
BytesRef surface = new BytesRef();
Int32sRef scratchInts = new Int32sRef();
var input = new ByteArrayDataInput();
// Used to remove duplicate surface forms (but we
// still index the hightest-weight one). We clear
// this when we see a new analyzed form, so it cannot
// grow unbounded (at most 256 entries):
var seenSurfaceForms = new JCG.HashSet <BytesRef>();
var dedup = 0;
while (reader.Read(scratch))
{
input.Reset(scratch.Bytes, scratch.Offset, scratch.Length);
ushort analyzedLength = (ushort)input.ReadInt16();
analyzed.Grow(analyzedLength + 2);
input.ReadBytes(analyzed.Bytes, 0, analyzedLength);
analyzed.Length = analyzedLength;
long cost = input.ReadInt32();
surface.Bytes = scratch.Bytes;
if (hasPayloads)
{
surface.Length = (ushort)input.ReadInt16();
surface.Offset = input.Position;
}
else
{
surface.Offset = input.Position;
surface.Length = scratch.Length - surface.Offset;
}
if (previousAnalyzed == null)
{
previousAnalyzed = new BytesRef();
previousAnalyzed.CopyBytes(analyzed);
seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface));
}
else if (analyzed.Equals(previousAnalyzed))
{
dedup++;
if (dedup >= maxSurfaceFormsPerAnalyzedForm)
{
// More than maxSurfaceFormsPerAnalyzedForm
// dups: skip the rest:
continue;
}
if (seenSurfaceForms.Contains(surface))
{
continue;
}
seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface));
}
else
{
dedup = 0;
previousAnalyzed.CopyBytes(analyzed);
seenSurfaceForms.Clear();
seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface));
}
// TODO: I think we can avoid the extra 2 bytes when
// there is no dup (dedup==0), but we'd have to fix
// the exactFirst logic ... which would be sort of
// hairy because we'd need to special case the two
// (dup/not dup)...
// NOTE: must be byte 0 so we sort before whatever
// is next
analyzed.Bytes[analyzed.Offset + analyzed.Length] = 0;
analyzed.Bytes[analyzed.Offset + analyzed.Length + 1] = (byte)dedup;
analyzed.Length += 2;
Util.Fst.Util.ToInt32sRef(analyzed, scratchInts);
//System.out.println("ADD: " + scratchInts + " -> " + cost + ": " + surface.utf8ToString());
if (!hasPayloads)
{
builder.Add(scratchInts, outputs.NewPair(cost, BytesRef.DeepCopyOf(surface)));
}
else
{
int payloadOffset = input.Position + surface.Length;
int payloadLength = scratch.Length - payloadOffset;
BytesRef br = new BytesRef(surface.Length + 1 + payloadLength);
Array.Copy(surface.Bytes, surface.Offset, br.Bytes, 0, surface.Length);
br.Bytes[surface.Length] = PAYLOAD_SEP;
Array.Copy(scratch.Bytes, payloadOffset, br.Bytes, surface.Length + 1, payloadLength);
br.Length = br.Bytes.Length;
builder.Add(scratchInts, outputs.NewPair(cost, br));
}
}
fst = builder.Finish();
//Util.dotToFile(fst, "/tmp/suggest.dot");
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(reader, writer);
}
else
{
IOUtils.DisposeWhileHandlingException(reader, writer);
}
tempInput.Delete();
tempSorted.Delete();
}
}
public virtual void TestRandomWithPrefix()
{
Directory dir = NewDirectory();
ISet <string> prefixes = new JCG.HashSet <string>();
int numPrefix = TestUtil.NextInt32(Random, 2, 7);
if (VERBOSE)
{
Console.WriteLine("TEST: use " + numPrefix + " prefixes");
}
while (prefixes.Count < numPrefix)
{
prefixes.Add(TestUtil.RandomRealisticUnicodeString(Random));
//prefixes.Add(TestUtil.RandomSimpleString(random));
}
string[] prefixesArray = prefixes.ToArray(/*new string[prefixes.Count]*/);
int NUM_TERMS = AtLeast(20);
ISet <BytesRef> terms = new JCG.HashSet <BytesRef>();
while (terms.Count < NUM_TERMS)
{
string s = prefixesArray[Random.Next(prefixesArray.Length)] + TestUtil.RandomRealisticUnicodeString(Random);
//final String s = prefixesArray[random.nextInt(prefixesArray.Length)] + TestUtil.RandomSimpleString(random);
if (s.Length > 0)
{
terms.Add(new BytesRef(s));
}
}
BytesRef[] termsArray = terms.ToArray();
Array.Sort(termsArray);
int NUM_DOCS = AtLeast(100);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
// Sometimes swap in codec that impls ord():
if (Random.Next(10) == 7)
{
Codec codec = TestUtil.AlwaysPostingsFormat(PostingsFormat.ForName("Lucene41WithOrds"));
conf.SetCodec(codec);
}
RandomIndexWriter w = new RandomIndexWriter(Random, dir, conf);
int[][] idToOrds = new int[NUM_DOCS][];
ISet <int?> ordsForDocSet = new JCG.HashSet <int?>();
for (int id = 0; id < NUM_DOCS; id++)
{
Document doc = new Document();
doc.Add(new Int32Field("id", id, Field.Store.NO));
int termCount = TestUtil.NextInt32(Random, 0, 20 * RANDOM_MULTIPLIER);
while (ordsForDocSet.Count < termCount)
{
ordsForDocSet.Add(Random.Next(termsArray.Length));
}
int[] ordsForDoc = new int[termCount];
int upto = 0;
if (VERBOSE)
{
Console.WriteLine("TEST: doc id=" + id);
}
foreach (int ord in ordsForDocSet)
{
ordsForDoc[upto++] = ord;
Field field = NewStringField("field", termsArray[ord].Utf8ToString(), Field.Store.NO);
if (VERBOSE)
{
Console.WriteLine(" f=" + termsArray[ord].Utf8ToString());
}
doc.Add(field);
}
ordsForDocSet.Clear();
Array.Sort(ordsForDoc);
idToOrds[id] = ordsForDoc;
w.AddDocument(doc);
}
DirectoryReader r = w.GetReader();
w.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: reader=" + r);
}
AtomicReader slowR = SlowCompositeReaderWrapper.Wrap(r);
foreach (string prefix in prefixesArray)
{
BytesRef prefixRef = prefix == null ? null : new BytesRef(prefix);
int[][] idToOrdsPrefix = new int[NUM_DOCS][];
for (int id = 0; id < NUM_DOCS; id++)
{
int[] docOrds = idToOrds[id];
IList <int?> newOrds = new List <int?>();
foreach (int ord in idToOrds[id])
{
if (StringHelper.StartsWith(termsArray[ord], prefixRef))
{
newOrds.Add(ord);
}
}
int[] newOrdsArray = new int[newOrds.Count];
int upto = 0;
foreach (int ord in newOrds)
{
newOrdsArray[upto++] = ord;
}
idToOrdsPrefix[id] = newOrdsArray;
}
foreach (AtomicReaderContext ctx in r.Leaves)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: sub=" + ctx.Reader);
}
Verify((AtomicReader)ctx.Reader, idToOrdsPrefix, termsArray, prefixRef);
}
// Also test top-level reader: its enum does not support
// ord, so this forces the OrdWrapper to run:
if (VERBOSE)
{
Console.WriteLine("TEST: top reader");
}
Verify(slowR, idToOrdsPrefix, termsArray, prefixRef);
}
FieldCache.DEFAULT.PurgeByCacheKey(slowR.CoreCacheKey);
r.Dispose();
dir.Dispose();
}
public virtual void TestRandom()
{
Directory dir = NewDirectory();
int NUM_TERMS = AtLeast(20);
ISet <BytesRef> terms = new JCG.HashSet <BytesRef>();
while (terms.Count < NUM_TERMS)
{
string s = TestUtil.RandomRealisticUnicodeString(Random);
//final String s = TestUtil.RandomSimpleString(random);
if (s.Length > 0)
{
terms.Add(new BytesRef(s));
}
}
BytesRef[] termsArray = terms.ToArray(/*new BytesRef[terms.Count]*/);
Array.Sort(termsArray);
int NUM_DOCS = AtLeast(100);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
// Sometimes swap in codec that impls ord():
if (Random.Next(10) == 7)
{
// Make sure terms index has ords:
Codec codec = TestUtil.AlwaysPostingsFormat(PostingsFormat.ForName("Lucene41WithOrds"));
conf.SetCodec(codec);
}
RandomIndexWriter w = new RandomIndexWriter(Random, dir, conf);
int[][] idToOrds = new int[NUM_DOCS][];
ISet <int?> ordsForDocSet = new JCG.HashSet <int?>();
for (int id = 0; id < NUM_DOCS; id++)
{
Document doc = new Document();
doc.Add(new Int32Field("id", id, Field.Store.NO));
int termCount = TestUtil.NextInt32(Random, 0, 20 * RandomMultiplier);
while (ordsForDocSet.Count < termCount)
{
ordsForDocSet.Add(Random.Next(termsArray.Length));
}
int[] ordsForDoc = new int[termCount];
int upto = 0;
if (Verbose)
{
Console.WriteLine("TEST: doc id=" + id);
}
foreach (int ord in ordsForDocSet)
{
ordsForDoc[upto++] = ord;
Field field = NewStringField("field", termsArray[ord].Utf8ToString(), Field.Store.NO);
if (Verbose)
{
Console.WriteLine(" f=" + termsArray[ord].Utf8ToString());
}
doc.Add(field);
}
ordsForDocSet.Clear();
Array.Sort(ordsForDoc);
idToOrds[id] = ordsForDoc;
w.AddDocument(doc);
}
DirectoryReader r = w.GetReader();
w.Dispose();
if (Verbose)
{
Console.WriteLine("TEST: reader=" + r);
}
foreach (AtomicReaderContext ctx in r.Leaves)
{
if (Verbose)
{
Console.WriteLine("\nTEST: sub=" + ctx.Reader);
}
Verify((AtomicReader)ctx.Reader, idToOrds, termsArray, null);
}
// Also test top-level reader: its enum does not support
// ord, so this forces the OrdWrapper to run:
if (Verbose)
{
Console.WriteLine("TEST: top reader");
}
AtomicReader slowR = SlowCompositeReaderWrapper.Wrap(r);
Verify(slowR, idToOrds, termsArray, null);
FieldCache.DEFAULT.PurgeByCacheKey(slowR.CoreCacheKey);
r.Dispose();
dir.Dispose();
}