public void TestExternalRefSorter()
{
ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
Check(s);
s.Dispose();
}
public static void Main(string[] args)
{
FileInfo input = new FileInfo("/home/dweiss/tmp/shuffled.dict");
int buckets = 20;
int shareMaxTail = 10;
ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);
TextReader reader =
new StreamReader(
new FileStream(input.FullName, FileMode.Open), Encoding.UTF8);
BytesRef scratch = new BytesRef();
string line;
int count = 0;
while ((line = reader.ReadLine()) != null)
{
scratch.CopyChars(line);
builder.Add(scratch, count % buckets);
if ((count++ % 100000) == 0)
{
Console.WriteLine("Line: " + count);
}
}
Console.WriteLine("Building FSTCompletion.");
FSTCompletion completion = builder.Build();
FileInfo fstFile = new FileInfo("completion.fst");
Console.WriteLine("Done. Writing automaton: " + fstFile.FullName);
completion.FST.Save(fstFile);
sorter.Dispose();
}
// LUCENENET specific - renaming from Main() because we must only have 1 entry point.
// Not sure why this utility is in a test project anyway - this seems like something that should
// be in Lucene.Net.Suggest so we can put it into the lucene-cli tool.
public static void Main2(string[] args)
{
FileInfo input = new FileInfo("/home/dweiss/tmp/shuffled.dict");
int buckets = 20;
int shareMaxTail = 10;
ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);
TextReader reader =
new StreamReader(
new FileStream(input.FullName, FileMode.Open), Encoding.UTF8);
BytesRef scratch = new BytesRef();
string line;
int count = 0;
while ((line = reader.ReadLine()) != null)
{
scratch.CopyChars(line);
builder.Add(scratch, count % buckets);
if ((count++ % 100000) == 0)
{
Console.WriteLine("Line: " + count);
}
}
Console.WriteLine("Building FSTCompletion.");
FSTCompletion completion = builder.Build();
FileInfo fstFile = new FileInfo("completion.fst");
Console.WriteLine("Done. Writing automaton: " + fstFile.FullName);
completion.FST.Save(fstFile);
sorter.Dispose();
}
