public static DictRadix<int> BuildPrefixTree(bool allowHeHasheela)
{
string[] prefixes;
int[] masks;
if (allowHeHasheela)
{
prefixes = Constants.prefixes_H;
masks = Constants.masks_H;
}
else
{
prefixes = Constants.prefixes_noH;
masks = Constants.masks_noH;
}
DictRadix<int> ret = new DictRadix<int>();
for (int i = 0; prefixes[i] != null; i++)
ret.AddNode(prefixes[i], masks[i]);
return ret;
}
static void AddAndIncrement <T>(DictRadix <T> d, string key, T obj, ref int counter)
{
// Only increment counter if the key doesn't already
bool hasKey = true;
if (object.Equals(d.Lookup(key), default(T)))
{
counter++;
hasKey = false;
}
d.AddNode(key, obj);
Assert.Equal(counter, d.Count);
// Only check insertion if there was one
if (d.AllowValueOverride || !hasKey)
{
Assert.Equal(d.Lookup(key), obj);
}
}
public static DictRadix <int> BuildPrefixTree(bool allowHeHasheela)
{
string[] prefixes;
int[] masks;
if (allowHeHasheela)
{
prefixes = Constants.prefixes_H;
masks = Constants.masks_H;
}
else
{
prefixes = Constants.prefixes_noH;
masks = Constants.masks_noH;
}
DictRadix <int> ret = new DictRadix <int>();
for (int i = 0; prefixes[i] != null; i++)
{
ret.AddNode(prefixes[i], masks[i]);
}
return(ret);
}
public static DictRadix<MorphData> LoadDictionaryFromHSpellFolder(string path, bool bLoadMorphData)
{
if (path[path.Length - 1] != Path.DirectorySeparatorChar)
path += Path.DirectorySeparatorChar;
if (bLoadMorphData)
{
// Load the count of morphological data slots required
string sizesFile = File.ReadAllText(path + HSpell.Constants.SizesFile);
int lookupLen = sizesFile.IndexOf(' ', sizesFile.IndexOf('\n'));
lookupLen = Convert.ToInt32(sizesFile.Substring(lookupLen + 1));
string[] lookup = new string[lookupLen + 1];
using (GZipStream fdict = new GZipStream(File.OpenRead(path + HSpell.Constants.DictionaryFile), CompressionMode.Decompress))
{
char[] sbuf = new char[HSpell.Constants.MaxWordLength];
int c = 0, n, slen = 0, i = 0;
while ((c = fdict.ReadByte()) > -1)
{
if (c >= '0' && c <= '9') // No conversion required for chars < 0xBE
{
/* new word - finalize and save old word */
lookup[i++] = new string(sbuf, 0, slen);
/* and read how much to go back */
n = 0;
do
{
/* base 10... */
n *= 10;
n += (c - '0');
} while ((c = fdict.ReadByte()) > -1 && c >= '0' && c <= '9');
slen -= n;
}
sbuf[slen++] = ISO8859_To_Unicode(c);
}
}
using (MorphDataLoader dataLoader = new MorphDataLoader(path + HSpell.Constants.DescFile,
path + HSpell.Constants.StemsFile))
{
using (GZipStream fprefixes = new GZipStream(File.OpenRead(path + HSpell.Constants.PrefixesFile), CompressionMode.Decompress))
{
DictRadix<MorphData> ret = new DictRadix<MorphData>();
for (int i = 0; lookup[i] != null; i++)
{
MorphData data = new MorphData();
data.Prefixes = Convert.ToByte(fprefixes.ReadByte()); // Read prefix hint byte
data.DescFlags = dataLoader.ReadDescFile();
List<int> stemReferences = dataLoader.ReadStemFile();
data.Lemmas = new string[stemReferences.Count];
int stemPosition = 0;
foreach (int r in stemReferences)
{
// This is a bypass for the psuedo-stem "שונות", as defined by hspell
// TODO: Try looking into changing this in hspell itself
if (lookup[r].Equals("שונות") && !lookup[r].Equals(lookup[i]))
{
data.Lemmas[stemPosition++] = null;
}
else
{
data.Lemmas[stemPosition++] = lookup[r];
}
}
ret.AddNode(lookup[i], data);
}
return ret;
}
}
}
else // Use optimized version for loading HSpell's dictionary files
{
using (GZipStream fdict = new GZipStream(File.OpenRead(path + HSpell.Constants.DictionaryFile), CompressionMode.Decompress))
{
using (GZipStream fprefixes = new GZipStream(File.OpenRead(path + HSpell.Constants.PrefixesFile), CompressionMode.Decompress))
{
DictRadix<MorphData> ret = new DictRadix<MorphData>();
char[] sbuf = new char[HSpell.Constants.MaxWordLength];
int c = 0, n, slen = 0;
while ((c = fdict.ReadByte()) > -1)
{
if (c >= '0' && c <= '9') // No conversion required for chars < 0xBE
{
/* new word - finalize old word first (set value) */
sbuf[slen] = '\0';
// TODO: Avoid creating new MorphData object, and enhance DictRadix to store
// the prefixes mask in the node itself
MorphData data = new MorphData();
data.Prefixes = Convert.ToByte(fprefixes.ReadByte()); // Read prefix hint byte
ret.AddNode(sbuf, data);
/* and read how much to go back */
n = 0;
do
{
/* base 10... */
n *= 10;
n += (c - '0');
} while ((c = fdict.ReadByte()) > -1 && c >= '0' && c <= '9');
slen -= n;
}
sbuf[slen++] = ISO8859_To_Unicode(c);
}
return ret;
}
}
}
}
public static DictRadix <MorphData> LoadDictionaryFromHSpellFolder(string path, bool bLoadMorphData)
{
if (path[path.Length - 1] != Path.DirectorySeparatorChar)
{
path += Path.DirectorySeparatorChar;
}
if (bLoadMorphData)
{
// Load the count of morphological data slots required
int lookupLen = GetWordCountInHSpellFolder(path);
var lookup = new string[lookupLen + 1];
using (GZipStream fdict = new GZipStream(File.OpenRead(path + Constants.DictionaryFile), CompressionMode.Decompress))
{
var sbuf = new char[Constants.MaxWordLength];
int c = 0, n, slen = 0, i = 0;
while ((c = fdict.ReadByte()) > -1)
{
if (c >= '0' && c <= '9') // No conversion required for chars < 0xBE
{
/* new word - finalize and save old word */
lookup[i++] = new string(sbuf, 0, slen);
/* and read how much to go back */
n = 0;
do
{
/* base 10... */
n *= 10;
n += (c - '0');
} while ((c = fdict.ReadByte()) > -1 && c >= '0' && c <= '9');
slen -= n;
}
sbuf[slen++] = ISO8859_To_Unicode(c);
}
}
using (var dataLoader = new MorphDataLoader(path + HSpell.Constants.DescFile, path + Constants.StemsFile))
using (var fprefixes = new GZipStream(File.OpenRead(path + HSpell.Constants.PrefixesFile), CompressionMode.Decompress))
{
DictRadix <MorphData> ret = new DictRadix <MorphData>();
for (int i = 0; lookup[i] != null; i++)
{
MorphData data = new MorphData();
data.Prefixes = Convert.ToByte(fprefixes.ReadByte()); // Read prefix hint byte
data.DescFlags = dataLoader.ReadDescFile();
var stemReferences = dataLoader.ReadStemFile();
data.Lemmas = new string[stemReferences.Count];
int stemPosition = 0;
foreach (int r in stemReferences)
{
// This is a bypass for the psuedo-stem "שונות", as defined by hspell
// TODO: Try looking into changing this in hspell itself
if (lookup[r].Equals("שונות") && !lookup[r].Equals(lookup[i]))
{
data.Lemmas[stemPosition++] = null;
}
else
{
data.Lemmas[stemPosition++] = lookup[r];
}
}
ret.AddNode(lookup[i], data);
}
return(ret);
}
}
else // Use optimized version for loading HSpell's dictionary files
{
using (var fdict = new GZipStream(File.OpenRead(path + HSpell.Constants.DictionaryFile), CompressionMode.Decompress))
using (var fprefixes = new GZipStream(File.OpenRead(path + HSpell.Constants.PrefixesFile), CompressionMode.Decompress))
{
var ret = new DictRadix <MorphData>();
var sbuf = new char[HSpell.Constants.MaxWordLength];
int c = 0, n, slen = 0;
while ((c = fdict.ReadByte()) > -1)
{
if (c >= '0' && c <= '9') // No conversion required for chars < 0xBE
{
/* new word - finalize old word first (set value) */
sbuf[slen] = '\0';
// TODO: Avoid creating new MorphData object, and enhance DictRadix to store
// the prefixes mask in the node itself
MorphData data = new MorphData();
data.Prefixes = Convert.ToByte(fprefixes.ReadByte()); // Read prefix hint byte
ret.AddNode(sbuf, data);
/* and read how much to go back */
n = 0;
do
{
/* base 10... */
n *= 10;
n += (c - '0');
} while ((c = fdict.ReadByte()) > -1 && c >= '0' && c <= '9');
slen -= n;
}
sbuf[slen++] = ISO8859_To_Unicode(c);
}
return(ret);
}
}
}
private void btnTestRadix_Click(object sender, EventArgs e)
{
DictRadix<object> r = new DictRadix<object>(); ;
r.AddNode("abcdef", 5);
r.AddNode("ab", 11);
r.AddNode("abcd", 115);
r.AddNode("aaa", 41);
r.AddNode("abc", 111);
r.AddNode("a", 101);
r.AddNode("bba", 22);
r.AddNode("bbc", 22);
r.AddNode("bb", 221);
r.AddNode("def", 22);
r.AddNode("deg", 33);
r.AddNode("deg", 33);
r.AddNode("cfg", 3222);
DictRadix<object>.RadixEnumerator en = r.GetEnumerator() as DictRadix<object>.RadixEnumerator;
while (en.MoveNext())
{
System.Diagnostics.Trace.WriteLine(string.Format("{0} {1}", en.CurrentKey, en.Current.ToString()));
}
}
private void btnTestRadix_Click(object sender, EventArgs e)
{
DictRadix <object> r = new DictRadix <object>();;
r.AddNode("abcdef", 5);
r.AddNode("ab", 11);
r.AddNode("abcd", 115);
r.AddNode("aaa", 41);
r.AddNode("abc", 111);
r.AddNode("a", 101);
r.AddNode("bba", 22);
r.AddNode("bbc", 22);
r.AddNode("bb", 221);
r.AddNode("def", 22);
r.AddNode("deg", 33);
r.AddNode("deg", 33);
r.AddNode("cfg", 3222);
DictRadix <object> .RadixEnumerator en = r.GetEnumerator() as DictRadix <object> .RadixEnumerator;
while (en.MoveNext())
{
System.Diagnostics.Trace.WriteLine(string.Format("{0} {1}", en.CurrentKey, en.Current.ToString()));
}
}
