public PrefixMap(string fPath) : this(ByLineFileIO.ReadFileNoWhitespace(fPath, zuio.GetEncUde(fPath)).ToArray())
{
}
static void Main(string[] args)
{
StopWatch w = new StopWatch();
List <string> mobypos;
SortedSet <string> scowl35 = new SortedSet <string>();
w.Click();
Console.WriteLine("Reading files...");
mobypos = ByLineFileIO.readFileNoWhitespace("mobyposi.i");
foreach (var s in ByLineFileIO.readFileNoWhitespace("english-words.35"))
{
scowl35.Add(s);
}
Console.WriteLine("Finished! Took {0:F} ms.", w.Click());
Console.WriteLine("Now parsing...");
Dictionary <string, string> mobyParsed = new Dictionary <string, string>();
foreach (string s in mobypos)
{
try
{
var pts = zusp.CutFirst(s, "*");
if (!mobyParsed.ContainsKey(pts.First))
{
mobyParsed.Add(pts.First, pts.Second);
}
else
{
mobyParsed[pts.First] += pts.Second;
}
}
catch (Exception e) //I've had enough with encoding
{ continue; }
}
Console.WriteLine("Finished! Took {0:F} ms.", w.Click());
Console.WriteLine("Filtering the list with SCOWL and remove uncommon words...");
Dictionary <string, string> filtered = new Dictionary <string, string>();
foreach (string s in mobyParsed.Keys)
{
if (scowl35.Contains(s))
{
filtered.Add(s, mobyParsed[s]);
}
}
Console.WriteLine("Finished! Took {0:F} ms.", w.Click());
Console.WriteLine("Filtered list contains {0:D} words with part-of-speech.", filtered.Count);
Console.WriteLine("Generating part-of-speech lists...");
Dictionary <string, List <string> > posLists = new Dictionary <string, List <string> >();
posLists.Add("noun", new List <string>());
posLists.Add("verb", new List <string>());
posLists.Add("adjective", new List <string>());
posLists.Add("adverb", new List <string>());
foreach (var s in filtered.Keys)
{
foreach (char c in filtered[s].ToCharArray())
{
if (c == 'N' || c == 'r')
{
posLists["noun"].Add(s);
}
if (c == 'V' || c == 't' || c == 'i')
{
posLists["verb"].Add(s);
}
if (c == 'A')
{
posLists["adjective"].Add(s);
}
if (c == 'v')
{
posLists["adverb"].Add(s);
}
}
}
Console.WriteLine("Finished! Took {0:F} ms.", w.Click());
Console.WriteLine("Writing lists into files...");
foreach (var s in posLists.Keys)
{
Console.WriteLine("There were {0:D} in list {1}", posLists[s].Count, s);
ByLineFileIO.writeFile(posLists[s], s + ".txt");
}
Console.WriteLine("Finished! Took {0:F} ms. The end :)", w.Click());
Console.ReadLine();
}
/// <summary>
/// Creates a ust from (absolute or relative) path given.
/// Note: Emperically, we found 8kb to be as good as whole file.
/// </summary>
/// <param name="fPath"></param>
public USTFile(string fPath) //
: this(ByLineFileIO
.ReadFileNoWhitespace(fPath, zuio.GetEncUde(fPath, 8192, Encoding.GetEncoding("Shift_JIS"))).ToArray())
{
}
/// <summary>
/// Load lists from a word set.
/// </summary>
/// <param name="SetName"></param>
private void LoadLists(string SetName)
{
Lists = new Dictionary <string, SortedSet <string> >();
foreach (var s in Directory.GetFiles(Path.Combine(Directory.GetCurrentDirectory(), "words", SetName)))
{
Lists.Add(Path.GetFileNameWithoutExtension(s), new SortedSet <string>(ByLineFileIO.readFileNoWhitespace(s)));
}
}
/// <summary>
/// Creates a ust from (absolute or relative) path given.
/// </summary>
/// <param name="fPath"></param>
public USTFile(string fPath) // For UDE, we found 8kb to be as good as whole file emperically.
: this(ByLineFileIO.ReadFileNoWhitespace(fPath, zuio.GetEncUde(fPath, 8192, Encoding.GetEncoding("Shift_JIS"))).ToArray())
{
}
public GeneratorTemplate(string file)
: this(ByLineFileIO.readFileVerbatim(file))
{
}
