protected internal virtual void loadDictionary(InputStream inputStream, bool isFillerDict)
{
InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
string text;
while ((text = bufferedReader.readLine()) != null)
{
text = String.instancehelper_trim(text);
if (!String.instancehelper_isEmpty(text))
{
int spaceIndex = this.getSpaceIndex(text);
if (spaceIndex < 0)
{
string text2 = new StringBuilder().append("Error loading word: ").append(text).toString();
throw new Error(text2);
}
string text3 = String.instancehelper_substring(text, 0, spaceIndex);
if (this.dictionary.containsKey(text3))
{
int num = 2;
string text5;
do
{
string text4 = "%s(%d)";
object[] array = new object[2];
array[0] = text3;
int num2 = 1;
int num3 = num;
num++;
array[num2] = Integer.valueOf(num3);
text5 = String.format(text4, array);
}while (this.dictionary.containsKey(text5));
text3 = text5;
}
if (isFillerDict)
{
this.dictionary.put(text3, new StringBuilder().append("-F-").append(text).toString());
this.fillerWords.add(text3);
}
else
{
this.dictionary.put(text3, text);
}
}
}
bufferedReader.close();
inputStreamReader.close();
inputStream.close();
}
protected internal virtual void loadMapping(InputStream inputStream)
{
InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
string text;
while ((text = bufferedReader.readLine()) != null)
{
StringTokenizer stringTokenizer = new StringTokenizer(text);
if (stringTokenizer.countTokens() != 2)
{
string text2 = "Wrong file format";
throw new IOException(text2);
}
this.mapping.put(stringTokenizer.nextToken(), stringTokenizer.nextToken());
}
bufferedReader.close();
inputStreamReader.close();
inputStream.close();
}
public override void run(string[] args)
{
Params @params = validateAndParseParams(args, typeof(Params));
File dictInFile = @params.InputFile;
File dictOutFile = @params.OutputFile;
Charset encoding = @params.Encoding;
CmdLineUtil.checkInputFile("dictionary input file", dictInFile);
CmdLineUtil.checkOutputFile("dictionary output file", dictOutFile);
InputStreamReader @in = null;
OutputStream @out = null;
try
{
@in = new InputStreamReader(new FileInputStream(dictInFile), encoding);
@out = new FileOutputStream(dictOutFile);
Dictionary dict = Dictionary.parseOneEntryPerLine(@in);
dict.serialize(@out);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
@in.close();
@out.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
}
