private void LoadVocab(string path, NsfwFilter filter)
{
if (_vocabulary != null) return;
if (String.IsNullOrEmpty(path))
{
_vocabulary = new Vocabulary(null);
return;
}
_vocabulary = Processus.Vocabulary.FromDirectory(path, filter);
}
private void LoadVocab(string path, NsfwFilter filter)
{
if (_vocabulary != null)
{
return;
}
if (!String.IsNullOrEmpty(path))
{
_vocabulary = RantVocabulary.FromDirectory(path, filter);
}
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directories and returns a Vocabulary object that contains the loaded dictionaries.
/// </summary>
/// <param name="directories">The directories from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <returns></returns>
public static RantVocabulary FromMultiDirectory(string[] directories, NsfwFilter filter)
{
return(new RantVocabulary(directories.SelectMany(path => Directory.GetFiles(path, "*.dic", SearchOption.AllDirectories)).Select(file => RantDictionary.FromFile(file, filter))));
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directory and returns a Vocabulary object that contains the loaded dictionaries.
/// </summary>
/// <param name="directory">The directory from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <returns></returns>
public static RantVocabulary FromDirectory(string directory, NsfwFilter filter)
{
return(new RantVocabulary(Directory.GetFiles(directory, "*.dic", SearchOption.AllDirectories).Select(file => RantDictionary.FromFile(file, filter)).ToList()));
}
/// <summary>
/// Loads a WordList from the file at the specified path.
/// </summary>
/// <param name="path">The path to the file to load.</param>
/// <param name="nsfwFilter">Specifies whether to allow or disallow NSFW entries.</param>
/// <returns></returns>
public static Dictionary FromFile(string path, NsfwFilter nsfwFilter = NsfwFilter.Disallow)
{
using (var reader = new StreamReader(path))
{
string name = null;
string[] subs = null;
var words = new List<DictionaryEntry>();
bool any = false; // Prevent multiple Any() calls
bool entryBlock = false;
var currentEntries = new List<string>();
var currentClasses = new List<string>();
var currentWeight = 1;
bool nsfw = false;
bool endNsfw = false;
while (!reader.EndOfStream)
{
var entry = ReadEntry(reader);
if (entry == null) continue;
switch (entry.Item1.ToLower())
{
case "nsfw":
nsfw = true;
break;
case "endnsfw":
endNsfw = true;
break;
case "name":
if (name != null)
{
throw new InvalidDataException("Multiple #name declarations in dictionary file.");
}
name = entry.Item2;
break;
case "subtypes":
if (subs != null)
{
throw new InvalidDataException("Multiple subtype declarations in dictionary file.");
}
subs = entry.Item2.Split(new[] {' '}, StringSplitOptions.RemoveEmptyEntries);
break;
case "one":
if (subs != null)
{
throw new InvalidDataException("Multiple subtype declarations in dictionary file.");
}
subs = new[] {"default"};
break;
case "entry":
if (name == null || subs == null)
{
throw new InvalidDataException("Missing name or subtype declarations before entry list.");
}
if (currentEntries.Count > 0)
{
if ((nsfwFilter == NsfwFilter.Disallow && !nsfw) || nsfwFilter == NsfwFilter.Allow)
{
words.Add(new DictionaryEntry(currentEntries.ToArray(), currentClasses.ToArray(),
currentWeight));
}
any = true;
}
else if (any)
{
throw new InvalidDataException("Attempted to add empty dictionary entry.");
}
if (endNsfw)
{
nsfw = false;
endNsfw = false;
}
currentEntries = new List<string>();
currentClasses = new List<string>();
currentWeight = 1;
entryBlock = true;
break;
case "values":
if (!entryBlock)
{
throw new InvalidDataException("Entry values defined before any #entry scope.");
}
currentEntries.AddRange(entry.Item2.Split(',').Select(s => s.Trim()));
break;
case "classes":
if (!entryBlock)
{
throw new InvalidDataException("Entry classes defined before any #entry scope.");
}
currentClasses.AddRange(entry.Item2.Split(',').Select(s => s.Trim().ToLower()));
break;
case "weight":
if (!entryBlock)
{
throw new InvalidDataException("Entry weight defined before any #entry scope.");
}
int weight;
if (!Int32.TryParse(entry.Item2.Trim(), out weight))
{
weight = 1;
}
currentWeight = weight;
break;
default:
continue;
}
}
if (name == null || subs == null)
{
throw new InvalidDataException("Missing name or subtype declarations.");
}
if (currentEntries.Count > 0)
{
words.Add(new DictionaryEntry(currentEntries.ToArray(), currentClasses.ToArray(), currentWeight));
any = true;
}
if (!any)
{
throw new InvalidDataException("No entries in dictionary.");
}
return new Dictionary(name, subs, words.ToArray());
}
}
/// <summary>
/// Creates a new Engine object that loads vocabulary from a path according to the specified filtering option.
/// </summary>
/// <param name="vocabularyPath">The path to the dictionary files to load.</param>
/// <param name="filter">The filtering option to apply when loading the files.</param>
public Engine(string vocabularyPath, NsfwFilter filter)
{
LoadVocab(vocabularyPath, filter);
_vars = new VarStore();
_subs = new SubStore();
}
/// <summary>
/// Creates a new Engine object that loads vocabulary from a path according to the specified filtering option.
/// </summary>
/// <param name="vocabularyPath">The path to the dictionary files to load.</param>
/// <param name="filter">The filtering option to apply when loading the files.</param>
public RantEngine(string vocabularyPath, NsfwFilter filter)
{
LoadVocab(vocabularyPath, filter);
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directory and returns a Vocabulary object that contains the loaded dictionaries.
/// </summary>
/// <param name="directory">The directory from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <returns></returns>
public static Vocabulary FromDirectory(string directory, NsfwFilter filter)
{
return(new Vocabulary(Directory.GetFiles(directory, "*.dic").Select(file => Dictionary.FromFile(file, filter)).ToList()));
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directories and returns a Vocabulary object that contains the loaded dictionaries.
/// </summary>
/// <param name="directories">The directories from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <returns></returns>
public static Vocabulary FromMultiDirectory(string[] directories, NsfwFilter filter)
{
return(new Vocabulary(directories.SelectMany(path => Directory.GetFiles("*.dic")).Select(file => Dictionary.FromFile(file, filter))));
}
/// <summary>
/// Loads a RantDictionary from the file at the specified path.
/// </summary>
/// <param name="path">The path to the file to load.</param>
/// <param name="nsfwFilter">Specifies whether to allow or disallow NSFW entries.</param>
/// <returns></returns>
public static RantDictionaryTable FromFile(string path, NsfwFilter nsfwFilter = NsfwFilter.Disallow)
{
var name = "";
var version = Version;
string[] subtypes = { "default" };
bool header = true;
bool nsfw = false;
var scopedClassSet = new HashSet <string>();
RantDictionaryEntry entry = null;
var entries = new List <RantDictionaryEntry>();
foreach (var token in DicLexer.Tokenize(File.ReadAllText(path)))
{
switch (token.ID)
{
case DicTokenType.Directive:
{
var parts = token.Value.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (!parts.Any())
{
continue;
}
switch (parts[0].ToLower())
{
case "name":
if (!header)
{
LoadError(path, token, "The #name directive may only be used in the file header.");
}
if (parts.Length != 2)
{
LoadError(path, token, "#name directive expected one word:\r\n\r\n" + token.Value);
}
if (!Util.ValidateName(parts[1]))
{
LoadError(path, token, "Invalid #name value: '\{parts[1]}'");
}
name = parts[1].ToLower();
break;
case "subs":
if (!header)
{
LoadError(path, token, "The #subs directive may only be used in the file header.");
}
subtypes = parts.Skip(1).Select(s => s.Trim().ToLower()).ToArray();
break;
case "version":
if (!header)
{
LoadError(path, token, "The #version directive may only be used in the file header.");
}
if (parts.Length != 2)
{
LoadError(path, token, "The #version directive requires a value.");
}
if (!int.TryParse(parts[1], out version))
{
LoadError(path, token, "Invalid version number '\{parts[1]}'");
}
if (version > Version)
{
LoadError(path, token, "Unsupported file version '\{version}'");
}
break;
case "nsfw":
nsfw = true;
break;
case "sfw":
nsfw = false;
break;
case "class":
{
if (parts.Length < 3)
{
LoadError(path, token, "The #class directive expects an operation and at least one value.");
}
switch (parts[1].ToLower())
{
case "add":
foreach (var cl in parts.Skip(2))
{
scopedClassSet.Add(cl.ToLower());
}
break;
case "remove":
foreach (var cl in parts.Skip(2))
{
scopedClassSet.Remove(cl.ToLower());
}
break;
}
}
break;
}
}
break;
case DicTokenType.Entry:
{
if (nsfwFilter == NsfwFilter.Disallow && nsfw)
{
continue;
}
if (Util.IsNullOrWhiteSpace(name))
{
LoadError(path, token, "Missing dictionary name before entry list.");
}
if (Util.IsNullOrWhiteSpace(token.Value))
{
LoadError(path, token, "Encountered empty dictionary entry.");
}
header = false;
entry = new RantDictionaryEntry(token.Value.Split('/').Select(s => s.Trim()).ToArray(), scopedClassSet, nsfw);
entries.Add(entry);
}
break;
case DicTokenType.DiffEntry:
{
if (nsfwFilter == NsfwFilter.Disallow && nsfw)
{
continue;
}
if (Util.IsNullOrWhiteSpace(name))
{
LoadError(path, token, "Missing dictionary name before entry list.");
}
if (Util.IsNullOrWhiteSpace(token.Value))
{
LoadError(path, token, "Encountered empty dictionary entry.");
}
header = false;
string first = null;
entry = new RantDictionaryEntry(token.Value.Split('/')
.Select((s, i) =>
{
if (i > 0)
{
return(Diff.Mark(first, s));
}
return(first = s.Trim());
}).ToArray(), scopedClassSet, nsfw);
entries.Add(entry);
}
break;
case DicTokenType.Property:
{
if (nsfwFilter == NsfwFilter.Disallow && nsfw)
{
continue;
}
var parts = token.Value.Split(new[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries);
if (!parts.Any())
{
LoadError(path, token, "Empty property field.");
}
switch (parts[0].ToLower())
{
case "class":
{
if (parts.Length < 2)
{
continue;
}
foreach (var cl in parts[1].Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
bool opt = cl.EndsWith("?");
entry.AddClass(VocabUtils.GetString(opt ? cl.Substring(0, cl.Length - 1) : cl), opt);
}
}
break;
case "weight":
{
if (parts.Length != 2)
{
LoadError(path, token, "'weight' property expected a value.");
}
int weight;
if (!Int32.TryParse(parts[1], out weight))
{
LoadError(path, token, "Invalid weight value: '" + parts[1] + "'");
}
entry.Weight = weight;
}
break;
case "pron":
{
if (parts.Length != 2)
{
LoadError(path, token, "'" + parts[0] + "' property expected a value.");
}
var pron =
parts[1].Split('/')
.Select(s => s.Trim())
.ToArray();
if (subtypes.Length != pron.Length)
{
LoadError(path, token, "Pronunciation list length must match subtype count.");
}
for (int i = 0; i < entry.Terms.Length; i++)
{
entry.Terms[i].Pronunciation = pron[i];
}
}
break;
}
}
break;
}
}
return(new RantDictionaryTable(name, subtypes, entries));
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directory and returns a Vocabulary object that contains the loaded dictionaries.
/// </summary>
/// <param name="directory">The directory from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <returns></returns>
public static Vocabulary FromDirectory(string directory, NsfwFilter filter)
{
return new Vocabulary(Directory.GetFiles(directory, "*.dic").Select(file => Dictionary.FromFile(file, filter)).ToList());
}
/// <summary>
/// Loads a WordList from the file at the specified path.
/// </summary>
/// <param name="path">The path to the file to load.</param>
/// <param name="nsfwFilter">Specifies whether to allow or disallow NSFW entries.</param>
/// <returns></returns>
public static Dictionary FromFile(string path, NsfwFilter nsfwFilter = NsfwFilter.Disallow)
{
using (var reader = new StreamReader(path))
{
string name = null;
string[] subs = null;
var words = new List <DictionaryEntry>();
bool any = false; // Prevent multiple Any() calls
bool entryBlock = false;
var currentEntries = new List <string>();
var currentClasses = new List <string>();
var currentWeight = 1;
bool nsfw = false;
bool endNsfw = false;
while (!reader.EndOfStream)
{
var entry = ReadEntry(reader);
if (entry == null)
{
continue;
}
switch (entry.Item1.ToLower())
{
case "nsfw":
nsfw = true;
break;
case "endnsfw":
endNsfw = true;
break;
case "name":
if (name != null)
{
throw new InvalidDataException("Multiple #name declarations in dictionary file.");
}
name = entry.Item2;
break;
case "subtypes":
if (subs != null)
{
throw new InvalidDataException("Multiple subtype declarations in dictionary file.");
}
subs = entry.Item2.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
break;
case "one":
if (subs != null)
{
throw new InvalidDataException("Multiple subtype declarations in dictionary file.");
}
subs = new[] { "default" };
break;
case "entry":
if (name == null || subs == null)
{
throw new InvalidDataException("Missing name or subtype declarations before entry list.");
}
if (currentEntries.Count > 0)
{
if ((nsfwFilter == NsfwFilter.Disallow && !nsfw) || nsfwFilter == NsfwFilter.Allow)
{
words.Add(new DictionaryEntry(currentEntries.ToArray(), currentClasses.ToArray(),
currentWeight));
}
any = true;
}
else if (any)
{
throw new InvalidDataException("Attempted to add empty dictionary entry.");
}
if (endNsfw)
{
nsfw = false;
endNsfw = false;
}
currentEntries = new List <string>();
currentClasses = new List <string>();
currentWeight = 1;
entryBlock = true;
break;
case "values":
if (!entryBlock)
{
throw new InvalidDataException("Entry values defined before any #entry scope.");
}
currentEntries.AddRange(entry.Item2.Split(',').Select(s => s.Trim()));
break;
case "classes":
if (!entryBlock)
{
throw new InvalidDataException("Entry classes defined before any #entry scope.");
}
currentClasses.AddRange(entry.Item2.Split(',').Select(s => s.Trim().ToLower()));
break;
case "weight":
if (!entryBlock)
{
throw new InvalidDataException("Entry weight defined before any #entry scope.");
}
int weight;
if (!Int32.TryParse(entry.Item2.Trim(), out weight))
{
weight = 1;
}
currentWeight = weight;
break;
default:
continue;
}
}
if (name == null || subs == null)
{
throw new InvalidDataException("Missing name or subtype declarations.");
}
if (currentEntries.Count > 0)
{
words.Add(new DictionaryEntry(currentEntries.ToArray(), currentClasses.ToArray(), currentWeight));
any = true;
}
if (!any)
{
throw new InvalidDataException("No entries in dictionary.");
}
return(new Dictionary(name, subs, words.ToArray()));
}
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directory and returns a Dictionary object that contains the loaded data.
/// </summary>
/// <param name="directory">The directory from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <param name="mergeBehavior">The merging strategy to employ.</param>
/// <returns></returns>
public static RantDictionary FromDirectory(string directory, NsfwFilter filter, TableMergeBehavior mergeBehavior = TableMergeBehavior.Naive)
{
return(new RantDictionary(Directory.GetFiles(directory, "*.dic", SearchOption.AllDirectories).Select(file => RantDictionaryTable.FromFile(file, filter)).ToList(), mergeBehavior));
}
/// <summary>
/// Loads all dictionary (.dic) files from the specified directories and returns a Dictionary object that contains the loaded data.
/// </summary>
/// <param name="directories">The directories from which to load dictionaries.</param>
/// <param name="filter">Indicates whether dictionary entries marked with the #nsfw flag should be loaded.</param>
/// <param name="mergeBehavior">The merging strategy to employ.</param>
/// <returns></returns>
public static RantDictionary FromMultiDirectory(string[] directories, NsfwFilter filter, TableMergeBehavior mergeBehavior)
{
return(new RantDictionary(directories.SelectMany(path => Directory.GetFiles(path, "*.dic", SearchOption.AllDirectories)).Select(file => RantDictionaryTable.FromFile(file, filter)), mergeBehavior));
}