/// <summary>
/// Creates a new job
/// </summary>
/// <param name="jobBaseDirectory"></param>
/// <param name="language"></param>
/// <param name="targetExtension"></param>
/// <param name="outputFolder"></param>
public NoccoJob(DirectoryInfo jobBaseDirectory, LanguageConfig language, string targetExtension, DirectoryInfo outputFolder)
{
if (jobBaseDirectory == null || !jobBaseDirectory.Exists)
throw new ArgumentException("JobBaseDirectory is not a valid directory");
if (language == null)
throw new ArgumentException("Language is not valid");
if (string.IsNullOrWhiteSpace(targetExtension))
throw new ArgumentException("TargetExtension is not valid");
this.JobBaseDirectory = jobBaseDirectory;
this.Language = language;
this.TargetExtension = targetExtension;
this.OuputFolder = outputFolder;
}
/// <summary>
/// Create separated and formatted sections for comments and code for use in documenation file
/// </summary>
/// <param name="source"></param>
/// <param name="language"></param>
/// <returns></returns>
private static IEnumerable<Section> ParseSections(FileInfo source, LanguageConfig language, DocumentSummary summary)
{
var sections = new List<Section>();
var hasCode = false;
var docsText = new StringBuilder();
var codeText = new StringBuilder();
bool OkayToReplaceSection = true;
//generates a section of comment and code
Func<StringBuilder, StringBuilder, Section> GenerateSection = (docs, code) =>
{
var docsString = docs.ToString();
//highlight comments if required
if (language.MarkdownMaps != null)
{
docsString = language.MarkdownMaps.Aggregate(docsString,
(currentDocs, map) =>
Regex.Replace(currentDocs, map.FindPattern, map.Replacement, RegexOptions.Multiline)
);
}
var ret = new Section
{
DocsHtml = MarkdownFormatter.Transform(docsString),
CodeHtml = System.Web.HttpUtility.HtmlEncode(code.ToString())
};
//set the top section for the summary, just in case, but ideally we want the second block
//or a block after the first which has both sections
if (summary.TopSection == null)
{
//easy winner
summary.TopSection = ret;
}
else if (OkayToReplaceSection)
{
summary.TopSection = ret;
OkayToReplaceSection = false;
}
else
{
if (string.IsNullOrWhiteSpace(summary.TopSection.CodeHtml) ||
string.IsNullOrWhiteSpace(summary.TopSection.DocsHtml))
{
if (!string.IsNullOrWhiteSpace(ret.CodeHtml) &&
!string.IsNullOrWhiteSpace(ret.DocsHtml))
{
//winner by content superiority
summary.TopSection = ret;
}
}
}
return ret;
};
foreach (var result in Parser.Process(source, language.CommentDefinitions))
{
//if this line matches a comment line
if (result.ResultType == ResultType.Comment)
{
//if we hit this comment line after already processing code, we need to make a new section
if (hasCode)
{
yield return GenerateSection(docsText, codeText);
hasCode = false;
docsText = new StringBuilder();
codeText = new StringBuilder();
}
//update the summary
summary.LinesOfComment++;
docsText.AppendLine(result.MatchingDefinition.CleanComment(result.Result));
}
else //hit code or unknown line
{
//update the summary
summary.LinesOfCode++;
hasCode = true;
codeText.AppendLine(result.Result);
}
}
yield return GenerateSection(docsText, codeText);
}
public static Lexer Build(LanguageConfig languageConfig)
{
return(new Lexer(languageConfig));
}
public TokenizersSource(LanguageConfig languageConfig)
{
// TODO: tests
var tokenizers = new List <ITokenizer>();
foreach (var type in Assembly.GetExecutingAssembly().DefinedTypes)
{
if (type.IsInterface || type.IsAbstract)
{
continue;
}
if (!typeof(ITokenizer).IsAssignableFrom(type))
{
continue;
}
const string suffix = "Tokenizer";
var index = type.Name.LastIndexOf(suffix, StringComparison.Ordinal);
if (index == -1)
{
throw new HandyQueryInitializationException(
$"Tokenizer {type.FullName} implements ITokenizer, but it does not follow naming " +
$"convention. Each tokenizer should end with \'{suffix}\' suffix.");
}
if (type.GetCustomAttribute <TokenizerAttribute>() == null)
{
throw new HandyQueryInitializationException(
$"Tokenizer {type.FullName} implements ITokenizer, but it does not follow attribute " +
$"convention. Each tokenizer should define TokenizerAttribute .");
}
var tokenizer = (ITokenizer)Activator.CreateInstance(type, languageConfig);
tokenizers.Add(tokenizer);
}
var waitingTokenizers = new List <ITokenizer>();
foreach (var tokenizer in tokenizers)
{
EvaluateWaiting();
var tokenizerAttribute = GetTokenizerAttribute(tokenizer);
if (tokenizerAttribute.ManualUsage)
{
continue;
}
var after = tokenizerAttribute.AfterTokenizer;
if (after == null || _orderedTokenizers.Any(x => x.GetType() == after))
{
_orderedTokenizers.Add(tokenizer);
continue;
}
waitingTokenizers.Add(tokenizer);
}
EvaluateWaiting();
if (waitingTokenizers.Any())
{
throw new HandyQueryInitializationException(
"Unable to create tokenizers in proper order. Inifnite loop via TokenizerAttribute.AfterTokenizer?");
}
void EvaluateWaiting()
{
var evaluatedTokenizers = new List <ITokenizer>();
foreach (var tokenizer in waitingTokenizers)
{
var after = GetTokenizerAttribute(tokenizer).AfterTokenizer;
if (_orderedTokenizers.Any(x => x.GetType() == after))
{
_orderedTokenizers.Add(tokenizer);
evaluatedTokenizers.Add(tokenizer);
}
}
foreach (var tokenizer in evaluatedTokenizers)
{
waitingTokenizers.Remove(tokenizer);
}
}
TokenizerAttribute GetTokenizerAttribute(ITokenizer tokenizer)
{
return(tokenizer.GetType().GetCustomAttribute <TokenizerAttribute>());
}
}
private void btCorrectIt_Click(object sender, EventArgs e)
{
if (cbMethod.SelectedItem.ToString() == "-- Choose method --")
{
MessageBox.Show("Choose method first", "", MessageBoxButtons.OK);
return;
}
if (!File.Exists(txOCR.Text.Trim()))
{
MessageBox.Show("Browse file first", "", MessageBoxButtons.OK);
return;
}
ResetControls(false);
articleFile = txOCR.Text.Trim();
Correction correction = new Correction();
Stemmer stemmer = new Stemmer();
// DeHyphenate and clean text:
string dehyphenatedText = correction.DeHyphenate(articleFile);
rtbOCR.Text = dehyphenatedText;
// for analysis:
string dehyphenatedTextGT = "";
if (File.Exists(articleFile.Substring(0, articleFile.Length - 4) + "GT.txt"))
{
articleFileGT = articleFile.Substring(0, articleFile.Length - 4) + "GT.txt";
}
articleFileName = Path.GetFileName(articleFile);
if (!string.IsNullOrEmpty(articleFileGT))
{
dehyphenatedTextGT = correction.DeHyphenate(articleFileGT);
}
// tokenize:
deHyphenateTokens = correction.GetTokensFromText(dehyphenatedText);
Regex rgx = new Regex("[^a-zA-Z]"); //omit all non alphabet word And clean word from non alphabet:
// for analysis:
Dictionary <int, string> deHyphenateTokensGT = new Dictionary <int, string>();
if (!string.IsNullOrEmpty(articleFileGT))
{
deHyphenateTokensGT = correction.GetTokensFromText(dehyphenatedTextGT);
foreach (KeyValuePair <int, string> token in deHyphenateTokens)
{
correction.InsertOCRAndTruth(articleFileName, token.Key, rgx.Replace(token.Value, ""), rgx.Replace(deHyphenateTokensGT[token.Key], ""));
}
}
// Omit non character,single char, All Capitals word, and clean word from non alphabet:
var tmp = deHyphenateTokens.Where(p => p.Value.Length > 1).ToDictionary(p => p.Key, p => p.Value);
tmp = tmp.Where(p => p.Value.Any(Char.IsLetter)).ToDictionary(p => p.Key, p => rgx.Replace(p.Value, ""));
Dictionary <int, string> cleanTokens = tmp.Where(p => !p.Value.All(Char.IsUpper)).ToDictionary(p => p.Key, p => p.Value);
// Find Suggestion:
if (cbMethod.SelectedItem.ToString().EndsWith("Hunspell"))
{
string hunspellLog = "";
// find Suggestion using Hunspell:
foreach (KeyValuePair <int, string> err in cleanTokens)
{
string errInNewSpell = correction.ChangeOldToNewSpell(err.Value).ToLowerInvariant();
List <string> hunspellSuggestions = new List <string>();
using (SpellEngine engine = new SpellEngine())
{
LanguageConfig idConfig = new LanguageConfig();
idConfig.LanguageCode = "id";
idConfig.HunspellAffFile = "id_ID.aff";
idConfig.HunspellDictFile = "id_ID.dic";
idConfig.HunspellKey = "";
engine.AddLanguage(idConfig);
bool correct = engine["id"].Spell(errInNewSpell);
if (!correct)
{
hunspellSuggestions = engine["id"].Suggest(errInNewSpell);
if (hunspellSuggestions.Count > 0 && err.Value != correction.ChangeNewToOldSpell(hunspellSuggestions[0]))
{
deHyphenateTokens[err.Key] = "[" + correction.ChangeNewToOldSpell(hunspellSuggestions[0]) + "]";
}
// for analysis:
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, err.Key, new Dictionary <string, string> {
{ getFieldNameFromOption(), rgx.Replace(deHyphenateTokens[err.Key], "") }, { getFieldNameFromOption().Replace("Correction", "Log"), hunspellLog }
});
}
}
else
{
// for analysis:
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, err.Key, new Dictionary <string, string> {
{ getFieldNameFromOption(), err.Value }, { getFieldNameFromOption().Replace("Correction", "Log"), err.Value + " is correct" }
});
}
}
}
}
ResetControls(true);
return;
}
//check only unique word (assumption:duplicate word is correct word) :
Dictionary <int, string> checkTokens = cleanTokens;
var duplicateValues = checkTokens.GroupBy(x => x.Value).Where(x => x.Count() > 1);
List <int> duplicateKeys = new List <int>();
foreach (var item in checkTokens)
{
foreach (var dup in duplicateValues)
{
if (item.Value == dup.Key)
{
duplicateKeys.Add(item.Key);
}
}
}
foreach (var dupkey in duplicateKeys)
{
// for analysis
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, dupkey, new Dictionary <string, string> {
{ "NCorrection", checkTokens[dupkey] }, { "NLog", "Duplicate" }, { "Correction", checkTokens[dupkey] }, { "Log", "Duplicate" }, { "WOSearchCorrection", checkTokens[dupkey] }, { "WOSearchLog", "Duplicate" }, { "WOStemCorrection", checkTokens[dupkey] }, { "WOStemLog", "Duplicate" }, { "WOStemSearchCorrection", checkTokens[dupkey] }, { "WOStemSearchLog", "Duplicate" }, { "GooglePureCorrection", checkTokens[dupkey] }, { "GooglePureLog", "Duplicate" }
});
}
checkTokens.Remove(dupkey);
}
//Check Word using Dictionary(kbbi+kompas pilihan, entitas kota,negara, nama pahlawan dari wiki ):
errors = new Dictionary <int, string>();
foreach (KeyValuePair <int, string> token in checkTokens)
{
// change Soewandi to Modern Spelling:
string wordToCheck = correction.ChangeOldToNewSpell(token.Value).ToLowerInvariant();
// check word in Dictionary and Add to Error list if not there:
int frequency;
if (!correction.CheckUnigram(wordToCheck, getSQLQueryToCheckUnigram(), out frequency))
{
if (cbMethod.SelectedItem.ToString().Contains("Stemmer"))
{
// check again its stem in dictionary :
string stem = stemmer.Stemming(wordToCheck);
if (wordToCheck != stem && stemmer.checkStem(stem))
{
// for analysis
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, token.Key, new Dictionary <string, string> {
{ getFieldNameFromOption(), token.Value }, { getFieldNameFromOption().Replace("Correction", "Log"), stem + " is word" }
});
}
}
else // jika tidak ada di kamus:
{
errors.Add(token.Key, wordToCheck);
}
}
else
{
errors.Add(token.Key, wordToCheck);
}
}
else // jika ada di kamus:
{
// for analysis
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, token.Key, new Dictionary <string, string> {
{ getFieldNameFromOption(), token.Value }, { getFieldNameFromOption().Replace("Correction", "Log"), wordToCheck + " is correct" }
});
}
}
}
// Find Suggestion:
if (cbMethod.SelectedItem.ToString().EndsWith("Google"))
{
timerGoogle.Enabled = true;
indexTimerGoogle = 0;
return;
}
else
{
foreach (KeyValuePair <int, string> err in errors)
{
//get suggestion:
string log; string suggestion;
suggestion = correction.GetSuggestion(getSPNameForGetCandidates(), deHyphenateTokens, err.Key, err.Value, out log, getWithStemAndAffixCorrParamFromOption(), getWithSearchParamFromOption());
// Change suggestion back to Old Spell if any suggestions:
if (log != "No candidates")
{
suggestion = correction.ChangeNewToOldSpell(suggestion);
}
// update token dic with suggestion:
if (!suggestion.Equals(deHyphenateTokens[err.Key], StringComparison.OrdinalIgnoreCase))
{
deHyphenateTokens[err.Key] = "[" + suggestion + "]";
}
// for analysis:
if (!string.IsNullOrEmpty(articleFileGT))
{
correction.UpdateFields(articleFileName, err.Key, new Dictionary <string, string> {
{ getFieldNameFromOption(), suggestion }, { getFieldNameFromOption().Replace("Correction", "Log"), log }
});
}
}
ResetControls(true);
}
}
public void AddLanguageConfig(LanguageConfig config)
{
config.ThrowIfNull("congif");
_Dictionary.Add(config.Language, config);
}
public static string Get(int id)
{
var config = LanguageConfig.Get(id);
return(config == null ? string.Empty : config.content);
}
public CommandPerformer(CashMachine cashMachine, LanguageConfig lang, StatsCounter statsCounter)
{
_cashMachine = cashMachine;
_lang = lang;
_statsCounter = statsCounter;
}
/// <summary>
/// Big fat language-setting-label-setter
/// </summary>
private void setLabels()
{
switch (LanguageConfig.GetCurrentLanguage())
{
case LanguageConfig.Language.NONE:
case LanguageConfig.Language.ENGLISH:
connAcceptLabel.Text = "Connections Accepted:";
connInitLabel.Text = "Connections Initiated:";
cumulativeConnLabel.Text = "Cumulative Connections:";
errRecLabel.Text = "Errors Received:";
failedConnAttLabel.Text = "Failed Connection Attempts:";
maxConLabel.Text = "Maximum Connections:";
resetConLabel.Text = "Reset Connections:";
resetsSentLabel.Text = "Resets Sent:";
segRecLabel.Text = "Segments Received:";
segResentLabel.Text = "Segments Resent:";
segSentLabel.Text = "Segments Sent:";
dataRecLabel.Text = "Datagrams Received:";
dataSentLabel.Text = "Datagrams Sent:";
incDataDiscLabel.Text = "Incoming Datagrams Discarded:";
incDataWErrLabel.Text = "Incoming Datagrams With Errors:";
break;
case LanguageConfig.Language.CHINESE:
connAcceptLabel.Text = "連接接受:";
connInitLabel.Text = "連接啟動:";
cumulativeConnLabel.Text = "累計連接:";
errRecLabel.Text = "錯誤時接收:";
failedConnAttLabel.Text = "連接嘗試失敗:";
maxConLabel.Text = "最大連接數:";
resetConLabel.Text = "重置連接:";
resetsSentLabel.Text = "復位發送:";
segRecLabel.Text = "分部收稿:";
segResentLabel.Text = "段反感:";
segSentLabel.Text = "段發送:";
dataRecLabel.Text = "數據報收稿:";
dataSentLabel.Text = "數據報發送:";
incDataDiscLabel.Text = "捨去傳入的數據報:";
incDataWErrLabel.Text = "有錯誤的傳入的數據報:";
break;
case LanguageConfig.Language.GERMAN:
connAcceptLabel.Text = "Verbindungen angenommen:";
connInitLabel.Text = "Verbindungen initiiert:";
cumulativeConnLabel.Text = "Kumulierte Connections:";
errRecLabel.Text = "Errors Received:";
failedConnAttLabel.Text = "Fehlgeschlagene Verbindungsversuche:";
maxConLabel.Text = "maximale Verbindungen:";
resetConLabel.Text = "Rücksetzanschlüsse:";
resetsSentLabel.Text = "Setzt Sent:";
segRecLabel.Text = "Segmente empfangen:";
segResentLabel.Text = "Segmente Resent:";
segSentLabel.Text = "Segmente Sent:";
dataRecLabel.Text = "Empfangene Datagramme:";
dataSentLabel.Text = "Datagramme:";
incDataDiscLabel.Text = "Eingehende Datagramme verworfen:";
incDataWErrLabel.Text = "Eingehende Datagramme mit Fehler:";
break;
case LanguageConfig.Language.RUSSIAN:
connAcceptLabel.Text = "Принято подключений:";
connInitLabel.Text = "соединения, инициированные:";
cumulativeConnLabel.Text = "Накопительное подключений:";
errRecLabel.Text = "Ошибки Поступила в редакцию:";
failedConnAttLabel.Text = "Неудачных попыток подключения:";
maxConLabel.Text = "Максимальная подключений:";
resetConLabel.Text = "Сброс подключений:";
resetsSentLabel.Text = "Сброс Отправленные:";
segRecLabel.Text = "Сегменты Поступила в редакцию:";
segResentLabel.Text = "Сегменты Resent:";
segSentLabel.Text = "Сегменты Отправленные:";
dataRecLabel.Text = "Датаграммы Поступило:";
dataSentLabel.Text = "датаграммы, посланные:";
incDataDiscLabel.Text = "Входящих датаграмм Выкинуть:";
incDataWErrLabel.Text = "Входящие пакеты с ошибками:";
break;
case LanguageConfig.Language.PORTUGUESE:
case LanguageConfig.Language.SPANISH:
connAcceptLabel.Text = "conexiones aceptadas:";
connInitLabel.Text = "conexiones que se inician:";
cumulativeConnLabel.Text = "Conexiones acumulada:";
errRecLabel.Text = "errores recibidos:";
failedConnAttLabel.Text = "Intentos fallidos de conexión:";
maxConLabel.Text = "máximo de conexiones:";
resetConLabel.Text = "Conexiones de reajuste:";
resetsSentLabel.Text = "restablece Enviado:";
segRecLabel.Text = "segmentos recibidos:";
segResentLabel.Text = "segmentos Reenviado:";
segSentLabel.Text = "segmentos enviados:";
dataRecLabel.Text = "datagramas recibidos:";
dataSentLabel.Text = "datagramas enviados:";
incDataDiscLabel.Text = "Datagramas de entrada desechados:";
incDataWErrLabel.Text = "Los datagramas entrantes con errores:";
break;
}
}
public static void Initialize()
{
LoadConfig(StringUtil.Contact(Localization.language, "/", "Language", Localization.language), s => { LanguageConfig.Parse(true, s, () => { CompleteLoad(); }); });
Game.Instance.StartCoroutine(OnLoadConfigs());
}
public MetadataRepository(Context.ConceptsContext context, IOptions <LanguageConfig> languageConfig)
{
_context = context;
_languageConfig = languageConfig.Value;
}
private void button1_Click(object sender, EventArgs e)
{
ai.NetAdapter.Enabled = !ai.NetAdapter.Enabled;
if (ai.NetAdapter.Enabled)
{
switch (LanguageConfig.GetCurrentLanguage())
{
case LanguageConfig.Language.NONE:
case LanguageConfig.Language.ENGLISH:
button1.Text = "Enabled";
break;
case LanguageConfig.Language.PORTUGUESE:
button1.Text = "Permitir";
break;
case LanguageConfig.Language.CHINESE:
button1.Text = "启用";
break;
case LanguageConfig.Language.GERMAN:
button1.Text = "ermöglichen";
break;
case LanguageConfig.Language.RUSSIAN:
button1.Text = "Включить";
break;
case LanguageConfig.Language.SPANISH:
button1.Text = "permitir";
break;
}
}
else
{
switch (LanguageConfig.GetCurrentLanguage())
{
case LanguageConfig.Language.NONE:
case LanguageConfig.Language.ENGLISH:
button1.Text = "Disabled";
break;
case LanguageConfig.Language.PORTUGUESE:
button1.Text = "desativado";
break;
case LanguageConfig.Language.CHINESE:
button1.Text = "禁用";
break;
case LanguageConfig.Language.GERMAN:
button1.Text = "deaktiviert";
break;
case LanguageConfig.Language.RUSSIAN:
button1.Text = "инвалид";
break;
case LanguageConfig.Language.SPANISH:
button1.Text = "discapacitado";
break;
}
}
}
/// <summary>
/// Конструктор
/// </summary>
/// <param name="configurator"> Конфигуратор для определенного языка </param>
/// <param name="fileName"> Имя файла </param>
public CodeEditor(Configurator configurator, string fileName)
{
this.fileName = fileName;
this.configurator = configurator;
this.configuration = configurator.createConfiguration();
}
public CodeAnalyzer(LanguageConfig languageConfig)
{
this.languageConfig = languageConfig;
}
private static void Main(string[] args)
{
//Путь по умолчанию к файлу с кассетами
var path = @"D:\Visual Studio\OOP\ATM\bin\Debug\data.json";
var cashMachine = new CashMachine(path);
var lang = new LanguageConfig("en-US");
var statsCounter = new StatsCounter();
var commandPerformer = new CommandPerformer(cashMachine, lang, statsCounter);
XmlConfigurator.Configure();
//Вывод текущего состояния счёта
Console.WriteLine(lang.Status + ":");
Console.WriteLine(cashMachine.Status() + '\n');
//Начало обработки пользовательского ввода
while (true)
{
Console.Write(lang.AskForMoney + ": ");
//Пользовательский запрос
var request = Console.ReadLine();
Log.Debug("Users request: " + request);
//Введённая пользователем сумма
decimal usersMoney = 0;
//Выполнение команды && проверка, команда ли это
var isCommand = request != null && commandPerformer.TryPerform(request.Trim().ToLower());
if (isCommand || decimal.TryParse(request, out usersMoney))
{
//Начало работы с введённой пользователем суммой
if (!isCommand && usersMoney < cashMachine.Balance && usersMoney >= 0)
{
//Выданная сумма
decimal withdrawnSum = 0;
//Вывод && подсчёт выданной суммы
Console.WriteLine('\n' + lang.YourMoney + ":");
foreach (var item in cashMachine.WithdrawMoney(usersMoney).Banknotes)
{
var banknoteNomimal = item.Key.Nominal;
var banknotesCount = item.Value;
withdrawnSum += banknoteNomimal * banknotesCount;
Console.WriteLine(lang.Banknote + ":" + item.Key.Nominal + " <-> " + lang.Number + ": " +
item.Value);
}
Console.WriteLine(lang.WithdrawnSum + ": " + withdrawnSum);
//Вызов события добавления записи статистики
statsCounter.Add(cashMachine.Balance, withdrawnSum);
//Вывод текущего состояния счёта
Console.WriteLine('\n' + lang.Status + ':');
Console.WriteLine(cashMachine.Status() + '\n');
}
else if (usersMoney > cashMachine.Balance)
{
Console.Write('\n' + lang.NotEnoughMoney + "\n\n");
Log.Error("Not enough money");
}
}
//Обработка некорректного ввода
else
{
Console.Write('\n' + lang.IncorrectInput + "\n\n");
Log.Error("Incorrect input");
}
}
}
public WhitespaceTokenizer(LanguageConfig languageConfig) : base(languageConfig)
{
}
public OptionDataMap(SystemConfig sys_conf, UserConfig user_conf, LanguageConfig lang_conf)
{
Load(sys_conf, user_conf, lang_conf);
}
private static MongoAppEntityLanguage FromLanguageConfig(LanguageConfig l)
{
return(new MongoAppEntityLanguage {
Iso2Code = l.Language, IsOptional = l.IsOptional, Fallback = l.Fallback.Select(x => x.Iso2Code).ToList()
});
}
public JsonLanguageConfig(LanguageConfig config)
{
SimpleMapper.Map(config, this);
Fallback = config.LanguageFallbacks.ToArray();
}
/// <summary>
/// @xis 验证错误码转文字 2020-2-19 20:06:50
/// </summary>
/// <returns></returns>
public static string GetErrorCode(this HttpRequest req, string code)
{
return(LanguageConfig.GetErrorCode(code, GetLang(req)));
}
public TextLiteralTokenizer(LanguageConfig languageConfig) : base(languageConfig)
{
}
public ActionResult SpellText(string htmlEncoded, string text, string langugage)
{
FileInfo affFile = SpellEngineManager.AllAffFiles.FirstOrDefault(f => f.Directory.Name.Equals(langugage, StringComparison.InvariantCultureIgnoreCase));
string nameWithoutExtension = affFile.Directory.Name.ToLower();// affFile.Name.Replace(".aff", string.Empty).ToLowerInvariant();
using (SpellEngine spellEngine = new SpellEngine())
{
LanguageConfig enConfig = new LanguageConfig();
enConfig.LanguageCode = nameWithoutExtension;
enConfig.HunspellAffFile = affFile.FullName;
enConfig.HunspellDictFile = affFile.FullName.Replace(".aff", ".dic");
enConfig.HunspellKey = "";
spellEngine.AddLanguage(enConfig);
var alphabetCharsOld = SpellEngineManager.GetAlphabetLetters(enConfig.HunspellAffFile);
SpellEngineManager.AlphabetLetters[nameWithoutExtension] = alphabetCharsOld;
string html = Server.HtmlDecode(htmlEncoded);
//HtmlDocument doc = new HtmlDocument();
//doc.LoadHtml(html);
////doc.Save(Console.Out); // show before
//RemoveComments(doc.DocumentNode);
//var test = doc.DocumentNode.InnerHtml;
//?? test
//var indexOfBase = html.IndexOf("<base");
//if (indexOfBase > 0)
//{
// html = html.Substring(indexOfBase);
// html = html.Replace("</body>", string.Empty);
// html = html.Replace("</html>", string.Empty);
//}
var testForClearText = StripTagsCharArray(html);
// ?? test
text = testForClearText;
var chars = text.ToCharArray();
List <string> emptyChars = new List <string>();
foreach (char c in chars)
{
if (string.IsNullOrWhiteSpace(c.ToString()))
{
emptyChars.Add(c.ToString());
}
}
emptyChars = emptyChars.Distinct().ToList();
var alphabetChars = SpellEngineManager.AlphabetLetters[langugage];
//var punctuationChars = chars.Where(c => char.IsPunctuation(c)).Distinct().ToList();
var nonAlphabetChars = chars.Distinct().Except(alphabetChars).ToList();
//var nonAlphabetChars = chars.Distinct().Except(alphabetChars).Except(punctuationChars).ToList();
//for (int i = 0; i < punctuationChars.Count; i++)
//{
// text = text.Replace(punctuationChars[i].ToString(), " ");
//}
//var words = text.Replace("\n", " ").Replace(",", " ").Replace(".", " ").Split(emptyChars.ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
var words = text.Split(emptyChars.ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
//var words = text.Replace("\n", " ").Replace(",", " ").Replace(".", " ").Split(emptyChars.ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
List <string> wrongWords = new List <string>();
//words = words.Select(w => w.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).Where(w => ContainsLetters(w)).ToList();
words = words.Select(w => w.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).Where(w => ContainsOnlyAlphabetLetters(w, alphabetChars)).ToList();
foreach (string word in words)
{
var res = spellEngine[langugage].Spell(word);
if (!res)
{
wrongWords.Add(word);
}
}
wrongWords = wrongWords.Distinct().ToList();
StringBuilder sbSummary = new StringBuilder();
string begin = @" <table id='grid'>
<colgroup>
<col />
<col />
</colgroup>
<thead>
<tr>
<th data-field='make'>Word</th>
<th data-field='model'>Suggestion</th>
</tr>
</thead>
<tbody>";
sbSummary.Append(begin);
foreach (string word in wrongWords)
{
var suggest = spellEngine[langugage].Suggest(word);
StringBuilder sb = new StringBuilder();
foreach (var item in suggest)
{
sb.Append(item + ", ");
}
string suggestions = sb.ToString();
//var wordsInHtml = html.Split(emptyChars.ToArray(), StringSplitOptions.None);
//StringBuilder newHtml = new StringBuilder();
//for (int i = 0; i < wordsInHtml.Count(); i++)
//{
// var currWord = wordsInHtml[i];
// if (currWord.Trim() == word.Trim())
// {
// newHtml.Append($"<span title='{suggestions}' style='color:blueviolet;background-color:yellow'>{word}</span>");
// }
// else
// {
// newHtml.Append(currWord);
// }
//}
//html = newHtml.ToString();
// заради Angstriom го правя това - съдържа не само букви
//if (word.ToCharArray().All(c => char.IsLetter(c)))
{
string input = html;
string pattern = $@"\b{word}\b";
string replace = $"<span title='{suggestions}' style='color:blueviolet;background-color:yellow'>{word}</span>";
var newhtml = Regex.Replace(input, pattern, replace);
if (newhtml != html)
{
// за да няма в грида повече думи отколкото сме подчертали в editor
sbSummary.Append(CreateGridRow(word, suggestions));
}
html = newhtml;
}
//html = html.Replace(word, $"<span title='{suggestions}' style='color:blueviolet;background-color:yellow'>{word}</span>");
}
sbSummary.Append("</tbody></table>");
var returnObject = new { html = html + "<div></div>", summary = sbSummary.ToString() };
return(Json(returnObject));
}
}
public IEnumerable <BuildResult> BuildMultipleVariants(LanguageConfig languageConfig)
{
// TODO: instead of receiving languageConfig build one here with different settings?
var variants = new BuildVariants();
foreach (var part in _queryParts)
{
switch (part)
{
case ColumnPart column:
{
// TODO: quoted column name, especially when with multiple words
variants.AddTokenToAll(column.ColumnName,
index => new TextLiteralToken(index, column.ColumnName.Length, column.ColumnName));
break;
}
case StatementPart statement:
{
variants.AppendToAll(" ");
var keyword = languageConfig.Syntax.KeywordsMap[statement.StatementKeyword];
variants.AddTokenToAll(keyword,
index => new KeywordToken(index, keyword.Length, statement.StatementKeyword));
break;
}
case CompareOperatorPart compareOperator:
{
var keyword = languageConfig.Syntax.KeywordsMap[compareOperator.CompareOperatorKeyword];
// produce variant without whitespace if comp op starts with special char
var newVariant = char.IsLetterOrDigit(keyword.First()) ? null : variants[0].Copy();
variants.AppendToAll(" ", exceptNewVariant: newVariant);
variants.AddTokenToAll(keyword,
index => new KeywordToken(index, keyword.Length, compareOperator.CompareOperatorKeyword));
// produce variant without whitespace if comp op ends with special char
newVariant = char.IsLetterOrDigit(keyword.Last()) ? null : variants[0].Copy();
variants.AppendToAll(" ", exceptNewVariant: newVariant);
string operand;
switch (compareOperator.Value)
{
case string s:
var noQuoteVariant = variants[0].Copy();
operand = $"\"{s}\"";
variants.AddTokenToAll(operand,
index => new TextLiteralToken(index, operand.Length, s));
if (s.Trim().Any(char.IsWhiteSpace) == false)
{
// produce variant without quote if single word
noQuoteVariant.AddToken(s, index => new TextLiteralToken(index, s.Length, s));
variants.Add(noQuoteVariant);
}
break;
case float v:
operand = v.ToString(languageConfig.Syntax.CultureInfo);
variants.AddTokenToAll(operand,
index => new NumberLiteralToken(index, operand.Length));
break;
case decimal v:
operand = v.ToString(languageConfig.Syntax.CultureInfo);
variants.AddTokenToAll(operand,
index => new NumberLiteralToken(index, operand.Length));
break;
case double v:
operand = v.ToString(languageConfig.Syntax.CultureInfo);
variants.AddTokenToAll(operand,
index => new NumberLiteralToken(index, operand.Length));
break;
// TODO: all types
default:
throw new NotSupportedException();
// TODO: all other cases
}
break;
}
default:
throw new NotSupportedException();
}
}
return(variants.CreateBuildResults());
}
protected abstract ITokenizer GetTokenizer(LanguageConfig config);
protected TokenizerBase(LanguageConfig languageConfig)
{
LanguageConfig = languageConfig;
}
protected void GivenConfig(LanguageConfig config)
{
var testCase = TestCase.RefreshCurrent(this);
testCase.Config = config;
}
private Lexer(LanguageConfig languageConfig)
{
_languageConfig = languageConfig;
_whitespaceTokenizer = new WhitespaceTokenizer(languageConfig);
}
private TestCase(LanguageConfig defaultConfig)
{
Config = defaultConfig;
}
void Awake()
{
m_LanguageConfig = this;
LoadXml();
}
protected override ITokenizer GetTokenizer(LanguageConfig config) => new TextLiteralTokenizer(config);
public static string Get(string key)
{
return(LanguageConfig.Get(key).text);
}
private LanguageModelFileReaderMMF(LanguageConfig languageConfig) : base(languageConfig.Language, languageConfig.ModelFilename)
{
}