public string Slugify(string text) { if (string.IsNullOrEmpty(text)) { return(text); } var sb = new StringBuilder(); var stFormKD = text.Trim().ToLower().Normalize(NormalizationForm.FormKD); foreach (var t in stFormKD) { // Allowed symbols if (t == '-' || t == '_' || t == '~') { sb.Append(t); continue; } var uc = CharUnicodeInfo.GetUnicodeCategory(t); switch (uc) { case UnicodeCategory.LowercaseLetter: case UnicodeCategory.OtherLetter: case UnicodeCategory.DecimalDigitNumber: // Keep letters and digits sb.Append(t); break; case UnicodeCategory.NonSpacingMark: // Remove diacritics break; default: // Replace all other chars with dash sb.Append('-'); break; } } var slug = sb.ToString().Normalize(NormalizationForm.FormC); // Simplifies dash groups for (var i = 0; i < slug.Length - 1; i++) { if (slug[i] == '-') { var j = 0; while (i + j + 1 < slug.Length && slug[i + j + 1] == '-') { j++; } if (j > 0) { slug = slug.Remove(i + 1, j); } } } if (slug.Length > 1000) { slug = slug.Substring(0, 1000); } slug = slug.Trim('-', '_', '.'); return(slug); }
private void WriteObject(string name, ProtectedString value, bool bIsEntryString) { Debug.Assert(name != null); Debug.Assert(value != null); if (value == null) { throw new ArgumentNullException("value"); } m_xmlWriter.WriteStartElement(ElemString); m_xmlWriter.WriteStartElement(ElemKey); m_xmlWriter.WriteString(StrUtil.SafeXmlString(name)); m_xmlWriter.WriteEndElement(); m_xmlWriter.WriteStartElement(ElemValue); bool bProtected = value.IsProtected; if (bIsEntryString) { // Adjust memory protection setting (which might be different // from the database default, e.g. due to an import which // didn't specify the correct setting) if (name == PwDefs.TitleField) { bProtected = m_pwDatabase.MemoryProtection.ProtectTitle; } else if (name == PwDefs.UserNameField) { bProtected = m_pwDatabase.MemoryProtection.ProtectUserName; } else if (name == PwDefs.PasswordField) { bProtected = m_pwDatabase.MemoryProtection.ProtectPassword; } else if (name == PwDefs.UrlField) { bProtected = m_pwDatabase.MemoryProtection.ProtectUrl; } else if (name == PwDefs.NotesField) { bProtected = m_pwDatabase.MemoryProtection.ProtectNotes; } } if (bProtected && (m_format == KdbxFormat.Default)) { m_xmlWriter.WriteAttributeString(AttrProtected, ValTrue); byte[] pbEnc = value.ReadXorredString(m_randomStream); if (pbEnc.Length > 0) { m_xmlWriter.WriteBase64(pbEnc, 0, pbEnc.Length); } } else { string strValue = value.ReadString(); // If names should be localized, we need to apply the language-dependent // string transformation here. By default, language-dependent conversions // should be applied, otherwise characters could be rendered incorrectly // (code page problems). if (g_bLocalizedNames) { StringBuilder sb = new StringBuilder(); foreach (char ch in strValue) { char chMapped = ch; // Symbols and surrogates must be moved into the correct code // page area if (char.IsSymbol(ch) || char.IsSurrogate(ch)) { UnicodeCategory cat = CharUnicodeInfo.GetUnicodeCategory(ch); // Map character to correct position in code page chMapped = (char)((int)cat * 32 + ch); } else if (char.IsControl(ch)) { if (ch >= 256) // Control character in high ANSI code page { // Some of the control characters map to corresponding ones // in the low ANSI range (up to 255) when calling // ToLower on them with invariant culture (see // http://lists.ximian.com/pipermail/mono-patches/2002-February/086106.html ) #if !KeePassLibSD chMapped = char.ToLowerInvariant(ch); #else chMapped = char.ToLower(ch); #endif } } sb.Append(chMapped); } strValue = sb.ToString(); // Correct string for current code page } if ((m_format == KdbxFormat.PlainXml) && bProtected) { m_xmlWriter.WriteAttributeString(AttrProtectedInMemPlainXml, ValTrue); } m_xmlWriter.WriteString(StrUtil.SafeXmlString(strValue)); } m_xmlWriter.WriteEndElement(); // ElemValue m_xmlWriter.WriteEndElement(); // ElemString }
public static string ElfRename(string name) // I probably missed a lot. { name = name.ToLower(); char[] nameArray = name.ToCharArray(); name = string.Empty; //string charStore = string.Empty; /// Remove diacritics for (int c = 0; c < nameArray.Length; c++) { switch (CharUnicodeInfo.GetUnicodeCategory(nameArray[c])) { case UnicodeCategory.NonSpacingMark: break; case UnicodeCategory.SpacingCombiningMark: break; case UnicodeCategory.EnclosingMark: break; default: name += nameArray[c].ToString(); break; } } nameArray = name.ToCharArray(); name = string.Empty; for (int c = 0; c < nameArray.Length; c++) { switch (nameArray[c]) { case 'ł': name += "l"; break; case 'æ': name += "ae"; break; case 'œ': name += "oe"; break; case 'ø': name += "oe"; break; case 'ð': name += "th"; break; case 'þ': name += "th"; break; case 'ß': name += "s"; break; default: name += nameArray[c].ToString(); break; } } name = name.Replace("qu", "kan").Replace("dr", "ryas").Replace("q", "k").Replace("tch", "s").Replace("era", "ana").Replace("er", "a").Replace("ph", "hen").Replace("bb", "de").Replace("ff", "far").Replace("ai", "ana").Replace("rr", "ran").Replace("lyn", "lan").Replace("atl", "atr").Replace("tl", "ani").Replace("yn", "an").Replace("wh", "h").Replace("oy", "ye").Replace("oi", "ola").Replace("bl", "lenal").Replace("b", "l").Replace("oo", "a").Replace("ou", "a").Replace("ch", "s").Replace("sh", "s").Replace("u", "a").Replace("wr", "r").Replace("z", "s").Replace("j", "y").Replace("p", "g").Replace("ee", "ena").Replace("io", "ia").Replace("nth", "nasæn").Replace("th", "s").Replace("t", "n").Replace("or", "ar").Replace("li", "en").Replace("w", "l").Replace("dd", "nian").Replace("se", "sa").Replace("d", "n").Replace("el", "as").Replace("sg", "se").Replace("asora", "asana").Replace("er", "en").Replace("x", "s").Replace("v", "ra").Replace("ay", "ayan").Replace("es", "en").Replace("ck", "kel").Replace("og", "aen").Replace("c", "k").Replace("rye", "rya").Replace("mm", "nan").Replace("m", "n").Replace("ll", "las").Replace("nn", "nen").Replace("ninen", "neyen").Replace("nk", "nas").Replace("enen", "enin").Replace("ae", "æ").Replace("oe", "œ").Replace("æa", "æ").Replace("rai", "raye").Replace("gh", "hb").Replace("hbl", "hay").Replace("hb", "he").Replace("ar", "ane").Replace("ayn", "ayen").Replace("enen", "enin"); name = new CultureInfo("en-US").TextInfo.ToTitleCase(name); return(name); }
/// <summary> /// Returns true if the Unicode character is a formatting character (Unicode class Cf). /// </summary> /// <param name="ch">The Unicode character.</param> internal static bool IsFormattingChar(char ch) { // There are no FormattingChars in ASCII range return(ch > 127 && IsFormattingChar(CharUnicodeInfo.GetUnicodeCategory(ch))); }
/// <summary> /// Is the character a valid first identifier character? /// </summary> private static bool IsValidIdFirstChar(char c) { return (char.IsLetter(c) || CharUnicodeInfo.GetUnicodeCategory(c) == UnicodeCategory.ConnectorPunctuation); }
private TSQLToken DetermineTokenType( string tokenValue, int startPosition, int endPosition) { if ( char.IsWhiteSpace(tokenValue[0])) { return (new TSQLWhitespace( startPosition, tokenValue)); } else if ( tokenValue[0] == '@') { if (TSQLVariables.IsVariable(tokenValue)) { return (new TSQLSystemVariable( startPosition, tokenValue)); } else { return (new TSQLVariable( startPosition, tokenValue)); } } else if (tokenValue.StartsWith("--")) { return (new TSQLSingleLineComment( startPosition, tokenValue)); } else if (tokenValue.StartsWith("/*")) { return (new TSQLMultilineComment( startPosition, tokenValue)); } else if ( tokenValue.StartsWith("\'") || tokenValue.StartsWith("N\'") || ( !UseQuotedIdentifiers && ( tokenValue.StartsWith("\"") || tokenValue.StartsWith("N\"") ) )) { return (new TSQLStringLiteral( startPosition, tokenValue)); } else if ( tokenValue[0] == '$') { // $IDENTITY if ( tokenValue.Length > 1 && char.IsLetter(tokenValue[1])) { return (new TSQLIdentifier( startPosition, tokenValue)); } // $45.56 else { return (new TSQLMoneyLiteral( startPosition, tokenValue)); } } else if (CharUnicodeInfo.GetUnicodeCategory(tokenValue[0]) == UnicodeCategory.CurrencySymbol) { return (new TSQLMoneyLiteral( startPosition, tokenValue)); } else if (tokenValue.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) { return (new TSQLBinaryLiteral( startPosition, tokenValue)); } else if ( char.IsDigit(tokenValue[0]) || ( tokenValue[0] == '.' && tokenValue.Length > 1 && char.IsDigit(tokenValue[1]) )) { return (new TSQLNumericLiteral( startPosition, tokenValue)); } else if ( tokenValue[0] == '=' || tokenValue[0] == '~' || tokenValue[0] == '-' || tokenValue[0] == '+' || tokenValue[0] == '*' || tokenValue[0] == '/' || tokenValue[0] == '<' || tokenValue[0] == '>' || tokenValue[0] == '!' || tokenValue[0] == '&' || tokenValue[0] == '|' || tokenValue[0] == '^' || tokenValue[0] == '%' || tokenValue[0] == ':') { return (new TSQLOperator( startPosition, tokenValue)); } else if (TSQLCharacters.IsCharacter(tokenValue)) { return (new TSQLCharacter( startPosition, tokenValue)); } else if (TSQLKeywords.IsKeyword(tokenValue)) { return (new TSQLKeyword( startPosition, tokenValue)); } else if (TSQLIdentifiers.IsIdentifier(tokenValue)) { return (new TSQLSystemIdentifier( startPosition, tokenValue)); } else { return (new TSQLIdentifier( startPosition, tokenValue)); } }
private static bool IsIdentifierStart(char c) { return(char.IsLetter(c) || c == '_' || CharUnicodeInfo.GetUnicodeCategory(c) == UnicodeCategory.LetterNumber); }
public static UniCatFlags GetUniCatFlags(char ch) { return((UniCatFlags)(1u << (int)CharUnicodeInfo.GetUnicodeCategory(ch))); }
public static string RemoveAccent(this string value) { if (string.IsNullOrEmpty(value)) { return(value); } return(string.Concat(value.Normalize(NormalizationForm.FormD).Where(ch => CharUnicodeInfo.GetUnicodeCategory(ch) != UnicodeCategory.NonSpacingMark)).Normalize(NormalizationForm.FormC)); }
/// <summary> /// Creates a parser with a grammar that matches each character within the specified /// <paramref name="category"/>. /// </summary> /// <param name="category">The unicode character in which to match characters.</param> /// <returns>A parser with a grammar that matches each character within the specified /// <paramref name="category"/>.</returns> protected IParser <char, char> Character(UnicodeCategory category) { Contract.Ensures(Contract.Result <IParser <char, char> >() != null); return(Next.Where(c => CharUnicodeInfo.GetUnicodeCategory(c) == category)); }
/// <summary> /// Normalize an input buffer of Sorani text /// </summary> /// <param name="s"> input buffer </param> /// <param name="len"> length of input buffer </param> /// <returns> length of input buffer after normalization </returns> public virtual int Normalize(char[] s, int len) { for (int i = 0; i < len; i++) { switch (s[i]) { case YEH: case DOTLESS_YEH: s[i] = FARSI_YEH; break; case KAF: s[i] = KEHEH; break; case ZWNJ: if (i > 0 && s[i - 1] == HEH) { s[i - 1] = AE; } len = StemmerUtil.Delete(s, i, len); i--; break; case HEH: if (i == len - 1) { s[i] = AE; } break; case TEH_MARBUTA: s[i] = AE; break; case HEH_DOACHASHMEE: s[i] = HEH; break; case REH: if (i == 0) { s[i] = RREH; } break; case RREH_ABOVE: s[i] = RREH; break; case TATWEEL: case KASRATAN: case DAMMATAN: case FATHATAN: case FATHA: case DAMMA: case KASRA: case SHADDA: case SUKUN: len = StemmerUtil.Delete(s, i, len); i--; break; default: if (CharUnicodeInfo.GetUnicodeCategory(s[i]) == UnicodeCategory.Format) { len = StemmerUtil.Delete(s, i, len); i--; } break; } } return(len); }
public static string FormatLiteral(string value, ObjectDisplayOptions options) { if (value == null) { throw new ArgumentNullException(nameof(value)); } StringBuilder builder = new StringBuilder(); bool flag1 = options.IncludesOption(ObjectDisplayOptions.UseQuotes); bool flag2 = options.IncludesOption(ObjectDisplayOptions.EscapeNonPrintableCharacters); bool flag3 = flag1 && !flag2 && ObjectDisplay.ContainsNewLine(value); if (flag1) { if (flag3) { builder.Append('@'); } builder.Append('"'); } for (int index = 0; index < value.Length; ++index) { char ch = value[index]; if (flag2 && CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.Surrogate) { UnicodeCategory unicodeCategory = CharUnicodeInfo.GetUnicodeCategory(value, index); if (unicodeCategory == UnicodeCategory.Surrogate) { builder.Append("\\u" + ((int)ch).ToString("x4")); } else if (ObjectDisplay.NeedsEscaping(unicodeCategory)) { int utf32 = char.ConvertToUtf32(value, index); builder.Append("\\U" + utf32.ToString("x8")); ++index; } else { builder.Append(ch); builder.Append(value[++index]); } } else { string replaceWith; if (flag2 && ObjectDisplay.TryReplaceChar(ch, out replaceWith)) { builder.Append(replaceWith); } else if (flag1 && ch == '"') { if (flag3) { builder.Append('"'); builder.Append('"'); } else { builder.Append('\\'); builder.Append('"'); } } else { builder.Append(ch); } } } if (flag1) { builder.Append('"'); } return(builder.ToString()); }
#pragma warning disable S3776 private static void CleanValue(string value, out CleaningResult result, bool isNormalized) { result = new CleaningResult(); if (value == null) { return; } var len = value.Length; // length of value var justProcessedSpace = false; // flag indicating whether we've just processed a space character StringBuilder sb = null; // String builder for result var lastCopiedPos = 0; // last position (excluding) copied to the result // String processing pattern: Iterate all characters and focus on runs of valid // characters that can simply be copied. If all characters are valid, no memory // is allocated. var pos = 0; while (pos < len) { var ch = value[pos]; // current character if (IsValidQrBillCharacter(ch)) { justProcessedSpace = ch == ' '; pos++; continue; } // Check for normalization if (ch > 0xff && !isNormalized) { isNormalized = value.IsNormalized(NormalizationForm.FormC); if (!isNormalized) { // Normalize string and start over value = value.Normalize(NormalizationForm.FormC); CleanValue(value, out result, true); return; } } if (sb == null) { sb = new StringBuilder(value.Length); } // copy processed characters to result before taking care of the invalid // character if (pos > lastCopiedPos) { sb.Append(value, lastCopiedPos, pos - lastCopiedPos); } if (char.IsHighSurrogate(ch)) { // Proper Unicode handling to prevent surrogates and combining characters // from being replaced with multiples periods. var category = CharUnicodeInfo.GetUnicodeCategory(value, pos); if (category != UnicodeCategory.SpacingCombiningMark) { sb.Append('.'); } justProcessedSpace = false; pos++; } else { if (ch <= ' ') { if (!justProcessedSpace) { sb.Append(' '); } justProcessedSpace = true; } else { sb.Append('.'); justProcessedSpace = false; } } pos++; lastCopiedPos = pos; } if (sb == null) { result.CleanedString = value.Trim(); return; } if (lastCopiedPos < len) { sb.Append(value, lastCopiedPos, len - lastCopiedPos); } result.CleanedString = sb.ToString().Trim(); result.ReplacedUnsupportedChars = true; }
public static void RegexUnicodeChar() { // Regex engine is Unicode aware now for the \w and \d character classes // \s is not - i.e. it still only recognizes the ASCII space separators, not Unicode ones // The new character classes for this: // [\p{L1}\p{Lu}\p{Lt}\p{Lo}\p{Nd}\p{Pc}] List <char> validChars = new List <char>(); List <char> invalidChars = new List <char>(); for (int i = 0; i < MaxUnicodeRange; i++) { char c = (char)i; switch (CharUnicodeInfo.GetUnicodeCategory(c)) { case UnicodeCategory.UppercaseLetter: //Lu case UnicodeCategory.LowercaseLetter: //Li case UnicodeCategory.TitlecaseLetter: // Lt case UnicodeCategory.ModifierLetter: // Lm case UnicodeCategory.OtherLetter: // Lo case UnicodeCategory.DecimalDigitNumber: // Nd // case UnicodeCategory.LetterNumber: // ?? // case UnicodeCategory.OtherNumber: // ?? case UnicodeCategory.NonSpacingMark: // case UnicodeCategory.SpacingCombiningMark: // Mc case UnicodeCategory.ConnectorPunctuation: // Pc validChars.Add(c); break; default: invalidChars.Add(c); break; } } // \w - we will create strings from valid characters that form \w and make sure that the regex engine catches this. // Build a random string with valid characters followed by invalid characters Random random = new Random(-55); Regex regex = new Regex(@"\w*"); int validCharLength = 10; int invalidCharLength = 15; for (int i = 0; i < 100; i++) { var builder1 = new StringBuilder(); var builder2 = new StringBuilder(); for (int j = 0; j < validCharLength; j++) { char c = validChars[random.Next(validChars.Count)]; builder1.Append(c); builder2.Append(c); } for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } string input = builder1.ToString(); Match match = regex.Match(input); Assert.True(match.Success); Assert.Equal(builder2.ToString(), match.Value); Assert.Equal(0, match.Index); Assert.Equal(validCharLength, match.Length); match = match.NextMatch(); do { // We get empty matches for each of the non-matching characters of input to match // the * wildcard in regex pattern. Assert.Equal(string.Empty, match.Value); Assert.Equal(0, match.Length); match = match.NextMatch(); } while (match.Success); } // Build a random string with invalid characters followed by valid characters and then again invalid random = new Random(-55); regex = new Regex(@"\w+"); validCharLength = 10; invalidCharLength = 15; for (int i = 0; i < 500; i++) { var builder1 = new StringBuilder(); var builder2 = new StringBuilder(); for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } for (int j = 0; j < validCharLength; j++) { char c = validChars[random.Next(validChars.Count)]; builder1.Append(c); builder2.Append(c); } for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } string input = builder1.ToString(); Match match = regex.Match(input); Assert.True(match.Success); Assert.Equal(builder2.ToString(), match.Value); Assert.Equal(invalidCharLength, match.Index); Assert.Equal(validCharLength, match.Length); match = match.NextMatch(); Assert.False(match.Success); } validChars = new List <char>(); invalidChars = new List <char>(); for (int i = 0; i < MaxUnicodeRange; i++) { char c = (char)i; if (CharUnicodeInfo.GetUnicodeCategory(c) == UnicodeCategory.DecimalDigitNumber) { validChars.Add(c); } else { invalidChars.Add(c); } } // \d - we will create strings from valid characters that form \d and make sure that the regex engine catches this. // Build a random string with valid characters and then again invalid regex = new Regex(@"\d+"); validCharLength = 10; invalidCharLength = 15; for (int i = 0; i < 100; i++) { var builder1 = new StringBuilder(); var builder2 = new StringBuilder(); for (int j = 0; j < validCharLength; j++) { char c = validChars[random.Next(validChars.Count)]; builder1.Append(c); builder2.Append(c); } for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } string input = builder1.ToString(); Match match = regex.Match(input); Assert.Equal(builder2.ToString(), match.Value); Assert.Equal(0, match.Index); Assert.Equal(validCharLength, match.Length); match = match.NextMatch(); Assert.False(match.Success); } // Build a random string with invalid characters, valid and then again invalid regex = new Regex(@"\d+"); validCharLength = 10; invalidCharLength = 15; for (int i = 0; i < 100; i++) { var builder1 = new StringBuilder(); var builder2 = new StringBuilder(); for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } for (int j = 0; j < validCharLength; j++) { char c = validChars[random.Next(validChars.Count)]; builder1.Append(c); builder2.Append(c); } for (int j = 0; j < invalidCharLength; j++) { builder1.Append(invalidChars[random.Next(invalidChars.Count)]); } string input = builder1.ToString(); Match match = regex.Match(input); Assert.True(match.Success); Assert.Equal(builder2.ToString(), match.Value); Assert.Equal(invalidCharLength, match.Index); Assert.Equal(validCharLength, match.Length); match = match.NextMatch(); Assert.False(match.Success); } }
public static string UrlFriendly(string text, int maxLength = 0) { // Return empty value if text is null if (text == null) { return(""); } var normalizedString = text // Make lowercase .ToLowerInvariant() // Normalize the text .Normalize(NormalizationForm.FormD); var stringBuilder = new StringBuilder(); var stringLength = normalizedString.Length; var prevdash = false; var trueLength = 0; char c; for (int i = 0; i < stringLength; i++) { c = normalizedString[i]; switch (CharUnicodeInfo.GetUnicodeCategory(c)) { // Check if the character is a letter or a digit if the character is a // international character remap it to an ascii valid character case UnicodeCategory.LowercaseLetter: case UnicodeCategory.UppercaseLetter: case UnicodeCategory.DecimalDigitNumber: if (c < 128) { stringBuilder.Append(c); } else { stringBuilder.Append(RemapInternationalCharToAscii(c)); } prevdash = false; trueLength = stringBuilder.Length; break; // Check if the character is to be replaced by a hyphen but only if the last character wasn't case UnicodeCategory.SpaceSeparator: case UnicodeCategory.ConnectorPunctuation: case UnicodeCategory.DashPunctuation: case UnicodeCategory.OtherPunctuation: case UnicodeCategory.MathSymbol: if (!prevdash) { stringBuilder.Append('-'); prevdash = true; trueLength = stringBuilder.Length; } break; } // If we are at max length, stop parsing if (maxLength > 0 && trueLength >= maxLength) { break; } } // Trim excess hyphens var result = stringBuilder.ToString().Trim('-'); // Remove any excess character to meet maxlength criteria return(maxLength <= 0 || result.Length <= maxLength ? result : result.Substring(0, maxLength)); }
public static void RegexUnicodeChar() { //////////// Global Variables used for all tests String strLoc = "Loc_000oo"; String strValue = String.Empty; int iCountErrors = 0; int iCountTestcases = 0; Char ch1; List <Char> alstValidChars; List <Char> alstInvalidChars; Int32 iCharCount; Int32 iNonCharCount; String pattern; String input; Regex regex; Match match; Int32 iNumLoop; Int32 iWordCharLength; Int32 iNonWordCharLength; StringBuilder sbldr1; StringBuilder sbldr2; Random random; try { ///////////////////////// START TESTS //////////////////////////// /////////////////////////////////////////////////////////////////// //[]Regex engine is Unicode aware now for the \w and \d character classes // \s is not - i.e. it still only recognizes the ASCII space separators, not Unicode ones // The new character classes for this: //[\p{L1}\p{Lu}\p{Lt}\p{Lo}\p{Nd}\p{Pc}] iCountTestcases++; alstValidChars = new List <Char>(); alstInvalidChars = new List <Char>(); for (int i = 0; i < MaxUnicodeRange; i++) { ch1 = (Char)i; switch (CharUnicodeInfo.GetUnicodeCategory(ch1)) { case UnicodeCategory.UppercaseLetter: //Lu case UnicodeCategory.LowercaseLetter: //Li case UnicodeCategory.TitlecaseLetter: // Lt case UnicodeCategory.ModifierLetter: // Lm case UnicodeCategory.OtherLetter: // Lo case UnicodeCategory.DecimalDigitNumber: // Nd // case UnicodeCategory.LetterNumber: // ?? // case UnicodeCategory.OtherNumber: // ?? case UnicodeCategory.NonSpacingMark: // case UnicodeCategory.SpacingCombiningMark: // Mc case UnicodeCategory.ConnectorPunctuation: // Pc alstValidChars.Add(ch1); break; default: alstInvalidChars.Add(ch1); break; } } //[]\w - we will create strings from valid characters that form \w and make sure that the regex engine catches this. //Build a random string with valid characters followed by invalid characters iCountTestcases++; random = new Random(-55); pattern = @"\w*"; regex = new Regex(pattern); iNumLoop = 100; iWordCharLength = 10; iCharCount = alstValidChars.Count; iNonCharCount = alstInvalidChars.Count; iNonWordCharLength = 15; for (int i = 0; i < iNumLoop; i++) { sbldr1 = new StringBuilder(); sbldr2 = new StringBuilder(); for (int j = 0; j < iWordCharLength; j++) { ch1 = alstValidChars[random.Next(iCharCount)]; sbldr1.Append(ch1); sbldr2.Append(ch1); } for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } input = sbldr1.ToString(); match = regex.Match(input); if (!match.Success || (match.Index != 0) || (match.Length != iWordCharLength) || !(match.Value.Equals(sbldr2.ToString())) ) { iCountErrors++; Console.WriteLine("Err_753wfgg_" + i + "! Error detected: input-{0}", input); Console.WriteLine("Match index={0}, length={1}, value={2}, match.Value.Equals(expected)={3}\n", match.Index, match.Length, match.Value, match.Value.Equals(sbldr2.ToString())); if (match.Length > iWordCharLength) { Console.WriteLine("FAIL!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n{0}={1}", (int)match.Value[iWordCharLength], CharUnicodeInfo.GetUnicodeCategory(match.Value[iWordCharLength])); } } match = match.NextMatch(); do { //This is tedious. But we report empty Matches for each of the non-matching characters!!! //duh!!! because we say so on the pattern - remember what * stands for :-) if (!match.Value.Equals(String.Empty) || (match.Length != 0) ) { iCountErrors++; Console.WriteLine("Err_2975sg_" + i + "! Error detected: input-{0}", input); Console.WriteLine("Match index={0}, length={1}, value={2}\n", match.Index, match.Length, match.Value); } match = match.NextMatch(); } while (match.Success); } //[]Build a random string with invalid characters followed by valid characters and then again invalid iCountTestcases++; random = new Random(-55); pattern = @"\w+"; regex = new Regex(pattern); iNumLoop = 500; iWordCharLength = 10; iCharCount = alstValidChars.Count; iNonCharCount = alstInvalidChars.Count; iNonWordCharLength = 15; for (int i = 0; i < iNumLoop; i++) { sbldr1 = new StringBuilder(); sbldr2 = new StringBuilder(); for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } for (int j = 0; j < iWordCharLength; j++) { ch1 = alstValidChars[random.Next(iCharCount)]; sbldr1.Append(ch1); sbldr2.Append(ch1); } for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } input = sbldr1.ToString(); match = regex.Match(input); if (!match.Success || (match.Index != iNonWordCharLength) || (match.Length != iWordCharLength) || !(match.Value.Equals(sbldr2.ToString())) ) { iCountErrors++; Console.WriteLine("Err_2975sgf_" + i + "! Error detected: input-{0}", input); } match = match.NextMatch(); if (match.Success) { iCountErrors++; Console.WriteLine("Err_246sdg_" + i + "! Error detected: input-{0}", input); } } alstValidChars = new List <Char>(); alstInvalidChars = new List <Char>(); for (int i = 0; i < MaxUnicodeRange; i++) { ch1 = (Char)i; switch (CharUnicodeInfo.GetUnicodeCategory(ch1)) { case UnicodeCategory.DecimalDigitNumber: // Nd alstValidChars.Add(ch1); break; default: alstInvalidChars.Add(ch1); break; } } //[]\d - we will create strings from valid characters that form \d and make sure that the regex engine catches this. //[]Build a random string with valid characters and then again invalid iCountTestcases++; pattern = @"\d+"; regex = new Regex(pattern); iNumLoop = 100; iWordCharLength = 10; iNonWordCharLength = 15; iCharCount = alstValidChars.Count; iNonCharCount = alstInvalidChars.Count; for (int i = 0; i < iNumLoop; i++) { sbldr1 = new StringBuilder(); sbldr2 = new StringBuilder(); for (int j = 0; j < iWordCharLength; j++) { ch1 = alstValidChars[random.Next(iCharCount)]; sbldr1.Append(ch1); sbldr2.Append(ch1); } for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } input = sbldr1.ToString(); match = regex.Match(input); if (!match.Success || (match.Index != 0) || (match.Length != iWordCharLength) || !(match.Value.Equals(sbldr2.ToString())) ) { iCountErrors++; Console.WriteLine("Err_245sfg_" + i + "! Error detected: input-{0}", input); } match = match.NextMatch(); if (match.Success) { iCountErrors++; Console.WriteLine("Err_29765sg_" + i + "! Error detected: input-{0}", input); } } //[]Build a random string with invalid characters, valid and then again invalid iCountTestcases++; pattern = @"\d+"; regex = new Regex(pattern); iNumLoop = 100; iWordCharLength = 10; iNonWordCharLength = 15; iCharCount = alstValidChars.Count; iNonCharCount = alstInvalidChars.Count; for (int i = 0; i < iNumLoop; i++) { sbldr1 = new StringBuilder(); sbldr2 = new StringBuilder(); for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } for (int j = 0; j < iWordCharLength; j++) { ch1 = alstValidChars[random.Next(iCharCount)]; sbldr1.Append(ch1); sbldr2.Append(ch1); } for (int j = 0; j < iNonWordCharLength; j++) { sbldr1.Append(alstInvalidChars[random.Next(iNonCharCount)]); } input = sbldr1.ToString(); match = regex.Match(input); if (!match.Success || (match.Index != iNonWordCharLength) || (match.Length != iWordCharLength) || !(match.Value.Equals(sbldr2.ToString())) ) { iCountErrors++; Console.WriteLine("Err_29756tsg_" + i + "! Error detected: input-{0}", input); } match = match.NextMatch(); if (match.Success) { iCountErrors++; Console.WriteLine("Err_27sjhr_" + i + "! Error detected: input-{0}", input); } } /////////////////////////////////////////////////////////////////// /////////////////////////// END TESTS ///////////////////////////// } catch (Exception exc_general) { ++iCountErrors; Console.WriteLine("Error Err_8888yyy! strLoc==" + strLoc + ", exc_general==" + exc_general.ToString()); } //// Finish Diagnostics Assert.Equal(0, iCountErrors); }
public static string ToSlug(this string s, bool lowercase = false, bool parenthetical = false, bool strict = false) { Contract.Requires <ArgumentNullException>(s != null); Contract.Ensures(Contract.Result <string>() != null); if (s.Length == 0) { return(string.Empty); } // Normalize to FormD in order to remove diacritics. var d = s.Normalize(NormalizationForm.FormD); var sb = new StringBuilder(s.Length); var wordbreak = false; var skipbreak = false; var parenCount = 0; foreach (var ch in d) { string append; var skipnext = false; if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_') { // Character is an alphanumeric if (lowercase && (ch >= 'A' && ch <= 'Z')) { // Append the lowercase form of this character append = char.ToLower(ch).ToString(); } else { // Append this character as-is append = ch.ToString(); } } else if (char.IsWhiteSpace(ch) || char.IsSeparator(ch) || ch == '-' || ch == '–' || ch == '—' || ch == '/' || ch == '\\') { // Break on this character. // Catches control-characters and other punctuations that UnicodeCategory does not. // It also catches certain common characters to avoid dictionary and category lookups. wordbreak = true; continue; } else if (ForeignCharacterSlugs.TryGetValue(ch, out append)) { // We have an (approximate) slug equivalent for this character. if (lowercase) { // We don't know if the value is lowercase or not, so we'll be safe. append = append.ToLowerInvariant(); } } else { var uc = CharUnicodeInfo.GetUnicodeCategory(ch); switch (uc) { case UnicodeCategory.SpaceSeparator: case UnicodeCategory.DashPunctuation: case UnicodeCategory.ConnectorPunctuation: // Character causes word-breaks when between two alphanumeric characters. wordbreak = true; continue; case UnicodeCategory.OpenPunctuation: if (parenthetical) { // Append parenthese and skip next wordbreak append = "("; parenCount++; skipnext = true; break; } wordbreak = true; goto default; case UnicodeCategory.ClosePunctuation: if (parenthetical && parenCount > 0) { // Skip current wordbreak, append parenthese, append = ")"; parenCount--; skipbreak = true; break; } wordbreak = true; goto default; case UnicodeCategory.MathSymbol: wordbreak = true; goto default; default: if (!strict) { continue; } var character = uc != UnicodeCategory.Control ? ch.ToString() : string.Empty; throw new ArgumentOutOfRangeException(nameof(s), ch, $"Unhandled character in UnicodeCategory.{uc}: '{character}' (0x{ch.ToHex()})."); } } if (wordbreak && !skipbreak && sb.Length != 0) { // We've added at least one alphanumeric character, we're // about to add another, and there was a break inbetween. sb.Append('-'); } // Append character and reset whether or not we're in a break. sb.Append(append); wordbreak = false; skipbreak = skipnext; } if (parenthetical) { for (; parenCount > 0; parenCount--) { sb.Append(")"); } } // Return slug re-normalized to FormC. return(sb.ToString().Normalize(NormalizationForm.FormC)); }
/// <summary> /// 返回指定字符的宽度。 /// </summary> /// <param name="ch">要获取宽度的字符。</param> /// <returns>指定字符的宽度,结果可能是 <c>0</c>、<c>1</c> 或 <c>2</c>。</returns> internal static int Width(int ch) { Contract.Requires(ch >= 0); Contract.Ensures(Contract.Result <int>() >= 0 && Contract.Result <int>() <= 2); UnicodeCategory category; if (ch <= char.MaxValue) { category = CharUnicodeInfo.GetUnicodeCategory((char)ch); } else { category = CharUnicodeInfo.GetUnicodeCategory(char.ConvertFromUtf32(ch), 0); } switch (category) { case UnicodeCategory.Control: case UnicodeCategory.NonSpacingMark: case UnicodeCategory.EnclosingMark: case UnicodeCategory.Format: return(0); } // 宽字符范围: // 0x1100 ~ 0x115F:Hangul Jamo init. consonants // 0x2329, 0x232A:左右尖括号〈〉 // 0x2E80 ~ 0xA4CF 除了 0x303F:CJK ... YI // 0xAC00 ~ 0xD7A3:Hangul Syllables // 0xF900 ~ 0xFAFF:CJK Compatibility Ideographs // 0xFE10 ~ 0xFE19:Vertical forms // 0xFE30 ~ 0xFE6F:CJK Compatibility Forms // 0xFF00 ~ 0xFF60:Fullwidth Forms // 0xFFE0 ~ 0xFFE6, 0x20000 ~ 0x2FFFD, 0x30000 ~ 0x3FFFD if (ch < 0x1100 || ch > 0x3FFFD) { return(1); } if (ch >= 0x20000) { return(ch <= 0x2FFFD || ch >= 0x30000 ? 2 : 1); } if (ch > 0xFFE6) { return(1); } if (ch <= 0xA4CF) { if (ch >= 0x2E80) { return(ch != 0x303F ? 2 : 1); } if (ch <= 0x115F) { return(2); } return(ch == 0x2329 || ch == 0x232A ? 2 : 1); } if (ch <= 0xD7A3) { return(ch >= 0xAC00 ? 2 : 1); } if (ch < 0xF900) { return(1); } if (ch <= 0xFAFF) { return(2); } if (ch < 0xFE10) { return(1); } if (ch < 0xFF00) { if (ch > 0xFE6F) { return(1); } if (ch >= 0xFE30 || ch <= 0xFE19) { return(2); } return(1); } if (ch <= 0xFF60 || ch >= 0xFFE0) { return(2); } return(1); }
// private methods private string EscapedString(string value) { if (value.All(c => !NeedsEscaping(c))) { return(value); } var sb = new StringBuilder(value.Length); foreach (char c in value) { switch (c) { case '"': sb.Append("\\\""); break; case '\\': sb.Append("\\\\"); break; case '\b': sb.Append("\\b"); break; case '\f': sb.Append("\\f"); break; case '\n': sb.Append("\\n"); break; case '\r': sb.Append("\\r"); break; case '\t': sb.Append("\\t"); break; default: switch (CharUnicodeInfo.GetUnicodeCategory(c)) { case UnicodeCategory.UppercaseLetter: case UnicodeCategory.LowercaseLetter: case UnicodeCategory.TitlecaseLetter: case UnicodeCategory.OtherLetter: case UnicodeCategory.DecimalDigitNumber: case UnicodeCategory.LetterNumber: case UnicodeCategory.OtherNumber: case UnicodeCategory.SpaceSeparator: case UnicodeCategory.ConnectorPunctuation: case UnicodeCategory.DashPunctuation: case UnicodeCategory.OpenPunctuation: case UnicodeCategory.ClosePunctuation: case UnicodeCategory.InitialQuotePunctuation: case UnicodeCategory.FinalQuotePunctuation: case UnicodeCategory.OtherPunctuation: case UnicodeCategory.MathSymbol: case UnicodeCategory.CurrencySymbol: case UnicodeCategory.ModifierSymbol: case UnicodeCategory.OtherSymbol: sb.Append(c); break; default: sb.AppendFormat("\\u{0:x4}", (int)c); break; } break; } } return(sb.ToString()); }
public override bool Execute() { string namespaceName; string className; if (string.IsNullOrEmpty(ResourceName)) { Log.LogError("ResourceName not specified"); return(false); } string resourceAccessName = string.IsNullOrEmpty(ResourceClassName) ? ResourceName : ResourceClassName; SplitName(resourceAccessName, out namespaceName, out className); string docCommentStart; Lang language; switch (Language.ToUpperInvariant()) { case "C#": language = Lang.CSharp; docCommentStart = "///"; break; case "VB": language = Lang.VisualBasic; docCommentStart = "'''"; break; default: Log.LogError($"GenerateResxSource doesn't support language: '{Language}'"); return(false); } string classIndent = (namespaceName == null ? "" : " "); string memberIndent = classIndent + " "; var strings = new StringBuilder(); foreach (var node in XDocument.Load(ResourceFile).Descendants("data")) { string name = node.Attribute("name")?.Value; if (name == null) { Log.LogError("Missing resource name"); return(false); } string value = node.Elements("value").FirstOrDefault()?.Value.Trim(); if (value == null) { Log.LogError($"Missing resource value: '{name}'"); return(false); } if (name == "") { Log.LogError($"Empty resource name"); return(false); } if (value.Length > maxDocCommentLength) { value = value.Substring(0, maxDocCommentLength) + " ..."; } string escapedTrimmedValue = new XElement("summary", value).ToString(); foreach (var line in escapedTrimmedValue.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None)) { strings.Append($"{memberIndent}{docCommentStart} "); strings.AppendLine(line); } string identifier = IsLetterChar(CharUnicodeInfo.GetUnicodeCategory(name[0])) ? name : "_" + name; string defaultValue = IncludeDefaultValues ? ", " + CreateStringLiteral(value, language) : string.Empty; switch (language) { case Lang.CSharp: strings.AppendLine($"{memberIndent}internal static string {identifier} => GetResourceString(\"{name}\"{defaultValue});"); break; case Lang.VisualBasic: strings.AppendLine($"{memberIndent}Friend Shared ReadOnly Property {identifier} As String"); strings.AppendLine($"{memberIndent} Get"); strings.AppendLine($"{memberIndent} Return GetResourceString(\"{name}\"{defaultValue})"); strings.AppendLine($"{memberIndent} End Get"); strings.AppendLine($"{memberIndent}End Property"); break; default: throw new InvalidOperationException(); } } string getStringMethod; if (OmitGetResourceString) { getStringMethod = null; } else { switch (language) { case Lang.CSharp: getStringMethod = $@"{memberIndent}internal static global::System.Globalization.CultureInfo Culture {{ get; set; }} {memberIndent}[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] {memberIndent}internal static string GetResourceString(string resourceKey, string defaultValue = null) => ResourceManager.GetString(resourceKey, Culture);"; break; case Lang.VisualBasic: getStringMethod = $@"{memberIndent}Friend Shared Property Culture As Global.System.Globalization.CultureInfo {memberIndent}<Global.System.Runtime.CompilerServices.MethodImpl(Global.System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)> {memberIndent}Friend Shared Function GetResourceString(ByVal resourceKey As String, Optional ByVal defaultValue As String = Nothing) As String {memberIndent} Get {memberIndent} Return ResourceManager.GetString(resourceKey, Culture) {memberIndent} End Get {memberIndent}End Function"; break; default: throw new InvalidOperationException(); } } string namespaceStart, namespaceEnd; if (namespaceName == null) { namespaceStart = namespaceEnd = null; } else { switch (language) { case Lang.CSharp: namespaceStart = $@"namespace {namespaceName}{Environment.NewLine}{{"; namespaceEnd = "}"; break; case Lang.VisualBasic: namespaceStart = $"Namespace {namespaceName}"; namespaceEnd = "End Namespace"; break; default: throw new InvalidOperationException(); } } string resourceTypeName; string resourceTypeDefinition; if (string.IsNullOrEmpty(ResourceClassName) || ResourceName == ResourceClassName) { // resource name is same as accessor, no need for a second type. resourceTypeName = className; resourceTypeDefinition = null; } else { // resource name differs from the access class, need a type for specifying the resources // this empty type must remain as it is required by the .NETNative toolchain for locating resources // once assemblies have been merged into the application resourceTypeName = ResourceName; string resourceNamespaceName; string resourceClassName; SplitName(resourceTypeName, out resourceNamespaceName, out resourceClassName); string resourceClassIndent = (resourceNamespaceName == null ? "" : " "); switch (language) { case Lang.CSharp: resourceTypeDefinition = $"{resourceClassIndent}internal static class {resourceClassName} {{ }}"; if (resourceNamespaceName != null) { resourceTypeDefinition = $@"namespace {resourceNamespaceName} {{ {resourceTypeDefinition} }}"; } break; case Lang.VisualBasic: resourceTypeDefinition = $@"{resourceClassIndent}Friend Class {resourceClassName} {resourceClassIndent}End Class"; if (resourceNamespaceName != null) { resourceTypeDefinition = $@"Namespace {resourceNamespaceName} {resourceTypeDefinition} End Namespace"; } break; default: throw new InvalidOperationException(); } } // The ResourceManager property being initialized lazily is an important optimization that lets .NETNative // completely remove the ResourceManager class if the disk space saving optimization to strip resources // (/DisableExceptionMessages) is turned on in the compiler. string result; switch (language) { case Lang.CSharp: result = $@"// <auto-generated> using System.Reflection; {resourceTypeDefinition} {namespaceStart} {classIndent}internal static partial class {className} {classIndent}{{ {memberIndent}private static global::System.Resources.ResourceManager s_resourceManager; {memberIndent}internal static global::System.Resources.ResourceManager ResourceManager => s_resourceManager ?? (s_resourceManager = new global::System.Resources.ResourceManager(typeof({resourceTypeName}))); {getStringMethod} {strings} {classIndent}}} {namespaceEnd} "; break; case Lang.VisualBasic: result = $@"' <auto-generated> Imports System.Reflection {resourceTypeDefinition} {namespaceStart} {classIndent}Friend Partial Class {className} {memberIndent}Private Sub New {memberIndent}End Sub {memberIndent} {memberIndent}Private Shared s_resourceManager As Global.System.Resources.ResourceManager {memberIndent}Friend Shared ReadOnly Property ResourceManager As Global.System.Resources.ResourceManager {memberIndent} Get {memberIndent} If s_resourceManager Is Nothing Then {memberIndent} s_resourceManager = New Global.System.Resources.ResourceManager(GetType({resourceTypeName})) {memberIndent} End If {memberIndent} Return s_resourceManager {memberIndent} End Get {memberIndent}End Property {getStringMethod} {strings} {classIndent}End Class {namespaceEnd} "; break; default: throw new InvalidOperationException(); } File.WriteAllText(OutputPath, result); return(true); }
/// <summary> /// Remplacer les caracteres spéciaux... /// </summary> /// <param name="sToClean"></param> /// <param name="cNewChar"></param> /// <returns></returns> public static string ReplaceSpecialCaracters(string sToClean, char?cChar) { try { // Vérification... if (string.IsNullOrEmpty(sToClean) == true) { return(string.Empty); } StringBuilder sOutString = new StringBuilder(); string sNormalized = WebUtility.HtmlDecode(sToClean);//.Normalize(NormalizationForm.FormD); /*string sNormalizedC = HttpUtility.HtmlDecode(sToClean).Normalize(NormalizationForm.FormC); * string sNormalizedKC = HttpUtility.HtmlDecode(sToClean).Normalize(NormalizationForm.FormKC); * string sNormalizedKD = HttpUtility.HtmlDecode(sToClean).Normalize(NormalizationForm.FormKD);*/ for (int i = 0; i < sNormalized.Length; i += 1) { Char c = sNormalized[i]; UnicodeCategory ucat = CharUnicodeInfo.GetUnicodeCategory(c); switch (ucat) { // Indicates that the character is an uppercase letter. Signified by the Unicode // designation "Lu" (letter, uppercase). The value is 0. case UnicodeCategory.UppercaseLetter: // Indicates that the character is a lowercase letter. Signified by the Unicode // designation "Ll" (letter, lowercase). The value is 1. case UnicodeCategory.LowercaseLetter: // Indicates that the character is a titlecase letter. Signified by the Unicode // designation "Lt" (letter, titlecase). The value is 2. case UnicodeCategory.TitlecaseLetter: // Indicates that the character is a decimal digit, that is, in the range 0 // through 9. Signified by the Unicode designation "Nd" (number, decimal digit). // The value is 8. case UnicodeCategory.DecimalDigitNumber: // Indicates that the character is a number represented by a letter, instead // of a decimal digit, for example, the Roman numeral for five, which is "V". // The indicator is signified by the Unicode designation "Nl" (number, letter). // The value is 9. case UnicodeCategory.LetterNumber: // Indicates that the character is a number that is neither a decimal digit // nor a letter number, for example, the fraction 1/2. The indicator is signified // by the Unicode designation "No" (number, other). The value is 10. case UnicodeCategory.OtherNumber: if ((cChar != null) || (i == (sNormalized.Length - 1)) || ((i + 1) < sNormalized.Length && sNormalized[i + 1] != '\'')) { sOutString.Append(c); } break; // Indicates that the character is a space character, which has no glyph but // is not a control or format character. Signified by the Unicode designation // "Zs" (separator, space). The value is 11. case UnicodeCategory.SpaceSeparator: // Indicates that the character is used to separate lines of text. Signified // by the Unicode designation "Zl" (separator, line). The value is 12. case UnicodeCategory.LineSeparator: // Indicates that the character is used to separate paragraphs. Signified by // the Unicode designation "Zp" (separator, paragraph). The value is 13. case UnicodeCategory.ParagraphSeparator: // Indicates that the character is a control code, with a Unicode value of U+007F // or in the range U+0000 through U+001F or U+0080 through U+009F. Signified // by the Unicode designation "Cc" (other, control). The value is 14. if (cChar != null) { sOutString.Append(cChar); } else { sOutString.Append(' '); } break; case UnicodeCategory.Control: // Indicates that the character is a format character, which is not normally // rendered but affects the layout of text or the operation of text processes. // Signified by the Unicode designation "Cf" (other, format). The value is 15. case UnicodeCategory.Format: // Indicates that the character is a high surrogate or a low surrogate. Surrogate // code values are in the range U+D800 through U+DFFF. Signified by the Unicode // designation "Cs" (other, surrogate). The value is 16. case UnicodeCategory.Surrogate: // Indicates that the character is a private-use character, with a Unicode value // in the range U+E000 through U+F8FF. Signified by the Unicode designation // "Co" (other, private use). The value is 17. case UnicodeCategory.PrivateUse: // Indicates that the character is a connector punctuation, which connects two // characters. Signified by the Unicode designation "Pc" (punctuation, connector). // The value is 18. case UnicodeCategory.ConnectorPunctuation: // Indicates that the character is a dash or a hyphen. Signified by the Unicode // designation "Pd" (punctuation, dash). The value is 19. case UnicodeCategory.DashPunctuation: // Indicates that the character is the opening character of one of the paired // punctuation marks, such as parentheses, square brackets, and braces. Signified // by the Unicode designation "Ps" (punctuation, open). The value is 20. case UnicodeCategory.OpenPunctuation: // Indicates that the character is the closing character of one of the paired // punctuation marks, such as parentheses, square brackets, and braces. Signified // by the Unicode designation "Pe" (punctuation, close). The value is 21. case UnicodeCategory.ClosePunctuation: // Indicates that the character is an opening or initial quotation mark. Signified // by the Unicode designation "Pi" (punctuation, initial quote). The value is // 22. case UnicodeCategory.InitialQuotePunctuation: // Indicates that the character is a closing or final quotation mark. Signified // by the Unicode designation "Pf" (punctuation, final quote). The value is // 23. case UnicodeCategory.FinalQuotePunctuation: // Indicates that the character is a punctuation that is not a connector punctuation, // a dash punctuation, an open punctuation, a close punctuation, an initial // quote punctuation, or a final quote punctuation. Signified by the Unicode // designation "Po" (punctuation, other). The value is 24. case UnicodeCategory.OtherPunctuation: // Indicates that the character is a mathematical symbol, such as "+" or "= // ". Signified by the Unicode designation "Sm" (symbol, math). The value is // 25. case UnicodeCategory.MathSymbol: // Indicates that the character is a currency symbol. Signified by the Unicode // designation "Sc" (symbol, currency). The value is 26. case UnicodeCategory.CurrencySymbol: // Indicates that the character is a modifier symbol, which indicates modifications // of surrounding characters. For example, the fraction slash indicates that // the number to the left is the numerator and the number to the right is the // denominator. The indicator is signified by the Unicode designation "Sk" (symbol, // modifier). The value is 27. case UnicodeCategory.ModifierSymbol: // Indicates that the character is a symbol that is not a mathematical symbol, // a currency symbol or a modifier symbol. Signified by the Unicode designation // "So" (symbol, other). The value is 28. case UnicodeCategory.OtherSymbol: // Indicates that the character is not assigned to any Unicode category. Signified // by the Unicode designation "Cn" (other, not assigned). The value is 29. case UnicodeCategory.OtherNotAssigned: default: if (cChar != null) { sOutString.Append(cChar); } break; } } return(sOutString.ToString()); } catch //(Exception e) { return(""); } }
private static bool IsInvalidPunctuationSymbol(char character) { return(character == '%' || CharUnicodeInfo.GetUnicodeCategory(character) == UnicodeCategory.CurrencySymbol); }
public static void ApplyControlBackspace(TextBox textBox) { if (textBox.SelectionLength == 0) { var text = textBox.Text; var deleteUpTo = textBox.SelectionStart; if (deleteUpTo > 0 && deleteUpTo <= text.Length) { text = text.Substring(0, deleteUpTo); var textElementIndices = StringInfo.ParseCombiningCharacters(text); var index = textElementIndices.Length; var textIndex = deleteUpTo; var deleteFrom = -1; while (index > 0) { index--; textIndex = textElementIndices[index]; if (!IsSpaceCategory(CharUnicodeInfo.GetUnicodeCategory(text, textIndex))) { break; } } if (index > 0) // HTML tag? { if (text[textIndex] == '>') { var openingBracketIndex = text.LastIndexOf('<', textIndex - 1); if (openingBracketIndex >= 0 && text.IndexOf('>', openingBracketIndex + 1) == textIndex) { deleteFrom = openingBracketIndex; // delete whole tag } } else if (text[textIndex] == '}') { var startIdx = text.LastIndexOf(@"{\", textIndex - 1, StringComparison.Ordinal); if (startIdx >= 0 && text.IndexOf('}', startIdx + 1) == textIndex) { deleteFrom = startIdx; } } } if (deleteFrom < 0) { if (BreakChars.Contains(text[textIndex])) { deleteFrom = -2; } while (index > 0) { index--; textIndex = textElementIndices[index]; if (IsSpaceCategory(CharUnicodeInfo.GetUnicodeCategory(text, textIndex))) { if (deleteFrom > -2) { if (deleteFrom < 0) { deleteFrom = textElementIndices[index + 1]; } break; } deleteFrom = textElementIndices[index + 1]; if (!":!?".Contains(text[deleteFrom])) { break; } } else if (BreakChars.Contains(text[textIndex])) { if (deleteFrom > -2) { if (deleteFrom < 0) { deleteFrom = textElementIndices[index + 1]; } break; } } else { deleteFrom = -1; } } } if (deleteFrom < deleteUpTo) { if (deleteFrom < 0) { deleteFrom = 0; } textBox.Select(deleteFrom, deleteUpTo - deleteFrom); textBox.Paste(string.Empty); } } } }
/// <summary> /// Returns whether or not a Unicode character represented by a codepoint is defined in Unicode. /// A character is considered to be defined if its Unicode designation is "Cn" (other, not assigned) OR if it is /// part of a surrogate pair. /// </summary> /// <param name="codepoint">The Unicode character's codepoint.</param> /// <returns>Whether or not the Unicode character represnted by codepoint is defined in Unicode.</returns> private static bool IsDefined(int codepoint) { return(IsSurrogateCodepoint(codepoint) || CharUnicodeInfo.GetUnicodeCategory(char.ConvertFromUtf32(codepoint), 0) != UnicodeCategory.OtherNotAssigned); }
public static bool IsWordSeparator(char c) { if (c == Document.NewLine || char.IsSeparator(c) || char.IsPunctuation(c) || CharUnicodeInfo.GetUnicodeCategory(c) == UnicodeCategory.MathSymbol) { return(true); } else { return(false); } }
public static string Create(string text, SlugOptions options) { if (text == null) { return(null); } if (options == null) { options = new SlugOptions(); } if (options.EarlyTruncate && options.MaximumLength <= 0 && text.Length <= options.MaximumLength) { text = text.Substring(0, options.MaximumLength); } text = text.Normalize(NormalizationForm.FormD); var sb = new StringBuilder(options.MaximumLength > 0 ? Math.Min(text.Length, options.MaximumLength) : text.Length); for (var index = 0; index < text.Length; index++) { var ch = text[index]; var unicodeCategory = CharUnicodeInfo.GetUnicodeCategory(ch); if (options.IsAllowed(ch)) { switch (unicodeCategory) { case UnicodeCategory.UppercaseLetter: if (options.ToLower) { ch = options.Culture != null?char.ToLower(ch, options.Culture) : char.ToLowerInvariant(ch); } sb.Append(options.Replace(ch)); break; case UnicodeCategory.LowercaseLetter: if (options.ToUpper) { ch = options.Culture != null?char.ToUpper(ch, options.Culture) : char.ToUpperInvariant(ch); } sb.Append(options.Replace(ch)); break; default: sb.Append(options.Replace(ch)); break; } } else if (unicodeCategory != UnicodeCategory.NonSpacingMark && options.Separator != null && !sb.EndsWith(options.Separator)) { sb.Append(options.Separator); } if (options.MaximumLength > 0 && sb.Length >= options.MaximumLength) { break; } } text = sb.ToString(); if (options.MaximumLength > 0 && text.Length > options.MaximumLength) { text = text.Substring(0, options.MaximumLength); } if (!options.CanEndWithSeparator && options.Separator != null && text.EndsWith(options.Separator, StringComparison.Ordinal)) { text = text.Substring(0, text.Length - options.Separator.Length); } return(text.Normalize(NormalizationForm.FormC)); }
public TSQLToken Parse( string tokenValue, int startPosition, int endPosition, bool useQuotedIdentifiers) { if ( char.IsWhiteSpace(tokenValue[0])) { return (new TSQLWhitespace( startPosition, tokenValue)); } else if ( tokenValue[0] == '@') { if (TSQLVariables.IsVariable(tokenValue)) { return (new TSQLSystemVariable( startPosition, tokenValue)); } else { return (new TSQLVariable( startPosition, tokenValue)); } } else if (tokenValue.StartsWith("--")) { return (new TSQLSingleLineComment( startPosition, tokenValue)); } else if (tokenValue.StartsWith("/*")) { if (tokenValue.EndsWith("*/")) { return (new TSQLMultilineComment( startPosition, tokenValue)); } else { return (new TSQLIncompleteCommentToken( startPosition, tokenValue)); } } else if ( tokenValue.StartsWith("'") || tokenValue.StartsWith("N'")) { // make sure there's an even number of quotes so that it's closed properly if ((tokenValue.Split('\'').Length - 1) % 2 == 0) { return (new TSQLStringLiteral( startPosition, tokenValue)); } else { return (new TSQLIncompleteStringToken( startPosition, tokenValue)); } } else if ( !useQuotedIdentifiers && tokenValue.StartsWith("\"")) { // make sure there's an even number of quotes so that it's closed properly if ((tokenValue.Split('\"').Length - 1) % 2 == 0) { return (new TSQLStringLiteral( startPosition, tokenValue)); } else { return (new TSQLIncompleteStringToken( startPosition, tokenValue)); } } else if ( tokenValue[0] == '$') { // $IDENTITY if ( tokenValue.Length > 1 && char.IsLetter(tokenValue[1])) { return (new TSQLSystemColumnIdentifier( startPosition, tokenValue)); } // $45.56 else { return (new TSQLMoneyLiteral( startPosition, tokenValue)); } } else if (CharUnicodeInfo.GetUnicodeCategory(tokenValue[0]) == UnicodeCategory.CurrencySymbol) { return (new TSQLMoneyLiteral( startPosition, tokenValue)); } else if (tokenValue.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) { return (new TSQLBinaryLiteral( startPosition, tokenValue)); } else if ( char.IsDigit(tokenValue[0]) || ( tokenValue[0] == '.' && tokenValue.Length > 1 && char.IsDigit(tokenValue[1]) )) { return (new TSQLNumericLiteral( startPosition, tokenValue)); } else if ( tokenValue[0] == '=' || tokenValue[0] == '~' || tokenValue[0] == '-' || tokenValue[0] == '+' || tokenValue[0] == '*' || tokenValue[0] == '/' || tokenValue[0] == '<' || tokenValue[0] == '>' || tokenValue[0] == '!' || tokenValue[0] == '&' || tokenValue[0] == '|' || tokenValue[0] == '^' || tokenValue[0] == '%' || tokenValue[0] == ':') { return (new TSQLOperator( startPosition, tokenValue)); } else if (TSQLCharacters.IsCharacter(tokenValue)) { return (new TSQLCharacter( startPosition, tokenValue)); } else if (TSQLKeywords.IsKeyword(tokenValue)) { return (new TSQLKeyword( startPosition, tokenValue)); } else if (TSQLIdentifiers.IsIdentifier(tokenValue)) { return (new TSQLSystemIdentifier( startPosition, tokenValue)); } else { if ( ( tokenValue.StartsWith("[") && !tokenValue.EndsWith("]") ) || ( useQuotedIdentifiers && tokenValue.StartsWith("\"") && // see if there's an odd number of quotes (tokenValue.Split('\"').Length - 1) % 2 == 1 )) { return (new TSQLIncompleteIdentifierToken( startPosition, tokenValue)); } else { return (new TSQLIdentifier( startPosition, tokenValue)); } } }
private static bool IsAllowedChar(char c) { return(!DisallowedCharacters.Contains(c) && CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark); }
public static string StripAccents(string text) { return(String.Concat(text.Normalize(NormalizationForm.FormD).Where(c => CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark))); }
/// <summary> /// Returns the <see cref="UnicodeCharacterClass"/> of the character at the specified offset in the given string. /// </summary> /// <param name="value">A <see cref="string"/>.</param> /// <param name="index">The character position in <paramref name="value"/>.</param> /// <returns>A <see cref="UnicodeCharacterClass"/> enumerated constant that identifies the character class of the character at position <paramref name="index"/> in <paramref name="value"/>.</returns> public static UnicodeCharacterClass GetCharacterClass(string value, int index) => GetCharacterClassFromCategory(CharUnicodeInfo.GetUnicodeCategory(value, index));