public override bool IsAlwaysNormalized (NormalizationForm form)
{
if (form != NormalizationForm.FormC)
return false;
if (isNormalized == null)
isNormalized = new byte [0x10000 / 8];
if (isNormalizedComputed == null)
isNormalizedComputed = new byte [0x10000 / 8];
if (normalization_bytes == null) {
normalization_bytes = new byte [0x100];
lock (normalization_bytes) {
for (int i = 0; i < 0x100; i++)
normalization_bytes [i] = (byte) i;
}
}
byte offset = (byte) (1 << (CodePage % 8));
if ((isNormalizedComputed [CodePage / 8] & offset) == 0) {
Encoding e = Clone () as Encoding;
e.DecoderFallback = new DecoderReplacementFallback ("");
string s = e.GetString (normalization_bytes);
// note that the flag only stores FormC information.
if (s != s.Normalize (form))
isNormalized [CodePage / 8] |= offset;
isNormalizedComputed [CodePage / 8] |= offset;
}
return (isNormalized [CodePage / 8] & offset) == 0;
}
internal unsafe Normalization(NormalizationForm form, String strDataFile)
{
// Remember which form we are
this.normalizationForm = form;
// Load the DLL
if (!nativeLoadNormalizationDLL())
{
// Unable to load the normalization DLL!
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
// Tell the DLL where to find our data
byte* pTables = GlobalizationAssembly.GetGlobalizationResourceBytePtr(
typeof(Normalization).Assembly, strDataFile);
if (pTables == null)
{
// Unable to load the specified normalizationForm,
// tables not loaded from file
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
// All we have to do is let the .dll know how to load it, then
// we can ignore the returned pointer.
byte* objNorm = nativeNormalizationInitNormalization(form, pTables);
if (objNorm == null)
{
// Unable to load the specified normalizationForm
// native library class not initialized correctly
throw new OutOfMemoryException(
Environment.GetResourceString("Arg_OutOfMemoryException"));
}
}
[System.Security.SecurityCritical] // auto-generated
static private unsafe void InitializeForm(NormalizationForm form, String strDataFile)
{
#if FEATURE_COREFX_GLOBALIZATION
//TODO: Implement this fully. We might need a PAL here.
throw new NotImplementedException();
#else
byte* pTables = null;
// Normalization uses OS on Win8
if (!Environment.IsWindows8OrAbove)
{
if (strDataFile == null)
{
// They were supposed to have a form that we know about!
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
// Tell the DLL where to find our data
pTables = GlobalizationAssembly.GetGlobalizationResourceBytePtr(
typeof(Normalization).Assembly, strDataFile);
if (pTables == null)
{
// Unable to load the specified normalizationForm,
// tables not loaded from file
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
}
nativeNormalizationInitNormalization(form, pTables);
#endif
}
public static string ClearForNormalize(this string value, NormalizationForm form)
{
if (!string.IsNullOrEmpty(value))
{
value = value.Replace(":", ""); // removed for solr
value = value.Replace("ΓΈ", "o");
var sb = new StringBuilder();
foreach (var c in value.Normalize(form))
switch (CharUnicodeInfo.GetUnicodeCategory(c))
{
case UnicodeCategory.NonSpacingMark:
case UnicodeCategory.SpacingCombiningMark:
case UnicodeCategory.EnclosingMark:
break;
default:
sb.Append(c);
break;
}
return sb.ToString().ToLower();
}
return string.Empty;
}
public static bool IsNormalized(this string value, NormalizationForm normalizationForm)
{
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
Contract.EndContractBlock();
// The only way to know if IsNormalizedString failed is through checking the Win32 last error
Interop.mincore.SetLastError(Interop.ERROR_SUCCESS);
bool result = Interop.mincore.IsNormalizedString((int)normalizationForm, value, value.Length);
int lastError = Marshal.GetLastWin32Error();
switch (lastError)
{
case Interop.ERROR_SUCCESS:
case Interop.LAST_ERROR_TRASH_VALUE:
break;
case Interop.ERROR_INVALID_PARAMETER:
case Interop.ERROR_NO_UNICODE_TRANSLATION:
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(value));
case Interop.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException(SR.Arg_OutOfMemoryException);
default:
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
return result;
}
public static string Normalize(this string strInput, NormalizationForm normalizationForm)
{
ValidateArguments(strInput, normalizationForm);
char[] buf = new char[strInput.Length];
for (int attempts = 2; attempts > 0; attempts--)
{
int realLen = Interop.GlobalizationNative.NormalizeString(normalizationForm, strInput, strInput.Length, buf, buf.Length);
if (realLen == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
if (realLen <= buf.Length)
{
return new string(buf, 0, realLen);
}
buf = new char[realLen];
}
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
static private unsafe void InitializeForm(NormalizationForm form, String strDataFile)
{
byte* pTables = null;
// Normalization uses OS on Win8
if (!Environment.IsWindows8OrAbove)
{
if (strDataFile == null)
{
// They were supposed to have a form that we know about!
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
// Tell the DLL where to find our data
pTables = GlobalizationAssembly.GetGlobalizationResourceBytePtr(
typeof(Normalization).Assembly, strDataFile);
if (pTables == null)
{
// Unable to load the specified normalizationForm,
// tables not loaded from file
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
}
nativeNormalizationInitNormalization(form, pTables);
}
public static String Normalize(this string value, NormalizationForm normalizationForm)
{
if (value == null)
throw new ArgumentNullException (nameof (value));
return value.Normalize (normalizationForm);
}
public void Normalize(string value, NormalizationForm normalizationForm, string expected)
{
if (normalizationForm == NormalizationForm.FormC)
{
Assert.Equal(expected, value.Normalize());
}
Assert.Equal(expected, value.Normalize(normalizationForm));
}
public void IsNormalized(string value, NormalizationForm normalizationForm, bool expected)
{
if (normalizationForm == NormalizationForm.FormC)
{
Assert.Equal(expected, value.IsNormalized());
}
Assert.Equal(expected, value.IsNormalized(normalizationForm));
}
public void IsNormalized(string value, NormalizationForm normalizationForm, bool expected)
{
if (normalizationForm == NormalizationForm.FormC)
{
Assert.Equal(expected, value.IsNormalized());
}
Assert.Equal(expected, value.IsNormalized(normalizationForm));
}
public void Normalize(string value, NormalizationForm normalizationForm, string expected)
{
if (normalizationForm == NormalizationForm.FormC)
{
Assert.Equal(expected, value.Normalize());
}
Assert.Equal(expected, value.Normalize(normalizationForm));
}
public static string Normalize(this string value, NormalizationForm normalizationForm)
{
if (IsNormalized(value, normalizationForm))
{
return value;
}
throw NotImplemented.ByDesign;
}
public static string Normalize(this string strInput, NormalizationForm normalizationForm)
{
if (strInput == null)
{
throw new ArgumentNullException(nameof(strInput));
}
return strInput.Normalize(normalizationForm);
}
/// <summary>
/// Initialize logging category
/// </summary>
/// <param name="r"></param>
/// <param name="unicodeNormalizeBool"></param>
public MarcTranslatedReader(IMarcReader r, bool unicodeNormalizeBool)
{
reader = r;
convert = new AnselToUnicode();
if (unicodeNormalizeBool)
{
this.unicodeNormalize = NormalizationForm.FormC;
}
}
public static string Normalize(this string strInput, NormalizationForm normalizationForm)
{
if (strInput == null)
{
throw new ArgumentNullException(nameof(strInput));
}
return(strInput.Normalize(normalizationForm));
}
public override bool IsAlwaysNormalized(NormalizationForm form)
{
// Latin-1 contains precomposed characters, so normal for Form C.
// Since some are composed, not normal for D & KD.
// Also some letters like 0x00A8 (spacing diarisis) have compatibility decompositions, so false for KD & KC.
// Only true for form C.
return(form == NormalizationForm.FormC);
}
public static string Normalize(this string value, NormalizationForm normalizationForm)
{
if (IsNormalized(value, normalizationForm))
{
return(value);
}
throw NotImplemented.ByDesign;
}
public static string StripZalgo(string displayName, ulong userId, NormalizationForm normalizationForm = NormalizationForm.FormD, int level = 0)
{
displayName = displayName?.Normalize(normalizationForm).TrimEager();
if (string.IsNullOrEmpty(displayName))
{
return("Rule #7 Breaker #" + userId.GetHashCode().ToString("x8"));
}
var builder = new StringBuilder();
bool skipLowSurrogate = false;
int consecutive = 0;
foreach (var c in displayName)
{
switch (char.GetUnicodeCategory(c))
{
case UnicodeCategory.ModifierSymbol:
case UnicodeCategory.NonSpacingMark:
if (++consecutive < level)
{
builder.Append(c);
}
break;
case UnicodeCategory.Control:
case UnicodeCategory.Format:
break;
case UnicodeCategory.OtherNotAssigned when c >= 0xdb40:
skipLowSurrogate = true;
break;
default:
if (char.IsLowSurrogate(c) && skipLowSurrogate)
{
skipLowSurrogate = false;
}
else
{
if (!OversizedChars.Contains(c))
{
builder.Append(c);
}
consecutive = 0;
}
break;
}
}
var result = builder.ToString().TrimEager();
if (string.IsNullOrEmpty(result))
{
return("Rule #7 Breaker #" + userId.GetHashCode().ToString("x8"));
}
return(result);
}
public static String Normalize(this string value, NormalizationForm normalizationForm)
{
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
return(value.Normalize(normalizationForm));
}
private static string[] NormalizeStrings(NormalizationForm nf, params string[] words)
{
for (int ctr = 0; ctr < words.Length; ctr++)
{
if (!words[ctr].IsNormalized(nf))
{
words[ctr] = words[ctr].Normalize(nf);
}
}
return(words);
}
//Method to normalize a given string in the normalization form specified
public string GetNormalizedString(string inputString)
{
if (normalizationForm != NormalizationNone)
{
NormalizationForm n = (NormalizationForm)Enum.Parse(typeof(NormalizationForm), normalizationForm);
//Normalize string
inputString = inputString.Normalize(n);
}
return(inputString);
}
public static string Normalize(this string str, NormalizationForm mode = NormalizationForm.FormC)
{
var it = CreateIterator(mode, str);
StringBuilder ret = new StringBuilder();
while (it.MoveNext())
{
ret.Append(it.Current);
}
return(ret.ToString());
}
/// <summary>
/// Safe version of normalize that doesn't crash on invalid code points in string.
/// Instead the points are replaced with question marks.
/// </summary>
public static string SafeNormalize(this string input, NormalizationForm normalizationForm = NormalizationForm.FormC)
{
try
{
return(StringTools.ReplaceNonCharacters(input, '?').Normalize(normalizationForm));
}
catch (ArgumentException e)
{
throw new InvalidDataException("String contains invalid characters. Data: " + Encoding.UTF32.GetBytes(input).ToHexString(), e);
}
}
public static bool IsNormalized(this string value, NormalizationForm normalizationForm)
{
for (int i = 0; i < value.Length; i++)
{
if (value[i] > 0x7F)
{
throw NotImplemented.ByDesign;
}
}
return(true);
}
public static bool IsNormalized(this string value, NormalizationForm normalizationForm)
{
for (int i = 0; i < value.Length; i++)
{
if (value[i] > 0x7F)
{
throw NotImplemented.ByDesign;
}
}
return true;
}
public static string Normalize (string source, NormalizationForm normalizationForm)
{
switch (normalizationForm) {
default:
return Normalization.Normalize (source, 0);
case NormalizationForm.FormD:
return Normalization.Normalize (source, 1);
case NormalizationForm.FormKC:
return Normalization.Normalize (source, 2);
case NormalizationForm.FormKD:
return Normalization.Normalize (source, 3);
}
}
public static bool IsNormalized(this string strInput, NormalizationForm normalizationForm)
{
ValidateArguments(strInput, normalizationForm);
int ret = Interop.GlobalizationNative.IsNormalized(normalizationForm, strInput, strInput.Length);
if (ret == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
return(ret == 1);
}
public static bool IsNormalized(this string value, NormalizationForm normalizationForm)
{
ValidateArguments(value, normalizationForm);
int ret = Interop.GlobalizationNative.IsNormalized(normalizationForm, value, value.Length);
if (ret == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, "value");
}
return(ret == 1);
}
public static bool IsNormalized(this string strInput, NormalizationForm normalizationForm)
{
ValidateArguments(strInput, normalizationForm);
int ret = Interop.GlobalizationInterop.IsNormalized(normalizationForm, strInput, strInput.Length);
if (ret == -1)
{
throw new ArgumentException(Environment.GetResourceString("Argument_InvalidCharSequenceNoIndex"), nameof(strInput));
}
return(ret == 1);
}
public static bool IsNormalized(this string strInput, NormalizationForm normalizationForm)
{
ValidateArguments(strInput, normalizationForm);
int ret = Interop.GlobalizationNative.IsNormalized(normalizationForm, strInput, strInput.Length);
if (ret == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
return ret == 1;
}
public static bool IsNormalized(this string strInput, NormalizationForm normalizationForm)
{
ValidateArguments(strInput, normalizationForm);
int ret = Interop.GlobalizationInterop.IsNormalized(normalizationForm, strInput, strInput.Length);
if (ret == -1)
{
throw new ArgumentException(Environment.GetResourceString("Argument_InvalidCharSequenceNoIndex"), nameof(strInput));
}
return ret == 1;
}
public static bool IsNormalized(this string value, NormalizationForm normalizationForm)
{
ValidateArguments(value, normalizationForm);
int ret = Interop.GlobalizationNative.IsNormalized(normalizationForm, value, value.Length);
if (ret == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, "value");
}
return ret == 1;
}
internal static string Normalize(string strInput, NormalizationForm normalizationForm)
{
if (GlobalizationMode.Invariant)
{
// In Invariant mode we assume all characters are normalized.
// This is because we don't support any linguistic operation on the strings
return(strInput);
}
return(GlobalizationMode.UseNls ?
NlsNormalize(strInput, normalizationForm) :
IcuNormalize(strInput, normalizationForm));
}
public static void Normalize(string utf16Source, string utf16Expected, NormalizationForm normalizationForm)
{
Utf8String utf8Source = u8(utf16Source);
// Quick IsNormalized tests
Assert.Equal(utf16Source == utf16Expected, utf8Source.IsNormalized(normalizationForm));
// Normalize and return new Utf8String instances
Utf8String utf8Normalized = utf8Source.Normalize(normalizationForm);
Assert.True(Utf8String.AreEquivalent(utf8Normalized, utf16Expected));
}
public bool IsNormalized(NormalizationForm normalizationForm)
{
if (this.IsFastSort())
{
// If its FastSort && one of the 4 main forms, then its already normalized
if (normalizationForm == NormalizationForm.FormC ||
normalizationForm == NormalizationForm.FormKC ||
normalizationForm == NormalizationForm.FormD ||
normalizationForm == NormalizationForm.FormKD)
{
return(true);
}
}
return(Normalization.IsNormalized(this, normalizationForm));
}
public String Normalize(NormalizationForm normalizationForm)
{
if (this.IsAscii())
{
// If its FastSort && one of the 4 main forms, then its already normalized
if (normalizationForm == NormalizationForm.FormC ||
normalizationForm == NormalizationForm.FormKC ||
normalizationForm == NormalizationForm.FormD ||
normalizationForm == NormalizationForm.FormKD)
{
return(this);
}
}
return(Normalization.Normalize(this, normalizationForm));
}
private static void ValidateArguments(string strInput, NormalizationForm normalizationForm)
{
Debug.Assert(strInput != null);
if (normalizationForm != NormalizationForm.FormC && normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC && normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
}
if (HasInvalidUnicodeSequence(strInput))
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
}
private static java.text.Normalizer.Form ToJavaNormalizeForm(NormalizationForm nf)
{
switch (nf)
{
case NormalizationForm.FormC: return(java.text.Normalizer.Form.NFC);
case NormalizationForm.FormD: return(java.text.Normalizer.Form.NFD);
case NormalizationForm.FormKC: return(java.text.Normalizer.Form.NFKC);
case NormalizationForm.FormKD: return(java.text.Normalizer.Form.NFKD);
default: return(java.text.Normalizer.Form.NFC);
}
}
internal static unsafe bool IsNormalized(string strInput, NormalizationForm normalizationForm)
{
if (GlobalizationMode.Invariant)
{
// In Invariant mode we assume all characters are normalized.
// This is because we don't support any linguistic operation on the strings
return(true);
}
Debug.Assert(strInput != null);
// The only way to know if IsNormalizedString failed is through checking the Win32 last error
// IsNormalizedString pinvoke has SetLastError attribute property which will set the last error
// to 0 (ERROR_SUCCESS) before executing the calls.
Interop.BOOL result;
fixed(char *pInput = strInput)
{
result = Interop.Normaliz.IsNormalizedString(normalizationForm, pInput, strInput.Length);
}
int lastError = Marshal.GetLastWin32Error();
switch (lastError)
{
case Interop.Errors.ERROR_SUCCESS:
break;
case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
if (normalizationForm != NormalizationForm.FormC &&
normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC &&
normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
}
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException();
default:
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
return(result != Interop.BOOL.FALSE);
}
protected override void loadTestFunction()
{
if (testName == "TestWin_NFC")
{
myTestFunction = new TestFunction(TestWin);
wmode = NormalizationForm.FormC;
}
else if (testName == "TestWin_NFD")
{
myTestFunction = new TestFunction(TestWin);
wmode = NormalizationForm.FormD;
}
else if (testName == "TestWin_NFKC")
{
myTestFunction = new TestFunction(TestWin);
wmode = NormalizationForm.FormKC;
}
else if (testName == "TestWin_NFKD")
{
myTestFunction = new TestFunction(TestWin);
wmode = NormalizationForm.FormKD;
}
else if (testName == "TestICU_NFC")
{
myTestFunction = new TestFunction(TestICU);
mode = (UNormalizationMode)4;
}
else if (testName == "TestICU_NFD")
{
myTestFunction = new TestFunction(TestICU);
mode = (UNormalizationMode)2;
}
else if (testName == "TestICU_NFKC")
{
myTestFunction = new TestFunction(TestICU);
mode = (UNormalizationMode)5;
}
else if (testName == "TestICU_NFKD")
{
myTestFunction = new TestFunction(TestICU);
mode = (UNormalizationMode)3;
}
else
{
Console.Out.WriteLine("Unknown test name: {0}", testName);
Environment.Exit(2);
}
}
/// <summary>
/// Converts this <see cref="Utf8Span"/> to the desired Unicode normalization form, writing the
/// UTF-8 result to the buffer <paramref name="destination"/>.
/// </summary>
/// <returns>
/// The number of bytes written to <paramref name="destination"/>, or -1 if <paramref name="destination"/>
/// is not large enough to hold the result of the normalization operation.
/// </returns>
/// <remarks>
/// The original <see cref="Utf8Span"/> is left unchanged by this operation. Note that the the required
/// length of <paramref name="destination"/> may be longer or shorter (in terms of UTF-8 byte count)
/// than the input <see cref="Utf8Span"/>.
/// </remarks>
public int Normalize(Span <byte> destination, NormalizationForm normalizationForm = NormalizationForm.FormC)
{
// TODO_UTF8STRING: Reduce allocations in this code path.
ReadOnlySpan <char> normalized = this.ToString().Normalize(normalizationForm).AsSpan();
OperationStatus status = Utf8.FromUtf16(normalized, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true);
Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "Normalize shouldn't have produced malformed Unicode string.");
if (status != OperationStatus.Done)
{
bytesWritten = -1; // "destination too small"
}
return(bytesWritten);
}
public static bool IsNormalized(this string src, NormalizationForm normalizationForm)
{
switch (normalizationForm)
{
default:
return(Normalization.IsNormalized(src, 0));
case NormalizationForm.FormD:
return(Normalization.IsNormalized(src, 1));
case NormalizationForm.FormKC:
return(Normalization.IsNormalized(src, 2));
case NormalizationForm.FormKD:
return(Normalization.IsNormalized(src, 3));
}
}
[System.Security.SecuritySafeCritical] // auto-generated
public String Normalize(NormalizationForm normalizationForm)
{
#if !FEATURE_NORM_IDNA_ONLY
if (this.IsAscii())
{
// If its FastSort && one of the 4 main forms, then its already normalized
if (normalizationForm == NormalizationForm.FormC ||
normalizationForm == NormalizationForm.FormKC ||
normalizationForm == NormalizationForm.FormD ||
normalizationForm == NormalizationForm.FormKD)
{
return(this);
}
}
#endif // !FEATURE_NORM_IDNA_ONLY
return(Normalization.Normalize(this, normalizationForm));
}
[System.Security.SecuritySafeCritical] // auto-generated
public bool IsNormalized(NormalizationForm normalizationForm)
{
#if !FEATURE_NORM_IDNA_ONLY
if (this.IsFastSort())
{
// If its FastSort && one of the 4 main forms, then its already normalized
if (normalizationForm == NormalizationForm.FormC ||
normalizationForm == NormalizationForm.FormKC ||
normalizationForm == NormalizationForm.FormD ||
normalizationForm == NormalizationForm.FormKD)
{
return(true);
}
}
#endif // !FEATURE_NORM_IDNA_ONLY
return(Normalization.IsNormalized(this, normalizationForm));
}
internal unsafe Normalization(NormalizationForm form, string strDataFile)
{
this.normalizationForm = form;
if (!nativeLoadNormalizationDLL())
{
throw new ArgumentException(Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
byte* globalizationResourceBytePtr = GlobalizationAssembly.GetGlobalizationResourceBytePtr(typeof(Normalization).Assembly, strDataFile);
if (globalizationResourceBytePtr == null)
{
throw new ArgumentException(Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
if (nativeNormalizationInitNormalization(form, globalizationResourceBytePtr) == null)
{
throw new OutOfMemoryException(Environment.GetResourceString("Arg_OutOfMemoryException"));
}
}
public override bool IsAlwaysNormalized(NormalizationForm form)
{
if (form != NormalizationForm.FormC)
{
return(false);
}
if (isNormalized == null)
{
isNormalized = new byte [0x10000 / 8];
}
if (isNormalizedComputed == null)
{
isNormalizedComputed = new byte [0x10000 / 8];
}
if (normalization_bytes == null)
{
normalization_bytes = new byte [0x100];
lock (normalization_bytes)
{
for (int i = 0; i < 0x100; i++)
{
normalization_bytes [i] = (byte)i;
}
}
}
byte offset = (byte)(1 << (CodePage % 8));
if ((isNormalizedComputed [CodePage / 8] & offset) == 0)
{
Encoding e = Clone() as Encoding;
e.DecoderFallback = new DecoderReplacementFallback("");
string s = e.GetString(normalization_bytes);
// note that the flag only stores FormC information.
if (s != s.Normalize(form))
{
isNormalized [CodePage / 8] |= offset;
}
isNormalizedComputed [CodePage / 8] |= offset;
}
return((isNormalized [CodePage / 8] & offset) == 0);
}
// -----------------------------
// ---- PAL layer ends here ----
// -----------------------------
private static void ValidateArguments(string value, NormalizationForm normalizationForm)
{
if (value == null)
{
throw new ArgumentNullException("value");
}
if (normalizationForm != NormalizationForm.FormC && normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC && normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, "normalizationForm");
}
if (HasInvalidUnicodeSequence(value))
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, "value");
}
}
// -----------------------------
// ---- PAL layer ends here ----
// -----------------------------
private static void ValidateArguments(string strInput, NormalizationForm normalizationForm)
{
if (strInput == null)
{
throw new ArgumentNullException(nameof(strInput));
}
if (normalizationForm != NormalizationForm.FormC && normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC && normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
}
if (HasInvalidUnicodeSequence(strInput))
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
}
}
void TestString (Testcase tc, string expected, NormalizationForm f)
{
string input = tc.Source;
string actual = null;
switch (f) {
default:
actual = Normalization.Normalize (input, 0); break;
case NormalizationForm.FormD:
actual = Normalization.Normalize (input, 1); break;
case NormalizationForm.FormKC:
actual = Normalization.Normalize (input, 2); break;
case NormalizationForm.FormKD:
actual = Normalization.Normalize (input, 3); break;
}
if (actual != expected)
Console.WriteLine ("Error: expected {0} but was {1} (for {2},type{3} form {4})",
expected, actual, tc.Source, tc.TestType, f);
}
unsafe private static extern byte* nativeNormalizationInitNormalization(
NormalizationForm NormForm, byte* pTableData);
unsafe private static extern int nativeNormalizationIsNormalizedString(
NormalizationForm NormForm, ref int iError,
String lpString, int cwLength);
unsafe private static extern int nativeNormalizationNormalizeString(
NormalizationForm NormForm, ref int iError,
String lpSrcString, int cwSrcLength,
char[] lpDstString, int cwDstLength);
internal static String Normalize(String strInput, NormalizationForm normForm)
{
return GetNormalization(normForm).Normalize(strInput);
}
internal static bool IsNormalized(String strInput, NormalizationForm normForm)
{
return GetNormalization(normForm).IsNormalized(strInput);
}
static internal Normalization GetNormalization(NormalizationForm form)
{
switch ((ExtendedNormalizationForms)form)
{
case ExtendedNormalizationForms.FormC:
return GetFormC();
case ExtendedNormalizationForms.FormD:
return GetFormD();
case ExtendedNormalizationForms.FormKC:
return GetFormKC();
case ExtendedNormalizationForms.FormKD:
return GetFormKD();
case ExtendedNormalizationForms.FormIdna:
return GetFormIDNA();
case ExtendedNormalizationForms.FormCDisallowUnassigned:
return GetFormCDisallowUnassigned();
case ExtendedNormalizationForms.FormDDisallowUnassigned:
return GetFormDDisallowUnassigned();
case ExtendedNormalizationForms.FormKCDisallowUnassigned:
return GetFormKCDisallowUnassigned();
case ExtendedNormalizationForms.FormKDDisallowUnassigned:
return GetFormKDDisallowUnassigned();
case ExtendedNormalizationForms.FormIdnaDisallowUnassigned:
return GetFormIDNADisallowUnassigned();
}
// They were supposed to have a form that we know about!
throw new ArgumentException(
Environment.GetResourceString("Argument_InvalidNormalizationForm"));
}
internal static extern int NormalizeString(NormalizationForm normalizationForm, string src, int srcLen, [Out] char[] dstBuffer, int dstBufferCapacity);
internal static extern int IsNormalized(NormalizationForm normalizationForm, string src, int srcLen);
public string Normalize(NormalizationForm normalizationForm)
{ return this.Value.Normalize(normalizationForm); }
public bool IsNormalized(NormalizationForm normalizationForm)
{ return this.Value.IsNormalized(normalizationForm); }
