private int FindPriorNonWhitespace(int from)
{
if (from > 0)
{
var cp = Codepoint.ReadAt(_text, from - 1, out var count);
var cls = cp.LineBreakClass;
if (cls == LineBreakClass.MandatoryBreak || cls == LineBreakClass.LineFeed || cls == LineBreakClass.CarriageReturn)
{
from -= count;
}
}
while (from > 0)
{
var cp = Codepoint.ReadAt(_text, from - 1, out var count);
var cls = cp.LineBreakClass;
if (cls == LineBreakClass.Space)
{
from -= count;
}
else
{
break;
}
}
return(from);
}
private static LineBreakClass MapClass(Codepoint cp)
{
if (cp.Value == 327685)
{
return(LineBreakClass.Alphabetic);
}
// LB 1
// ==========================================
// Resolved Original General_Category
// ==========================================
// AL AI, SG, XX Any
// CM SA Only Mn or Mc
// AL SA Any except Mn and Mc
// NS CJ Any
switch (cp.LineBreakClass)
{
case LineBreakClass.Ambiguous:
case LineBreakClass.Surrogate:
case LineBreakClass.Unknown:
return(LineBreakClass.Alphabetic);
case LineBreakClass.ComplexContext:
return(cp.GeneralCategory == GeneralCategory.NonspacingMark || cp.GeneralCategory == GeneralCategory.SpacingMark
? LineBreakClass.CombiningMark
: LineBreakClass.Alphabetic);
case LineBreakClass.ConditionalJapaneseStarter:
return(LineBreakClass.Nonstarter);
default:
return(cp.LineBreakClass);
}
}
// Get the next character class
private LineBreakClass ReadCharClass()
{
var cp = Codepoint.ReadAt(_text, _pos, out var count);
_pos += count;
return(MapClass(cp.LineBreakClass));
}
public bool TryGetPairedBracket(out Codepoint codepoint)
{
if (PairedBracketType == BidiPairedBracketType.None)
{
codepoint = default;
return(false);
}
codepoint = UnicodeData.GetBiDiPairedBracket(_value);
return(true);
}
internal static Codepoint GetCanonicalType(Codepoint codePoint)
{
if (codePoint._value == 0x3008)
{
return(new Codepoint(0x2329));
}
if (codePoint._value == 0x3009)
{
return(new Codepoint(0x232A));
}
return(codePoint);
}
/// <summary>
/// Moves to the next <see cref="Codepoint"/>.
/// </summary>
/// <returns></returns>
public bool MoveNext()
{
if (_text.IsEmpty)
{
Current = Codepoint.ReplacementCodepoint;
return(false);
}
Current = Codepoint.ReadAt(_text, 0, out var count);
_text = _text.Skip(count);
return(true);
}
public void MoveNext()
{
// For ill-formed subsequences (like unpaired UTF-16 surrogate code points), we rely on
// the decoder's default behavior of interpreting these ill-formed subsequences as
// equivalent to U+FFFD REPLACEMENT CHARACTER. This code point has a boundary property
// of Other (XX), which matches the modifications made to UAX#29, Rev. 35.
// See: https://www.unicode.org/reports/tr29/tr29-35.html#Modifications
// This change is also reflected in the UCD files. For example, Unicode 11.0's UCD file
// https://www.unicode.org/Public/11.0.0/ucd/auxiliary/GraphemeBreakProperty.txt
// has the line "D800..DFFF ; Control # Cs [2048] <surrogate-D800>..<surrogate-DFFF>",
// but starting with Unicode 12.0 that line has been removed.
//
// If a later version of the Unicode Standard further modifies this guidance we should reflect
// that here.
if (CurrentCodeUnitOffset == _buffer.Length)
{
CurrentCodepoint = Codepoint.ReplacementCodepoint;
}
else
{
CurrentCodeUnitOffset += _codeUnitLengthOfCurrentScalar;
if (CurrentCodeUnitOffset < _buffer.Length)
{
CurrentCodepoint = Codepoint.ReadAt(_buffer, CurrentCodeUnitOffset,
out _codeUnitLengthOfCurrentScalar);
}
else
{
CurrentCodepoint = Codepoint.ReplacementCodepoint;
}
}
CurrentType = CurrentCodepoint.GraphemeBreakClass;
}
public bool MoveNext()
{
// get the first char if we're at the beginning of the string
if (!_curClass.HasValue)
{
_curClass = PeekCharClass() == LineBreakClass.Space ? LineBreakClass.WordJoiner : MapFirst(ReadCharClass());
}
while (_pos < _text.Length)
{
_lastPos = _pos;
var lastClass = _nextClass;
_nextClass = ReadCharClass();
// explicit newline
if (_curClass.HasValue && (_curClass == LineBreakClass.MandatoryBreak || _curClass == LineBreakClass.CarriageReturn && _nextClass != LineBreakClass.LineFeed))
{
_curClass = MapFirst(MapClass(_nextClass.Value));
Current = new LineBreak(FindPriorNonWhitespace(_lastPos), _lastPos, true);
return(true);
}
// handle classes not handled by the pair table
LineBreakClass?cur = null;
switch (_nextClass.Value)
{
case LineBreakClass.Space:
cur = _curClass;
break;
case LineBreakClass.MandatoryBreak:
case LineBreakClass.LineFeed:
case LineBreakClass.NextLine:
cur = LineBreakClass.MandatoryBreak;
break;
case LineBreakClass.CarriageReturn:
cur = LineBreakClass.CarriageReturn;
break;
case LineBreakClass.ContingentBreak:
cur = LineBreakClass.BreakAfter;
break;
}
if (cur != null)
{
_curClass = cur;
if (_nextClass.Value == LineBreakClass.MandatoryBreak)
{
_lastPos = _pos;
Current = new LineBreak(FindPriorNonWhitespace(_lastPos), _lastPos, true);
return(true);
}
continue;
}
// if not handled already, use the pair table
var shouldBreak = false;
switch (BreakPairTable.Map(_curClass.Value, _nextClass.Value))
{
case PairBreakType.DI: // Direct break
shouldBreak = true;
_lastPos = _pos;
break;
case PairBreakType.IN: // possible indirect break
shouldBreak = lastClass.HasValue && lastClass.Value == LineBreakClass.Space;
break;
case PairBreakType.CI:
shouldBreak = lastClass.HasValue && lastClass.Value == LineBreakClass.Space;
if (!shouldBreak)
{
continue;
}
break;
case PairBreakType.CP: // prohibited for combining marks
if (!lastClass.HasValue || lastClass.Value != LineBreakClass.Space)
{
continue;
}
break;
}
_curClass = _nextClass;
if (shouldBreak)
{
Current = new LineBreak(FindPriorNonWhitespace(_lastPos), _lastPos);
return(true);
}
}
if (_pos >= _text.Length)
{
if (_lastPos < _text.Length)
{
_lastPos = _text.Length;
var cls = Codepoint.ReadAt(_text, _text.Length - 1, out _).LineBreakClass;
bool required = cls == LineBreakClass.MandatoryBreak || cls == LineBreakClass.LineFeed || cls == LineBreakClass.CarriageReturn;
Current = new LineBreak(FindPriorNonWhitespace(_text.Length), _text.Length, required);
return(true);
}
}
return(false);
}
private LineBreakClass PeekCharClass()
{
return(MapClass(Codepoint.ReadAt(_text, _pos, out _).LineBreakClass));
}
public static bool IsInRangeInclusive(Codepoint cp, uint lowerBound, uint upperBound) => (cp._value - lowerBound) <= (upperBound - lowerBound);
// Get the next character class
private LineBreakClass NextCharClass()
{
var cp = Codepoint.ReadAt(_text, _position, out var count);
var cls = MapClass(cp);
_position += count;
// Keep track of alphanumeric + any combining marks.
// This is used for LB22 and LB30.
if (IsAlphaNumeric(_currentClass) || _alphaNumericCount > 0 && cls == LineBreakClass.CombiningMark)
{
_alphaNumericCount++;
}
// Track combining mark exceptions. LB22
if (cls == LineBreakClass.CombiningMark)
{
switch (_currentClass)
{
case LineBreakClass.MandatoryBreak:
case LineBreakClass.ContingentBreak:
case LineBreakClass.Exclamation:
case LineBreakClass.LineFeed:
case LineBreakClass.NextLine:
case LineBreakClass.Space:
case LineBreakClass.ZWSpace:
case LineBreakClass.CarriageReturn:
_lb22ex = true;
break;
}
}
// Track combining mark exceptions. LB31
if (_first && cls == LineBreakClass.CombiningMark)
{
_lb31 = true;
}
if (cls == LineBreakClass.CombiningMark)
{
switch (_currentClass)
{
case LineBreakClass.MandatoryBreak:
case LineBreakClass.ContingentBreak:
case LineBreakClass.Exclamation:
case LineBreakClass.LineFeed:
case LineBreakClass.NextLine:
case LineBreakClass.Space:
case LineBreakClass.ZWSpace:
case LineBreakClass.CarriageReturn:
case LineBreakClass.ZWJ:
_lb31 = true;
break;
}
}
if (_first &&
(cls == LineBreakClass.PostfixNumeric || cls == LineBreakClass.PrefixNumeric || cls == LineBreakClass.Space))
{
_lb31 = true;
}
if (_currentClass == LineBreakClass.Alphabetic &&
(cls == LineBreakClass.PostfixNumeric || cls == LineBreakClass.PrefixNumeric || cls == LineBreakClass.Space))
{
_lb31 = true;
}
// Reset LB31 if next is U+0028 (Left Opening Parenthesis)
if (_lb31 &&
_currentClass != LineBreakClass.PostfixNumeric &&
_currentClass != LineBreakClass.PrefixNumeric &&
cls == LineBreakClass.OpenPunctuation && cp.Value == 0x0028)
{
_lb31 = false;
}
// Rule LB24
if (_first && (cls == LineBreakClass.ClosePunctuation || cls == LineBreakClass.CloseParenthesis))
{
_lb24ex = true;
}
// Rule LB25
if (_first &&
(cls == LineBreakClass.ClosePunctuation || cls == LineBreakClass.InfixNumeric || cls == LineBreakClass.BreakSymbols))
{
_lb25ex = true;
}
if (cls == LineBreakClass.Space || cls == LineBreakClass.WordJoiner || cls == LineBreakClass.Alphabetic)
{
var next = PeekNextCharClass();
if (next == LineBreakClass.ClosePunctuation || next == LineBreakClass.InfixNumeric || next == LineBreakClass.BreakSymbols)
{
_lb25ex = true;
}
}
// AlphaNumeric + and combining marks can break for OP except.
// - U+0028 (Left Opening Parenthesis)
// - U+005B (Opening Square Bracket)
// - U+007B (Left Curly Bracket)
// See custom columns|rules in the text pair table.
// https://www.unicode.org/Public/13.0.0/ucd/auxiliary/LineBreakTest.html
_lb30 = _alphaNumericCount > 0 &&
cls == LineBreakClass.OpenPunctuation &&
cp.Value != 0x0028 &&
cp.Value != 0x005B &&
cp.Value != 0x007B;
return(cls);
}
private LineBreakClass PeekNextCharClass()
{
var cp = Codepoint.ReadAt(_text, _position, out _);
return(MapClass(cp));
}
/// <summary>
/// Appends text to the bidi data.
/// </summary>
/// <param name="text">The text to process.</param>
public void Append(ReadOnlySlice <char> text)
{
_classes.Add(text.Length);
_pairedBracketTypes.Add(text.Length);
_pairedBracketValues.Add(text.Length);
// Resolve the BidiCharacterType, paired bracket type and paired
// bracket values for all code points
HasBrackets = false;
HasEmbeddings = false;
HasIsolates = false;
int i = Length;
var codePointEnumerator = new CodepointEnumerator(text);
while (codePointEnumerator.MoveNext())
{
var codepoint = codePointEnumerator.Current;
// Look up BiDiClass
var dir = codepoint.BiDiClass;
_classes[i] = dir;
switch (dir)
{
case BidiClass.LeftToRightEmbedding:
case BidiClass.LeftToRightOverride:
case BidiClass.RightToLeftEmbedding:
case BidiClass.RightToLeftOverride:
case BidiClass.PopDirectionalFormat:
{
HasEmbeddings = true;
break;
}
case BidiClass.LeftToRightIsolate:
case BidiClass.RightToLeftIsolate:
case BidiClass.FirstStrongIsolate:
case BidiClass.PopDirectionalIsolate:
{
HasIsolates = true;
break;
}
}
// Lookup paired bracket types
var pbt = codepoint.PairedBracketType;
_pairedBracketTypes[i] = pbt;
if (pbt == BidiPairedBracketType.Open)
{
// Opening bracket types can never have a null pairing.
codepoint.TryGetPairedBracket(out var paired);
_pairedBracketValues[i] = (int)Codepoint.GetCanonicalType(paired).Value;
HasBrackets = true;
}
else if (pbt == BidiPairedBracketType.Close)
{
_pairedBracketValues[i] = (int)Codepoint.GetCanonicalType(codepoint).Value;
HasBrackets = true;
}
i++;
}
Length = i;
Classes = _classes.AsSlice(0, Length);
PairedBracketTypes = _pairedBracketTypes.AsSlice(0, Length);
PairedBracketValues = _pairedBracketValues.AsSlice(0, Length);
}
private bool GetPairTableBreak(LineBreakClass lastClass)
{
// If not handled already, use the pair table
bool shouldBreak = false;
switch (LineBreakPairTable.Table[(int)_currentClass][(int)_nextClass])
{
case LineBreakPairTable.DIBRK: // Direct break
shouldBreak = true;
break;
// TODO: Rewrite this so that it defaults to true and rules are set as exceptions.
case LineBreakPairTable.INBRK: // Possible indirect break
// LB31
if (_lb31 && _nextClass == LineBreakClass.OpenPunctuation)
{
shouldBreak = true;
_lb31 = false;
break;
}
// LB30
if (_lb30)
{
shouldBreak = true;
_lb30 = false;
_alphaNumericCount = 0;
break;
}
// LB25
if (_lb25ex && (_nextClass == LineBreakClass.PrefixNumeric || _nextClass == LineBreakClass.Numeric))
{
shouldBreak = true;
_lb25ex = false;
break;
}
// LB24
if (_lb24ex && (_nextClass == LineBreakClass.PostfixNumeric || _nextClass == LineBreakClass.PrefixNumeric))
{
shouldBreak = true;
_lb24ex = false;
break;
}
// LB18
shouldBreak = lastClass == LineBreakClass.Space;
break;
case LineBreakPairTable.CIBRK:
shouldBreak = lastClass == LineBreakClass.Space;
if (!shouldBreak)
{
return(false);
}
break;
case LineBreakPairTable.CPBRK: // prohibited for combining marks
if (lastClass != LineBreakClass.Space)
{
return(false);
}
break;
case LineBreakPairTable.PRBRK:
break;
}
// Rule LB22
if (_nextClass == LineBreakClass.Inseparable)
{
switch (lastClass)
{
case LineBreakClass.MandatoryBreak:
case LineBreakClass.ContingentBreak:
case LineBreakClass.Exclamation:
case LineBreakClass.LineFeed:
case LineBreakClass.NextLine:
case LineBreakClass.Space:
case LineBreakClass.ZWSpace:
// Allow break
break;
case LineBreakClass.CombiningMark:
if (_lb22ex)
{
// Allow break
_lb22ex = false;
break;
}
shouldBreak = false;
break;
default:
shouldBreak = false;
break;
}
}
if (_lb8a)
{
shouldBreak = false;
}
// Rule LB21a
if (_lb21a && (_currentClass == LineBreakClass.Hyphen || _currentClass == LineBreakClass.BreakAfter))
{
shouldBreak = false;
_lb21a = false;
}
else
{
_lb21a = _currentClass == LineBreakClass.HebrewLetter;
}
// Rule LB30a
if (_currentClass == LineBreakClass.RegionalIndicator)
{
_lb30a++;
if (_lb30a == 2 && _nextClass == LineBreakClass.RegionalIndicator)
{
shouldBreak = true;
_lb30a = 0;
}
}
else
{
_lb30a = 0;
}
// Rule LB30b
if (_nextClass == LineBreakClass.EModifier && _lastPosition > 0)
{
// Mahjong Tiles (Unicode block) are extended pictographics but have a class of ID
// Unassigned codepoints with Line_Break=ID in some blocks are also assigned the Extended_Pictographic property.
// Those blocks are intended for future allocation of emoji characters.
var cp = Codepoint.ReadAt(_text, _lastPosition - 1, out int _);
if (Codepoint.IsInRangeInclusive(cp, 0x1F000, 0x1F02F))
{
shouldBreak = false;
}
}
_currentClass = _nextClass;
return(shouldBreak);
}
public Grapheme(Codepoint firstCodepoint, ReadOnlySlice <char> text)
{
FirstCodepoint = firstCodepoint;
Text = text;
}