public List <Token> Tokenize(string input)
{
_input = input;
input = input.ToLower();
List <Token> tokens = new List <Token>();
CharGroup currentCharaterType = CharGroup.Undefined;
ExpressionType tokenizerContext = ExpressionType.Undefined;
int index = 0;
string rawExpression = string.Empty;
tokens.Add(new Token(TokenType.Start, 0, string.Empty));
for (int i = 0; i < input.Length; i++)
{
var t = GetCharGroup(input[i]);
if (currentCharaterType != t)
{
ProcessRawExpression(rawExpression, currentCharaterType, index, tokens, ref tokenizerContext);
rawExpression = string.Empty;
currentCharaterType = t;
index = i;
}
rawExpression += input[i];
}
ProcessRawExpression(rawExpression, currentCharaterType, index, tokens, ref tokenizerContext);
tokens.Add(new Token(TokenType.End, input.Length, string.Empty));
return(tokens);
}
/// 文字列、デコレーション、オフセットを指定するコンストラクタ。
public GWord(TextDecoration d, int o, CharGroup chargroup) {
Debug.Assert(d != null);
_offset = o;
_decoration = d;
_next = null;
_charGroup = chargroup;
}
public static GLine CreateSimpleGLine(string content)
{
TextDecoration dec = TextDecoration.Default;
CharGroup cg = content.Length > 0? GLine.CalcCharGroup(content[0]) : CharGroup.Hankaku;
return(new GLine(GLine.ToCharArray(content), new GWord(dec, 0, cg)));
}
/// 文字列、デコレーション、オフセットを指定するコンストラクタ。
public GWord(TextDecoration d, int o, CharGroup chargroup)
{
Debug.Assert(d != null);
_offset = o;
_decoration = d;
_next = null;
_charGroup = chargroup;
}
/// <summary>
/// 文字列、デコレーション、オフセットを指定するコンストラクタ。TypeはNormalになる。
/// </summary>
internal GWord(TextDecoration d, int o, CharGroup chargroup)
{
Debug.Assert(d != null);
_offset = o;
_decoration = d;
_next = null;
_charGroup = chargroup;
nextOffsetCache = -1;
displayLengthCache = -1;
}
public static GLine CreateSimpleGLine(string text, TextDecoration dec)
{
char[] buff = new char[text.Length * 2];
int offset = 0;
int start = 0;
CharGroup prevType = CharGroup.LatinHankaku;
GWord firstWord = null;
GWord lastWord = null;
for (int i = 0; i < text.Length; i++)
{
char originalChar = text[i];
char privateChar = Unicode.ToPrivate(originalChar);
CharGroup nextType = Unicode.GetCharGroup(privateChar);
int size = CharGroupUtil.GetColumnsPerCharacter(nextType);
if (nextType != prevType)
{
if (offset > start)
{
GWord newWord = new GWord(dec, start, prevType);
if (lastWord == null)
{
firstWord = lastWord = newWord;
}
else
{
lastWord.Next = newWord;
lastWord = newWord;
}
}
prevType = nextType;
start = offset;
}
buff[offset++] = originalChar;
if (size == 2)
{
buff[offset++] = WIDECHAR_PAD;
}
}
GWord w = new GWord(dec, start, prevType);
if (lastWord == null)
{
firstWord = w;
}
else
{
lastWord.Next = w;
}
return(new GLine(buff, offset, firstWord));
}
public static int GetColumnsPerCharacter(CharGroup cg)
{
if (cg == CharGroup.CJKZenkaku)
{
return(2);
}
else
{
return(1);
}
}
//テキストファイルから読み出す。装飾はムリだけど
public void LoadForTest(string filename)
{
StreamReader r = null;
try {
r = new StreamReader(filename, Encoding.Default);
TextDecoration dec = TextDecoration.Default;
string line = r.ReadLine();
while (line != null)
{
//GWordへの分割はしてない、注意
CharGroup cg = line.Length > 0? GLine.CalcCharGroup(line[0]) : CharGroup.Hankaku;
this.AddLine(new GLine(GLine.ToCharArray(line), new GWord(dec, 0, cg)));
line = r.ReadLine();
}
} finally {
if (r != null)
{
r.Close();
}
}
}
public CharGroup AddCharGroup(float originX)
{
int lo = 0;
int hi = m_charGroups.Count - 1;
int i = -1;
CharGroup charGroup2;
while (lo <= hi)
{
int mi = lo + ((hi - lo) >> 1);
var charGroup1 = m_charGroups[mi];
if (charGroup1.OriginX < originX)
{
lo = mi + 1;
}
else if (originX < charGroup1.OriginX)
{
hi = mi - 1;
}
else
{
if (charGroup1.Value.IsWhiteSpace())
{
charGroup1.AlwaysInsertSpaceBefore = true;
return(charGroup1);
}
throw new NotImplementedException();
}
}
i = m_charGroups.Count == 0 ? 0 : lo < 0 ? ~lo : lo;
charGroup2 = new CharGroup()
{
OriginX = originX,
};
m_charGroups.Insert(i, charGroup2);
return(charGroup2);
}
internal Font CalcFont(TextDecoration dec, CharGroup cg)
{
return CalcFontInternal(dec, cg, false).Font;
}
/// <summary>
/// <ja>
/// 指定位置に1文字書き込みます。
/// </ja>
/// <en>
/// Write one character to specified position.
/// </en>
/// </summary>
/// <param name="ch"><ja>書き込む文字</ja><en>Character to write.</en></param>
/// <param name="dec"><ja>テキスト書式を指定するTextDecorationオブジェクト</ja>
/// <en>TextDecoration object that specifies text format
/// </en></param>
public void PutChar(char ch, TextDecoration dec)
{
Debug.Assert(dec != null);
Debug.Assert(_caretColumn >= 0);
Debug.Assert(_caretColumn < _text.Length);
char originalChar = Unicode.ToOriginal(ch);
CharGroup charGroup = Unicode.GetCharGroup(ch);
bool onZenkaku = (_attrs[_caretColumn].CharGroup == CharGroup.CJKZenkaku);
bool onZenkakuRight = (_text[_caretColumn] == GLine.WIDECHAR_PAD);
if (onZenkaku)
{
//全角の上に書く
if (!onZenkakuRight)
{
_text[_caretColumn] = originalChar;
_attrs[_caretColumn] = new CharAttr(dec, charGroup);
if (CharGroupUtil.GetColumnsPerCharacter(charGroup) == 1)
{
//全角の上に半角を書いた場合、隣にスペースを入れないと表示が乱れる
_caretColumn++;
if (_caretColumn < _text.Length)
{
_text[_caretColumn] = ' ';
_attrs[_caretColumn].CharGroup = CharGroup.LatinHankaku;
}
}
else
{
_attrs[_caretColumn + 1] = new CharAttr(dec, charGroup);
_caretColumn += 2;
}
}
else
{
_text[_caretColumn - 1] = ' ';
_attrs[_caretColumn - 1].CharGroup = CharGroup.LatinHankaku;
_text[_caretColumn] = originalChar;
_attrs[_caretColumn] = new CharAttr(dec, charGroup);
if (CharGroupUtil.GetColumnsPerCharacter(charGroup) == 2)
{
if (CharGroupUtil.GetColumnsPerCharacter(_attrs[_caretColumn + 1].CharGroup) == 2)
{
if (_caretColumn + 2 < _text.Length)
{
_text[_caretColumn + 2] = ' ';
_attrs[_caretColumn + 2].CharGroup = CharGroup.LatinHankaku;
}
}
_text[_caretColumn + 1] = GLine.WIDECHAR_PAD;
_attrs[_caretColumn + 1] = _attrs[_caretColumn];
_caretColumn += 2;
}
else
{
_caretColumn++;
}
}
}
else //半角の上に書く
{
_text[_caretColumn] = originalChar;
_attrs[_caretColumn] = new CharAttr(dec, charGroup);
if (CharGroupUtil.GetColumnsPerCharacter(charGroup) == 2)
{
if (CharGroupUtil.GetColumnsPerCharacter(_attrs[_caretColumn + 1].CharGroup) == 2) //半角、全角となっているところに全角を書いたら
{
if (_caretColumn + 2 < _text.Length)
{
_text[_caretColumn + 2] = ' ';
_attrs[_caretColumn + 2].CharGroup = CharGroup.LatinHankaku;
}
}
_text[_caretColumn + 1] = GLine.WIDECHAR_PAD;
_attrs[_caretColumn + 1] = _attrs[_caretColumn];
_caretColumn += 2;
}
else
{
_caretColumn++; //これが最もcommonなケースだが
}
}
}
public CharAttr(TextDecoration dec, CharGroup cg) {
Decoration = dec;
CharGroup = cg;
}
public static bool IsCJK(CharGroup cg) {
return (cg == CharGroup.CJKHankaku || cg == CharGroup.CJKZenkaku);
}
public List <CharGroup> split()
{
List <CharGroup> groupsToAdd = new List <CharGroup>();
Stack <CharGroup> toAddStacked = new Stack <CharGroup>();
if (repr.Length == 1)
{
return(groupsToAdd);
}
bool split = false;
if (type == enumGroupType.CONSONANT)
{
if (repr.Length >= 3)
{
split = repr[0] != 's';
}
else
{
split = toBeSeparated(repr[0], repr[1]);
}
}
else
{
while (repr.Length > 3)
{
int lenToRemove = 1;
if (TRIPHTONGS.Contains(repr.Substring(repr.Length - 3)))
{
lenToRemove = 3;
}
else if (isDiphtong(repr[repr.Length - 2], repr[repr.Length - 1]))
{
lenToRemove = 2;
}
toAddStacked.Push(new CharGroup(repr.Substring(repr.Length - lenToRemove), enumGroupType.VOWEL));
repr = repr.Substring(0, repr.Length - lenToRemove);
}
if (repr.Length == 3)
{
split = !TRIPHTONGS.Contains(repr);
}
else if (repr.Length == 2)
{
split = !isDiphtong(repr[0], repr[1]);
}
}
if (split)
{
CharGroup ret = new CharGroup(repr.Substring(1), this.type);
repr = repr[0].ToString();
groupsToAdd.Add(ret);
}
while (toAddStacked.Count > 0)
{
groupsToAdd.Add(toAddStacked.Pop());
}
return(groupsToAdd);
}
private FontHandle CalcFontInternal(TextDecoration dec, CharGroup cg, bool ignore_underline)
{
if (_font == null)
{
CreateFonts();
}
if (CharGroupUtil.IsCJK(cg))
{
if (dec == null)
{
return(_cjkFont);
}
if (CalcBold(dec))
{
if (!ignore_underline && dec.Underline)
{
return(_cjkBoldUnderlinefont);
}
else
{
return(_cjkBoldfont);
}
}
else if (!ignore_underline && dec.Underline)
{
return(_cjkUnderlinefont);
}
else
{
return(_cjkFont);
}
}
else
{
if (dec == null)
{
return(_font);
}
if (CalcBold(dec))
{
if (!ignore_underline && dec.Underline)
{
return(_boldunderlinefont);
}
else
{
return(_boldfont);
}
}
else if (!ignore_underline && dec.Underline)
{
return(_underlinefont);
}
else
{
return(_font);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="dec"></param>
/// <param name="cg"></param>
/// <returns></returns>
/// <exclude/>
public Font CalcFont(TextDecoration dec, CharGroup cg)
{
return(CalcFontInternal(dec, cg, false).Font);
}
/// <summary>
/// ������A�f�R���[�V�����A�I�t�Z�b�g��w�肷��R���X�g���N�^�BType��Normal�ɂȂ�B
/// </summary>
internal GWord(TextDecoration d, int o, CharGroup chargroup)
{
Debug.Assert(d!=null);
_offset = o;
_decoration = d;
_next = null;
_charGroup = chargroup;
nextOffsetCache = -1;
displayLengthCache = -1;
}
public CharAttr(TextDecoration dec, CharGroup cg)
{
Decoration = dec;
CharGroup = cg;
}
private FontHandle CalcFontInternal(TextDecoration dec, CharGroup cg, bool ignore_underline) {
if (_font == null)
CreateFonts();
if (CharGroupUtil.IsCJK(cg)) {
if (dec == null)
return _cjkFont;
if (CalcBold(dec)) {
if (!ignore_underline && dec.Underline)
return _cjkBoldUnderlinefont;
else
return _cjkBoldfont;
}
else if (!ignore_underline && dec.Underline)
return _cjkUnderlinefont;
else
return _cjkFont;
}
else {
if (dec == null)
return _font;
if (CalcBold(dec)) {
if (!ignore_underline && dec.Underline)
return _boldunderlinefont;
else
return _boldfont;
}
else if (!ignore_underline && dec.Underline)
return _underlinefont;
else
return _font;
}
}
internal CharGroupCharGroup(CharGroup group, bool negative)
{
if (group == null)
throw new ArgumentNullException(nameof(group));
_group = group;
Negative = negative;
}
private FontHandle CalcFontInternal(TextDecoration dec, CharGroup cg, bool ignore_underline)
{
if (_font == null)
{
CreateFonts();
}
if (cg == CharGroup.TwoBytes)
{
if (dec == null)
{
return(_japaneseFont);
}
if (dec.Bold)
{
if (!ignore_underline && dec.Underline)
{
return(_japaneseBoldunderlinefont);
}
else
{
return(_japaneseBoldfont);
}
}
else if (!ignore_underline && dec.Underline)
{
return(_japaneseUnderlinefont);
}
else
{
return(_japaneseFont);
}
}
else
{
if (dec == null)
{
return(_font);
}
if (dec.Bold)
{
if (!ignore_underline && dec.Underline)
{
return(_boldunderlinefont);
}
else
{
return(_boldfont);
}
}
else if (!ignore_underline && dec.Underline)
{
return(_underlinefont);
}
else
{
return(_font);
}
}
}
private FontHandle CalcFontInternal(TextDecoration dec, CharGroup cg, bool ignore_underline)
{
if(_font==null) CreateFonts();
if(cg==CharGroup.TwoBytes) {
if(dec==null) return _japaneseFont;
if(dec.Bold) {
if(!ignore_underline && dec.Underline)
return _japaneseBoldunderlinefont;
else
return _japaneseBoldfont;
}
else if(!ignore_underline && dec.Underline)
return _japaneseUnderlinefont;
else
return _japaneseFont;
}
else {
if(dec==null) return _font;
if(dec.Bold) {
if(!ignore_underline && dec.Underline)
return _boldunderlinefont;
else
return _boldfont;
}
else if(!ignore_underline && dec.Underline)
return _underlinefont;
else
return _font;
}
}
public static bool IsCJK(CharGroup cg)
{
return(cg == CharGroup.CJKHankaku || cg == CharGroup.CJKZenkaku);
}
internal IntPtr CalcHFONT_NoUnderline(TextDecoration dec, CharGroup cg)
{
return CalcFontInternal(dec, cg, true).HFONT;
}
private void ProcessRawExpression(string expression, CharGroup charsType, int index, List <Token> tokens, ref ExpressionType tokenizerContext)
{
switch (charsType)
{
case CharGroup.Letter:
if (expression == TEMP_LITERAL)
{
tokenizerContext = ExpressionType.Tempo;
tokens.Add(new Token(TokenType.TempLiteral, index, expression));
return;
}
if (expression.Length == 2)
{
if (tokenizerContext == ExpressionType.Sound && expression == "ms")
{
tokens.Add(new Token(TokenType.TimeUnit, index, expression));
return;
}
//Detect aliasID
if (tokenizerContext == ExpressionType.Undefined)
{
if (!_degrees.Contains(expression[0]) || !_accidentals.Contains(expression[1]))
{
if (_input.Skip(index + 2).Take(1).FirstOrDefault() == '[')
{
tokens.Add(new Token(TokenType.AliasDeclaration, index, expression));
}
else
{
tokens.Add(new Token(TokenType.AliasReference, index, expression));
}
return;
}
}
ProcessRawExpression(expression[0].ToString(), CharGroup.Letter, index, tokens, ref tokenizerContext);
ProcessRawExpression(expression[1].ToString(), CharGroup.Letter, index + 1, tokens, ref tokenizerContext);
return;
}
if (expression.Length == 1)
{
//Detect the transposition marker
if (tokenizerContext == ExpressionType.Tempo && expression == TRANSPOSITION_MARKER)
{
tokens.Add(new Token(TokenType.TranspositionMarker, index, expression));
tokenizerContext = ExpressionType.Operator;
return;
}
//Detect a note
if (tokenizerContext == ExpressionType.Undefined && _degrees.Contains(expression))
{
tokens.Add(new Token(TokenType.DegreeLiteral, index, expression));
tokenizerContext = ExpressionType.Note;
return;
}
//Detrect pause
if (tokenizerContext == ExpressionType.Undefined && expression == "p")
{
tokens.Add(new Token(TokenType.PauseLiteral, index, expression));
tokenizerContext = ExpressionType.Note;
return;
}
//Detect an accidential sign
if (tokenizerContext == ExpressionType.Note && _accidentals.Contains(expression))
{
tokens.Add(new Token(TokenType.AccidentalLiteral, index, expression));
return;
}
//Detect the triplet sign
if (tokenizerContext == ExpressionType.Note && expression == TRANSPOSITION_MARKER)
{
tokens.Add(new Token(TokenType.TripletMarker, index, expression));
return;
}
if ((tokenizerContext == ExpressionType.Note || tokenizerContext == ExpressionType.Sound) && expression == PROLONG_OPERATOR)
{
tokens.Add(new Token(TokenType.ProlongOperator, index, expression));
return;
}
if (tokenizerContext == ExpressionType.Sound && _timeUnits.Contains(expression))
{
tokens.Add(new Token(TokenType.TimeUnit, index, expression));
return;
}
if (tokenizerContext == ExpressionType.Undefined && _input.Skip(index + expression.Length).Take(1).FirstOrDefault() == '[' && !_degrees.Contains(expression))
{
tokens.Add(new Token(TokenType.AliasDeclaration, index, expression));
return;
}
if (tokenizerContext == ExpressionType.Undefined && !_degrees.Contains(expression))
{
tokens.Add(new Token(TokenType.AliasReference, index, expression));
return;
}
tokens.Add(new Token(TokenType.Unknown, index, expression));
return;
}
if (tokenizerContext == ExpressionType.Undefined && _input.Skip(index + expression.Length).Take(1).FirstOrDefault() == '[')
{
tokens.Add(new Token(TokenType.AliasDeclaration, index, expression));
return;
}
tokens.Add(new Token(TokenType.AliasReference, index, expression));
return;
case CharGroup.Digit:
tokens.Add(new Token(TokenType.Number, index, expression));
if (tokenizerContext == ExpressionType.Undefined)
{
tokenizerContext = ExpressionType.Sound;
}
break;
case CharGroup.Other:
if (expression.Length > 1 && expression.Count(ch => ch == '.') != expression.Length)
{
for (int i = 0; i < expression.Length; i++)
{
ProcessRawExpression(expression[i].ToString(), CharGroup.Other, index + i, tokens, ref tokenizerContext);
}
return;
}
if (expression == REPEAT_OPERATOR)
{
tokens.Add(new Token(TokenType.RepeatOperator, index, expression));
return;
}
if ((tokenizerContext == ExpressionType.Note || tokenizerContext == ExpressionType.Sound) && expression == DIVIDER)
{
tokens.Add(new Token(TokenType.Divider, index, expression));
return;
}
if (tokenizerContext == ExpressionType.Operator && expression == DIVIDER)
{
tokens.Add(new Token(TokenType.MinusSign, index, expression));
return;
}
if (expression == "[")
{
tokens.Add(new Token(TokenType.OpenBracket, index, expression));
return;
}
if (expression == "]")
{
tokens.Add(new Token(TokenType.CloseBracket, index, expression));
return;
}
if (_accidentals.Contains(expression))
{
tokens.Add(new Token(TokenType.AccidentalLiteral, index, expression));
return;
}
if (expression.Count(ch => ch == '.') == expression.Length && (tokenizerContext == ExpressionType.Note || tokenizerContext == ExpressionType.Sound))
{
tokens.Add(new Token(TokenType.DotOperator, index, expression));
return;
}
tokens.Add(new Token(TokenType.Unknown, index, expression));
break;
case CharGroup.WhiteSpace:
tokens.Add(new Token(TokenType.WhiteSpace, index, expression));
tokenizerContext = ExpressionType.Undefined;
break;
case CharGroup.Undefined:
return;
}
}
/// <summary>
///
/// </summary>
/// <param name="dec"></param>
/// <param name="cg"></param>
/// <returns></returns>
/// <exclude/>
public IntPtr CalcHFONT_NoUnderline(TextDecoration dec, CharGroup cg)
{
return(CalcFontInternal(dec, cg, true).HFONT);
}
public static int GetColumnsPerCharacter(CharGroup cg) {
if (cg == CharGroup.CJKZenkaku)
return 2;
else
return 1;
}
/// <summary>
/// Gets number of columns used to display a character.
/// </summary>
/// <param name="ch">Private character code.</param>
/// <returns>1 (half-width character) or 2 (full-width character)</returns>
public static int GetCharacterWidth(char ch)
{
CharGroup charGroup = GetCharGroup(ch);
return(CharGroupUtil.GetColumnsPerCharacter(charGroup));
}
private List <CharGroup> getCharGroups(string word)
{
List <CharGroup> temp = new List <CharGroup>();
enumGroupType curType = enumGroupType.NONE;
CharGroup curGroup = null;
int i = 0;
char?prev = null;
char?next = null;
for (i = 0; i < word.Length; i++)
{
char c = word[i];
next = (i < word.Length - 1) ? (char?)word[i + 1] : null;
enumGroupType t = getTypeFromChar(c, prev, next);
if (t != curType)
{
curGroup = new CharGroup(c.ToString(), t);
curType = t;
temp.Add(curGroup);
}
else
{
curGroup.repr += c;
}
prev = c;
}
List <CharGroup> ret = new List <CharGroup>()
{
new CharGroup("", enumGroupType.NONE)
};
foreach (var group in temp)
{
ret.Add(group);
var groupsToAdd = group.split();
foreach (var toAdd in groupsToAdd)
{
ret.Add(toAdd);
}
}
ret.Add(new CharGroup("", enumGroupType.NONE));
temp = new List <CharGroup>();
i = 1;
while (i < ret.Count - 1)
{
CharGroup g = new CharGroup(ret[i].repr, ret[i].type);
while (g.type == enumGroupType.CONSONANT)
{
g.repr += ret[++i].repr;
if (ret[i].type != enumGroupType.CONSONANT)
{
g.type = ret[i].type;
}
}
if (ret[i].type == enumGroupType.VOWEL)
{
if (ret[i + 1].type == enumGroupType.CONSONANT && ret[i + 2].type != enumGroupType.VOWEL)
{
g.repr += ret[++i].repr;
if (ret[i + 1].type == enumGroupType.CONSONANT && ret[i + 2].type == enumGroupType.NONE)
{
g.repr += ret[++i].repr;
}
}
}
i++;
temp.Add(g);
}
return(temp);
}
