private static unsafe void GetLinesBurst(ParseLinesWorker.Info *info, ref UnsafeRefToNativeList <Char16> continueBuff, ref UnsafeRefToNativeHeadRemovableList <Char16> charBuff, ref UnsafeRefToNativeStringList lines, out int line_count) { GetLinesImpl(info, continueBuff, charBuff, lines, out line_count); }
internal static unsafe void GetLinesImpl(ParseLinesWorker.Info *info, UnsafeRefToNativeList <Char16> continueBuff, UnsafeRefToNativeHeadRemovableList <Char16> charBuff, UnsafeRefToNativeStringList lines, out int line_count) { // move continue buffer data into head of new charBuff if (continueBuff.Length > 0) { charBuff.InsertHead((Char16 *)continueBuff.GetUnsafePtr(), continueBuff.Length); continueBuff.Clear(); } line_count = 0; while (true) { // read charBuff by line bool detect_line_factor = ParseLineImpl(info, charBuff, lines); if (detect_line_factor) { line_count++; } else { break; } } // move left charBuff data into continue buffer if (charBuff.Length > 0) { continueBuff.Clear(); continueBuff.AddRange(charBuff.GetUnsafePtr(), charBuff.Length); /* * var sb = new StringBuilder(); * sb.Append("TextDecoder >> continueBuff:\n"); * for (int m = 0; m < _continueBuff.Length; m++) sb.Append(_continueBuff[m]); * Debug.Log(sb.ToString()); */ } charBuff.Clear(); }
private static unsafe bool ParseLineImpl(ParseLinesWorker.Info *info, UnsafeRefToNativeHeadRemovableList <Char16> charBuff, UnsafeRefToNativeStringList lines) { // check '\r\n' is overlap between previous buffer and current buffer if (info->check_CR && charBuff.Length > 0) { if (charBuff[0] == UTF16CodeSet.code_LF) { charBuff.RemoveHead(); } } int len_chars = charBuff.Length; Char16 *ptr_chars = (Char16 *)charBuff.GetUnsafePtr(); if (len_chars == 0) { return(false); } for (int i = 0; i < len_chars; i++) { Char16 ch = ptr_chars[i]; // detect ch = '\n' (unix), '\r\n' (DOS), or '\r' (Mac) if (ch == UTF16CodeSet.code_LF || ch == UTF16CodeSet.code_CR) { /* * //Debug.Log(" ** found LF = " + ((int)ch).ToString() + ", i=" + i.ToString() + "/" + charBuff.Length.ToString()); * if (_charBuff[i] == '\n' && i > 0) * { * //Debug.Log(" ** before LF = " + ((int)charBuff[i-1]).ToString()); * } * //*/ lines.Add(ptr_chars, i); if (ch == UTF16CodeSet.code_CR) { if (i + 1 < len_chars) { if (ptr_chars[i + 1] == UTF16CodeSet.code_LF) { i++; //Debug.Log(" ** found CRLF"); } } else { // check '\r\n' or not on the head of next buffer //Debug.LogWarning(" >> checking overlap CRLF"); info->check_CR = true; } } else { info->check_CR = false; } charBuff.RemoveHead(i + 1); return(true); } } return(false); }