internal static unsafe void SplitCharImpl <Ts, Tr>(Ts source, Char16 delim, Tr result)
where Ts : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <Ts>
where Tr : IAddSlice <Ts>
{
int len_source = source.Length;
Char16 *ptr_source = (Char16 *)source.GetUnsafePtr();
int start = 0;
for (int i = 0; i < len_source; i++)
{
if (ptr_source[i] == delim)
{
int len = (i - start);
if (len > 0)
{
result.Add(source.Slice(start, i));
}
start = i + 1;
}
}
// final part
if (start < len_source)
{
result.Add(source.Slice(start, len_source));
}
}
private static void GetBytesBurst(ref UnsafeRefToNativeBase64Decoder decoder,
ref UnsafeRefToNativeList <byte> bytes,
Char16 *ptr_chars, int len_chars,
out bool success)
{
success = decoder.GetBytes(bytes, ptr_chars, len_chars);
}
internal static unsafe void SplitStringImpl <Ts, Tdel, Tr>(Ts source, Tdel delim, Tr result)
where Ts : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <Ts>
where Tdel : IJaggedArraySliceBase <Char16>
where Tr : IAddSlice <Ts>
{
int len_source = source.Length;
Char16 *ptr_source = (Char16 *)source.GetUnsafePtr();
int len_delim = delim.Length;
Char16 *ptr_delim = (Char16 *)delim.GetUnsafePtr();
if (len_delim == 0)
{
result.Add(source);
return;
}
int start = 0;
for (int i = 0; i < len_source; i++)
{
//if (i < start) continue;
if (ptr_source[i].Equals(ptr_delim[0]))
{
if (len_source - i < len_delim)
{
break;
}
// delim check
bool is_delim = true;
for (int j = 1; j < len_delim; j++)
{
if (!ptr_source[i + j].Equals(ptr_delim[j]))
{
is_delim = false;
break;
}
}
if (is_delim)
{
int len = (i - start);
if (len > 0)
{
result.Add(source.Slice(start, i));
}
start = i + len_delim;
i = start;
}
}
}
// final part
if (start < len_source)
{
result.Add(source.Slice(start, len_source));
}
}
public StringEntity(Char16 *ptr, int Length, long *gen_ptr, long gen_entity)
{
_ptr = ptr;
_len = Length;
_gen_ptr = gen_ptr;
_gen_entity = gen_entity;
}
internal unsafe static void TryParseHex32Impl(Char16 *ptr_source,
int length,
out bool success,
out uint buf,
Endian endian)
{
const int n_digits = 8; // accepts 8 digits set (4bit * 8)
int i_start = 0;
buf = 0;
if (!(length == n_digits || length == n_digits + 2))
{
success = false;
return;
}
if (ptr_source[0] == UTF16CodeSet.code_0 && ptr_source[1] == UTF16CodeSet.code_x)
{
i_start = 2;
}
if (endian == Endian.Big)
{
for (int i = i_start; i < length; i++)
{
if (ptr_source[i].IsHex(out uint h))
{
buf = (buf << 4) | h;
}
else
{
success = false;
return;
}
}
}
else if (endian == Endian.Little)
{
int n_byte = length / 2;
int i_last = i_start / 2;
for (int i = n_byte - 1; i >= i_last; i--)
{
Char16 c0 = ptr_source[2 * i];
Char16 c1 = ptr_source[2 * i + 1];
if (c0.IsHex(out uint h0) && c1.IsHex(out uint h1))
{
buf = (buf << 4) | h0;
buf = (buf << 4) | h1;
}
else
{
success = false;
return;
}
}
internal unsafe static void TryParseFloat64Impl(Char16 *ptr_source, int length,
out bool success, out double result)
{
result = 0.0;
if (length <= 0)
{
success = false;
return;
}
if (!TryParseFloatFormat(ptr_source, length,
out int sign, out int i_start, out int dot_pos,
out int exp_pos, out int n_pow))
//if (!TryParseFloatFormat_v2(ptr_source, length,
// out int sign, out int i_start, out int dot_pos,
// out int exp_pos, out int n_pow))
{
success = false;
return;
}
//UnityEngine.Debug.Log("i_start=" + i_start.ToString() + ", dot_pos=" + dot_pos.ToString() + ", exp_pos=" + exp_pos.ToString() + ", n_pow=" + n_pow.ToString());
double mantissa = 0.0;
for (int i = exp_pos - 1; i >= i_start; i--)
{
if (i == dot_pos)
{
continue;
}
mantissa = mantissa * 0.1 + (double)ptr_source[i].ToInt();
}
int m_pow = 0;
if (dot_pos > i_start + 1)
{
m_pow = dot_pos - i_start - 1;
}
if (dot_pos == i_start)
{
m_pow = -1;
}
double tmp = mantissa * (sign * math.pow(10.0, n_pow + m_pow));
if (double.IsInfinity(tmp))
{
success = false;
return;
}
result = tmp;
success = true;
//UnityEngine.Debug.Log($">> parsed value = {result}");
return;
}
public static bool GetBytes(NativeBase64Decoder decoder,
NativeList <byte> bytes,
Char16 *ptr, int len)
{
var dec = decoder.GetUnsafeRef();
var res = bytes.GetUnsafeRef();
Base64EncodingBurstCompile._getBytesDelegate(ref dec, ref res, ptr, len, out bool success);
return(success);
}
public StringEntity(NativeJaggedArraySlice <Char16> slice)
{
_ptr = slice._ptr;
_len = slice._len;
#if ENABLE_UNITY_COLLECTIONS_CHECKS
_gen_ptr = slice._gen_ptr;
_gen_entity = slice._gen_entity;
#endif
}
private unsafe void SplitImpl(Char16 *source_ptr, int source_len, NativeStringList result)
{
// parse
int start = 0;
for (int i = 0; i < source_len; i++)
{
int match_len = 0;
foreach (var d in _delims)
{
// skip shorter delim
if (match_len >= d.Length)
{
continue;
}
bool match = true;
for (int j = 0; j < d.Length; j++)
{
if (source_ptr[i + j] != d[j])
{
match = false;
break;
}
}
if (match)
{
match_len = d.Length;
}
}
if (match_len > 0)
{
int elem_len = i - start;
if (elem_len > 0)
{
Char16 *st_ptr = source_ptr + start;
result.Add(st_ptr, elem_len);
}
start = i + match_len;
i += match_len;
}
}
// final part
if (start < source_len)
{
int len = source_len - start;
Char16 *st_ptr = source_ptr + start;
result.Add(st_ptr, len);
}
}
private unsafe int IndexOfSliceImpl(Char16 *tgt_ptr, int tgt_len, int start)
{
this.CheckReallocate();
this.CheckElemIndex(start);
if (tgt_len > _len - start)
{
return(-1);
}
if (tgt_len <= 0)
{
return(-1);
}
for (int i = start; i < _len - (tgt_len - 1); i++)
{
if (_ptr[i].Equals(tgt_ptr[0]))
{
bool is_match = true;
for (int j = 1; j < tgt_len; j++)
{
if (!_ptr[i + j].Equals(tgt_ptr[j]))
{
is_match = false;
break;
}
}
if (is_match)
{
return(i);
}
}
}
return(-1);
}
public StringEntity(Char16 *ptr, int Length)
{
_ptr = ptr;
_len = Length;
}
public unsafe void Add(Char16 *ptr, int Length)
{
_jarr.Add(ptr, Length);
}
internal unsafe static void TryParseBoolImpl(Char16 *ptr_source, int len_source,
out bool success, out bool result)
{
if (len_source == 5)
{
// match "False", "false", or "FALSE"
if (ptr_source[0] == UTF16CodeSet.code_F)
{
if (ptr_source[1] == UTF16CodeSet.code_a &&
ptr_source[2] == UTF16CodeSet.code_l &&
ptr_source[3] == UTF16CodeSet.code_s &&
ptr_source[4] == UTF16CodeSet.code_e)
{
result = false;
success = true;
return;
}
else if (ptr_source[1] == UTF16CodeSet.code_A &&
ptr_source[2] == UTF16CodeSet.code_L &&
ptr_source[3] == UTF16CodeSet.code_S &&
ptr_source[4] == UTF16CodeSet.code_E)
{
result = false;
success = true;
return;
}
}
else if (ptr_source[0] == UTF16CodeSet.code_f &&
ptr_source[1] == UTF16CodeSet.code_a &&
ptr_source[2] == UTF16CodeSet.code_l &&
ptr_source[3] == UTF16CodeSet.code_s &&
ptr_source[4] == UTF16CodeSet.code_e)
{
result = false;
success = true;
return;
}
}
else if (len_source == 4)
{
// match "True", "true", or "TRUE"
if (ptr_source[0] == UTF16CodeSet.code_T)
{
if (ptr_source[1] == UTF16CodeSet.code_r &&
ptr_source[2] == UTF16CodeSet.code_u &&
ptr_source[3] == UTF16CodeSet.code_e)
{
result = true;
success = true;
return;
}
else if (ptr_source[1] == UTF16CodeSet.code_R &&
ptr_source[2] == UTF16CodeSet.code_U &&
ptr_source[3] == UTF16CodeSet.code_E)
{
result = true;
success = true;
return;
}
}
else if (ptr_source[0] == UTF16CodeSet.code_t &&
ptr_source[1] == UTF16CodeSet.code_r &&
ptr_source[2] == UTF16CodeSet.code_u &&
ptr_source[3] == UTF16CodeSet.code_e)
{
result = true;
success = true;
return;
}
}
result = false;
success = false;
return;
}
private static void TryParseHex64Burst(Char16 *ptr_source, int length,
out bool success, out ulong result,
Endian endian)
{
StringParserExt.TryParseHex64Impl(ptr_source, length, out success, out result, endian);
}
private static void TryParseFloat64Burst(Char16 *ptr_source, int length,
out bool success, out double result)
{
StringParserExt.TryParseFloat64Impl(ptr_source, length, out success, out result);
}
internal unsafe static void TryParseInt64Impl(Char16 *ptr_source, int len_source,
out bool success, out long result)
{
const int max_len = 19;
result = 0;
if (len_source <= 0)
{
success = false;
return;
}
int i_start = 0;
if (ptr_source[0].IsSign(out int sign))
{
i_start = 1;
}
int digit_count = 0;
int MSD = 0;
Int64 tmp = 0;
for (int i = i_start; i < len_source; i++)
{
if (ptr_source[i].IsDigit(out int d))
{
if (digit_count > 0)
{
digit_count++;
}
if (MSD == 0 && d > 0)
{
MSD = d;
digit_count = 1;
}
tmp = tmp * 10 + d;
if (digit_count == max_len)
{
if ((sign == 1 && tmp < 0L) ||
(sign == -1 && (tmp - 1L) < 0))
{
success = false;
return;
}
}
if (digit_count > max_len)
{
success = false;
return;
}
}
else
{
success = false;
return;
}
}
result = sign * tmp;
success = true;
return;
}
internal static unsafe bool GetBytesImpl(Base64Info *info_ptr,
UnsafeRefToNativeList <byte> buff,
Char16 *char_ptr,
int char_len)
{
CheckLengthIsPositive(char_len);
uint store = info_ptr->store;
int bytePos = info_ptr->bytePos;
for (int i = 0; i < char_len; i++)
{
Char16 c = char_ptr[i];
if (IsWhiteSpace(c))
{
continue;
}
if (c == '=')
{
switch (bytePos)
{
case 0:
case 1:
/*
* var sb = new StringBuilder();
* sb.Append("bytePos = " + bytePos.ToString() + '\n');
* sb.Append("i = " + i.ToString() + "\n\n");
* sb.Append("decoded:\n");
* for(int j=0; j<buff.Length; j++)
* {
* sb.Append(buff[j].ToString() + ' ');
* }
* sb.Append('\n');
* sb.Append("currect char: " + c + '\n');
* sb.Append("left char: " + c + '\n');
* int c_count = 0;
* for (int j=i+1; j<char_len; j++)
* {
* sb.Append(char_ptr[j]);
* c_count++;
* if(c_count == 16)
* {
* sb.Append('\n');
* c_count = 0;
* }
* }
* sb.Append('\n');
* UnityEngine.Debug.Log(sb);
*/
#if UNITY_EDITOR
throw new ArgumentException("invalid padding was detected.");
#else
return(false);
#endif
case 2:
// pick 1 byte from "**==" code
buff.Add((byte)((store & 0x0ff0) >> 4));
bytePos = 0;
break;
case 3:
// pick 2 byte from "***=" code
buff.Add((byte)((store & 0x03fc00) >> 10));
buff.Add((byte)((store & 0x0003fc) >> 2));
bytePos = 0;
break;
}
return(true);
}
else
{
uint b = Base64DecodeTable.Get(c);
if (b != 255)
{
store = (store << 6) | (b & 0x3f);
bytePos++;
}
else
{
// invalid char for Base64
#if UNITY_EDITOR
throw new ArgumentException($"invalid code was detected. char={c}");
#else
return(false);
#endif
}
}
if (bytePos == 4)
{
buff.Add((byte)((store & 0xff0000) >> 16));
buff.Add((byte)((store & 0x00ff00) >> 8));
buff.Add((byte)((store & 0x0000ff)));
store = 0;
bytePos = 0;
}
}
info_ptr->store = store;
info_ptr->bytePos = bytePos;
return(true);
}
private unsafe static bool TryParseFloatFormat_v2(Char16 *ptr_source,
int length,
out int sign,
out int i_start,
out int dot_pos,
out int exp_pos,
out int n_pow)
{
// check sign on front
i_start = 0;
if (ptr_source[0].IsSign(out sign))
{
i_start = 1;
}
// eat zeros on the head
bool is_zero_head = false;
for (int i = i_start; i < length; i++)
{
UInt16 c = ptr_source[i];
if (c == UTF16CodeSet.code_0)
{
is_zero_head = true;
i_start++;
}
else
{
break;
}
}
// initialize value
dot_pos = -1;
exp_pos = -1;
n_pow = 0;
if (length > 2048)
{
return(false);
}
// check dot pos
// check exp pos
int dummy;
bool is_digit_found = false;
for (int i = i_start; i < length; i++)
{
Char16 c = ptr_source[i];
if (!is_digit_found && c.IsDigit(out dummy))
{
is_digit_found = true;
}
else if (c.IsDot())
{
if (dot_pos >= 0)
{
return(false); // # of dots must be 0 or 1 in string.
}
dot_pos = i;
}
else if (c.IsExp())
{
if (!is_digit_found &&
!is_zero_head)
{
return(false); // must have digits before [E/e] mark.
}
if (exp_pos > 0)
{
return(false); // # of [E/e] marks must be 0 or 1 in string.
}
exp_pos = i;
}
}
// check exp sign
int exp_start = exp_pos + 1;
int exp_sign = 1;
if (exp_pos > 0)
{
if (ptr_source[exp_start].IsSign(out exp_sign))
{
exp_start += 1;
}
if (exp_start == length)
{
return(false); // no digits in exp
}
}
else
{
exp_pos = (short)length;
exp_start = exp_pos;
}
// check mantissa region is digits only or not
for (int i = i_start; i < exp_pos; i++)
{
Char16 c = ptr_source[i];
if (!c.IsDigit(out dummy) && !c.IsDot())
{
return(false);
}
}
// check exp region and extract n_pow
if (exp_start > 0)
{
for (int i = 0; i < math.min(length - exp_start, 64); i++) // up to 64 digits for pow
{
Char16 c = ptr_source[i + exp_start];
if (!c.IsDigit(out int d))
{
return(false); // exp region must be digits only
}
n_pow = n_pow * 10 + (int)d;
if (n_pow > 350)
{
if (exp_sign > 0)
{
// overflow
return(false);
}
break;
}
}
n_pow *= exp_sign;
}
// parsing manntissa search [i_start, exp_pos] and use dot_pos to calc 10^m in mantissa.
if (dot_pos < 0)
{
dot_pos = exp_pos;
}
return(true);
}
private unsafe static bool TryParseFloatFormat(Char16 *ptr_source,
int length,
out int sign,
out int i_start,
out int dot_pos,
out int exp_pos,
out int n_pow)
{
i_start = 0;
if (ptr_source[0].IsSign(out sign))
{
i_start = 1;
}
bool zero_head = false;
// eat zeros on the head
for (int i = i_start; i < length; i++)
{
UInt16 c = ptr_source[i];
if (c == UTF16CodeSet.code_0)
{
i_start++;
zero_head = true;
}
else
{
break;
}
}
dot_pos = -1;
exp_pos = -1;
n_pow = 0;
int exp_sign = 1;
int digit_count = 0;
int dummy;
int dummy_sign;
int exp_start = -1;;
// format check
for (int i = i_start; i < length; i++)
{
Char16 c = ptr_source[i];
//UnityEngine.Debug.Log($"parse format: i={i}, c={c}");
if (c.IsDigit(out dummy))
{
if (exp_pos == -1)
{
digit_count++;
}
// do nothing
//UnityEngine.Debug.Log("c is number");
}
else
{
if (exp_start > 0 && i >= exp_start)
{
return(false); // after [e/E] region must be digits.
}
if (c.IsDot())
{
if (dot_pos != -1 || dot_pos == i_start + 1)
{
return(false);
}
if (exp_pos > 0 && i >= exp_pos)
{
return(false);
}
dot_pos = i;
//UnityEngine.Debug.Log("c is dot");
}
else if (c.IsExp())
{
//UnityEngine.Debug.Log("c is EXP");
if (exp_pos != -1)
{
return(false); // 2nd [e/E] found
}
if (digit_count == 0 && !zero_head)
{
return(false); // no digits before [e/E]
}
if (length - i == 0)
{
return(false); // no digits after [e/E]
}
if ((length - i == 1) &&
!ptr_source[i + 1].IsDigit(out dummy))
{
return(false);
}
exp_pos = i;
exp_start = i + 1;
if (ptr_source[i + 1].IsSign(out exp_sign))
{
if (exp_start + 1 >= length)
{
return(false); // ended by [e/E][+/-]. no pow digits.
}
exp_start += 1;
}
}
else if (c.IsSign(out dummy_sign))
{
if (i != exp_pos + 1)
{
return(false);
}
exp_start = i + 1;
}
else
{
//UnityEngine.Debug.Log("failure to parse");
return(false);
}
}
}
// decode exp part
if (exp_start > 0)
{
for (int i = 0; i < math.min(length - exp_start, 32); i++) // up to 32 digits for pow
{
n_pow = n_pow * 10 + ptr_source[i + exp_start].ToInt();
//UnityEngine.Debug.Log("n_pow(1)=" + n_pow.ToString());
if (n_pow > 350)
{
if (exp_sign < 0)
{
n_pow = -400;
if (dot_pos < 0)
{
dot_pos = exp_pos;
}
return(true); // underflow, round to zero.
}
else
{
return(false); // overflow
}
}
}
n_pow *= exp_sign;
}
else
{
// no [e/E+-[int]] part
exp_pos = length;
}
if (dot_pos < 0)
{
dot_pos = exp_pos;
}
//UnityEngine.Debug.Log("ParseFloatFormat=" + true.ToString());
return(true);
}
/// <summary>
/// convert Base64 format chars into bytes.
/// </summary>
/// <param name="buff">output</param>
/// <param name="char_ptr">source ptr</param>
/// <param name="char_len">source length</param>
/// <returns>convert successfull or not</returns>
public unsafe bool GetBytes(NativeList <byte> buff, Char16 *char_ptr, int char_len)
{
return(GetBytesImpl(_info.Target,
buff.GetUnsafeRef(),
char_ptr, char_len));
}
private static void TryParseInt32Burst(Char16 *ptr_source, int length,
out bool success, out int result)
{
StringParserExt.TryParseInt32Impl(ptr_source, length, out success, out result);
}
public bool GetBytes(UnsafeRefToNativeList <byte> buff, Char16 *char_ptr, int char_len)
{
return(NativeBase64Decoder.GetBytesImpl(info_ptr, buff, char_ptr, char_len));
}
private static unsafe bool ParseLineImpl(ParseLinesWorker.Info *info,
UnsafeRefToNativeHeadRemovableList <Char16> charBuff,
UnsafeRefToNativeStringList lines)
{
// check '\r\n' is overlap between previous buffer and current buffer
if (info->check_CR && charBuff.Length > 0)
{
if (charBuff[0] == UTF16CodeSet.code_LF)
{
charBuff.RemoveHead();
}
}
int len_chars = charBuff.Length;
Char16 *ptr_chars = (Char16 *)charBuff.GetUnsafePtr();
if (len_chars == 0)
{
return(false);
}
for (int i = 0; i < len_chars; i++)
{
Char16 ch = ptr_chars[i];
// detect ch = '\n' (unix), '\r\n' (DOS), or '\r' (Mac)
if (ch == UTF16CodeSet.code_LF || ch == UTF16CodeSet.code_CR)
{
/*
* //Debug.Log(" ** found LF = " + ((int)ch).ToString() + ", i=" + i.ToString() + "/" + charBuff.Length.ToString());
* if (_charBuff[i] == '\n' && i > 0)
* {
* //Debug.Log(" ** before LF = " + ((int)charBuff[i-1]).ToString());
* }
* //*/
lines.Add(ptr_chars, i);
if (ch == UTF16CodeSet.code_CR)
{
if (i + 1 < len_chars)
{
if (ptr_chars[i + 1] == UTF16CodeSet.code_LF)
{
i++;
//Debug.Log(" ** found CRLF");
}
}
else
{
// check '\r\n' or not on the head of next buffer
//Debug.LogWarning(" >> checking overlap CRLF");
info->check_CR = true;
}
}
else
{
info->check_CR = false;
}
charBuff.RemoveHead(i + 1);
return(true);
}
}
return(false);
}
