public void CheckAPI_ReallocateTracer()
{
#if ENABLE_UNITY_COLLECTIONS_CHECKS
string str = "1234567890--@@";
var NSL = new NativeStringList(Allocator.TempJob);
NSL.Add(str);
StringEntity entity = NSL[0];
bool effective_ref = true;
for (int i = 0; i < 100; i++)
{
NSL.Add(str);
try
{
Debug.Log($"add: {i}, entity: {entity.ToString()}, NSL [Size/Capacity] = [{NSL.Size}/{NSL.Capacity}]");
}
catch (InvalidOperationException e)
{
effective_ref = false;
Debug.Log("the reallocation of NativeStringList is detected by StringEntity. exception: " + e.Message);
break;
}
}
Assert.IsFalse(effective_ref);
NSL.Dispose();
#else
Debug.Log("this feature will be enabled when the macro 'ENABLE_UNITY_COLLECTIONS_CHECKS' is defined.");
Debug.Log("it is not tested in current setting.");
#endif
}
private void CheckParseHexFloat64(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// double hex
double[] double_list = new double[]
{
0.0, 128.0, 512.0, 12345.67, -12345678,
-9223372036854775808d, 9223372036854775807d,
-3.40281e+38, 3.40281E+38,
-1.797693134862e+308, 1.797693134862E+308
};
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < double_list.Length; i++)
{
double double_v = double_list[i];
byte[] bytes = BitConverter.GetBytes(double_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
double value_ref = double_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out double value_lit);
bool success_big = str_big.TryParseHex(out double value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
// must be bit-complete convertion in hex data format
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
public void Init()
{
EnableBurst = true;
EnableBurstFunc = true;
Data = new NativeList <CharaData>(Allocator.Persistent);
IndexSeq = new NativeList <int>(Allocator.Persistent);
_name = new NativeStringList(Allocator.Persistent);
_tmp_name = new NativeStringList(Allocator.Persistent);
_str_list = new NativeList <ReadOnlyStringEntity>(Allocator.Persistent);
_mark_list = new NativeStringList(Allocator.Persistent);
_mark_list.Add("#");
_mark_list.Add("<@MARK>");
_mark_list.Add("Header");
_mark_list.Add("ExtData");
_mark_list.Add("ExtDataEnd");
_mark_list.Add("Body");
_mark_list.Add("n_total");
_mark_list.Add("d");
_mark_list.Add("r");
pack.mark_comment = _mark_list[0];
pack.mark_tag = _mark_list[1];
pack.mark_header = _mark_list[2];
pack.mark_ext = _mark_list[3];
pack.mark_ext_end = _mark_list[4];
pack.mark_body = _mark_list[5];
pack.mark_n_total = _mark_list[6];
pack.mark_d = _mark_list[7];
pack.mark_r = _mark_list[8];
_b64_decoder = new NativeBase64Decoder(Allocator.Persistent);
_b64_decoded_bytes = new NativeList <byte>(Allocator.Persistent);
// initialize references
pack.str_list = _str_list.GetUnsafeRef();
pack.b64_decoder = _b64_decoder.GetUnsafeRef();
pack.b64_decoded_bytes = _b64_decoded_bytes.GetUnsafeRef();
pack.tmp_name = _tmp_name.GetUnsafeRef();
pack.IndexSeq = IndexSeq.GetUnsafeRef();
pack.Data = Data.GetUnsafeRef();
pack.name = _name.GetUnsafeRef();
_data_size = 0;
_timer = new System.Diagnostics.Stopwatch();
_timer.Start();
_allocated = true;
}
// helper functions
private void GenCollectionTestString(int n_char)
{
if (n_char <= 0)
{
return;
}
Debug.Log(" == generate numbered string ==");
str_native.Clear();
str_list.Clear();
int char_count = 0;
int line_count = 0;
int n_lines = n_char / 10;
for (int i = 0; i < n_lines; i++)
{
int n_term = random.Next(3, 7);
string word = i.ToString() + '_';
string str = "";
for (int jj = 1; jj < n_term; jj++)
{
str += word;
}
str_native.Add(str);
str_list.Add(str);
char_count++;
line_count++;
}
Debug.Log(" generated: " + line_count.ToString() + " strings, " + char_count.ToString() + " charactors.");
}
private void CheckParseHexFloat32(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// float hex
float[] float_list = new float[] { 0.0f, 128.0f, 512.0f, 12345.67f, -12345678f, -3.40281e+38f, 3.40281E+38f };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < float_list.Length; i++)
{
float float_v = float_list[i];
byte[] bytes = BitConverter.GetBytes(float_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
float value_ref = float_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out float value_lit);
bool success_big = str_big.TryParseHex(out float value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
// must be bit-complete convertion in hex data format
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
private void CheckParseHexInt64(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// long hex
long[] long_list = new long[] { 0, 128, 512, 10000, -12345678, -9223372036854775808, 9223372036854775807 };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < long_list.Length; i++)
{
long long_v = long_list[i];
byte[] bytes = BitConverter.GetBytes(long_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
long value_ref = long_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out long value_lit);
bool success_big = str_big.TryParseHex(out long value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
private void CheckParseHexInt32(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// int hex
int[] int_list = new int[] { 0, 128, 512, 10000, -12345678, -2147483648, 2147483647 };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < int_list.Length; i++)
{
int int_v = int_list[i];
byte[] bytes = BitConverter.GetBytes(int_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
int value_ref = int_list[i / 2]; // stored 2x elems as [hex data] and 0x[hex data]
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out int value_lit);
bool success_big = str_big.TryParseHex(out int value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
private unsafe void ParseLinesFromBuffer()
{
// decode byte buffer
long byte_pos = _info.Target->blockPos * _info.Target->decodeBlockSize;
int decode_len = (int)(_byteReader.Length - byte_pos);
if (decode_len <= 0)
{
return;
}
int byte_len = Math.Min(_info.Target->decodeBlockSize, decode_len);
byte *byte_ptr = (byte *)_byteReader.GetUnsafePtr() + byte_pos;
_worker.DecodeTextIntoBuffer(byte_ptr, byte_len, _decoder);
// parse lines from buffer
if (_info.Target->enableBurst)
{
LineParserBurst.GetLines(_worker, _lines); // with Burst version
}
else
{
_worker.GetLines(_lines); // without Burst version
}
_info.Target->blockPos++;
// termination of file
if (_info.Target->blockPos == _info.Target->blockNum)
{
if (!_worker.IsEmpty)
{
using (var buff = new NativeList <Char16>(Allocator.Temp))
{
_worker.GetInternalBuffer(buff);
_lines.Add((Char16 *)buff.GetUnsafePtr(), buff.Length);
}
}
}
/*
* var sb = new StringBuilder();
* sb.Append("decoded lines:\n");
* foreach(var se in _lines)
* {
* sb.Append(se.ToString());
* sb.Append('\n');
* }
* Debug.Log(sb.ToString());
*/
}
public void CheckParseInt32_Boundary()
{
// boundary value check
str_native.Clear();
str_list.Clear();
string in_range_lo = "-2147483648";
string in_range_hi = "2147483647";
string out_range_lo = "-2147483649";
string out_range_hi = "2147483648";
string out_sp_1 = "4400000000"; // overflow to plus region
str_list.Add(in_range_lo.ToString());
str_list.Add(in_range_hi.ToString());
str_list.Add(out_range_lo.ToString());
str_list.Add(out_range_hi.ToString());
str_list.Add(out_sp_1.ToString());
for (int i = 0; i < str_list.Count; i++)
{
str_native.Add(str_list[i]);
}
for (int i = 0; i < str_native.Length; i++)
{
var str = str_list[i];
ReadOnlyStringEntity str_e = str_native[i];
bool success = int.TryParse(str, out int value);
bool success_e = str_e.TryParse(out int value_e);
Assert.AreEqual(success, success_e);
Assert.AreEqual(value, value_e);
if (success != success_e || value != value_e)
{
Debug.LogError("failed to parse. string [str/entity] = [" + str + "/" + str_e.ToString() + "]"
+ "bool [str/entity] = [" + success.ToString() + "/" + success_e.ToString() + "], "
+ "value [str/entity] = [" + value.ToString() + "/" + value_e.ToString() + "]");
}
}
}
public static CharaData Generate(long id, NativeStringList NSL)
{
var tmp = new CharaData();
float vv = id;
var name_tmp = "Chara" + (vv * vv).GetHashCode().ToString();
NSL.Add(name_tmp);
vv *= 0.1f;
vv = vv * vv;
tmp.ID = id;
tmp.Name = NSL.Last;
tmp.HP = (int)id * 100;
tmp.MP = (int)id * 50;
tmp.Attack = vv * 4;
tmp.Defence = vv * 3;
//tmp.Attack = (int)id * 4;
//tmp.Defence = (int)id * 3;
return(tmp);
}
public void CheckAPI_StringEntity_IndexOf()
{
string str = "1234567890--@@G1234567890--@@@xxmhr1234567890--@@";
var NSL = new NativeStringList(Allocator.TempJob);
NSL.Add(str);
StringEntity entity = NSL[0];
// StringEntity.IndexOf(Char16)
Assert.AreEqual(entity.IndexOf('6'), 5);
Assert.AreEqual(entity.IndexOf('6', 10), 20);
Assert.AreEqual(entity.IndexOf('x'), 30);
Assert.AreEqual(entity.IndexOf('W'), -1); // not found
// StringEntity.IndexOf(string) (= same implementation of IndexOf(StringEntity))
Assert.AreEqual(entity.IndexOf("@@x"), 28);
Assert.AreEqual(entity.IndexOf("890-"), 7);
Assert.AreEqual(entity.IndexOf("890-", 12), 22);
Assert.AreEqual(entity.IndexOf("99"), -1); // not found
NSL.Dispose();
}
public unsafe void PostReadProc()
{
// switch to Burst version PostReadProc()
if (EnableBurst)
{
ParserFuncBurst.PostReadProc(ref pack);
return;
}
// below is normal implementation of PostReadProc()
//--- check reading was success or not
if (pack.read_mode != ReadMode.Body)
{
return;
}
//--- store Base64 ext data
int index_len = _b64_decoded_bytes.Length / UnsafeUtility.SizeOf <int>();
if (_b64_decoded_bytes.Length % UnsafeUtility.SizeOf <int>() != 0)
{
pack.read_mode = ReadMode.Base64DataError;
return;
}
IndexSeq.ResizeUninitialized(index_len);
UnsafeUtility.MemCpy(IndexSeq.GetUnsafePtr(), _b64_decoded_bytes.GetUnsafePtr(), _b64_decoded_bytes.Length);
_b64_decoded_bytes.Clear();
//--- check Base64 ext data
if (IndexSeq.Length != Data.Length)
{
pack.read_mode = ReadMode.PostProcError;
// var sb = new System.Text.StringBuilder();
// sb.Append("ERROR in PostReadProc().\n");
// sb.Append($"IndexSeq.Length = {IndexSeq.Length}\n");
// sb.Append($"Data.Length = {Data.Length}\n");
// UnityEngine.Debug.LogError(sb.ToString());
return;
}
for (int i = 0; i < IndexSeq.Length; i++)
{
if (IndexSeq[i] != Data[i].ID)
{
pack.read_mode = ReadMode.PostProcError;
// var sb = new System.Text.StringBuilder();
// sb.Append("ERROR in PostReadProc().\n");
// for(int j=0; j<IndexSeq.Length; j++)
// {
// sb.Append($" Elems[{j}]: [IndexSeq[i] / Data[i].ID] = [{IndexSeq[j]} / {Data[j].ID}]");
// if (IndexSeq[j] != Data[j].ID) sb.Append(" -- differ.");
// sb.Append('\n');
// }
// UnityEngine.Debug.LogError(sb.ToString());
return;
}
}
//--- overwright CharaData.Name StringEntity to _name from _tmp_name.
// (the reference of CharaData.Name maybe became invalid because of reallocating in _tmp.)
_name.Clear();
for (int i = 0; i < _tmp_name.Length; i++)
{
_name.Add(_tmp_name[i]);
var chara = Data[i];
chara.Name = _name.Last;
Data[i] = chara;
}
pack.read_mode = ReadMode.Complete;
return;
}
private bool ParseLineImpl(ReadOnlyStringEntity line)
{
pack.Lines++;
if (line.Length < 1)
{
return(true);
}
if (EnableBurstFunc)
{
BurstFunc.Split(line, '\t', _str_list);
}
else
{
line.Split('\t', _str_list);
}
//--- check data block
if (_str_list.Length >= 2 && _str_list[0] == pack.mark_tag)
{
if (_str_list[1] == pack.mark_header)
{
pack.read_mode = ReadMode.Header;
}
else if (_str_list[1] == pack.mark_ext)
{
pack.read_mode = ReadMode.ExtData;
}
else if (_str_list[1] == pack.mark_ext_end)
{
pack.read_mode = ReadMode.None;
}
else if (_str_list[1] == pack.mark_body)
{
//--- check header info was read correctly or not
if (pack.N <= 0 || pack.D <= 0 || pack.R < 0.0 || pack.R >= 1.0)
{
pack.read_mode = ReadMode.HeaderError;
return(false);
}
pack.read_mode = ReadMode.Body;
}
return(true);
}
if (pack.read_mode == ReadMode.None)
{
return(true);
}
//--- ignore comment line
if (_str_list[0].Length >= 1 && _str_list[0].Slice(0, 1) == pack.mark_comment)
{
return(true);
}
//--- store data
if (pack.read_mode == ReadMode.Header)
{
bool success = true;
if (EnableBurstFunc)
{
// using BurstCompiler applied TryParse
if (_str_list[0] == pack.mark_n_total)
{
success = BurstFunc.TryParse(_str_list[1], out pack.N);
}
else if (_str_list[0] == pack.mark_d)
{
success = BurstFunc.TryParse(_str_list[1], out pack.D);
}
else if (_str_list[0] == pack.mark_r)
{
success = BurstFunc.TryParse(_str_list[1], out pack.R);
}
}
else
{
// using normal TryParse()
if (_str_list[0] == pack.mark_n_total)
{
success = _str_list[1].TryParse(out pack.N);
}
else if (_str_list[0] == pack.mark_d)
{
success = _str_list[1].TryParse(out pack.D);
}
else if (_str_list[0] == pack.mark_r)
{
success = _str_list[1].TryParse(out pack.R);
}
}
if (!success)
{
pack.read_mode = ReadMode.HeaderError;
return(false);
}
}
else if (pack.read_mode == ReadMode.ExtData)
{
if (_str_list.Length > 1)
{
// must be 1 element in line
pack.read_mode = ReadMode.Base64DataError;
return(false);
}
else if (_str_list.Length == 1)
{
bool success = true;
if (EnableBurst)
{
// using BurstCompiler applied GetChars()
success = BurstFunc.GetBytes(_b64_decoder, _b64_decoded_bytes, _str_list[0]);
}
else
{
// using normal GetChars()
success = _b64_decoder.GetBytes(_b64_decoded_bytes, _str_list[0]);
}
if (!success)
{
pack.read_mode = ReadMode.Base64DataError;
return(false);
}
}
}
else if (pack.read_mode == ReadMode.Body)
{
if (_str_list.Length < 6)
{
return(true);
}
var tmp = new CharaData();
bool success = true;
if (EnableBurstFunc)
{
// using BurstCompiler applied TryParse
success = success && BurstFunc.TryParse(_str_list[0], out tmp.ID);
success = success && BurstFunc.TryParse(_str_list[2], out tmp.HP);
success = success && BurstFunc.TryParse(_str_list[3], out tmp.MP);
success = success && BurstFunc.TryParse(_str_list[4], out tmp.Attack);
success = success && BurstFunc.TryParse(_str_list[5], out tmp.Defence);
}
else
{
// using normal TryParse()
success = success && _str_list[0].TryParse(out tmp.ID);
success = success && _str_list[2].TryParse(out tmp.HP);
success = success && _str_list[3].TryParse(out tmp.MP);
success = success && _str_list[4].TryParse(out tmp.Attack);
success = success && _str_list[5].TryParse(out tmp.Defence);
}
if (!success)
{
pack.read_mode = ReadMode.FormatError;
return(false);
}
_tmp_name.Add(_str_list[1]);
tmp.Name = _tmp_name.Last;
Data.Add(tmp);
if (Data.Length > pack.N)
{
pack.read_mode = ReadMode.FormatError;
return(false);
}
}
return(true);
}