public void CheckAPI_ReallocateTracer() { #if ENABLE_UNITY_COLLECTIONS_CHECKS string str = "1234567890--@@"; var NSL = new NativeStringList(Allocator.TempJob); NSL.Add(str); StringEntity entity = NSL[0]; bool effective_ref = true; for (int i = 0; i < 100; i++) { NSL.Add(str); try { Debug.Log($"add: {i}, entity: {entity.ToString()}, NSL [Size/Capacity] = [{NSL.Size}/{NSL.Capacity}]"); } catch (InvalidOperationException e) { effective_ref = false; Debug.Log("the reallocation of NativeStringList is detected by StringEntity. exception: " + e.Message); break; } } Assert.IsFalse(effective_ref); NSL.Dispose(); #else Debug.Log("this feature will be enabled when the macro 'ENABLE_UNITY_COLLECTIONS_CHECKS' is defined."); Debug.Log("it is not tested in current setting."); #endif }
public ParseJob(Allocator alloc, bool enableBurst = true) { _byteReader = new AsyncByteReader(alloc); _worker = new ParseLinesWorker(alloc); _lines = new NativeStringList(alloc); _data = new GCHandle <Tdata>(); _state_ptr = new PtrHandle <ReadStateImpl>(); // do not allocate (this will be assigned). used as reference. _info = new PtrHandle <ParseJobInfo>(alloc); _info.Target->decodeBlockSize = Define.DefaultDecodeBlock; _info.Target->blockNum = 0; _info.Target->blockPos = 0; _info.Target->allocated = true; _info.Target->disposeHandle = false; _info.Target->enableBurst = enableBurst; _info.Target->jobHandle = new JobHandle(); _timer = new GCHandle <System.Diagnostics.Stopwatch>(); _timer_ms_coef = 1.0f; }
/// <summary> /// split into NativeStringList by single char delimiter. /// </summary> /// <param name="source"></param> /// <param name="delim"></param> /// <param name="alloc"></param> /// <returns>result</returns> public static NativeStringList Split(this NativeList <Char16> source, Char16 delim, Allocator alloc) { var tmp = new NativeStringList(alloc); source.Split(delim, tmp, false); return(tmp); }
public void OneTimeSetUp() { this.seed = 123456; this.random = new System.Random(this.seed); this.str_native = new NativeStringList(Allocator.Persistent); this.str_list = new List <string>(); }
/// <summary> /// split into NativeStringList by Char16.IsWhiteSpace() delimiter. /// </summary> /// <param name="source"></param> /// <param name="alloc"></param> /// <returns></returns> public static NativeStringList Split <T>(this T source, Allocator alloc) where T : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <T> { var tmp = new NativeStringList(alloc); source.Split(tmp, false); return(tmp); }
public void Init() { EnableBurst = true; EnableBurstFunc = true; Data = new NativeList <CharaData>(Allocator.Persistent); IndexSeq = new NativeList <int>(Allocator.Persistent); _name = new NativeStringList(Allocator.Persistent); _tmp_name = new NativeStringList(Allocator.Persistent); _str_list = new NativeList <ReadOnlyStringEntity>(Allocator.Persistent); _mark_list = new NativeStringList(Allocator.Persistent); _mark_list.Add("#"); _mark_list.Add("<@MARK>"); _mark_list.Add("Header"); _mark_list.Add("ExtData"); _mark_list.Add("ExtDataEnd"); _mark_list.Add("Body"); _mark_list.Add("n_total"); _mark_list.Add("d"); _mark_list.Add("r"); pack.mark_comment = _mark_list[0]; pack.mark_tag = _mark_list[1]; pack.mark_header = _mark_list[2]; pack.mark_ext = _mark_list[3]; pack.mark_ext_end = _mark_list[4]; pack.mark_body = _mark_list[5]; pack.mark_n_total = _mark_list[6]; pack.mark_d = _mark_list[7]; pack.mark_r = _mark_list[8]; _b64_decoder = new NativeBase64Decoder(Allocator.Persistent); _b64_decoded_bytes = new NativeList <byte>(Allocator.Persistent); // initialize references pack.str_list = _str_list.GetUnsafeRef(); pack.b64_decoder = _b64_decoder.GetUnsafeRef(); pack.b64_decoded_bytes = _b64_decoded_bytes.GetUnsafeRef(); pack.tmp_name = _tmp_name.GetUnsafeRef(); pack.IndexSeq = IndexSeq.GetUnsafeRef(); pack.Data = Data.GetUnsafeRef(); pack.name = _name.GetUnsafeRef(); _data_size = 0; _timer = new System.Diagnostics.Stopwatch(); _timer.Start(); _allocated = true; }
// helper f unctions private bool CheckSplitterResult(NativeStringList result, List <string> ref_data) { var sb = new StringBuilder(); bool check = true; if (result.Length != ref_data.Count) { sb.Append(" !! the element number was differ." + " result: " + result.Length.ToString() + ", ref: " + ref_data.Count.ToString() + "\n"); check = false; } int len = Mathf.Max(result.Length, ref_data.Count); sb.Append(" elements [result/ref] = {\n"); for (int i = 0; i < len; i++) { bool local_check = true; if (i < result.Length && i < ref_data.Count) { if (result[i] != ref_data[i]) { check = local_check = false; } } sb.Append(" [ "); if (i < result.Length) { sb.Append(result[i]); } sb.Append(" / "); if (i < ref_data.Count) { sb.Append(ref_data[i]); } sb.Append(" ]"); if (!local_check || i >= result.Length || i >= ref_data.Count) { sb.Append(" - differ."); } sb.Append("\n"); } sb.Append("}\n"); if (check) { Debug.Log(sb.ToString()); } else { Debug.LogWarning(sb.ToString()); } return(check); }
/// <summary> /// split into NativeStringList by Char16.IsWhiteSpace() delimiter. /// </summary> /// <param name="source"></param> /// <param name="result"></param> /// <param name="append"></param> public static unsafe void Split <T>(this T source, NativeStringList result, bool append = false) where T : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <T> { if (!append) { result.Clear(); } source.Split(result.GetUnsafeRef()); }
private unsafe int GetLinesImpl(NativeStringList lines) { LineParserBurst.GetLinesImpl(_info.Target, _continueBuff.GetUnsafeRef(), _charBuff.GetUnsafeRef(), lines.GetUnsafeRef(), out int line_count); return(line_count); }
/// <summary> /// split into NativeStringList by StringEntity delimiter. /// </summary> /// <param name="source"></param> /// <param name="delim"></param> /// <param name="alloc"></param> /// <returns>result</returns> public static NativeStringList Split <T>(this NativeList <Char16> source, T delim, Allocator alloc) where T : IJaggedArraySliceBase <Char16> { var tmp = new NativeStringList(alloc); source.Split(delim, tmp, false); return(tmp); }
/// <summary> /// split into NativeStringList by Char16.IsWhiteSpace() delimiter. /// </summary> /// <param name="source"></param> /// <param name="result"></param> /// <param name="append"></param> public static unsafe void Split(this NativeList <Char16> source, NativeStringList result, bool append = false) { if (!append) { result.Clear(); } var se = source.ToStringEntity(); se.Split(result.GetUnsafeRef()); }
private void CheckParseHexFloat64(NativeStringList str_native_big, NativeStringList str_native_lit) { // double hex double[] double_list = new double[] { 0.0, 128.0, 512.0, 12345.67, -12345678, -9223372036854775808d, 9223372036854775807d, -3.40281e+38, 3.40281E+38, -1.797693134862e+308, 1.797693134862E+308 }; str_native_big.Clear(); str_native_lit.Clear(); for (int i = 0; i < double_list.Length; i++) { double double_v = double_list[i]; byte[] bytes = BitConverter.GetBytes(double_v); string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86. string str_0x = "0x" + str; str_native_lit.Add(str); str_native_lit.Add(str_0x); str_native_big.Add(this.ConvertEndian(str)); str_native_big.Add(this.ConvertEndian(str_0x)); } for (int i = 0; i < str_native_lit.Length; i++) { double value_ref = double_list[i / 2]; ReadOnlyStringEntity str_lit = str_native_lit[i]; ReadOnlyStringEntity str_big = str_native_big[i]; bool success_lit = str_lit.TryParseHex(out double value_lit); bool success_big = str_big.TryParseHex(out double value_big, Endian.Big); Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); Assert.IsTrue(success_lit); Assert.IsTrue(success_big); Assert.AreEqual(value_ref, value_lit); Assert.AreEqual(value_ref, value_big); // must be bit-complete convertion in hex data format if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big) { Debug.LogError("failed to parse. i = " + i.ToString() + " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); } } }
public void CheckParseHex() { var str_native_big = new NativeStringList(Allocator.Temp); var str_native_lit = new NativeStringList(Allocator.Temp); this.CheckParseHexInt32(str_native_big, str_native_lit); this.CheckParseHexInt64(str_native_big, str_native_lit); this.CheckParseHexFloat32(str_native_big, str_native_lit); this.CheckParseHexFloat64(str_native_big, str_native_lit); str_native_big.Dispose(); str_native_lit.Dispose(); }
public static int GetLines(ParseLinesWorker worker, NativeStringList lines) { var ref_continueBuff = worker._continueBuff.GetUnsafeRef(); var ref_charBuff = worker._charBuff.GetUnsafeRef(); var ref_lines = lines.GetUnsafeRef(); _parseLinesDelegate(worker._info.Target, ref ref_continueBuff, ref ref_charBuff, ref ref_lines, out int line_count); return(line_count); }
/// <summary> /// split into NativeStringList by NativeList(Char16) delimiter. /// </summary> /// <param name="source"></param> /// <param name="delim"></param> /// <param name="result"></param> /// <param name="append"></param> public unsafe static void Split <T>(this T source, NativeList <Char16> delim, NativeStringList result, bool append = false) where T : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <T> { if (!append) { result.Clear(); } var de = delim.ToStringEntity(); source.Split(de, result.GetUnsafeRef()); }
/// <summary> /// split into NativeStringList by StringEntity delimiter. /// </summary> /// <param name="source"></param> /// <param name="delim"></param> /// <param name="result"></param> /// <param name="append"></param> public unsafe static void Split <T>(this NativeList <Char16> source, T delim, NativeStringList result, bool append = false) where T : IJaggedArraySliceBase <Char16> { if (!append) { result.Clear(); } var se = source.ToStringEntity(); se.Split(delim, result.GetUnsafeRef()); }
public void OneTimeSetUp() { this.str_source = " 1234567890@@0987654321^^ 1234567\t890 # "; this.NL_source = new NativeList <Char16>(Allocator.TempJob); foreach (Char16 c in str_source) { NL_source.Add(c); } this.SE_source = NL_source.ToStringEntity(); this.ref_list = new List <string>(); this.NSL_result = new NativeStringList(Allocator.TempJob); this.NL_SE_result = new NativeList <StringEntity>(Allocator.TempJob); }
private void CheckParseHexFloat32(NativeStringList str_native_big, NativeStringList str_native_lit) { // float hex float[] float_list = new float[] { 0.0f, 128.0f, 512.0f, 12345.67f, -12345678f, -3.40281e+38f, 3.40281E+38f }; str_native_big.Clear(); str_native_lit.Clear(); for (int i = 0; i < float_list.Length; i++) { float float_v = float_list[i]; byte[] bytes = BitConverter.GetBytes(float_v); string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86. string str_0x = "0x" + str; str_native_lit.Add(str); str_native_lit.Add(str_0x); str_native_big.Add(this.ConvertEndian(str)); str_native_big.Add(this.ConvertEndian(str_0x)); } for (int i = 0; i < str_native_lit.Length; i++) { float value_ref = float_list[i / 2]; ReadOnlyStringEntity str_lit = str_native_lit[i]; ReadOnlyStringEntity str_big = str_native_big[i]; bool success_lit = str_lit.TryParseHex(out float value_lit); bool success_big = str_big.TryParseHex(out float value_big, Endian.Big); Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); Assert.IsTrue(success_lit); Assert.IsTrue(success_big); Assert.AreEqual(value_ref, value_lit); Assert.AreEqual(value_ref, value_big); // must be bit-complete convertion in hex data format if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big) { Debug.LogError("failed to parse. i = " + i.ToString() + " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); } } }
private void CheckParseHexInt32(NativeStringList str_native_big, NativeStringList str_native_lit) { // int hex int[] int_list = new int[] { 0, 128, 512, 10000, -12345678, -2147483648, 2147483647 }; str_native_big.Clear(); str_native_lit.Clear(); for (int i = 0; i < int_list.Length; i++) { int int_v = int_list[i]; byte[] bytes = BitConverter.GetBytes(int_v); string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86. string str_0x = "0x" + str; str_native_lit.Add(str); str_native_lit.Add(str_0x); str_native_big.Add(this.ConvertEndian(str)); str_native_big.Add(this.ConvertEndian(str_0x)); } for (int i = 0; i < str_native_lit.Length; i++) { int value_ref = int_list[i / 2]; // stored 2x elems as [hex data] and 0x[hex data] ReadOnlyStringEntity str_lit = str_native_lit[i]; ReadOnlyStringEntity str_big = str_native_big[i]; bool success_lit = str_lit.TryParseHex(out int value_lit); bool success_big = str_big.TryParseHex(out int value_big, Endian.Big); Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); Assert.IsTrue(success_lit); Assert.IsTrue(success_big); Assert.AreEqual(value_ref, value_lit); Assert.AreEqual(value_ref, value_big); if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big) { Debug.LogError("failed to parse. i = " + i.ToString() + " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); } } }
private void CheckParseHexInt64(NativeStringList str_native_big, NativeStringList str_native_lit) { // long hex long[] long_list = new long[] { 0, 128, 512, 10000, -12345678, -9223372036854775808, 9223372036854775807 }; str_native_big.Clear(); str_native_lit.Clear(); for (int i = 0; i < long_list.Length; i++) { long long_v = long_list[i]; byte[] bytes = BitConverter.GetBytes(long_v); string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86. string str_0x = "0x" + str; str_native_lit.Add(str); str_native_lit.Add(str_0x); str_native_big.Add(this.ConvertEndian(str)); str_native_big.Add(this.ConvertEndian(str_0x)); } for (int i = 0; i < str_native_lit.Length; i++) { long value_ref = long_list[i / 2]; ReadOnlyStringEntity str_lit = str_native_lit[i]; ReadOnlyStringEntity str_big = str_native_big[i]; bool success_lit = str_lit.TryParseHex(out long value_lit); bool success_big = str_big.TryParseHex(out long value_big, Endian.Big); Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); Assert.IsTrue(success_lit); Assert.IsTrue(success_big); Assert.AreEqual(value_ref, value_lit); Assert.AreEqual(value_ref, value_big); if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big) { Debug.LogError("failed to parse. i = " + i.ToString() + " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString() + "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString() + "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]"); } } }
public bool ParseLines(NativeStringList lines) { if (EnableBurst) { var lines_input = lines.GetUnsafeRef(); ParserFuncBurst.ParseLines(ref pack, ref lines_input, out bool success); return(success); } else { for (int i = 0; i < lines.Length; i++) { bool success = ParseLineImpl(lines[i]); if (!success) { return(false); } } } return(true); }
public static CharaData Generate(long id, NativeStringList NSL) { var tmp = new CharaData(); float vv = id; var name_tmp = "Chara" + (vv * vv).GetHashCode().ToString(); NSL.Add(name_tmp); vv *= 0.1f; vv = vv * vv; tmp.ID = id; tmp.Name = NSL.Last; tmp.HP = (int)id * 100; tmp.MP = (int)id * 50; tmp.Attack = vv * 4; tmp.Defence = vv * 3; //tmp.Attack = (int)id * 4; //tmp.Defence = (int)id * 3; return(tmp); }
public void CheckAPI_StringEntity_IndexOf() { string str = "1234567890--@@G1234567890--@@@xxmhr1234567890--@@"; var NSL = new NativeStringList(Allocator.TempJob); NSL.Add(str); StringEntity entity = NSL[0]; // StringEntity.IndexOf(Char16) Assert.AreEqual(entity.IndexOf('6'), 5); Assert.AreEqual(entity.IndexOf('6', 10), 20); Assert.AreEqual(entity.IndexOf('x'), 30); Assert.AreEqual(entity.IndexOf('W'), -1); // not found // StringEntity.IndexOf(string) (= same implementation of IndexOf(StringEntity)) Assert.AreEqual(entity.IndexOf("@@x"), 28); Assert.AreEqual(entity.IndexOf("890-"), 7); Assert.AreEqual(entity.IndexOf("890-", 12), 22); Assert.AreEqual(entity.IndexOf("99"), -1); // not found NSL.Dispose(); }
public unsafe bool ParseLines(NativeStringList lines) { Lines += lines.Length; return(true); }
private unsafe void GenerateImpl(Encoding encoding, int n, int d, float r) { _i_chara = 0; var sb = new StringBuilder(); Debug.Log(" >> write header >> "); //--- header sb.Append("#======================================="); sb.Append(_lf); sb.Append("# the belong area is a header data."); sb.Append(_lf); sb.Append("#======================================="); sb.Append(_lf); sb.Append("<@MARK>" + _delim + "Header"); sb.Append(_lf); sb.Append("n_total" + _delim + n.ToString()); sb.Append(_lf); sb.Append("d" + _delim + d.ToString()); sb.Append(_lf); sb.Append("r" + _delim + r.ToString()); sb.Append(_lf); sb.Append(_lf); // insert empty lines sb.Append(_lf); sb.Append(_lf); //--- generate random id sequence var id_Sequence = new NativeList <int>(n, Allocator.Persistent); id_Sequence.Clear(); for (int id = 0; id < n; id++) { id_Sequence.Add(id * d); } //int n_swap = (int)Math.Sqrt(n); int n_swap = n; var random = new System.Random(n_swap * d); for (int i_swap = 0; i_swap < n_swap; i_swap++) { int index = (int)(n * random.NextDouble() * 0.95); var tmp = id_Sequence[index]; id_Sequence.RemoveAtSwapBack(index); id_Sequence.Add(tmp); } Debug.Log(" >> write Ex Data >> "); //--- Ex data sb.Append("#======================================="); sb.Append(_lf); sb.Append("# the belong area is a Base64 encorded ext data."); sb.Append(_lf); sb.Append("#======================================="); sb.Append(_lf); sb.Append("<@MARK>" + _delim + "ExtData"); sb.Append(_lf); int byte_len = id_Sequence.Length * UnsafeUtility.SizeOf <int>(); var byte_Sequence = new NativeArray <byte>(byte_len, Allocator.Persistent); UnsafeUtility.MemCpy(byte_Sequence.GetUnsafePtr(), id_Sequence.GetUnsafePtr(), byte_len); var b64_Sequence = new NativeList <Char16>(Allocator.Persistent); var b64_Encoder = new NativeBase64Encoder(Allocator.Persistent); b64_Encoder.GetChars(b64_Sequence, byte_Sequence); for (int i = 0; i < b64_Sequence.Length; i++) { sb.Append(b64_Sequence[i]); } sb.Append(_lf); sb.Append("<@MARK>" + _delim + "ExtDataEnd"); sb.Append(_lf); sb.Append(_lf); byte_Sequence.Dispose(); b64_Sequence.Dispose(); b64_Encoder.Dispose(); Debug.Log(" >> write body >> "); //--- body sb.Append("#======================================="); sb.Append(_lf); sb.Append("# the belong area is a data body."); sb.Append(_lf); sb.Append("#======================================="); sb.Append(_lf); sb.Append("<@MARK>" + _delim + "Body"); sb.Append(_lf); var dummy_line = "# ---" + _delim + "dummy" + _delim + "line" + _delim + "--- #"; //--- write CharaData var NSL_tmp = new NativeStringList(Allocator.Persistent); using (StreamWriter writer = new StreamWriter(_path, false, encoding)) { int i = 0; _i_chara = i; while (i < n) { if (random.NextDouble() > r) { NSL_tmp.Clear(); int id = id_Sequence[i]; var chara_tmp = CharaDataExt.Generate(id, NSL_tmp); sb.Append(chara_tmp.ToString(_delim)); sb.Append(_lf); i++; _i_chara = i; //Debug.Log(" -- body [" + i.ToString() + '/' + n.ToString() + ']'); } else { sb.Append(dummy_line); sb.Append(_lf); } if (sb.Length > 4096) // 4k: page size { writer.Write(sb); sb.Clear(); } } if (sb.Length > 0) { writer.Write(sb); } } id_Sequence.Dispose(); NSL_tmp.Dispose(); _standby = true; }
public static UnsafeRefToNativeStringList GetUnsafeRef(this NativeStringList target) { return(new UnsafeRefToNativeStringList(target)); }
/// <summary> /// Create the unsafe reference to NativeStringList. /// </summary> /// <param name="list"></param> public UnsafeRefToNativeStringList(NativeStringList list) { _jarr = new UnsafeRefToNativeJaggedArray <Char16>(list._jarr); }
public unsafe int GetLines(NativeStringList lines) { return(this.GetLinesImpl(lines)); }