public void Clear()
{
Data.Clear();
IndexSeq.Clear();
pack.N = 0;
pack.D = 0;
pack.R = 0;
pack.Lines = 0;
_name.Clear();
_tmp_name.Clear();
_str_list.Clear();
_b64_decoder.Clear();
_b64_decoded_bytes.Clear();
// pre-reallocating for after 2nd loadings
if (_data_size > 0)
{
Data.Capacity = _data_size;
IndexSeq.Capacity = _data_size;
_name.Capacity = 8 * _data_size; // estimate 8 charactors for name of 1 CharaData
_tmp_name.Capacity = 8 * _data_size;
_name.IndexCapacity = _data_size;
_tmp_name.Capacity = _data_size;
_b64_decoded_bytes.Capacity = 4 * _data_size; // convrt to Int32 array
}
pack.read_mode = ReadMode.None;
}
// helper functions
private void GenCollectionTestString(int n_char)
{
if (n_char <= 0)
{
return;
}
Debug.Log(" == generate numbered string ==");
str_native.Clear();
str_list.Clear();
int char_count = 0;
int line_count = 0;
int n_lines = n_char / 10;
for (int i = 0; i < n_lines; i++)
{
int n_term = random.Next(3, 7);
string word = i.ToString() + '_';
string str = "";
for (int jj = 1; jj < n_term; jj++)
{
str += word;
}
str_native.Add(str);
str_list.Add(str);
char_count++;
line_count++;
}
Debug.Log(" generated: " + line_count.ToString() + " strings, " + char_count.ToString() + " charactors.");
}
/// <summary>
/// split into NativeStringList by Char16.IsWhiteSpace() delimiter.
/// </summary>
/// <param name="source"></param>
/// <param name="result"></param>
/// <param name="append"></param>
public static unsafe void Split <T>(this T source, NativeStringList result, bool append = false)
where T : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <T>
{
if (!append)
{
result.Clear();
}
source.Split(result.GetUnsafeRef());
}
/// <summary>
/// split into NativeStringList by single Char16 delimiter.
/// </summary>
/// <param name="source"></param>
/// <param name="delim"></param>
/// <param name="result"></param>
/// <param name="append"></param>
public static unsafe void Split(this NativeList <Char16> source, Char16 delim, NativeStringList result, bool append = false)
{
if (!append)
{
result.Clear();
}
var se = source.ToStringEntity();
se.Split(delim, result.GetUnsafeRef());
}
private void CheckParseHexFloat64(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// double hex
double[] double_list = new double[]
{
0.0, 128.0, 512.0, 12345.67, -12345678,
-9223372036854775808d, 9223372036854775807d,
-3.40281e+38, 3.40281E+38,
-1.797693134862e+308, 1.797693134862E+308
};
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < double_list.Length; i++)
{
double double_v = double_list[i];
byte[] bytes = BitConverter.GetBytes(double_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
double value_ref = double_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out double value_lit);
bool success_big = str_big.TryParseHex(out double value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
// must be bit-complete convertion in hex data format
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
/// <summary>
/// split into NativeStringList by NativeList(Char16) delimiter.
/// </summary>
/// <param name="source"></param>
/// <param name="delim"></param>
/// <param name="result"></param>
/// <param name="append"></param>
public unsafe static void Split <T>(this T source,
NativeList <Char16> delim,
NativeStringList result,
bool append = false)
where T : unmanaged, IJaggedArraySliceBase <Char16>, ISlice <T>
{
if (!append)
{
result.Clear();
}
var de = delim.ToStringEntity();
source.Split(de, result.GetUnsafeRef());
}
/// <summary>
/// split into NativeStringList by StringEntity delimiter.
/// </summary>
/// <param name="source"></param>
/// <param name="delim"></param>
/// <param name="result"></param>
/// <param name="append"></param>
public unsafe static void Split <T>(this NativeList <Char16> source,
T delim,
NativeStringList result,
bool append = false)
where T : IJaggedArraySliceBase <Char16>
{
if (!append)
{
result.Clear();
}
var se = source.ToStringEntity();
se.Split(delim, result.GetUnsafeRef());
}
public void SplitByCharIsWhiteSpace()
{
// split by char.IsWhiteSpace()
ref_list.Clear();
ref_list.Add("1234567890@@0987654321^^");
ref_list.Add("1234567");
ref_list.Add("890");
ref_list.Add("#");
Debug.Log(" >> try NativeList<char>.Split(result) >>");
NSL_result.Clear();
NL_source.Split(NSL_result);
Assert.IsTrue(this.CheckSplitterResult(NSL_result, ref_list));
Debug.Log(" >> try StringEntity.Split(result) >>");
NSL_result.Clear();
SE_source.Split(NSL_result);
Assert.IsTrue(this.CheckSplitterResult(NSL_result, ref_list));
Debug.Log(" >> try StringEntity.Split(result) (result: NativeList<StringEntity>) >>");
NL_SE_result.Clear();
SE_source.Split(NL_SE_result);
Assert.IsTrue(this.CheckSplitterResult(NL_SE_result, ref_list));
}
private void CheckParseHexFloat32(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// float hex
float[] float_list = new float[] { 0.0f, 128.0f, 512.0f, 12345.67f, -12345678f, -3.40281e+38f, 3.40281E+38f };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < float_list.Length; i++)
{
float float_v = float_list[i];
byte[] bytes = BitConverter.GetBytes(float_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
float value_ref = float_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out float value_lit);
bool success_big = str_big.TryParseHex(out float value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
// must be bit-complete convertion in hex data format
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
private void CheckParseHexInt64(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// long hex
long[] long_list = new long[] { 0, 128, 512, 10000, -12345678, -9223372036854775808, 9223372036854775807 };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < long_list.Length; i++)
{
long long_v = long_list[i];
byte[] bytes = BitConverter.GetBytes(long_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
long value_ref = long_list[i / 2];
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out long value_lit);
bool success_big = str_big.TryParseHex(out long value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
private void CheckParseHexInt32(NativeStringList str_native_big, NativeStringList str_native_lit)
{
// int hex
int[] int_list = new int[] { 0, 128, 512, 10000, -12345678, -2147483648, 2147483647 };
str_native_big.Clear();
str_native_lit.Clear();
for (int i = 0; i < int_list.Length; i++)
{
int int_v = int_list[i];
byte[] bytes = BitConverter.GetBytes(int_v);
string str = BitConverter.ToString(bytes).Replace("-", ""); // BitConverter returns little endian code on x86.
string str_0x = "0x" + str;
str_native_lit.Add(str);
str_native_lit.Add(str_0x);
str_native_big.Add(this.ConvertEndian(str));
str_native_big.Add(this.ConvertEndian(str_0x));
}
for (int i = 0; i < str_native_lit.Length; i++)
{
int value_ref = int_list[i / 2]; // stored 2x elems as [hex data] and 0x[hex data]
ReadOnlyStringEntity str_lit = str_native_lit[i];
ReadOnlyStringEntity str_big = str_native_big[i];
bool success_lit = str_lit.TryParseHex(out int value_lit);
bool success_big = str_big.TryParseHex(out int value_big, Endian.Big);
Debug.Log("parse str[big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
Assert.IsTrue(success_lit);
Assert.IsTrue(success_big);
Assert.AreEqual(value_ref, value_lit);
Assert.AreEqual(value_ref, value_big);
if ((value_ref != value_lit || value_ref != value_big) || !success_lit || !success_big)
{
Debug.LogError("failed to parse. i = " + i.ToString()
+ " string [big/little] = [" + str_big.ToString() + "/" + str_lit.ToString()
+ "], try[big/little] = [" + success_big.ToString() + "/" + success_lit.ToString()
+ "], value[ref/big/little] = [" + value_ref.ToString() + "/" + value_big.ToString() + "/" + value_lit.ToString() + "]");
}
}
}
public void CheckParseInt32_Boundary()
{
// boundary value check
str_native.Clear();
str_list.Clear();
string in_range_lo = "-2147483648";
string in_range_hi = "2147483647";
string out_range_lo = "-2147483649";
string out_range_hi = "2147483648";
string out_sp_1 = "4400000000"; // overflow to plus region
str_list.Add(in_range_lo.ToString());
str_list.Add(in_range_hi.ToString());
str_list.Add(out_range_lo.ToString());
str_list.Add(out_range_hi.ToString());
str_list.Add(out_sp_1.ToString());
for (int i = 0; i < str_list.Count; i++)
{
str_native.Add(str_list[i]);
}
for (int i = 0; i < str_native.Length; i++)
{
var str = str_list[i];
ReadOnlyStringEntity str_e = str_native[i];
bool success = int.TryParse(str, out int value);
bool success_e = str_e.TryParse(out int value_e);
Assert.AreEqual(success, success_e);
Assert.AreEqual(value, value_e);
if (success != success_e || value != value_e)
{
Debug.LogError("failed to parse. string [str/entity] = [" + str + "/" + str_e.ToString() + "]"
+ "bool [str/entity] = [" + success.ToString() + "/" + success_e.ToString() + "], "
+ "value [str/entity] = [" + value.ToString() + "/" + value_e.ToString() + "]");
}
}
}
private void ParseText()
{
_decoder.Target.Reset();
_worker.Clear();
_data.Target.Clear();
Boolean continue_flag = true;
for (int pos = 0; pos < _info.Target->blockNum; pos++)
{
_lines.Clear();
this.ParseLinesFromBuffer();
continue_flag = _data.Target.ParseLines(_lines);
if (!continue_flag)
{
return;
}
_state_ptr.Target->Read = pos;
}
}
private unsafe void GenerateImpl(Encoding encoding, int n, int d, float r)
{
_i_chara = 0;
var sb = new StringBuilder();
Debug.Log(" >> write header >> ");
//--- header
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("# the belong area is a header data.");
sb.Append(_lf);
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("<@MARK>" + _delim + "Header");
sb.Append(_lf);
sb.Append("n_total" + _delim + n.ToString());
sb.Append(_lf);
sb.Append("d" + _delim + d.ToString());
sb.Append(_lf);
sb.Append("r" + _delim + r.ToString());
sb.Append(_lf);
sb.Append(_lf); // insert empty lines
sb.Append(_lf);
sb.Append(_lf);
//--- generate random id sequence
var id_Sequence = new NativeList <int>(n, Allocator.Persistent);
id_Sequence.Clear();
for (int id = 0; id < n; id++)
{
id_Sequence.Add(id * d);
}
//int n_swap = (int)Math.Sqrt(n);
int n_swap = n;
var random = new System.Random(n_swap * d);
for (int i_swap = 0; i_swap < n_swap; i_swap++)
{
int index = (int)(n * random.NextDouble() * 0.95);
var tmp = id_Sequence[index];
id_Sequence.RemoveAtSwapBack(index);
id_Sequence.Add(tmp);
}
Debug.Log(" >> write Ex Data >> ");
//--- Ex data
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("# the belong area is a Base64 encorded ext data.");
sb.Append(_lf);
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("<@MARK>" + _delim + "ExtData");
sb.Append(_lf);
int byte_len = id_Sequence.Length * UnsafeUtility.SizeOf <int>();
var byte_Sequence = new NativeArray <byte>(byte_len, Allocator.Persistent);
UnsafeUtility.MemCpy(byte_Sequence.GetUnsafePtr(), id_Sequence.GetUnsafePtr(), byte_len);
var b64_Sequence = new NativeList <Char16>(Allocator.Persistent);
var b64_Encoder = new NativeBase64Encoder(Allocator.Persistent);
b64_Encoder.GetChars(b64_Sequence, byte_Sequence);
for (int i = 0; i < b64_Sequence.Length; i++)
{
sb.Append(b64_Sequence[i]);
}
sb.Append(_lf);
sb.Append("<@MARK>" + _delim + "ExtDataEnd");
sb.Append(_lf);
sb.Append(_lf);
byte_Sequence.Dispose();
b64_Sequence.Dispose();
b64_Encoder.Dispose();
Debug.Log(" >> write body >> ");
//--- body
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("# the belong area is a data body.");
sb.Append(_lf);
sb.Append("#=======================================");
sb.Append(_lf);
sb.Append("<@MARK>" + _delim + "Body");
sb.Append(_lf);
var dummy_line = "# ---" + _delim + "dummy" + _delim + "line" + _delim + "--- #";
//--- write CharaData
var NSL_tmp = new NativeStringList(Allocator.Persistent);
using (StreamWriter writer = new StreamWriter(_path, false, encoding))
{
int i = 0;
_i_chara = i;
while (i < n)
{
if (random.NextDouble() > r)
{
NSL_tmp.Clear();
int id = id_Sequence[i];
var chara_tmp = CharaDataExt.Generate(id, NSL_tmp);
sb.Append(chara_tmp.ToString(_delim));
sb.Append(_lf);
i++;
_i_chara = i;
//Debug.Log(" -- body [" + i.ToString() + '/' + n.ToString() + ']');
}
else
{
sb.Append(dummy_line);
sb.Append(_lf);
}
if (sb.Length > 4096) // 4k: page size
{
writer.Write(sb);
sb.Clear();
}
}
if (sb.Length > 0)
{
writer.Write(sb);
}
}
id_Sequence.Dispose();
NSL_tmp.Dispose();
_standby = true;
}