public unsafe void ReadBuffer(NativeList <char> result)
{
var tmp_buff = new NativeList <Char16>(Allocator.Temp);
if (_blockPos < _blockNum)
{
this.ReadStream();
fixed(byte *byte_ptr = _byteBuffer)
{
var handle = new GCHandle <Decoder>();
handle.Create(_decoder);
_worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle);
handle.Dispose();
_worker.GetChars(tmp_buff);
}
}
if (!_worker.IsEmpty)
{
_worker.GetInternalBuffer(tmp_buff);
}
result.AddRange(tmp_buff.GetUnsafePtr(), tmp_buff.Length);
tmp_buff.Dispose();
}
private unsafe void ReadLinesImpl(IAddResult result, int max_lines)
{
int line_count = 0;
line_count += this.PickLinesImpl(result, max_lines);
if (line_count == max_lines)
{
return;
}
_lines.Clear();
while (line_count < max_lines)
{
if (_blockPos < _blockNum && _worker.IsEmpty)
{
this.ReadStream();
fixed(byte *byte_ptr = _byteBuffer)
{
var handle = new GCHandle <Decoder>();
handle.Create(_decoder);
_worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle);
handle.Dispose();
_worker.GetLines(_lines);
}
}
line_count += this.PickLinesImpl(result, max_lines - line_count);
if (line_count == max_lines)
{
return;
}
if (_blockPos == _blockNum)
{
// if final part do not have LF.
if (!_worker.IsEmpty)
{
var buff = new NativeList <Char16>(Allocator.Temp);
_worker.GetInternalBuffer(buff);
result.AddResult((char *)buff.GetUnsafePtr(), buff.Length);
buff.Dispose();
}
// reach EOF
break;
}
}
}
/// <summary>
/// initializer of NativeByteStreamReader.
/// this function must be called by main thread.
/// use Init(Utility.GCHandle<string>, buffersize) for calling in JobSystem.
/// </summary>
/// <param name="path">file path.</param>
/// <param name="bufferSize">internal buffer size (if < 0, buffering entire the file).</param>
public void Init(string path, int bufferSize = Define.DefaultBufferSize)
{
var fileInfo = new System.IO.FileInfo(path);
if (!fileInfo.Exists)
{
throw new ArgumentException("file is not exists.");
}
long fileSize = fileInfo.Length;
var pathHandle = new GCHandle <string>();
pathHandle.Create(path);
this.InitState(pathHandle, true, fileSize, bufferSize);
}
private void InitializeReadJob(Encoding encoding, Tdata data, PtrHandle <ReadStateImpl> state_ptr)
{
this.DisposeHandle();
_decoder.Create(encoding.GetDecoder());
_worker.SetPreamble(encoding.GetPreamble());
_data.Create(data);
_state_ptr = state_ptr;
_state_ptr.Target->JobState = ReadJobState.ReadAsync;
_state_ptr.Target->DelayReadAsync = -1;
_state_ptr.Target->DelayParseText = -1;
_state_ptr.Target->DelayPostProc = -1;
_info.Target->disposeHandle = true;
_timer.Create(new System.Diagnostics.Stopwatch());
_timer.Target.Start();
_timer_ms_coef = 1000.0f / System.Diagnostics.Stopwatch.Frequency;
}
/// <summary>
/// the constructor must be called by main thread only.
/// </summary>
/// <param name="path"></param>
public NativeByteStreamReader(Allocator alloc)
{
_info = new PtrHandle <ByteStreamReaderInfo>(alloc);
_byteBuffer = new NativeArray <byte>(Define.MinBufferSize, alloc);
_readCommands = new NativeArray <ReadCommand>(1, alloc);
_path.Create("");
_info.Target->disposePathHandle = true;
_info.Target->fileSize = 0;
_alloc = alloc;
_info.Target->dataLength = 0;
_info.Target->blockSize = 0;
_info.Target->blockNum = 0;
_info.Target->blockPos = 0;
_info.Target->allocated = true;
}
public unsafe void ReadLinesFromBuffer(NativeStringList result)
{
var ret = new Result_NSL();
ret.result = result;
if (_blockPos < _blockNum)
{
this.ReadStream();
fixed(byte *byte_ptr = _byteBuffer)
{
var handle = new GCHandle <Decoder>();
handle.Create(_decoder);
_worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle);
handle.Dispose();
_worker.GetLines(_lines);
}
}
// read all lines in buffer
this.PickLinesImpl(ret, int.MaxValue);
_lines.Clear();
if (_blockPos == _blockNum)
{
// if final part do not have LF.
if (!_worker.IsEmpty)
{
using (var buff = new NativeList <Char16>(Allocator.Temp))
{
_worker.GetInternalBuffer(buff);
result.Add((char *)buff.GetUnsafePtr(), buff.Length);
}
}
}
}
