public unsafe void ReadBuffer(NativeList <char> result) { var tmp_buff = new NativeList <Char16>(Allocator.Temp); if (_blockPos < _blockNum) { this.ReadStream(); fixed(byte *byte_ptr = _byteBuffer) { var handle = new GCHandle <Decoder>(); handle.Create(_decoder); _worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle); handle.Dispose(); _worker.GetChars(tmp_buff); } } if (!_worker.IsEmpty) { _worker.GetInternalBuffer(tmp_buff); } result.AddRange(tmp_buff.GetUnsafePtr(), tmp_buff.Length); tmp_buff.Dispose(); }
private unsafe void ReadLinesImpl(IAddResult result, int max_lines) { int line_count = 0; line_count += this.PickLinesImpl(result, max_lines); if (line_count == max_lines) { return; } _lines.Clear(); while (line_count < max_lines) { if (_blockPos < _blockNum && _worker.IsEmpty) { this.ReadStream(); fixed(byte *byte_ptr = _byteBuffer) { var handle = new GCHandle <Decoder>(); handle.Create(_decoder); _worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle); handle.Dispose(); _worker.GetLines(_lines); } } line_count += this.PickLinesImpl(result, max_lines - line_count); if (line_count == max_lines) { return; } if (_blockPos == _blockNum) { // if final part do not have LF. if (!_worker.IsEmpty) { var buff = new NativeList <Char16>(Allocator.Temp); _worker.GetInternalBuffer(buff); result.AddResult((char *)buff.GetUnsafePtr(), buff.Length); buff.Dispose(); } // reach EOF break; } } }
/// <summary> /// initializer of NativeByteStreamReader. /// this function must be called by main thread. /// use Init(Utility.GCHandle<string>, buffersize) for calling in JobSystem. /// </summary> /// <param name="path">file path.</param> /// <param name="bufferSize">internal buffer size (if < 0, buffering entire the file).</param> public void Init(string path, int bufferSize = Define.DefaultBufferSize) { var fileInfo = new System.IO.FileInfo(path); if (!fileInfo.Exists) { throw new ArgumentException("file is not exists."); } long fileSize = fileInfo.Length; var pathHandle = new GCHandle <string>(); pathHandle.Create(path); this.InitState(pathHandle, true, fileSize, bufferSize); }
private void InitializeReadJob(Encoding encoding, Tdata data, PtrHandle <ReadStateImpl> state_ptr) { this.DisposeHandle(); _decoder.Create(encoding.GetDecoder()); _worker.SetPreamble(encoding.GetPreamble()); _data.Create(data); _state_ptr = state_ptr; _state_ptr.Target->JobState = ReadJobState.ReadAsync; _state_ptr.Target->DelayReadAsync = -1; _state_ptr.Target->DelayParseText = -1; _state_ptr.Target->DelayPostProc = -1; _info.Target->disposeHandle = true; _timer.Create(new System.Diagnostics.Stopwatch()); _timer.Target.Start(); _timer_ms_coef = 1000.0f / System.Diagnostics.Stopwatch.Frequency; }
/// <summary> /// the constructor must be called by main thread only. /// </summary> /// <param name="path"></param> public NativeByteStreamReader(Allocator alloc) { _info = new PtrHandle <ByteStreamReaderInfo>(alloc); _byteBuffer = new NativeArray <byte>(Define.MinBufferSize, alloc); _readCommands = new NativeArray <ReadCommand>(1, alloc); _path.Create(""); _info.Target->disposePathHandle = true; _info.Target->fileSize = 0; _alloc = alloc; _info.Target->dataLength = 0; _info.Target->blockSize = 0; _info.Target->blockNum = 0; _info.Target->blockPos = 0; _info.Target->allocated = true; }
public unsafe void ReadLinesFromBuffer(NativeStringList result) { var ret = new Result_NSL(); ret.result = result; if (_blockPos < _blockNum) { this.ReadStream(); fixed(byte *byte_ptr = _byteBuffer) { var handle = new GCHandle <Decoder>(); handle.Create(_decoder); _worker.DecodeTextIntoBuffer(byte_ptr, _byteLength, handle); handle.Dispose(); _worker.GetLines(_lines); } } // read all lines in buffer this.PickLinesImpl(ret, int.MaxValue); _lines.Clear(); if (_blockPos == _blockNum) { // if final part do not have LF. if (!_worker.IsEmpty) { using (var buff = new NativeList <Char16>(Allocator.Temp)) { _worker.GetInternalBuffer(buff); result.Add((char *)buff.GetUnsafePtr(), buff.Length); } } } }