//2021.01.19 cwj protected internal BasicStringVector(DATA_FORM df, ExtendedDataInput @in, bool symbol, bool blob) : base(df) { isBlob = blob; isSymbol = symbol; int rows = @in.readInt(); int columns = @in.readInt(); int size = rows * columns; values = new List <string>(size); values.AddRange(new string[size]); if (!blob) { for (int i = 0; i < size; ++i) { values[i] = @in.readString(); } } else { for (int i = 0; i < size; ++i) { values[i] = @in.readBlob(); } } }
protected internal BasicAnyVector(ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR) { int rows = @in.readInt(); int cols = @in.readInt(); int size = rows * cols; values = new IEntity[size]; BasicEntityFactory factory = new BasicEntityFactory(); for (int i = 0; i < size; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; bool extended = type >= 128; if (type >= 128) { type -= 128; } //if (form != 1) //assert (form == 1); //if (type != 4) //assert(type == 4); IEntity obj = factory.createEntity((DATA_FORM)form, (DATA_TYPE)type, @in, extended); values[i] = obj; } }
protected internal AbstractMatrix(ExtendedDataInput @in) { byte hasLabels = @in.readByte(); BasicEntityFactory factory = null; DATA_TYPE[] types = Enum.GetValues(typeof(DATA_TYPE)) as DATA_TYPE[]; if (hasLabels > 0) { factory = new BasicEntityFactory(); } short flag; int form; int type; if ((hasLabels & 1) == 1) { //contain row labels flag = @in.readShort(); form = flag >> 8; type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of matrix row labels must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix row labels: " + type); } rowLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); } if ((hasLabels & 2) == 2) { //contain columns labels flag = @in.readShort(); form = flag >> 8; type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of matrix columns labels must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix column labels: " + type); } columnLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); } flag = @in.readShort(); type = flag & 0xff; if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix: " + type); } _rows = @in.readInt(); _columns = @in.readInt(); readMatrixFromInputStream(_rows, _columns, @in); }
public EntityBlockReader(ExtendedDataInput input) { int rows = input.readInt(); int cols = input.readInt(); size = rows * cols; currentIndex = 0; instream = input; }
protected internal BasicBooleanVector(DATA_FORM df, ExtendedDataInput @in) : base(df) { int rows = @in.readInt(); int cols = @in.readInt(); int size = rows * cols; values = new byte[size]; for (int i = 0; i < size; ++i) { values[i] = @in.readByte(); } }
protected internal BasicStringVector(DATA_FORM df, ExtendedDataInput @in) : base(df) { int rows = @in.readInt(); int columns = @in.readInt(); int size = rows * columns; values = new string[size]; for (int i = 0; i < size; ++i) { values[i] = @in.readString(); } }
protected internal BasicByteVector(DATA_FORM df, ExtendedDataInput @in) : base(df) { int rows = @in.readInt(); int cols = @in.readInt(); int size = rows * cols; values = new List <byte>(size); values.AddRange(new byte[size]); for (int i = 0; i < size; ++i) { values[i] = @in.readByte(); } }
protected internal BasicSymbolVector(DATA_FORM df, ExtendedDataInput @in, SymbolBaseCollection collection) : base(df) { int rows = @in.readInt(); int columns = @in.readInt(); int size = rows * columns; values = new List <int>(new int[rows]); @base = collection.add(@in); for (int i = 0; i < size; ++i) { values[i] = @in.readInt(); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: protected BasicIntVector(Entity_DATA_FORM df, dolphindb.io.ExtendedDataInput in) throws java.io.IOException protected internal BasicIntVector(DATA_FORM df, ExtendedDataInput @in) : base(df) { int rows = @in.readInt(); //if (rows != 1024) //assert(rows == 1024); int cols = @in.readInt(); int size = rows * cols; values = new int[size]; for (int i = 0; i < size; ++i) { values[i] = @in.readInt(); } }
public BasicChunkMeta(ExtendedDataInput @in) { @in.readShort(); //skip the length of the data path = @in.readString(); id = new sbyte[16]; @in.readFully(id); version = @in.readInt(); size_Renamed = @in.readInt(); flag = @in.readByte(); sites = new List <>(); int copyCount = @in.readByte(); for (int i = 0; i < copyCount; ++i) { sites.Add(@in.readString()); } }
public SymbolBase(int id, ExtendedDataInput @in) { this.id = id; int size = @in.readInt(); for (int i = 0; i < size; ++i) { syms.Add(@in.readString()); } }
internal BasicInt128Vector(DATA_FORM df, ExtendedDataInput @in) : base(df) { int rows = @in.readInt(); int cols = @in.readInt(); int size = rows * cols; values = new List <Long2>(); values.AddRange(new Long2[size]); int totalBytes = size * 16, off = 0; bool littleEndian = @in.isLittleEndian(); while (off < totalBytes) { int len = System.Math.Min(BUF_SIZE, totalBytes - off); @in.readFully(buf, 0, len); int start = off / 16, end = len / 16; Byte[] dst = new Byte[len]; Buffer.BlockCopy(buf, 0, dst, 0, len); if (!littleEndian) { Array.Reverse(dst); } if (littleEndian) { for (int i = 0; i < end; i++) { long low = BitConverter.ToInt64(dst, i * 16); long high = BitConverter.ToInt64(dst, i * 16 + 8); values[i + start] = new Long2(high, low); } } else { for (int i = 0; i < end; i++) { long high = BitConverter.ToInt64(dst, i * 16); long low = BitConverter.ToInt64(dst, i * 16 + 8); values[i + start] = new Long2(high, low); } } off += len; } }
protected internal override void readMatrixFromInputStream(int rows, int columns, ExtendedDataInput @in) { int size = rows * columns; values = new int[size]; for (int i = 0; i < size; ++i) { values[i] = @in.readInt(); } }
public override void deserialize(int start, int count, ExtendedDataInput @in) { if (start + count > values.Count) { values.AddRange(new int[start + count - values.Count]); } for (int i = 0; i < count; ++i) { values[start + i] = @in.readInt(); } }
public virtual SymbolBase add(ExtendedDataInput @in) { int id = @in.readInt(); if (symbaseMap.ContainsKey(id)) { int size = @in.readInt(); if (size != 0) { throw new IOException("Invalid symbol base."); } lastSymbase = symbaseMap[id]; } else { SymbolBase cur = new SymbolBase(id, @in); symbaseMap[id] = cur; lastSymbase = cur; } return(lastSymbase); }
public BasicTable(ExtendedDataInput @in) { int rows = @in.readInt(); int cols = @in.readInt(); _tableName = @in.readString(); //read column names for (int i = 0; i < cols; ++i) { string name = @in.readString(); name2index_[name] = name2index_.Count; names_.Add(name); } BasicEntityFactory factory = new BasicEntityFactory(); //read columns for (int i = 0; i < cols; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; DATA_FORM df = (DATA_FORM)form; DATA_TYPE dt = (DATA_TYPE)type; if (df != DATA_FORM.DF_VECTOR) { throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName); } Console.WriteLine("vector create " + i + ":" + DateTime.Now); IVector vector = (IVector)factory.createEntity(df, dt, @in); Console.WriteLine("vector end create " + i + ":" + DateTime.Now); if (vector.rows() != rows && vector.rows() != 1) { throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns"); } columns_.Add(vector); } }
public IVector Decompress(IEntityFactory factory, ExtendedDataInput input, bool extended, bool isLittleEndian) { int compressedBytes = input.readInt(); //((LittleEndianDataInputStream)input).skipBytes(7); for (int i = 0; i < 7; ++i) { input.readByte(); } int compression = input.readByte(); int dataType = input.readByte(); int unitLength = input.readByte(); //((LittleEndianDataInputStream)input).skipBytes(6); for (int i = 0; i < 2; ++i) { input.readByte(); } int etra = input.readInt(); int elementCount = input.readInt(); if (dataType < (int)DATA_TYPE.DT_BOOL_ARRAY) { etra = 1; } input.readInt(); ExtendedDataInput decompressedIn = DecoderFactory.get(compression).Decompress(input, compressedBytes - 20, unitLength, elementCount, etra, isLittleEndian); bool extend = dataType >= 128; if (dataType >= 128) { dataType -= 128; } DATA_TYPE dt = (DATA_TYPE)dataType; return((IVector)factory.createEntity(DATA_FORM.DF_VECTOR, dt, decompressedIn, extend)); }
public override ExtendedDataInput Decompress(ExtendedDataInput input, int length, int unitLength, int elementCount, int extra, bool isLittleEndian) { int offset = 8; byte[] output = CreateColumnVector(elementCount, extra, unitLength, isLittleEndian, 0).array(); ILZ4Decompressor t = LZ4DecompressorFactory.CreateNew(); while (length > 0) { int blockSize = input.readInt(); if (blockSize < 0) { blockSize = blockSize & 2147483647; } length -= sizeof(int); blockSize = Math.Min(blockSize, length); if (blockSize == 0) { break; } byte[] src = new byte[blockSize]; int index = 0; while (index < blockSize) { index += ((BinaryReader)input).Read(src, index, blockSize - index); } byte[] ret = t.Decompress(src); if (offset + ret.Length > output.Length) { byte[] tmp = new byte[Math.Max(offset + ret.Length, output.Length) * 2]; Array.Copy(output, tmp, offset); output = tmp; } Buffer.BlockCopy(ret, 0, output, offset, ret.Length); offset += ret.Length; length -= blockSize; } if (isLittleEndian) { return(new LittleEndianDataInputStream(new MemoryStream(output))); } return(new BigEndianDataInputStream(new MemoryStream(output))); }
public override ExtendedDataInput Decompress(ExtendedDataInput input, int length, int unitLength, int elementCount, int extra, bool isLittleEndian) { int offset = 8; ByteBuffer dest = CreateColumnVector(elementCount, extra, unitLength, isLittleEndian, 0); byte[] output = dest.array(); int outLength = output.Length - offset; int count = 0; DeltaOfDeltaBlockDecoder blockDecoder = new DeltaOfDeltaBlockDecoder(unitLength); while (length > 0 && count < outLength) { int blockSize = input.readInt(); if (blockSize < 0) { blockSize = blockSize & 2147483647; } length -= sizeof(int); blockSize = Math.Min(blockSize, length); if (blockSize == 0) { break; } long[] src = new long[blockSize / sizeof(long)]; for (int i = 0; i < src.Length; i++) { src[i] = input.readLong(); } count += blockDecoder.decompress(src, dest) * unitLength; length -= blockSize; } if (isLittleEndian) { return(new LittleEndianDataInputStream(new MemoryStream(output, 0, offset + count))); } return (new BigEndianDataInputStream(new MemoryStream(output, 0, offset + count))); }
public BasicTable(ExtendedDataInput @in) { int rows = @in.readInt(); int cols = @in.readInt(); _tableName = @in.readString(); //read column names for (int i = 0; i < cols; ++i) { string name = @in.readString(); name2index_[name] = name2index_.Count; names_.Add(name); } BasicEntityFactory factory = new BasicEntityFactory(); VectorDecompressor decompressor = null; SymbolBaseCollection collection = null; //read columns for (int i = 0; i < cols; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; bool extended = type >= 128; if (type >= 128) { type -= 128; } DATA_FORM df = (DATA_FORM)form; DATA_TYPE dt = (DATA_TYPE)type; if (df != DATA_FORM.DF_VECTOR) { throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName); } IVector vector; if (dt == DATA_TYPE.DT_SYMBOL && extended) { if (collection == null) { collection = new SymbolBaseCollection(); } vector = new BasicSymbolVector(df, @in, collection); } else if (dt == DATA_TYPE.DT_COMPRESS) { if (decompressor == null) { decompressor = new VectorDecompressor(); } vector = decompressor.Decompress(factory, @in, false, true); } else { vector = (IVector)factory.createEntity(df, dt, @in, extended); } if (vector.rows() != rows && vector.rows() != 1) { int tmp = vector.rows(); throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns"); } columns_.Add(vector); } }
public SymbolBase(ExtendedDataInput @in) : this(@in.readInt(), @in) { }
public BasicArrayVector(DATA_TYPE type, ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR) { this.type = type; int rows = @in.readInt(); int cols = @in.readInt(); rowIndices = new List <int>(new int[rows]); DATA_TYPE valueType = type - 64; valueVec = (AbstractVector)BasicEntityFactory.instance().createVectorWithDefaultValue(valueType, cols); this.baseUnitLength_ = valueVec.getUnitLength(); int rowsRead = 0; int rowsReadInBlock = 0; int prevIndex = 0; int totalBytes = 0; while (rowsRead < rows) { //read block header int blockRows = @in.readShort(); int countBytes = @in.readChar(); @in.skipBytes(1); //read array of count totalBytes = blockRows * countBytes; rowsReadInBlock = 0; int offect = 0; while (offect < totalBytes) { int len = Math.Min(BUF_SIZE, totalBytes - offect); @in.readFully(buf, 0, len); int curRows = len / countBytes; if (countBytes == 1) { for (int i = 0; i < curRows; i++) { int curRowCells = buf[i]; rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } else if (countBytes == 2) { for (int i = 0; i < curRows; ++i) { int curRowCells = BitConverter.ToInt16(buf, i * 2); rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } else { for (int i = 0; i < curRows; ++i) { int curRowCells = BitConverter.ToInt32(buf, i * 4); rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } rowsReadInBlock += curRows; offect += len; } //read array of values int rowStart = rowsRead == 0 ? 0 : rowIndices[rowsRead - 1]; int valueCount = rowIndices[rowsRead + rowsReadInBlock - 1] - rowStart; valueVec.deserialize(rowStart, valueCount, @in); rowsRead += rowsReadInBlock; } }
public BasicInt(ExtendedDataInput @in) { value = @in.readInt(); }