//2021.01.19 cwj
protected internal BasicStringVector(DATA_FORM df, ExtendedDataInput @in, bool symbol, bool blob) : base(df)
{
isBlob = blob;
isSymbol = symbol;
int rows = @in.readInt();
int columns = @in.readInt();
int size = rows * columns;
values = new List <string>(size);
values.AddRange(new string[size]);
if (!blob)
{
for (int i = 0; i < size; ++i)
{
values[i] = @in.readString();
}
}
else
{
for (int i = 0; i < size; ++i)
{
values[i] = @in.readBlob();
}
}
}
protected internal BasicAnyVector(ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR)
{
int rows = @in.readInt();
int cols = @in.readInt();
int size = rows * cols;
values = new IEntity[size];
BasicEntityFactory factory = new BasicEntityFactory();
for (int i = 0; i < size; ++i)
{
short flag = @in.readShort();
int form = flag >> 8;
int type = flag & 0xff;
bool extended = type >= 128;
if (type >= 128)
{
type -= 128;
}
//if (form != 1)
//assert (form == 1);
//if (type != 4)
//assert(type == 4);
IEntity obj = factory.createEntity((DATA_FORM)form, (DATA_TYPE)type, @in, extended);
values[i] = obj;
}
}
protected internal AbstractMatrix(ExtendedDataInput @in)
{
byte hasLabels = @in.readByte();
BasicEntityFactory factory = null;
DATA_TYPE[] types = Enum.GetValues(typeof(DATA_TYPE)) as DATA_TYPE[];
if (hasLabels > 0)
{
factory = new BasicEntityFactory();
}
short flag;
int form;
int type;
if ((hasLabels & 1) == 1)
{
//contain row labels
flag = @in.readShort();
form = flag >> 8;
type = flag & 0xff;
if (form != (int)DATA_FORM.DF_VECTOR)
{
throw new IOException("The form of matrix row labels must be vector");
}
if (type < 0 || type >= types.Length)
{
throw new IOException("Invalid data type for matrix row labels: " + type);
}
rowLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in);
}
if ((hasLabels & 2) == 2)
{
//contain columns labels
flag = @in.readShort();
form = flag >> 8;
type = flag & 0xff;
if (form != (int)DATA_FORM.DF_VECTOR)
{
throw new IOException("The form of matrix columns labels must be vector");
}
if (type < 0 || type >= types.Length)
{
throw new IOException("Invalid data type for matrix column labels: " + type);
}
columnLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in);
}
flag = @in.readShort();
type = flag & 0xff;
if (type < 0 || type >= types.Length)
{
throw new IOException("Invalid data type for matrix: " + type);
}
_rows = @in.readInt();
_columns = @in.readInt();
readMatrixFromInputStream(_rows, _columns, @in);
}
public EntityBlockReader(ExtendedDataInput input)
{
int rows = input.readInt();
int cols = input.readInt();
size = rows * cols;
currentIndex = 0;
instream = input;
}
protected internal BasicBooleanVector(DATA_FORM df, ExtendedDataInput @in) : base(df)
{
int rows = @in.readInt();
int cols = @in.readInt();
int size = rows * cols;
values = new byte[size];
for (int i = 0; i < size; ++i)
{
values[i] = @in.readByte();
}
}
protected internal BasicStringVector(DATA_FORM df, ExtendedDataInput @in) : base(df)
{
int rows = @in.readInt();
int columns = @in.readInt();
int size = rows * columns;
values = new string[size];
for (int i = 0; i < size; ++i)
{
values[i] = @in.readString();
}
}
protected internal BasicByteVector(DATA_FORM df, ExtendedDataInput @in) : base(df)
{
int rows = @in.readInt();
int cols = @in.readInt();
int size = rows * cols;
values = new List <byte>(size);
values.AddRange(new byte[size]);
for (int i = 0; i < size; ++i)
{
values[i] = @in.readByte();
}
}
protected internal BasicSymbolVector(DATA_FORM df, ExtendedDataInput @in, SymbolBaseCollection collection) : base(df)
{
int rows = @in.readInt();
int columns = @in.readInt();
int size = rows * columns;
values = new List <int>(new int[rows]);
@base = collection.add(@in);
for (int i = 0; i < size; ++i)
{
values[i] = @in.readInt();
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: protected BasicIntVector(Entity_DATA_FORM df, dolphindb.io.ExtendedDataInput in) throws java.io.IOException
protected internal BasicIntVector(DATA_FORM df, ExtendedDataInput @in) : base(df)
{
int rows = @in.readInt();
//if (rows != 1024)
//assert(rows == 1024);
int cols = @in.readInt();
int size = rows * cols;
values = new int[size];
for (int i = 0; i < size; ++i)
{
values[i] = @in.readInt();
}
}
public BasicChunkMeta(ExtendedDataInput @in)
{
@in.readShort(); //skip the length of the data
path = @in.readString();
id = new sbyte[16];
@in.readFully(id);
version = @in.readInt();
size_Renamed = @in.readInt();
flag = @in.readByte();
sites = new List <>();
int copyCount = @in.readByte();
for (int i = 0; i < copyCount; ++i)
{
sites.Add(@in.readString());
}
}
public SymbolBase(int id, ExtendedDataInput @in)
{
this.id = id;
int size = @in.readInt();
for (int i = 0; i < size; ++i)
{
syms.Add(@in.readString());
}
}
internal BasicInt128Vector(DATA_FORM df, ExtendedDataInput @in) : base(df)
{
int rows = @in.readInt();
int cols = @in.readInt();
int size = rows * cols;
values = new List <Long2>();
values.AddRange(new Long2[size]);
int totalBytes = size * 16, off = 0;
bool littleEndian = @in.isLittleEndian();
while (off < totalBytes)
{
int len = System.Math.Min(BUF_SIZE, totalBytes - off);
@in.readFully(buf, 0, len);
int start = off / 16, end = len / 16;
Byte[] dst = new Byte[len];
Buffer.BlockCopy(buf, 0, dst, 0, len);
if (!littleEndian)
{
Array.Reverse(dst);
}
if (littleEndian)
{
for (int i = 0; i < end; i++)
{
long low = BitConverter.ToInt64(dst, i * 16);
long high = BitConverter.ToInt64(dst, i * 16 + 8);
values[i + start] = new Long2(high, low);
}
}
else
{
for (int i = 0; i < end; i++)
{
long high = BitConverter.ToInt64(dst, i * 16);
long low = BitConverter.ToInt64(dst, i * 16 + 8);
values[i + start] = new Long2(high, low);
}
}
off += len;
}
}
protected internal override void readMatrixFromInputStream(int rows, int columns, ExtendedDataInput @in)
{
int size = rows * columns;
values = new int[size];
for (int i = 0; i < size; ++i)
{
values[i] = @in.readInt();
}
}
public override void deserialize(int start, int count, ExtendedDataInput @in)
{
if (start + count > values.Count)
{
values.AddRange(new int[start + count - values.Count]);
}
for (int i = 0; i < count; ++i)
{
values[start + i] = @in.readInt();
}
}
public virtual SymbolBase add(ExtendedDataInput @in)
{
int id = @in.readInt();
if (symbaseMap.ContainsKey(id))
{
int size = @in.readInt();
if (size != 0)
{
throw new IOException("Invalid symbol base.");
}
lastSymbase = symbaseMap[id];
}
else
{
SymbolBase cur = new SymbolBase(id, @in);
symbaseMap[id] = cur;
lastSymbase = cur;
}
return(lastSymbase);
}
public BasicTable(ExtendedDataInput @in)
{
int rows = @in.readInt();
int cols = @in.readInt();
_tableName = @in.readString();
//read column names
for (int i = 0; i < cols; ++i)
{
string name = @in.readString();
name2index_[name] = name2index_.Count;
names_.Add(name);
}
BasicEntityFactory factory = new BasicEntityFactory();
//read columns
for (int i = 0; i < cols; ++i)
{
short flag = @in.readShort();
int form = flag >> 8;
int type = flag & 0xff;
DATA_FORM df = (DATA_FORM)form;
DATA_TYPE dt = (DATA_TYPE)type;
if (df != DATA_FORM.DF_VECTOR)
{
throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName);
}
Console.WriteLine("vector create " + i + ":" + DateTime.Now);
IVector vector = (IVector)factory.createEntity(df, dt, @in);
Console.WriteLine("vector end create " + i + ":" + DateTime.Now);
if (vector.rows() != rows && vector.rows() != 1)
{
throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns");
}
columns_.Add(vector);
}
}
public IVector Decompress(IEntityFactory factory, ExtendedDataInput input, bool extended, bool isLittleEndian)
{
int compressedBytes = input.readInt();
//((LittleEndianDataInputStream)input).skipBytes(7);
for (int i = 0; i < 7; ++i)
{
input.readByte();
}
int compression = input.readByte();
int dataType = input.readByte();
int unitLength = input.readByte();
//((LittleEndianDataInputStream)input).skipBytes(6);
for (int i = 0; i < 2; ++i)
{
input.readByte();
}
int etra = input.readInt();
int elementCount = input.readInt();
if (dataType < (int)DATA_TYPE.DT_BOOL_ARRAY)
{
etra = 1;
}
input.readInt();
ExtendedDataInput decompressedIn = DecoderFactory.get(compression).Decompress(input, compressedBytes - 20, unitLength, elementCount, etra, isLittleEndian);
bool extend = dataType >= 128;
if (dataType >= 128)
{
dataType -= 128;
}
DATA_TYPE dt = (DATA_TYPE)dataType;
return((IVector)factory.createEntity(DATA_FORM.DF_VECTOR, dt, decompressedIn, extend));
}
public override ExtendedDataInput Decompress(ExtendedDataInput input, int length, int unitLength, int elementCount, int extra, bool isLittleEndian)
{
int offset = 8;
byte[] output = CreateColumnVector(elementCount, extra, unitLength, isLittleEndian, 0).array();
ILZ4Decompressor t = LZ4DecompressorFactory.CreateNew();
while (length > 0)
{
int blockSize = input.readInt();
if (blockSize < 0)
{
blockSize = blockSize & 2147483647;
}
length -= sizeof(int);
blockSize = Math.Min(blockSize, length);
if (blockSize == 0)
{
break;
}
byte[] src = new byte[blockSize];
int index = 0;
while (index < blockSize)
{
index += ((BinaryReader)input).Read(src, index, blockSize - index);
}
byte[] ret = t.Decompress(src);
if (offset + ret.Length > output.Length)
{
byte[] tmp = new byte[Math.Max(offset + ret.Length, output.Length) * 2];
Array.Copy(output, tmp, offset);
output = tmp;
}
Buffer.BlockCopy(ret, 0, output, offset, ret.Length);
offset += ret.Length;
length -= blockSize;
}
if (isLittleEndian)
{
return(new LittleEndianDataInputStream(new MemoryStream(output)));
}
return(new BigEndianDataInputStream(new MemoryStream(output)));
}
public override ExtendedDataInput Decompress(ExtendedDataInput input, int length, int unitLength, int elementCount, int extra, bool isLittleEndian)
{
int offset = 8;
ByteBuffer dest = CreateColumnVector(elementCount, extra, unitLength, isLittleEndian, 0);
byte[] output = dest.array();
int outLength = output.Length - offset;
int count = 0;
DeltaOfDeltaBlockDecoder blockDecoder = new DeltaOfDeltaBlockDecoder(unitLength);
while (length > 0 && count < outLength)
{
int blockSize = input.readInt();
if (blockSize < 0)
{
blockSize = blockSize & 2147483647;
}
length -= sizeof(int);
blockSize = Math.Min(blockSize, length);
if (blockSize == 0)
{
break;
}
long[] src = new long[blockSize / sizeof(long)];
for (int i = 0; i < src.Length; i++)
{
src[i] = input.readLong();
}
count += blockDecoder.decompress(src, dest) * unitLength;
length -= blockSize;
}
if (isLittleEndian)
{
return(new LittleEndianDataInputStream(new MemoryStream(output, 0, offset + count)));
}
return
(new BigEndianDataInputStream(new MemoryStream(output, 0, offset + count)));
}
public BasicTable(ExtendedDataInput @in)
{
int rows = @in.readInt();
int cols = @in.readInt();
_tableName = @in.readString();
//read column names
for (int i = 0; i < cols; ++i)
{
string name = @in.readString();
name2index_[name] = name2index_.Count;
names_.Add(name);
}
BasicEntityFactory factory = new BasicEntityFactory();
VectorDecompressor decompressor = null;
SymbolBaseCollection collection = null;
//read columns
for (int i = 0; i < cols; ++i)
{
short flag = @in.readShort();
int form = flag >> 8;
int type = flag & 0xff;
bool extended = type >= 128;
if (type >= 128)
{
type -= 128;
}
DATA_FORM df = (DATA_FORM)form;
DATA_TYPE dt = (DATA_TYPE)type;
if (df != DATA_FORM.DF_VECTOR)
{
throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName);
}
IVector vector;
if (dt == DATA_TYPE.DT_SYMBOL && extended)
{
if (collection == null)
{
collection = new SymbolBaseCollection();
}
vector = new BasicSymbolVector(df, @in, collection);
}
else if (dt == DATA_TYPE.DT_COMPRESS)
{
if (decompressor == null)
{
decompressor = new VectorDecompressor();
}
vector = decompressor.Decompress(factory, @in, false, true);
}
else
{
vector = (IVector)factory.createEntity(df, dt, @in, extended);
}
if (vector.rows() != rows && vector.rows() != 1)
{
int tmp = vector.rows();
throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns");
}
columns_.Add(vector);
}
}
public SymbolBase(ExtendedDataInput @in) : this(@in.readInt(), @in)
{
}
public BasicArrayVector(DATA_TYPE type, ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR)
{
this.type = type;
int rows = @in.readInt();
int cols = @in.readInt();
rowIndices = new List <int>(new int[rows]);
DATA_TYPE valueType = type - 64;
valueVec = (AbstractVector)BasicEntityFactory.instance().createVectorWithDefaultValue(valueType, cols);
this.baseUnitLength_ = valueVec.getUnitLength();
int rowsRead = 0;
int rowsReadInBlock = 0;
int prevIndex = 0;
int totalBytes = 0;
while (rowsRead < rows)
{
//read block header
int blockRows = @in.readShort();
int countBytes = @in.readChar();
@in.skipBytes(1);
//read array of count
totalBytes = blockRows * countBytes;
rowsReadInBlock = 0;
int offect = 0;
while (offect < totalBytes)
{
int len = Math.Min(BUF_SIZE, totalBytes - offect);
@in.readFully(buf, 0, len);
int curRows = len / countBytes;
if (countBytes == 1)
{
for (int i = 0; i < curRows; i++)
{
int curRowCells = buf[i];
rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells;
prevIndex += curRowCells;
}
}
else if (countBytes == 2)
{
for (int i = 0; i < curRows; ++i)
{
int curRowCells = BitConverter.ToInt16(buf, i * 2);
rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells;
prevIndex += curRowCells;
}
}
else
{
for (int i = 0; i < curRows; ++i)
{
int curRowCells = BitConverter.ToInt32(buf, i * 4);
rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells;
prevIndex += curRowCells;
}
}
rowsReadInBlock += curRows;
offect += len;
}
//read array of values
int rowStart = rowsRead == 0 ? 0 : rowIndices[rowsRead - 1];
int valueCount = rowIndices[rowsRead + rowsReadInBlock - 1] - rowStart;
valueVec.deserialize(rowStart, valueCount, @in);
rowsRead += rowsReadInBlock;
}
}
public BasicInt(ExtendedDataInput @in)
{
value = @in.readInt();
}
