public virtual void write(ExtendedDataOutput @out) { if (valueType == DATA_TYPE.DT_DICTIONARY) { throw new IOException("Can't streamlize the dictionary with value type " + valueType.name()); } BasicEntityFactory factory = new BasicEntityFactory(); Vector keys = (Vector)factory.createVectorWithDefaultValue(keyType, dict.Count); Vector values = (Vector)factory.createVectorWithDefaultValue(valueType, dict.Count); int index = 0; try { foreach (KeyValuePair <Scalar, Entity> entry in dict.SetOfKeyValuePairs()) { keys.set(index, entry.Key); values.set(index, (Scalar)entry.Value); ++index; } } catch (Exception ex) { throw new IOException(ex.Message); } int flag = ((int)DATA_FORM.DF_DICTIONARY << 8) + DataType.ordinal(); @out.writeShort(flag); keys.write(@out); values.write(@out); }
public virtual void write(ExtendedDataOutput @out) { if (valueType == DATA_TYPE.DT_DICTIONARY) { throw new IOException("Can't streamlize the dictionary with value type " + valueType.ToString()); } BasicEntityFactory factory = new BasicEntityFactory(); IVector keys = (IVector)factory.createScalarWithDefaultValue(keyType); IVector values = (IVector)factory.createVectorWithDefaultValue(valueType, dict.Count); int index = 0; try { IEnumerator <KeyValuePair <IScalar, IEntity> > itr = dict.GetEnumerator(); while (itr.MoveNext()) { keys.set(index, itr.Current.Key); values.set(index, (IScalar)itr.Current.Value); ++index; } } catch (Exception ex) { throw new IOException(ex.Message); } int flag = ((int)DATA_FORM.DF_DICTIONARY << 8) + (int)getDataType(); @out.writeShort(flag); keys.write(@out); values.write(@out); }
protected internal BasicAnyVector(ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR) { int rows = @in.readInt(); int cols = @in.readInt(); int size = rows * cols; values = new IEntity[size]; BasicEntityFactory factory = new BasicEntityFactory(); for (int i = 0; i < size; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; bool extended = type >= 128; if (type >= 128) { type -= 128; } //if (form != 1) //assert (form == 1); //if (type != 4) //assert(type == 4); IEntity obj = factory.createEntity((DATA_FORM)form, (DATA_TYPE)type, @in, extended); values[i] = obj; } }
public BasicSet(DATA_TYPE keyType, ExtendedDataInput @in) { this.keyType = keyType; BasicEntityFactory factory = new BasicEntityFactory(); DATA_TYPE[] types = Enum.GetValues(typeof(DATA_TYPE)) as DATA_TYPE[]; //read key vector short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of set keys must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid key type: " + type); } IVector keys = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); int size = keys.rows(); set = new HashSet <IScalar>(); for (int i = 0; i < size; ++i) { set.Add(keys.get(i)); } }
protected internal AbstractMatrix(ExtendedDataInput @in) { byte hasLabels = @in.readByte(); BasicEntityFactory factory = null; DATA_TYPE[] types = Enum.GetValues(typeof(DATA_TYPE)) as DATA_TYPE[]; if (hasLabels > 0) { factory = new BasicEntityFactory(); } short flag; int form; int type; if ((hasLabels & 1) == 1) { //contain row labels flag = @in.readShort(); form = flag >> 8; type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of matrix row labels must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix row labels: " + type); } rowLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); } if ((hasLabels & 2) == 2) { //contain columns labels flag = @in.readShort(); form = flag >> 8; type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of matrix columns labels must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix column labels: " + type); } columnLabels = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); } flag = @in.readShort(); type = flag & 0xff; if (type < 0 || type >= types.Length) { throw new IOException("Invalid data type for matrix: " + type); } _rows = @in.readInt(); _columns = @in.readInt(); readMatrixFromInputStream(_rows, _columns, @in); }
public BasicArrayVector(DATA_TYPE type) : base(DATA_FORM.DF_VECTOR) { this.type = type; this.rowIndices = new List <int>(); DATA_TYPE valueType = type - 64; valueVec = (AbstractVector)BasicEntityFactory.instance().createVectorWithDefaultValue(valueType, 0); this.baseUnitLength_ = valueVec.getUnitLength(); }
public BasicArrayVector(List <IVector> value) : base(DATA_FORM.DF_VECTOR) { DATA_TYPE valueType = value[0].getDataType(); this.type = valueType + 64; int count = 0; int indexCount = value.Count; foreach (IVector temp in value) { if (temp.rows() == 0) { count += 1; } else { count += temp.rows(); } } int indexPos = 0; this.rowIndices = new List <int>(new int[indexCount]); this.valueVec = (AbstractVector)BasicEntityFactory.instance().createVectorWithDefaultValue(valueType, count); int index = 0; int curRows = 0; for (int valuePos = 0; valuePos < indexCount; ++valuePos) { IVector temp = value[valuePos]; int size = temp.rows(); for (int i = 0; i < size; ++i) { this.valueVec.set(index, temp.get(i)); index++; } if (size == 0) { size = 1; this.valueVec.setNull(index); index++; } curRows += size; this.rowIndices[indexPos++] = curRows; } this.baseUnitLength_ = (this.valueVec).getUnitLength(); }
public BasicTable(ExtendedDataInput @in) { int rows = @in.readInt(); int cols = @in.readInt(); _tableName = @in.readString(); //read column names for (int i = 0; i < cols; ++i) { string name = @in.readString(); name2index_[name] = name2index_.Count; names_.Add(name); } BasicEntityFactory factory = new BasicEntityFactory(); //read columns for (int i = 0; i < cols; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; DATA_FORM df = (DATA_FORM)form; DATA_TYPE dt = (DATA_TYPE)type; if (df != DATA_FORM.DF_VECTOR) { throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName); } Console.WriteLine("vector create " + i + ":" + DateTime.Now); IVector vector = (IVector)factory.createEntity(df, dt, @in); Console.WriteLine("vector end create " + i + ":" + DateTime.Now); if (vector.rows() != rows && vector.rows() != 1) { throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns"); } columns_.Add(vector); } }
private IVector keys(int top) { BasicEntityFactory factory = new BasicEntityFactory(); int size = Math.Min(top, set.Count); IVector keys = (IVector)factory.createVectorWithDefaultValue(keyType, size); IEnumerator <IScalar> it = set.GetEnumerator(); int count = 0; try { while (count < size) { keys.set(count++, it.Current); it.MoveNext(); } } catch (Exception) { return(null); } return(keys); }
public override void append(IVector value) { if (value is IVector) { int indexCount = rowIndices.Count; int prev = indexCount == 0 ? 0 : rowIndices[indexCount - 1]; if (((IVector)value).rows() != 0) { rowIndices.Add(prev + value.rows()); valueVec.append(value); } else { rowIndices.Add(prev + 1); IScalar scalar = BasicEntityFactory.instance().createScalarWithDefaultValue(value.getDataType()); scalar.setNull(); valueVec.append(scalar); } } else { throw new Exception("Append to arrayctor must be a vector. "); } }
public BasicDictionary(DATA_TYPE valueType, ExtendedDataInput @in) { this.valueType = valueType; BasicEntityFactory factory = new BasicEntityFactory(); DATA_TYPE[] types = Enum.GetValues(typeof(DATA_TYPE)) as DATA_TYPE[]; //read key vector short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of dictionary keys must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid key type: " + type); } keyType = types[type]; IVector keys = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); //read value vector flag = @in.readShort(); form = flag >> 8; type = flag & 0xff; if (form != (int)DATA_FORM.DF_VECTOR) { throw new IOException("The form of dictionary values must be vector"); } if (type < 0 || type >= types.Length) { throw new IOException("Invalid value type: " + type); } IVector values = (IVector)factory.createEntity(DATA_FORM.DF_VECTOR, types[type], @in); if (keys.rows() != values.rows()) { throw new IOException("The key size doesn't equate to value size."); } int size = keys.rows(); int capacity = (int)(size / 0.75); dict = new Dictionary <IScalar, IEntity>(capacity); if (values.getDataType() == DATA_TYPE.DT_ANY) { BasicAnyVector entityValues = (BasicAnyVector)values; for (int i = 0; i < size; ++i) { dict[keys.get(i)] = entityValues.getEntity(i); } } else { for (int i = 0; i < size; ++i) { dict[keys.get(i)] = values.get(i); } } }
public BasicTable(ExtendedDataInput @in) { int rows = @in.readInt(); int cols = @in.readInt(); _tableName = @in.readString(); //read column names for (int i = 0; i < cols; ++i) { string name = @in.readString(); name2index_[name] = name2index_.Count; names_.Add(name); } BasicEntityFactory factory = new BasicEntityFactory(); VectorDecompressor decompressor = null; SymbolBaseCollection collection = null; //read columns for (int i = 0; i < cols; ++i) { short flag = @in.readShort(); int form = flag >> 8; int type = flag & 0xff; bool extended = type >= 128; if (type >= 128) { type -= 128; } DATA_FORM df = (DATA_FORM)form; DATA_TYPE dt = (DATA_TYPE)type; if (df != DATA_FORM.DF_VECTOR) { throw new IOException("Invalid form for column [" + names_[i] + "] for table " + _tableName); } IVector vector; if (dt == DATA_TYPE.DT_SYMBOL && extended) { if (collection == null) { collection = new SymbolBaseCollection(); } vector = new BasicSymbolVector(df, @in, collection); } else if (dt == DATA_TYPE.DT_COMPRESS) { if (decompressor == null) { decompressor = new VectorDecompressor(); } vector = decompressor.Decompress(factory, @in, false, true); } else { vector = (IVector)factory.createEntity(df, dt, @in, extended); } if (vector.rows() != rows && vector.rows() != 1) { int tmp = vector.rows(); throw new IOException("The number of rows for column " + names_[i] + " is not consistent with other columns"); } columns_.Add(vector); } }
public BasicArrayVector(DATA_TYPE type, ExtendedDataInput @in) : base(DATA_FORM.DF_VECTOR) { this.type = type; int rows = @in.readInt(); int cols = @in.readInt(); rowIndices = new List <int>(new int[rows]); DATA_TYPE valueType = type - 64; valueVec = (AbstractVector)BasicEntityFactory.instance().createVectorWithDefaultValue(valueType, cols); this.baseUnitLength_ = valueVec.getUnitLength(); int rowsRead = 0; int rowsReadInBlock = 0; int prevIndex = 0; int totalBytes = 0; while (rowsRead < rows) { //read block header int blockRows = @in.readShort(); int countBytes = @in.readChar(); @in.skipBytes(1); //read array of count totalBytes = blockRows * countBytes; rowsReadInBlock = 0; int offect = 0; while (offect < totalBytes) { int len = Math.Min(BUF_SIZE, totalBytes - offect); @in.readFully(buf, 0, len); int curRows = len / countBytes; if (countBytes == 1) { for (int i = 0; i < curRows; i++) { int curRowCells = buf[i]; rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } else if (countBytes == 2) { for (int i = 0; i < curRows; ++i) { int curRowCells = BitConverter.ToInt16(buf, i * 2); rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } else { for (int i = 0; i < curRows; ++i) { int curRowCells = BitConverter.ToInt32(buf, i * 4); rowIndices[rowsRead + rowsReadInBlock + i] = prevIndex + curRowCells; prevIndex += curRowCells; } } rowsReadInBlock += curRows; offect += len; } //read array of values int rowStart = rowsRead == 0 ? 0 : rowIndices[rowsRead - 1]; int valueCount = rowIndices[rowsRead + rowsReadInBlock - 1] - rowStart; valueVec.deserialize(rowStart, valueCount, @in); rowsRead += rowsReadInBlock; } }