// Note: the below constructor isn't used for rindex. public QaDataReader(QaCommand cm, System.Net.Sockets.NetworkStream ns) { cmd = cm; netstm = ns; isClosed = false; eof = false; isRindexEnabled = cm.conn.isRIndexEnabled(); //Get metadata try { //column count int bc = 0; cmd.buf = XContent.ReceiveXBytes(netstm, out bc, cmd.buf); if (bc != 4) { throw new Exception("Column count expected 4 bytes."); } int columnCount = Utils.BytesToInt(cmd.buf); metadata = new MetaData[columnCount]; ordinals = new Dictionary <string, int>(); recordsize = 0; for (int i = 0; i < columnCount; i++) { string cname = XContent.ReceiveXString(netstm, cmd.buf); string ctype = XContent.ReceiveXString(netstm, cmd.buf); cmd.buf = XContent.ReceiveXBytes(netstm, out bc, cmd.buf); if (bc != 4) { throw new Exception("Front bytes expected 4 bytes."); } int frontbytes = Utils.BytesToInt(cmd.buf); cmd.buf = XContent.ReceiveXBytes(netstm, out bc, cmd.buf); if (bc != 4) { throw new Exception("Size expected 4 bytes."); } int csize = Utils.BytesToInt(cmd.buf); cmd.buf = XContent.ReceiveXBytes(netstm, out bc, cmd.buf); if (bc != 4) { throw new Exception("Back bytes expected 4 bytes."); } int backbytes = Utils.BytesToInt(cmd.buf); MetaData md = MetaData.Prepare(cname, Type.GetType(ctype), csize, frontbytes, backbytes); metadata[i] = md; recordsize += csize + frontbytes + backbytes; ordinals[cname.ToLower()] = i; } if (null == cmd.buf || cmd.buf.Length < recordsize) { cmd.buf = new byte[recordsize]; } currow = new object[columnCount]; } catch { cmd.Abort(); throw; } }
void _OpenRIndex() { try { if (state == ConnectionState.Open) { throw new Exception("Connnection is already open."); } if (connstr.RIndex == QaConnectionString.RIndexType.POOLED) { string[] hosts = connstr.DataSource; Random rnd = new Random(unchecked (System.DateTime.Now.Millisecond + System.Threading.Thread.CurrentThread.ManagedThreadId)); for (int hi = 0; hi < hosts.Length; hi++) { int swapindex = rnd.Next() % hosts.Length; string oval = hosts[hi]; hosts[hi] = hosts[swapindex]; hosts[swapindex] = oval; } for (int hi = 0; hi < hosts.Length; hi++) { _OpenSocketRIndex(hosts[hi]); } } else { _OpenSocketRIndex(null); } netstm.WriteByte((byte)'i'); //get all master indexes. sysindexes = new Dictionary <string, Index>(); { string xml = XContent.ReceiveXString(netstm, buf); if (xml.Length > 0) { System.Xml.XmlDocument xi = new System.Xml.XmlDocument(); xi.LoadXml(xml); System.Xml.XmlNodeList xnIndexes = xi.SelectNodes("/indexes/index"); foreach (System.Xml.XmlNode xnIndex in xnIndexes) { string indName = xnIndex["name"].InnerText; System.Xml.XmlNode xnUpdatememoryonly = xnIndex.SelectSingleNode("updatememoryonly"); System.Xml.XmlElement xePinHash = xnIndex["pinHash"]; System.Xml.XmlElement xeTable = xnIndex["table"]; System.Xml.XmlNodeList xnCols = xeTable.SelectNodes("column"); Column[] cols = new Column[xnCols.Count]; for (int ci = 0; ci < xnCols.Count; ci++) { System.Xml.XmlNode xnCol = xnCols[ci]; cols[ci].Name = xnCol["name"].InnerText; cols[ci].Type = xnCol["type"].InnerText; cols[ci].Bytes = Int32.Parse(xnCol["bytes"].InnerText); } Table tab; tab.Name = xeTable["name"].InnerText; tab.Columns = cols; Index ind; ind.Name = indName; ind.Ordinal = Int32.Parse(xnIndex["ordinal"].InnerText); ind.Table = tab; ind.UpdateMemoryOnly = (xnUpdatememoryonly != null && xnUpdatememoryonly.InnerText == "1"); ind.PinHash = (xePinHash != null && xePinHash.InnerText == "1"); ind.Hash = ind.PinHash ? new Position[256 * 256] : null; ind.MaxKey = new byte[tab.Columns[ind.Ordinal].Bytes]; sysindexes.Add(indName.ToLower(), ind); } } } int micnt = 0; XContent.ReceiveXBytes(netstm, out micnt, buf); micnt = Utils.BytesToInt(buf, 0); mindexes = new Dictionary <string, List <KeyValuePair <byte[], string> > >(micnt); for (int mi = 0; mi < micnt; mi++) { string indexname = XContent.ReceiveXString(netstm, buf).ToLower(); List <KeyValuePair <byte[], string> > lines = new List <KeyValuePair <byte[], string> >(); int keylen = 9; bool pinhash = false; Position[] hash = null; byte[] maxkey = null; if (sysindexes.ContainsKey(indexname)) { Index thisindex = sysindexes[indexname]; int ordinal = thisindex.Ordinal; keylen = thisindex.Table.Columns[ordinal].Bytes; pinhash = thisindex.PinHash; hash = thisindex.Hash; maxkey = thisindex.MaxKey; } else { throw new Exception("Index version conflict, need to recreate indexes"); } byte[] lastkeybuf = new byte[3]; int filelen = 0; XContent.ReceiveXBytesNoCap(netstm, out filelen, ref buf); if (filelen > 0) { int pos = 0; int hoffset = 0; int hlen = 0; for (int ki = 0; ki < keylen; ki++) { maxkey[ki] = buf[pos++]; } while (pos < filelen) { byte[] keybuf = new byte[keylen]; for (int ki = 0; ki < keylen; ki++) { keybuf[ki] = buf[pos++]; } /*bool samekey = true; * for (int ki = 0; ki < keybuf.Length; ki++) * { * if (lastkeybuf[ki] != keybuf[ki]) * { * samekey = false; * break; * } * }*/ int chunknamestartpos = pos; while (buf[pos++] != (byte)'\0') { } //samekey = false; //if (!samekey) { string chunkname = System.Text.Encoding.UTF8.GetString(buf, chunknamestartpos, pos - chunknamestartpos - 1); if (pinhash) { if (lines.Count > 0) { if (keybuf[1] != lastkeybuf[1] || keybuf[2] != lastkeybuf[2]) { int shortkey = TwoBytesToInt(lastkeybuf[1], lastkeybuf[2]); hash[shortkey].Offset = hoffset; hash[shortkey].Length = hlen; hoffset = lines.Count; hlen = 0; } } hlen++; } lines.Add(new KeyValuePair <byte[], string>(keybuf, chunkname)); Buffer.BlockCopy(keybuf, 0, lastkeybuf, 0, 3); } } if (pinhash) { //last flush if (hlen > 0) { int shortkey = TwoBytesToInt(lastkeybuf[1], lastkeybuf[2]); hash[shortkey].Offset = hoffset; hash[shortkey].Length = hlen; } //fill in the gap int prevoffset = 0; int prevlen = 0; for (int hi = 0; hi < hash.Length; hi++) { Position thispos = hash[hi]; if (thispos.Length == 0) { thispos.Length = prevlen; thispos.Offset = prevoffset; hash[hi] = thispos; } else { prevoffset = thispos.Offset; prevlen = thispos.Length; } } } } mindexes.Add(indexname, lines); } if (connstr.RIndex == QaConnectionString.RIndexType.NOPOOL) { _CloseSocketRIndex(); } state = ConnectionState.Open; } catch { Cleanup(); throw; } }
public override bool Read() { try { if (recordbufpos >= recordbuflen) { int ib = netstm.ReadByte(); if (ib == -1 || ib == (byte)'.') { eof = true; return(false); } if (ib != (byte)'+') { throw new Exception("Read() handshake is not received."); } recordbuf = XContent.ReceiveXBytes(netstm, out recordbuflen, recordbuf); recordbufpos = 0; if (recordbuflen < recordsize) { throw new Exception("Read() error: returned buffer is smaller than a record"); } } int curpos = recordbufpos; recordbufpos += recordsize; // Prepare for next row first in case of exception. if (recordbuflen - curpos < recordsize) { throw new Exception("Row received from service has fewer bytes than expected."); } //Read a row. for (int i = 0; i < metadata.Length; i++) { MetaData column = metadata[i]; int bc = column.FrontBytes + column.Size + column.BackBytes; Buffer.BlockCopy(recordbuf, curpos, cmd.buf, 0, bc); curpos += bc; if (0 != cmd.buf[0]) { //currow[i] = null; currow[i] = DBNull.Value; if (isRindexEnabled) { throw new Exception("TCP distributed memory buffer corrupted, one or more machines may be out of memory"); } } else { switch (column.Type.FullName) { case "System.String": { //trim padding int stringend = 0; for (int si = column.FrontBytes + column.Size - 1; si > column.FrontBytes; si = si - 2) { if (cmd.buf[si] != 0 || cmd.buf[si - 1] != 0) { stringend = si; break; } } currow[i] = System.Text.Encoding.Unicode.GetString(cmd.buf, column.FrontBytes, stringend - column.FrontBytes + 1); } break; case "System.Int16": currow[i] = Utils.BytesToInt16(cmd.buf, column.FrontBytes); break; case "System.UInt16": currow[i] = Utils.BytesToUInt16(cmd.buf, column.FrontBytes); break; case "System.Int32": currow[i] = Utils.ToInt32((UInt32)Utils.BytesToInt(cmd.buf, column.FrontBytes)); break; case "System.UInt32": currow[i] = Utils.BytesToUInt32(cmd.buf, column.FrontBytes); break; case "System.Int64": currow[i] = Utils.ToInt64((UInt64)Utils.BytesToLong(cmd.buf, column.FrontBytes)); break; case "System.UInt64": currow[i] = Utils.BytesToULong(cmd.buf, column.FrontBytes); break; case "System.Double": currow[i] = Utils.BytesToDouble(cmd.buf, column.FrontBytes); break; case "System.DateTime": { Int64 ticks = Utils.BytesToLong(cmd.buf, column.FrontBytes); currow[i] = new DateTime(ticks); } break; default: throw new Exception("Type not supported yet"); } } } return(true); } catch { cmd.Abort(); throw; } }