internal T Read <T>(DataRowMessage row, FieldDescription fieldDescription, int len) { Contract.Requires(row.PosInColumn == 0); Contract.Ensures(row.PosInColumn == row.ColumnLen); T result; var asSimpleReader = this as ISimpleTypeReader <T>; if (asSimpleReader != null) { var buf = row.Buffer.EnsureOrAllocateTemp(len); result = asSimpleReader.Read(buf, fieldDescription, row.ColumnLen); } else { var asChunkingReader = this as IChunkingTypeReader <T>; if (asChunkingReader == null) { throw new InvalidCastException(String.Format("Can't cast database type {0} to {1}", fieldDescription.Handler.PgName, typeof(T).Name)); } asChunkingReader.PrepareRead(row.Buffer, fieldDescription, len); while (!asChunkingReader.Read(out result)) { row.Buffer.ReadMore(); } } row.PosInColumn += row.ColumnLen; return(result); }
protected override void ProcessRow(DataRowMessage message) { dynamic row = Convert.ChangeType(message.Row, message.RowType); Analyzer.ConsumeRow(row); _referenceDataWriter.UpdateReferenceData(row); }
internal override void ProcessDataMessage(DataRowMessage dataMsg) { // The connector's buffer can actually change between DataRows: // If a large DataRow exceeding the connector's current read buffer arrives, and we're // reading in non-sequential mode, a new oversize buffer is allocated. We thus have to // recapture the connector's buffer on each new DataRow. Buffer = Connector.ReadBuffer; _numColumns = Buffer.ReadInt16(); _column = -1; Debug.Assert(RowDescription.NumFields == _numColumns); if (_columnOffsets == null) { _columnOffsets = new List <int>(_numColumns); } else { _columnOffsets.Clear(); } for (var i = 0; i < _numColumns; i++) { _columnOffsets.Add(Buffer.ReadPosition); var len = Buffer.ReadInt32(); if (len != -1) { Buffer.Seek(len, SeekOrigin.Current); } } _endOffset = Buffer.ReadPosition; }
private MemoryStream modifyDataRow(DataRowMessage dataRow, RowDescriptionMessage rowDescription) { Func <string, string> maskingFunction; var ms = new MemoryStream(); ms.Write(EndianHelpers.ToBigE(dataRow.NumFields)); int i = 0; foreach (var f in dataRow.DataRowFields) { uint tableOid = rowDescription.FieldDescriptions[i].ObjectId; string columnName = rowDescription.FieldDescriptions[i].FieldName; uint dataTypeOid = rowDescription.FieldDescriptions[i].DataTypeObjectId; //If the column vale is null or we are preserving keyed column values and the column is either a primary or foreign key. if (f.ColumnValueLength == -1 || (_state._maskingModel.PreserveKeys && _state.isColumnKeyed(tableOid, columnName))) { ms.Write(EndianHelpers.ToBigE(f.ColumnValueLength)); ms.Write(f.ColumnValue); } else { maskingFunction = getMaskingFunction(tableOid, dataTypeOid, columnName); string rowValue = System.Text.Encoding.UTF8.GetString(f.ColumnValue); byte[] newBytes = System.Text.Encoding.UTF8.GetBytes(maskingFunction(rowValue)); byte[] newLength = EndianHelpers.ToBigE(newBytes.Length); ms.Write(newLength); ms.Write(newBytes); } i++; } return(ms); }
internal override object ReadValueAsObject(DataRowMessage row, FieldDescription fieldDescription) { var result = fieldDescription.TypeModifier == 1 ? (object)((ISimpleTypeReader <bool>) this).Read(row.Buffer, row.ColumnLen, fieldDescription) : Read <BitArray>(row, row.ColumnLen, fieldDescription); row.PosInColumn += row.ColumnLen; return(result); }
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) { PrepareRead(row.Buffer, row.ColumnLen, fieldDescription); Array result; while (!Read <TPsv>(out result)) { row.Buffer.ReadMore(); } return(result); }
internal override void ProcessDataMessage(DataRowMessage dataMsg) { // When reading sequentially, we never allocate oversize buffers for data rows since they don't have // to fit in the buffer (that's the point of sequential). However, if the row description message is bigger // than the buffer, an oversize buffer will be allocated (see #2003). This is hacky and needs to be redone. Buffer = Connector.ReadBuffer; _column = -1; ColumnLen = -1; PosInColumn = 0; }
public void ParseCsvFile() { while (_csvReader.Read()) { var record = _csvReader.GetRecord <TRowType>() as IDataRow; var dataRowMessage = new DataRowMessage(typeof(TRowType), record); Sender.Tell(dataRowMessage, Self); } Sender.Tell(new FileAnalysisFinishedMessage()); _logger.Info($"Finished parsing {_sourcePath}"); }
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) { try { return(Read <TPsv>(row.Buffer, false).Result); } finally { // Important in case a SafeReadException was thrown, position must still be updated row.PosInColumn += row.ColumnLen; } }
public long GetChars(DataRowMessage row, int charOffset, char[] output, int outputOffset, int charsCount, FieldDescription field) { if (row.PosInColumn == 0) { _charPos = 0; } if (output == null) { // Note: Getting the length of a text column means decoding the entire field, // very inefficient and also consumes the column in sequential mode. But this seems to // be SqlClient's behavior as well. int bytesSkipped, charsSkipped; row.Buffer.SkipChars(int.MaxValue, row.ColumnLen - row.PosInColumn, out bytesSkipped, out charsSkipped); Contract.Assume(bytesSkipped == row.ColumnLen - row.PosInColumn); row.PosInColumn += bytesSkipped; _charPos += charsSkipped; return(_charPos); } if (charOffset < _charPos) { row.SeekInColumn(0); _charPos = 0; } if (charOffset > _charPos) { var charsToSkip = charOffset - _charPos; int bytesSkipped, charsSkipped; row.Buffer.SkipChars(charsToSkip, row.ColumnLen - row.PosInColumn, out bytesSkipped, out charsSkipped); row.PosInColumn += bytesSkipped; _charPos += charsSkipped; if (charsSkipped < charsToSkip) { // TODO: What is the actual required behavior here? throw new IndexOutOfRangeException(); } } int bytesRead, charsRead; row.Buffer.ReadChars(output, outputOffset, charsCount, row.ColumnLen - row.PosInColumn, out bytesRead, out charsRead); row.PosInColumn += bytesRead; _charPos += charsRead; return(charsRead); }
internal async ValueTask <T> Read <T>(DataRowMessage row, int len, bool async, FieldDescription fieldDescription = null) { Debug.Assert(row.PosInColumn == 0); T result; try { result = await Read <T>(row.Buffer, len, async, fieldDescription); } finally { // Important in case a SafeReadException was thrown, position must still be updated row.PosInColumn += row.ColumnLen; } return(result); }
internal T ReadFully <T>(DataRowMessage row, int len, FieldDescription fieldDescription = null) { Contract.Requires(row.PosInColumn == 0); Contract.Ensures(row.PosInColumn == row.ColumnLen); T result; try { result = ReadFully <T>(row.Buffer, len, fieldDescription); } finally { // Important in case a SafeReadException was thrown, position must still be updated row.PosInColumn += row.ColumnLen; } return(result); }
private byte[] handleDataRow(byte[] buffer, int messageLength) { var dataRow = new DataRowMessage(); int offset = 5; //1 byte for message type and 4 bytes for the int32 that states the length. ushort numFields = EndianHelpers.SwapEndianness(BitConverter.ToUInt16(buffer, 5)); dataRow.NumFields = numFields; if (numFields != 0) { dataRow.DataRowFields = new List <DataRowField>(numFields); } offset += 2; for (int i = 0; i < numFields; i++) { var dataRowField = new DataRowField(); int columnValueLength = IPAddress.NetworkToHostOrder(BitConverter.ToInt32(buffer, offset)); dataRowField.ColumnValueLength = columnValueLength; offset += 4; if (columnValueLength != -1) { dataRowField.ColumnValue = new byte[columnValueLength]; Buffer.BlockCopy(buffer, offset, dataRowField.ColumnValue, 0, columnValueLength); var colValue = Encoding.ASCII.GetString(buffer, offset, columnValueLength); offset += columnValueLength; } dataRow.DataRowFields.Add(dataRowField); } var modifiedResultStream = _modifyDataRow(dataRow, currentRowDescription); modifiedResultStream.Seek(0, SeekOrigin.Begin); var message = new byte[modifiedResultStream.Length + 5]; message[0] = buffer[0]; var newLength = EndianHelpers.ToBigE((int)modifiedResultStream.Length + 4); message[1] = newLength[0]; message[2] = newLength[1]; message[3] = newLength[2]; message[4] = newLength[3]; modifiedResultStream.Read(message, 5, (int)modifiedResultStream.Length); return(message); }
public long GetBytes(DataRowMessage row, int offset, [CanBeNull] byte[] output, int outputOffset, int len, FieldDescription field) { if (output == null) { return(row.ColumnLen); } row.SeekInColumn(offset); // Attempt to read beyond the end of the column if (offset + len > row.ColumnLen) { len = row.ColumnLen - offset; } row.Buffer.ReadAllBytes(output, outputOffset, len, false); row.PosInColumn += len; return(len); }
internal virtual object ReadPsvAsObjectFully(DataRowMessage row, FieldDescription fieldDescription) { return(ReadValueAsObjectFully(row, fieldDescription)); }
internal ByteaStream(DataRowMessage row) { Row = row; }
internal Stream GetStream(DataRowMessage row, FieldDescription fieldDescription) { Contract.Requires(row.PosInColumn == 0); return(new ByteaStream(row)); }
internal virtual object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) => ReadAsObject(row, fieldDescription);
internal abstract object ReadAsObject(DataRowMessage row, FieldDescription fieldDescription = null);
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) => Read <TPsv>(row, row.ColumnLen, false, fieldDescription).Result;
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription = null) => fieldDescription?.TypeModifier == 1 ? (object)Read <bool>(row, row.ColumnLen, false, fieldDescription).Result : Read <BitArray>(row, row.ColumnLen, false, fieldDescription);
private void PopulateReferenceData(DataRowMessage message) { _postcodeReferenceDataWriter.UpdateReferenceData(message.Row); }
internal SequentialByteaStream(DataRowMessage row) { _row = row; }
private void PopulateLocationReferenceData(DataRowMessage message) { _locationReferenceDataWriter.UpdateReferenceData(message.Row); }
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) { return(ReadValueAsObject(row, fieldDescription)); }
internal override object ReadValueAsObjectFully(DataRowMessage row, FieldDescription fieldDescription = null) { return(fieldDescription?.TypeModifier == 1 ? (object)ReadFully <bool>(row, row.ColumnLen, fieldDescription) : ReadFully <BitArray>(row, row.ColumnLen, fieldDescription)); }
internal abstract object ReadValueAsObjectFully(DataRowMessage row, FieldDescription fieldDescription);
internal override object ReadPsvAsObjectFully(DataRowMessage row, FieldDescription fieldDescription) { return(ReadFully <TPsv>(row, row.ColumnLen, fieldDescription)); }
internal override void ProcessDataMessage(DataRowMessage dataMsg) { _column = -1; ColumnLen = -1; PosInColumn = 0; }
protected abstract void ProcessRow(DataRowMessage message);