internal T Read <T>(DataRowMessage row, FieldDescription fieldDescription, int len)
{
Contract.Requires(row.PosInColumn == 0);
Contract.Ensures(row.PosInColumn == row.ColumnLen);
T result;
var asSimpleReader = this as ISimpleTypeReader <T>;
if (asSimpleReader != null)
{
var buf = row.Buffer.EnsureOrAllocateTemp(len);
result = asSimpleReader.Read(buf, fieldDescription, row.ColumnLen);
}
else
{
var asChunkingReader = this as IChunkingTypeReader <T>;
if (asChunkingReader == null)
{
throw new InvalidCastException(String.Format("Can't cast database type {0} to {1}", fieldDescription.Handler.PgName, typeof(T).Name));
}
asChunkingReader.PrepareRead(row.Buffer, fieldDescription, len);
while (!asChunkingReader.Read(out result))
{
row.Buffer.ReadMore();
}
}
row.PosInColumn += row.ColumnLen;
return(result);
}
protected override void ProcessRow(DataRowMessage message)
{
dynamic row = Convert.ChangeType(message.Row, message.RowType);
Analyzer.ConsumeRow(row);
_referenceDataWriter.UpdateReferenceData(row);
}
internal override void ProcessDataMessage(DataRowMessage dataMsg)
{
// The connector's buffer can actually change between DataRows:
// If a large DataRow exceeding the connector's current read buffer arrives, and we're
// reading in non-sequential mode, a new oversize buffer is allocated. We thus have to
// recapture the connector's buffer on each new DataRow.
Buffer = Connector.ReadBuffer;
_numColumns = Buffer.ReadInt16();
_column = -1;
Debug.Assert(RowDescription.NumFields == _numColumns);
if (_columnOffsets == null)
{
_columnOffsets = new List <int>(_numColumns);
}
else
{
_columnOffsets.Clear();
}
for (var i = 0; i < _numColumns; i++)
{
_columnOffsets.Add(Buffer.ReadPosition);
var len = Buffer.ReadInt32();
if (len != -1)
{
Buffer.Seek(len, SeekOrigin.Current);
}
}
_endOffset = Buffer.ReadPosition;
}
private MemoryStream modifyDataRow(DataRowMessage dataRow, RowDescriptionMessage rowDescription)
{
Func <string, string> maskingFunction;
var ms = new MemoryStream();
ms.Write(EndianHelpers.ToBigE(dataRow.NumFields));
int i = 0;
foreach (var f in dataRow.DataRowFields)
{
uint tableOid = rowDescription.FieldDescriptions[i].ObjectId;
string columnName = rowDescription.FieldDescriptions[i].FieldName;
uint dataTypeOid = rowDescription.FieldDescriptions[i].DataTypeObjectId;
//If the column vale is null or we are preserving keyed column values and the column is either a primary or foreign key.
if (f.ColumnValueLength == -1 || (_state._maskingModel.PreserveKeys && _state.isColumnKeyed(tableOid, columnName)))
{
ms.Write(EndianHelpers.ToBigE(f.ColumnValueLength));
ms.Write(f.ColumnValue);
}
else
{
maskingFunction = getMaskingFunction(tableOid, dataTypeOid, columnName);
string rowValue = System.Text.Encoding.UTF8.GetString(f.ColumnValue);
byte[] newBytes = System.Text.Encoding.UTF8.GetBytes(maskingFunction(rowValue));
byte[] newLength = EndianHelpers.ToBigE(newBytes.Length);
ms.Write(newLength);
ms.Write(newBytes);
}
i++;
}
return(ms);
}
internal override object ReadValueAsObject(DataRowMessage row, FieldDescription fieldDescription)
{
var result = fieldDescription.TypeModifier == 1
? (object)((ISimpleTypeReader <bool>) this).Read(row.Buffer, row.ColumnLen, fieldDescription)
: Read <BitArray>(row, row.ColumnLen, fieldDescription);
row.PosInColumn += row.ColumnLen;
return(result);
}
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription)
{
PrepareRead(row.Buffer, row.ColumnLen, fieldDescription);
Array result;
while (!Read <TPsv>(out result))
{
row.Buffer.ReadMore();
}
return(result);
}
internal override void ProcessDataMessage(DataRowMessage dataMsg)
{
// When reading sequentially, we never allocate oversize buffers for data rows since they don't have
// to fit in the buffer (that's the point of sequential). However, if the row description message is bigger
// than the buffer, an oversize buffer will be allocated (see #2003). This is hacky and needs to be redone.
Buffer = Connector.ReadBuffer;
_column = -1;
ColumnLen = -1;
PosInColumn = 0;
}
public void ParseCsvFile()
{
while (_csvReader.Read())
{
var record = _csvReader.GetRecord <TRowType>() as IDataRow;
var dataRowMessage = new DataRowMessage(typeof(TRowType), record);
Sender.Tell(dataRowMessage, Self);
}
Sender.Tell(new FileAnalysisFinishedMessage());
_logger.Info($"Finished parsing {_sourcePath}");
}
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription)
{
try
{
return(Read <TPsv>(row.Buffer, false).Result);
}
finally
{
// Important in case a SafeReadException was thrown, position must still be updated
row.PosInColumn += row.ColumnLen;
}
}
public long GetChars(DataRowMessage row, int charOffset, char[] output, int outputOffset, int charsCount, FieldDescription field)
{
if (row.PosInColumn == 0)
{
_charPos = 0;
}
if (output == null)
{
// Note: Getting the length of a text column means decoding the entire field,
// very inefficient and also consumes the column in sequential mode. But this seems to
// be SqlClient's behavior as well.
int bytesSkipped, charsSkipped;
row.Buffer.SkipChars(int.MaxValue, row.ColumnLen - row.PosInColumn, out bytesSkipped, out charsSkipped);
Contract.Assume(bytesSkipped == row.ColumnLen - row.PosInColumn);
row.PosInColumn += bytesSkipped;
_charPos += charsSkipped;
return(_charPos);
}
if (charOffset < _charPos)
{
row.SeekInColumn(0);
_charPos = 0;
}
if (charOffset > _charPos)
{
var charsToSkip = charOffset - _charPos;
int bytesSkipped, charsSkipped;
row.Buffer.SkipChars(charsToSkip, row.ColumnLen - row.PosInColumn, out bytesSkipped, out charsSkipped);
row.PosInColumn += bytesSkipped;
_charPos += charsSkipped;
if (charsSkipped < charsToSkip)
{
// TODO: What is the actual required behavior here?
throw new IndexOutOfRangeException();
}
}
int bytesRead, charsRead;
row.Buffer.ReadChars(output, outputOffset, charsCount, row.ColumnLen - row.PosInColumn, out bytesRead, out charsRead);
row.PosInColumn += bytesRead;
_charPos += charsRead;
return(charsRead);
}
internal async ValueTask <T> Read <T>(DataRowMessage row, int len, bool async, FieldDescription fieldDescription = null)
{
Debug.Assert(row.PosInColumn == 0);
T result;
try
{
result = await Read <T>(row.Buffer, len, async, fieldDescription);
}
finally
{
// Important in case a SafeReadException was thrown, position must still be updated
row.PosInColumn += row.ColumnLen;
}
return(result);
}
internal T ReadFully <T>(DataRowMessage row, int len, FieldDescription fieldDescription = null)
{
Contract.Requires(row.PosInColumn == 0);
Contract.Ensures(row.PosInColumn == row.ColumnLen);
T result;
try
{
result = ReadFully <T>(row.Buffer, len, fieldDescription);
}
finally
{
// Important in case a SafeReadException was thrown, position must still be updated
row.PosInColumn += row.ColumnLen;
}
return(result);
}
private byte[] handleDataRow(byte[] buffer, int messageLength)
{
var dataRow = new DataRowMessage();
int offset = 5; //1 byte for message type and 4 bytes for the int32 that states the length.
ushort numFields = EndianHelpers.SwapEndianness(BitConverter.ToUInt16(buffer, 5));
dataRow.NumFields = numFields;
if (numFields != 0)
{
dataRow.DataRowFields = new List <DataRowField>(numFields);
}
offset += 2;
for (int i = 0; i < numFields; i++)
{
var dataRowField = new DataRowField();
int columnValueLength = IPAddress.NetworkToHostOrder(BitConverter.ToInt32(buffer, offset));
dataRowField.ColumnValueLength = columnValueLength;
offset += 4;
if (columnValueLength != -1)
{
dataRowField.ColumnValue = new byte[columnValueLength];
Buffer.BlockCopy(buffer, offset, dataRowField.ColumnValue, 0, columnValueLength);
var colValue = Encoding.ASCII.GetString(buffer, offset, columnValueLength);
offset += columnValueLength;
}
dataRow.DataRowFields.Add(dataRowField);
}
var modifiedResultStream = _modifyDataRow(dataRow, currentRowDescription);
modifiedResultStream.Seek(0, SeekOrigin.Begin);
var message = new byte[modifiedResultStream.Length + 5];
message[0] = buffer[0];
var newLength = EndianHelpers.ToBigE((int)modifiedResultStream.Length + 4);
message[1] = newLength[0];
message[2] = newLength[1];
message[3] = newLength[2];
message[4] = newLength[3];
modifiedResultStream.Read(message, 5, (int)modifiedResultStream.Length);
return(message);
}
public long GetBytes(DataRowMessage row, int offset, [CanBeNull] byte[] output, int outputOffset, int len, FieldDescription field)
{
if (output == null)
{
return(row.ColumnLen);
}
row.SeekInColumn(offset);
// Attempt to read beyond the end of the column
if (offset + len > row.ColumnLen)
{
len = row.ColumnLen - offset;
}
row.Buffer.ReadAllBytes(output, outputOffset, len, false);
row.PosInColumn += len;
return(len);
}
internal virtual object ReadPsvAsObjectFully(DataRowMessage row, FieldDescription fieldDescription)
{
return(ReadValueAsObjectFully(row, fieldDescription));
}
internal ByteaStream(DataRowMessage row)
{
Row = row;
}
internal Stream GetStream(DataRowMessage row, FieldDescription fieldDescription)
{
Contract.Requires(row.PosInColumn == 0);
return(new ByteaStream(row));
}
internal virtual object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) => ReadAsObject(row, fieldDescription);
internal abstract object ReadAsObject(DataRowMessage row, FieldDescription fieldDescription = null);
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription) => Read <TPsv>(row, row.ColumnLen, false, fieldDescription).Result;
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription = null)
=> fieldDescription?.TypeModifier == 1
? (object)Read <bool>(row, row.ColumnLen, false, fieldDescription).Result
: Read <BitArray>(row, row.ColumnLen, false, fieldDescription);
private void PopulateReferenceData(DataRowMessage message)
{
_postcodeReferenceDataWriter.UpdateReferenceData(message.Row);
}
internal SequentialByteaStream(DataRowMessage row)
{
_row = row;
}
private void PopulateLocationReferenceData(DataRowMessage message)
{
_locationReferenceDataWriter.UpdateReferenceData(message.Row);
}
internal override object ReadPsvAsObject(DataRowMessage row, FieldDescription fieldDescription)
{
return(ReadValueAsObject(row, fieldDescription));
}
internal override object ReadValueAsObjectFully(DataRowMessage row, FieldDescription fieldDescription = null)
{
return(fieldDescription?.TypeModifier == 1
? (object)ReadFully <bool>(row, row.ColumnLen, fieldDescription)
: ReadFully <BitArray>(row, row.ColumnLen, fieldDescription));
}
internal abstract object ReadValueAsObjectFully(DataRowMessage row, FieldDescription fieldDescription);
internal override object ReadPsvAsObjectFully(DataRowMessage row, FieldDescription fieldDescription)
{
return(ReadFully <TPsv>(row, row.ColumnLen, fieldDescription));
}
internal override void ProcessDataMessage(DataRowMessage dataMsg)
{
_column = -1;
ColumnLen = -1;
PosInColumn = 0;
}
protected abstract void ProcessRow(DataRowMessage message);
