/// <summary>
/// Commit changes maded in headers data
/// </summary>
/// <returns></returns>
public DataPackage UpdateHeaders()
{
_bw.Position = header_pos;
int limit = header_pos + _headerSpaceSize;
BinaryDataBuffer b = new BinaryDataBuffer(_headerSpaceSize);
BinaryDataWriter bw = new BinaryDataWriter(b);
bw.Write(_headers);
if (bw.Length <= _headerSpaceSize)
{
_bw.WriteBlock(b, b.Length);
}
else if (_dataPos >= _b.Length)
{
// No data rewrite package header only
WritePackageHeader(b, b.Length);
}
else
{
// rewrite package header and shift data
byte[] data = _b.Slice(_dataPos, _b.Length - _dataPos).GetBytes();
WritePackageHeader(b, b.Length);
_bw.WriteBlock(data);
}
return(this);
}
/// <summary>
/// Create a new instance of <see cref="DataPackage"/> from <see cref="IDataReader"/>
/// </summary>
/// <param name="dr"><see cref="IDataReader"/></param>
/// <param name="headerSpaceSize">Header size (default 512 bytes)</param>
/// <param name="allowDublicate"><see cref="bool"/></param>
/// <returns></returns>
public DataPackage(IDataReader dr, int headerSpaceSize, bool allowDublicate) : base(null)
{
_headerSpaceSize = headerSpaceSize <= 0 ? header_space_size_default : headerSpaceSize;
_b = new BinaryDataBuffer(_headerSpaceSize * 2);
_bw = new BinaryDataWriter(_b);
_br = new BinaryDataReader(_b);
_headers = new MapTable <string, object>(StringComparer.Ordinal);
using (DataTable dt = dr.GetSchemaTable())
{
_colCount = dr.FieldCount;
_indexes = new string[_colCount];
_colInfo = new ColumnInfo[_colCount];
for (int i = 0; i < dt.Rows.Count; i++)
{
DataRow drCol = dt.Rows[i];
string name = (string)drCol["ColumnName"];
if (allowDublicate)
{
string nameOrig = name;
for (int j = 1; _indexes.Contains(name, StringComparer.OrdinalIgnoreCase); j++)
{
name = nameOrig + j.ToString();
}
}
//else if (_indexes.Contains(name, StringComparer.OrdinalIgnoreCase))
// continue;
ColumnInfo ci = new ColumnInfo()
{
DataType = (Type)drCol["DataType"],
ColumnSize = (int)drCol["ColumnSize"],
AllowDBNull = (bool)drCol["AllowDBNull"]
};
_indexes[i] = name;
_colInfo[i] = ci;
}
WritePackageHeader();
//write data
for (; dr.Read();)
{
object[] values = new object[_colCount];
dr.GetValues(values);
//_bw.Write(values);
AddNew();
SetValues(values);
Update();
}
GoDataTop();
}
}
public BinaryDataBuffer GetBinaryDataBufferFromBase64BinaryDataBuffer(BinaryDataBuffer buffer)
{
try
{
var enc = new UTF8Encoding(false, true);
var str = enc.GetString(buffer.Bytes, 0, buffer.Bytes.Length);
return(new BinaryDataBuffer(Convert.FromBase64String(str)));
}
catch
{
return(new BinaryDataBuffer(new byte[0]));
}
}
/// <summary>
/// Create a new instance of <see cref="DataPackage"/> with headerSpaceSize from list of columns definitions and list of values
/// </summary>
/// <param name="headerSpaceSize">Header size (default 512 bytes)</param>
/// <param name="columns">list of columns definitions strings in format ColimnName[.ColumnType][.ColumnWidth]</param>
/// <param name="values">list of values</param>
public DataPackage(int headerSpaceSize, string[] columns, object[] values) : base(null)
{
_headerSpaceSize = headerSpaceSize <= 0 ? header_space_size_default : headerSpaceSize;
_b = new BinaryDataBuffer(_headerSpaceSize * 2);
_bw = new BinaryDataWriter(_b);
_br = new BinaryDataReader(_b);
_colCount = columns.Length;
bool hasValues = values != null && values.Length == _colCount;
_headers = new MapTable <string, object>(StringComparer.Ordinal);
//Create col info
_indexes = new string[_colCount];
_colInfo = new ColumnInfo[_colCount];
for (int i = 0; i < _colCount; i++)
{
ColumnInfo info;
string name;
if (columns[i].IndexOf('.') > 0)
{
info = ColumnInfo.FromName(columns[i]);
name = columns[i].GetToken(0, ".");
}
else if (hasValues)
{
name = columns[i];
info = ColumnInfo.FromValue(values[i]);
}
else
{
name = columns[i];
info = ColumnInfo.FromType(typeof(string));
}
_indexes[i] = name;
_colInfo[i] = info;
}
WritePackageHeader();
if (hasValues)
{
AddNew();
SetValues(values);
Update();
}
else if (values != null)
{
throw new ArgumentException("The length of the data array must be equal to the length of the array of field names");
}
}
public void BufferPerformanceTest()
{
using (FileLog _logger = new FileLog("MetaStackBuffers.PerformanceTest", new FileLogSettings()
{
DateFolderMask = "yyyy-MM-dd"
}))
using (_b = new BinaryDataBuffer(1024))
{
_w = new BinaryDataWriter(_b);
_r = new BinaryDataReader(_b);
WriterPerformanceTest(_logger);
ReaderPerformanceTest(_logger);
}
}
/// <summary>
/// Create a new instance of <see cref="DataPackage"/> from <see cref="byte"/> array
/// using when deserialise <see cref="DataPackage"/> from array created with <see cref="DataPackage.ToArray()"/> methos
/// </summary>
/// <param name="data"><see cref="byte"/> array created with <see cref="DataPackage.ToArray()"/> methos</param>
public DataPackage(byte[] source) : base(null)
{
_b = (BinaryDataBuffer)source;
_br = new BinaryDataReader(_b);
_bw = new BinaryDataWriter(_b);
//Read sizes
_headerSpaceSize = _br.Read <int>();
if (_headerSpaceSize < 0)
{
throw new DataPackageMessageFormatException("Invalid header size");
}
_colCount = _br.Read <int>();
if (_colCount <= 0 || _colCount > 64192)
{
throw new DataPackageMessageFormatException("Invalid columns count, must be more thrn 0 and less then 64192");
}
//Read headers
_headers = new MapTable <string, object>(StringComparer.Ordinal);
_br.ReadNext(); //Must be Object
_br.ReadRaw(_headers);
_br.Position = _headerSpaceSize + header_pos;
//Read columns
_indexes = new string[_colCount];
_colInfo = new ColumnInfo[_colCount];
for (int i = 0; i < _colCount; i++)
{
_br.ReadNext();
string name = _br.ReadString();
if (string.IsNullOrEmpty(name))
{
throw new DataPackageMessageFormatException($"Invalid column name for column index {i}");
}
_indexes[i] = name;
_colInfo[i] = ReadColumnInfo(_br);
}
_dataPos = _br.Position;
}
public BinaryDataContext GetBinaryDataFromBinaryDataBuffer(BinaryDataBuffer buffer)
{
return(new BinaryDataContext(buffer.Bytes));
}
public string GetHexStringFromBinaryDataBuffer(BinaryDataBuffer buffer)
{
return(BitConverter.ToString(buffer.Bytes).Replace("-", ""));
}
public long Read(BinaryDataBuffer buffer, int positionInBuffer, int number)
{
return(_underlyingStream.Read(buffer.Bytes, positionInBuffer, number));
}
public void Write(BinaryDataBuffer buffer, int positionInBuffer, int number)
{
buffer.ThrowIfReadonly();
_underlyingStream.Write(buffer.Bytes, positionInBuffer, number);
}
public BinaryDataBuffer GetHexBinaryDataBufferFromBinaryDataBuffer(BinaryDataBuffer buffer)
{
var str = GetHexStringFromBinaryDataBuffer(buffer);
return(GetBinaryDataBufferFromString(str));
}
public BinaryDataBuffer GetBinaryDataBufferFromHexBinaryDataBuffer(BinaryDataBuffer buffer)
{
return(new BinaryDataBuffer(HexArrayToByteArray(buffer.Bytes)));
}
public BinaryDataBuffer GetBase64BinaryDataBufferFromBinaryDataBuffer(BinaryDataBuffer buffer)
{
var base64str = Convert.ToBase64String(buffer.Bytes, Base64FormattingOptions.InsertLineBreaks);
return(new BinaryDataBuffer(Encoding.ASCII.GetBytes(base64str)));
}
public string GetBase64StringFromBinaryDataBuffer(BinaryDataBuffer buffer)
{
return(Convert.ToBase64String(buffer.Bytes, Base64FormattingOptions.InsertLineBreaks));
}
public string GetStringFromBinaryDataBuffer(BinaryDataBuffer buffer, IValue encoding = null)
{
var enc = (encoding != null) ? TextEncodingEnum.GetEncoding(encoding) : Encoding.UTF8;
return(enc.GetString(buffer.Bytes));
}
/// <summary>
/// Create <see cref="DataPackage"/> from full json format
/// </summary>
/// <param name="source">json string whit Headers, Columns and Data rows </param>
/// <param name="jsonFormat"><see cref="TsJsonFormat"/></param>
/// <param name="headerSpaceSize">For no headers package set headerSpaceSize = 8</param>
internal DataPackage(JsonValue source, TsJsonFormat jsonFormat = TsJsonFormat.Full, int headerSpaceSize = 0) : base(source)
{
if (jsonFormat != TsJsonFormat.Simple)
{
var j = source as JsonObject;
if (headerSpaceSize == 0)
{
headerSpaceSize = (int)j.GetIntOrDefault("HeaderSize", header_space_size_default);
}
JsonArray columns = j.GetArray("Columns");
JsonArray rows = j.GetArray("Rows");
_b = new BinaryDataBuffer(headerSpaceSize * 2);
_bw = new BinaryDataWriter(_b);
_br = new BinaryDataReader(_b);
_colCount = columns.Count;
bool hasValues = rows != null;
//headers
_headerSpaceSize = headerSpaceSize;
_headers = new MapTable <string, object>(StringComparer.Ordinal);
JsonObject headers = (JsonObject)j["Headers"];
foreach (var pair in headers)
{
_headers.Add(pair.Key, pair.Value?.GetValue());
}
//Create col info
_indexes = new string[_colCount];
_colInfo = new ColumnInfo[_colCount];
for (int i = 0; i < _colCount; i++)
{
var c = columns[i];
_indexes[i] = (string)c["Name"];
_colInfo[i] = new ColumnInfo()
{
DataType = MdbTypeMap.GetType((string)c["Type"], typeof(string)),
ColumnSize = (int)c["Size"],
AllowDBNull = (bool)c["AllowNull"]
};
}
WritePackageHeader();
//values
if (hasValues)
{
if (jsonFormat == TsJsonFormat.Full)
{
foreach (JsonObject r in rows)
{
AddNew();
foreach (var o in r)
{
SetValue(o.Key, o.Value?.GetValue());
}
Update();
}
}
else
{
//!!! Not tested
foreach (JsonArray a in rows)
{
AddNew();
foreach (var o in a)
{
for (int i = 0; i < _colCount; i++)
{
SetValue(_indexes[i], o?.GetValue());
}
}
Update();
}
}
}
}
else //simple
{
if (headerSpaceSize == 0)
{
headerSpaceSize = header_space_size_default;
}
_b = new BinaryDataBuffer(headerSpaceSize * 2);
_bw = new BinaryDataWriter(_b);
_br = new BinaryDataReader(_b);
JsonObject dataRow;
bool isArray = false;
if (source is JsonObject j)
{
dataRow = j;
}
else if (source is JsonArray a)
{
if (a.Count == 0)
{
throw new ArgumentException("Parameter array cannot be empty");
}
dataRow = (JsonObject)a[0];
isArray = true;
}
else
{
throw new ArgumentException("The constructor of DataPackage requires JsonObject or JsonArray parameter type.");
}
_headerSpaceSize = headerSpaceSize;
_headers = new MapTable <string, object>(StringComparer.Ordinal);
_colCount = dataRow.Count;
_indexes = new string[_colCount];
_colInfo = new ColumnInfo[_colCount];
int i = 0;
foreach (var kvp in dataRow)
{
_indexes[i] = kvp.Key;
_colInfo[i] = ColumnInfo.FromValue(kvp.Value?.GetValue());
i++;
}
WritePackageHeader();
if (isArray)
{
foreach (var r in (source as JsonArray))
{
AddNew();
foreach (var o in (JsonObject)r)
{
SetValue(o.Key, o.Value?.GetValue());
}
Update();
}
}
else
{
AddNew();
foreach (var o in dataRow)
{
SetValue(o.Key, o.Value?.GetValue());
}
Update();
}
}
GoDataTop();
}
