// from bytes
public override void Unpack(byte[] data, int startIndex)
{
uint tmpUInt;
VarintType type;
// remaining size
int startOffsetOfRemainingSize = 0;
(tmpUInt, type) = VarintBitConverter.FromVarint(data, startIndex + startOffsetOfRemainingSize);
// header size
int startOffsetOfHeaderSize = startOffsetOfRemainingSize + VarintSize.GetVarintSize(tmpUInt);
(tmpUInt, type) = VarintBitConverter.FromVarint(data, startIndex + startOffsetOfHeaderSize);
// key-idx size
int startOffsetOfKeyIdxSize = startOffsetOfHeaderSize + VarintSize.GetVarintSize(tmpUInt);
(tmpUInt, type) = VarintBitConverter.FromVarint(data, startIndex + startOffsetOfKeyIdxSize);
// key-pk size
int startOffsetOfKeyPKSize = startOffsetOfKeyIdxSize + VarintSize.GetVarintSize(tmpUInt);
(tmpUInt, type) = VarintBitConverter.FromVarint(data, startIndex + startOffsetOfKeyPKSize);
// key-idx
int startOffsetOfKeyIdx = startOffsetOfKeyPKSize + VarintSize.GetVarintSize(tmpUInt);
this.Key = new DBRecord(data, startOffsetOfKeyIdx + startIndex);
// key-pk
int startOffsetOfKeyPK = startOffsetOfKeyIdx + this.Key.RecordSize;
this.PrimaryKey = new DBRecord(data, startOffsetOfKeyPK + startIndex);
}
public void VarintEncodeAndDecodeUInt64Read()
{
const ulong number = UInt64.MaxValue / 2;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ReadUInt64(new BinaryReader(new MemoryStream(encoded))));
}
public void VarintEncodeAndDecodeUInt32Zero()
{
const uint number = 0;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToUInt32(encoded));
}
// from this object to raw data
public byte[] Pack()
{
List <byte> pack = new List <byte>();
// header
pack.Add(this.HeaderSize);
foreach (int headerValue in this.HeaderList)
{
if (headerValue == (int)HeaderValue.NULL ||
headerValue == (int)HeaderValue.INTEGER ||
headerValue == (int)HeaderValue.FloatingPoint)
{
pack.AddRange(VarintBitConverter.ToVarint((uint)headerValue, VarintType.Varint8));
}
else if ((headerValue - (int)HeaderValue.TEXT) % 2 == 0)
{
pack.AddRange(VarintBitConverter.ToVarint((uint)headerValue, VarintType.Varint32));
}
else
{
throw new Exception($"Header value {headerValue} does not exists");
}
}
// field
pack.AddRange(this.FieldData);
return(pack.ToArray());
}
public void VarintEncodeAndDecodeInt64MinValue()
{
const long number = Int64.MinValue;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToInt64(encoded));
}
//[Fact(Skip = "Skipped")]
public void VariantBitConverterIntTest()
{
Util.Log("VariantBitConverterIntTest");
Random random = new Random();
MemoryStream stream = new MemoryStream();
long size = 0;
for (int i = 0; i < 1000; i++)
{
stream.Position = 0;
int src = (int)random.Next(int.MinValue, int.MaxValue);
int s = 0;
VarintBitConverter.SerializeInt(src, stream, out s);
size = stream.Position;
stream.Position = 0;
int dst = VarintBitConverter.ToInt(stream, out s);
//Util.Log("src:" + src + ", dst:" + dst + ", size:" + size);
if (src != dst)
{
throw new InvalidProgramException("index:" + i + ", src:" + src + ", dst:" + dst + ", size:" + size);
}
}
}
public void VarintEncodeAndDecodeInt16Zero()
{
const short number = 0;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToInt16(encoded));
}
public void VarintEncodeAndDecodeUInt64()
{
const ulong number = UInt64.MaxValue / 2;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToUInt64(encoded));
}
private void WriteTags(IOSMElement element, List <byte> bytes)
{
if (element.Tags.Count == 0)
{
return;
}
foreach (string tagKey in element.Tags)
{
var tagKeyBytes = Encoding.UTF8.GetBytes(tagKey);
var tagValueBytes = Encoding.UTF8.GetBytes(element.Tags[tagKey]);
var elementPosition = this._storedStringPairs.GetElementPosition(tagKeyBytes, tagValueBytes);
if (elementPosition != -1)
{
var positionBytes = VarintBitConverter.GetVarintBytes((uint)elementPosition);
bytes.AddRange(positionBytes);
}
else
{
var stringPair = new KeyValuePair <byte[], byte[]>(tagKeyBytes, tagValueBytes);
bytes.Add(0x00);
bytes.AddRange(tagKeyBytes);
bytes.Add(0x00);
bytes.AddRange(tagValueBytes);
bytes.Add(0x00);
if (stringPair.Key.Length + stringPair.Value.Length <= 250)
{
this._storedStringPairs.Insert(0, stringPair);
}
}
}
}
public void VarintEncodeAndDecodeByteMaxValue()
{
const byte number = Byte.MaxValue;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToByte(encoded));
}
public void VarintEncodeAndDecodeZero()
{
const ushort number = UInt16.MaxValue / 2;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToUInt16(encoded));
}
public void VarintEncodeAndDecodeInt32()
{
const int number = Int32.MaxValue / 2;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToInt32(encoded));
}
public bool TryGetVarInt64(out long v)
{
var(result, newoffset) = VarintBitConverter.ToInt64(Buffer, Offset);
v = result;
Offset = newoffset;
return(true);
}
public override byte[] Encode(BufferPlus bp, uint value, Encoding encoding = null, int position = -1, int length = -1)
{
bp.SanitizeParameter(ref position, ref length, ref encoding);
var bytes = VarintBitConverter.GetVarintBytes(value);
bp.Memory.Write(bytes, 0, bytes.Length);
return(bytes);
}
public override uint Decode(BufferPlus bp, Encoding encoding = null, int position = -1, int length = -1)
{
bp.SanitizeParameter(ref position, ref length, ref encoding);
var bytes = bp.Buffer.Skip(position).ToArray();
var value = VarintBitConverter.ToUInt32(bytes);
return(value);
}
private void WriteVersionData(IOSMElement element, List <byte> bytes)
{
if (element.Version == 0)
{
bytes.Add(0x00);
return;
}
var versionBytes = VarintBitConverter.GetVarintBytes(element.Version);
bytes.AddRange(versionBytes);
var timestamp = (long)element.Timestamp.Subtract(UNIX_START).TotalSeconds;
var timestampDiff = timestamp - this._lastTimestamp;
var timestampBytes = VarintBitConverter.GetVarintBytes(timestampDiff);
this._lastTimestamp = timestamp;
bytes.AddRange(timestampBytes);
if (timestamp == 0)
{
return;
}
var changesetDiff = (long)element.Changeset - this._lastChangeset;
var changesetBytes = VarintBitConverter.GetVarintBytes(changesetDiff);
this._lastChangeset = (long)element.Changeset;
bytes.AddRange(changesetBytes);
var uidBytes = VarintBitConverter.GetVarintBytes(element.UserId);
if (element.UserId == 0)
{
uidBytes = new byte[0];
}
var userBytes = Encoding.UTF8.GetBytes(element.UserName);
var elementPosition = this._storedStringPairs.GetElementPosition(uidBytes, userBytes);
if (elementPosition != -1)
{
var positionBytes = VarintBitConverter.GetVarintBytes((uint)elementPosition);
bytes.AddRange(positionBytes);
}
else
{
bytes.Add(0x00);
bytes.AddRange(uidBytes);
bytes.Add(0x00);
bytes.AddRange(userBytes);
bytes.Add(0x00);
var uidUserPair = new KeyValuePair <byte[], byte[]>(uidBytes, userBytes);
if (uidUserPair.Key.Length + uidUserPair.Value.Length <= 250)
{
this._storedStringPairs.Insert(0, uidUserPair);
}
}
}
/// <summary>
/// Writes the timestamp.
/// </summary>
/// <param name="timestamp">Timestamp.</param>
public void WriteTimestamp(DateTime timestamp)
{
this._stream.WriteByte((byte)O5MFileByteMarker.FileTimestamp);
var unixTimestamp = (long)timestamp.Subtract(UNIX_START).TotalSeconds;
var timestampBytes = VarintBitConverter.GetVarintBytes(unixTimestamp);
var timestampBytesLength = VarintBitConverter.GetVarintBytes((ulong)timestampBytes.Length);
this._stream.Write(timestampBytesLength, 0, timestampBytesLength.Length);
this._stream.Write(timestampBytes, 0, timestampBytes.Length);
}
public void VarintEncodeAndDecodeUInt64MoreBytes()
{
const ulong number = UInt64.MaxValue / 2;
var encoded = VarintBitConverter.GetVarintBytes(number);
Assert.Equal(number, VarintBitConverter.ToUInt64(encoded));
var encoded2 = encoded.Concat(new byte[] { 0xa, 0xb }).ToArray();
Assert.Equal(number, VarintBitConverter.ToUInt64(encoded2));
}
public static byte [] GetNearbyCellIds(double longitude, double latitude)
{
var cellIds = new List <byte>();
var cells = _GetNearbyCellIds(longitude, latitude);
foreach (var cellId in cells.OrderBy(c => c))
{
cellIds.AddRange(VarintBitConverter.GetVarintBytes(cellId));
}
return(cellIds.ToArray());
}
public static byte[] EncodeUlongList(List <ulong> integers)
{
var output = new List <byte>();
foreach (var integer in integers.OrderBy(c => c))
{
output.AddRange(VarintBitConverter.GetVarintBytes(integer));
}
return(output.ToArray());
}
public bool TryGetVarInt32(out int v)
{
if (Avail() < sizeof(int))
{
v = 0;
return(false);
}
var(result, newoffset) = VarintBitConverter.ToInt32(Buffer, Offset);
v = result;
Offset = newoffset;
return(true);
}
// to bytes
public override byte[] Pack()
{
List <byte> pack = new List <byte>();
pack.AddRange(VarintBitConverter.ToVarint(this.RemainingSize).Item1);
pack.AddRange(VarintBitConverter.ToVarint(this.HeaderSize).Item1);
pack.AddRange(VarintBitConverter.ToVarint(this.KeyIdxSize).Item1);
pack.AddRange(VarintBitConverter.ToVarint(this.KeyPKSize).Item1);
pack.AddRange(this.Key.Pack());
pack.AddRange(this.PrimaryKey.Pack());
return(pack.ToArray());
}
private void Init(Stream stream, O5MHeader header = O5MHeader.O5M2)
{
this._stream = stream;
this._stream.WriteByte((byte)O5MFileByteMarker.StartByte);
this._stream.WriteByte((byte)O5MFileByteMarker.Header);
var headerLengthBytes = VarintBitConverter.GetVarintBytes(4U);
var headerBuffer = Encoding.UTF8.GetBytes(header.ToString().ToLower());
this._stream.Write(headerLengthBytes, 0, headerLengthBytes.Length);
this._stream.Write(headerBuffer, 0, headerBuffer.Length);
}
private void sendMessage(IExtensible proto)
{
byte[] x;
using (var ms = new MemoryStream()) {
Serializer.Serialize(ms, proto);
x = ms.ToArray();
}
var _buf = VarintBitConverter.GetVarintBytes((uint)x.Length);
util.debugWriteLine("sendmessage mcs " + _buf + " x len " + x.Length + " " + proto);
sslStream.Write(_buf);
sslStream.Write(x);
sslStream.Flush();
}
/// <summary>
/// Writes the boundings.
/// </summary>
/// <param name="latitudeMin">Minimum Latitude.</param>
/// <param name="latitudeMax">Maximum Latitude.</param>
/// <param name="longitudeMin">Minimum Longitude.</param>
/// <param name="longitudeMax">Maximum Longitude.</param>
public void WriteBoundings(double latitudeMin, double latitudeMax, double longitudeMin, double longitudeMax)
{
this._stream.WriteByte((byte)O5MFileByteMarker.BoundingBox);
var longitudeMinLongBytes = VarintBitConverter.GetVarintBytes((long)(longitudeMin * POINT_DIVIDER));
var latitudeMinLongBytes = VarintBitConverter.GetVarintBytes((long)(latitudeMin * POINT_DIVIDER));
var longitudeMaxLongBytes = VarintBitConverter.GetVarintBytes((long)(longitudeMax * POINT_DIVIDER));
var latitudeMaxLongBytes = VarintBitConverter.GetVarintBytes((long)(latitudeMax * POINT_DIVIDER));
var boundingBytesLength = VarintBitConverter.GetVarintBytes((ulong)(longitudeMinLongBytes.Length + latitudeMinLongBytes.Length + longitudeMaxLongBytes.Length + latitudeMaxLongBytes.Length));
this._stream.Write(boundingBytesLength, 0, boundingBytesLength.Length);
this._stream.Write(longitudeMinLongBytes, 0, longitudeMinLongBytes.Length);
this._stream.Write(latitudeMinLongBytes, 0, latitudeMinLongBytes.Length);
this._stream.Write(longitudeMaxLongBytes, 0, longitudeMaxLongBytes.Length);
this._stream.Write(latitudeMaxLongBytes, 0, latitudeMaxLongBytes.Length);
}
private void WriteWay(OSMWay way, out byte[] writtenData)
{
if (!this._waysStarted)
{
this.Reset();
this._stream.WriteByte((byte)O5MFileByteMarker.Reset);
this._waysStarted = true;
}
this._stream.WriteByte((byte)O5MFileByteMarker.Way);
var bytes = new List <byte>();
var diffId = (long)way.Id - this._lastWayId;
var idBytes = VarintBitConverter.GetVarintBytes(diffId);
bytes.AddRange(idBytes);
this._lastWayId = (long)way.Id;
this.WriteVersionData(way, bytes);
if (way.NodeRefs.Count == 0)
{
bytes.Add(0x00);
}
else
{
var nodeRefsBytes = new List <byte>();
foreach (var nodeRef in way.NodeRefs)
{
var nextReferenceDiff = (long)nodeRef - this._lastReferenceId;
var nodeRefBytes = VarintBitConverter.GetVarintBytes(nextReferenceDiff);
this._lastReferenceId = (long)nodeRef;
nodeRefsBytes.AddRange(nodeRefBytes);
}
var nodeRefsBytesCountBytes = VarintBitConverter.GetVarintBytes((ulong)nodeRefsBytes.Count);
bytes.AddRange(nodeRefsBytesCountBytes);
bytes.AddRange(nodeRefsBytes);
}
this.WriteTags(way, bytes);
var length = VarintBitConverter.GetVarintBytes((ulong)bytes.Count);
this._stream.Write(length, 0, length.Length);
writtenData = bytes.ToArray();
this._stream.Write(writtenData, 0, bytes.Count);
}
private void WriteNode(OSMNode node, out byte[] writtenData)
{
if (!this._nodesStarted)
{
this.Reset();
this._stream.WriteByte((byte)O5MFileByteMarker.Reset);
this._nodesStarted = true;
}
this._stream.WriteByte((byte)O5MFileByteMarker.Node);
var bytes = new List <byte>();
var diffId = (long)node.Id - this._lastNodeId;
var idBytes = VarintBitConverter.GetVarintBytes(diffId);
bytes.AddRange(idBytes);
this._lastNodeId = (long)node.Id;
this.WriteVersionData(node, bytes);
var varIntLongitude = (int)Math.Round(node.Longitude * POINT_DIVIDER);
var diffLongitude = varIntLongitude - this._lastLongitude;
var longitudeBytes = VarintBitConverter.GetVarintBytes(diffLongitude);
bytes.AddRange(longitudeBytes);
this._lastLongitude = varIntLongitude;
var varIntLatitude = (int)Math.Round(node.Latitude * POINT_DIVIDER);
var diffLatitude = varIntLatitude - this._lastLatitude;
var latitudeBytes = VarintBitConverter.GetVarintBytes(diffLatitude);
bytes.AddRange(latitudeBytes);
this._lastLatitude = varIntLatitude;
this.WriteTags(node, bytes);
var length = VarintBitConverter.GetVarintBytes((ulong)bytes.Count);
this._stream.Write(length, 0, length.Length);
writtenData = bytes.ToArray();
this._stream.Write(writtenData, 0, bytes.Count);
}
/// <summary>
/// Returns 64-bit signed value from varint encoded array of bytes.
/// </summary>
/// <param name="reader">Varint encoded array of bytes.</param>
/// <returns>64-bit signed value</returns>
public static ulong ReadVarUInt64(this BinaryReader reader)
{
return(VarintBitConverter.ReadUInt64(reader));
}
/// <summary>
/// Returns 64-bit signed value from varint encoded array of bytes.
/// </summary>
/// <param name="writer">Varint encoded array of bytes.</param>
/// <param name="value"></param>
/// <returns>64-bit signed value</returns>
public static void WriteVarint(this BinaryWriter writer, ulong value)
{
var encoded = VarintBitConverter.GetVarintBytes(value);
writer.Write(encoded);
}
/// <summary>
/// Returns 16-bit signed value from varint encoded array of bytes.
/// </summary>
/// <param name="reader">Varint encoded array of bytes.</param>
/// <returns>16-bit signed value</returns>
public static ushort ReadVarUInt16(this BinaryReader reader)
{
return(VarintBitConverter.ReadUInt16(reader));
}
