public static FeatureCollection FromFlatGeobuf(byte[] bytes)
{
var fc = new NetTopologySuite.Features.FeatureCollection();
var bb = new ByteBuffer(bytes);
var headerSize = (ulong)ByteBufferUtil.GetSizePrefix(bb);
bb.Position = FlatBufferConstants.SizePrefixLength;
var header = Header.GetRootAsHeader(bb);
var count = header.FeaturesCount;
var featuresSize = header.FeaturesSize;
var nodeSize = header.IndexNodeSize;
bb.Position += (int)headerSize;
var index = new PackedHilbertRTree(count, nodeSize);
var indexData = bytes.Skip((int)(headerSize + featuresSize)).Take((int)index.Size).ToArray();
index.Load(indexData);
while (count-- > 0)
{
var featureLength = ByteBufferUtil.GetSizePrefix(bb);
bb.Position += FlatBufferConstants.SizePrefixLength;
var feature = FeatureConversions.FromByteBuffer(bb, header);
fc.Add(feature);
bb.Position += featureLength;
}
return(fc);
}
public static byte[] ToFlatGeobuf(FeatureCollection fc, GeometryType?geometryType = null, byte dimensions = 2, IList <ColumnMeta> columns = null)
{
ulong count = (ulong)fc.Features.LongCount();
if (count == 0)
{
return(new byte[0]);
}
var index = new PackedHilbertRTree(count);
foreach (var f in fc.Features)
{
var b = f.Geometry.EnvelopeInternal;
index.Add(b.MinX, b.MinY, b.MaxX, b.MaxY);
}
index.Finish();
var featureFirst = fc.Features.First();
if (geometryType == null)
{
geometryType = GeometryConversions.ToGeometryType(featureFirst.Geometry);
}
if (columns == null && featureFirst.Attributes != null)
{
columns = featureFirst.Attributes.GetNames()
.Select(n => new ColumnMeta()
{
Name = n, Type = ToColumnType(featureFirst.Attributes.GetType(n))
})
.ToList();
}
using (var memoryStream = new MemoryStream())
{
using (var featuresStream = new MemoryStream())
using (var offsetsStream = new MemoryStream())
using (var offetsWriter = new BinaryWriter(offsetsStream))
{
ulong offset = 0;
for (ulong i = 0; i < count; i++)
{
var feature = fc.Features[(int)index.Indices[i]];
var buffer = FeatureConversions.ToByteBuffer(feature, geometryType.Value, dimensions, columns);
featuresStream.Write(buffer, 0, buffer.Length);
offetsWriter.Write(offset);
offset += (ulong)buffer.Length;
}
var header = BuildHeader(count, geometryType.Value, dimensions, columns, index);
memoryStream.Write(header, 0, header.Length);
var indexBytes = index.ToBytes();
memoryStream.Write(indexBytes, 0, indexBytes.Length);
offsetsStream.WriteTo(memoryStream);
featuresStream.WriteTo(memoryStream);
}
return(memoryStream.ToArray());
}
}
public void SingleItemTest()
{
var tree = new PackedHilbertRTree(1);
tree.Add(0, 0, 1, 1);
tree.Finish();
var list = tree.Search(0, 0, 1, 1);
Assert.AreEqual(1, list.Count);
}
public void TwoItemsTest()
{
var tree = new PackedHilbertRTree(2);
tree.Add(0, 0, 1, 1);
tree.Add(2, 2, 3, 3);
tree.Finish();
var result = tree.Search(1, 1, 2, 2);
Assert.AreEqual(2, result.Count);
}
public static FeatureCollection FromFlatGeobuf(byte[] bytes)
{
var fc = new NetTopologySuite.Features.FeatureCollection();
var bb = new ByteBuffer(bytes);
var headerSize = ByteBufferUtil.GetSizePrefix(bb);
bb.Position = FlatBufferConstants.SizePrefixLength;
var header = Header.GetRootAsHeader(bb);
var count = header.FeaturesCount;
var nodeSize = header.IndexNodeSize;
var geometryType = header.GeometryType;
var dimensions = header.Dimensions;
IList <ColumnMeta> columns = null;
if (header.ColumnsLength > 0)
{
columns = new List <ColumnMeta>();
for (int i = 0; i < header.ColumnsLength; i++)
{
var column = header.Columns(i).Value;
columns.Add(new ColumnMeta()
{
Name = column.Name, Type = column.Type
});
}
}
bb.Position += headerSize;
if (nodeSize > 0)
{
var index = new PackedHilbertRTree(count, nodeSize);
var indexData = bytes.Skip(headerSize).Take((int)index.Size).ToArray();
index.Load(indexData);
bb.Position += (int)index.Size + (int)count * 8;
}
while (bb.Position < bb.Length)
{
var featureLength = ByteBufferUtil.GetSizePrefix(bb);
bb.Position += FlatBufferConstants.SizePrefixLength;
var feature = FeatureConversions.FromByteBuffer(bb, geometryType, dimensions, columns);
fc.Add(feature);
bb.Position += featureLength;
}
return(fc);
}
public static byte[] ToFlatGeobuf(FeatureCollection fc, IList <LayerMeta> layers = null)
{
ulong count = (ulong)fc.Features.LongCount();
if (count == 0)
{
throw new ApplicationException("Empty feature collection is not allowed as input");
}
var index = new PackedHilbertRTree(count);
foreach (var f in fc.Features)
{
var b = f.Geometry.EnvelopeInternal;
index.Add(b.MinX, b.MinY, b.MaxX, b.MaxY);
}
index.Finish();
if (layers == null)
{
layers = IntrospectLayers(fc);
}
using (var memoryStream = new MemoryStream())
{
using (var featuresStream = new MemoryStream())
using (var offsetsStream = new MemoryStream())
using (var offetsWriter = new BinaryWriter(offsetsStream))
{
ulong offset = 0;
for (ulong i = 0; i < count; i++)
{
var feature = fc.Features[(int)index.Indices[i]];
var buffer = FeatureConversions.ToByteBuffer(feature, layers);
featuresStream.Write(buffer, 0, buffer.Length);
offetsWriter.Write(offset);
offset += (ulong)buffer.Length;
}
var header = BuildHeader(count, offset, layers, index);
memoryStream.Write(header, 0, header.Length);
featuresStream.WriteTo(memoryStream);
var indexBytes = index.ToBytes();
memoryStream.Write(indexBytes, 0, indexBytes.Length);
offsetsStream.WriteTo(memoryStream);
}
return(memoryStream.ToArray());
}
}
public void SingleItemRoundtripTest()
{
var tree = new PackedHilbertRTree(1);
tree.Add(0, 0, 1, 1);
tree.Finish();
var data = tree.ToBytes();
Assert.AreEqual(tree.Size, (ulong)data.LongLength);
var tree2 = new PackedHilbertRTree(1, 16);
tree2.Load(data);
var list = tree2.Search(0, 0, 1, 1);
Assert.AreEqual(1, list.Count);
}
private static byte[] BuildHeader(ulong count, GeometryType geometryType, byte dimensions, IList <ColumnMeta> columns, PackedHilbertRTree index)
{
var builder = new FlatBufferBuilder(4096);
VectorOffset?columnsOffset = null;
if (columns != null)
{
var columnsArray = columns
.Select(c => Column.CreateColumn(builder, builder.CreateString(c.Name), c.Type))
.ToArray();
columnsOffset = Column.CreateSortedVectorOfColumn(builder, columnsArray);
}
Header.StartHeader(builder);
Header.AddGeometryType(builder, geometryType);
Header.AddDimensions(builder, dimensions);
if (columnsOffset.HasValue)
{
Header.AddColumns(builder, columnsOffset.Value);
}
if (index != null)
{
Header.AddIndexNodeSize(builder, 16);
}
Header.AddFeaturesCount(builder, count);
var offset = Header.EndHeader(builder);
builder.FinishSizePrefixed(offset.Value);
return(builder.DataBuffer.ToSizedArray());
}
private static byte[] BuildHeader(ulong count, ulong featuresSize, IList <LayerMeta> layers, PackedHilbertRTree index)
{
// TODO: size might not be enough, need to be adaptive
var builder = new FlatBufferBuilder(1024);
// TODO: can be different per layer...
var columns = layers.First().Columns;
VectorOffset?columnsOffset = null;
if (columns != null)
{
var columnsArray = columns
.Select(c => Column.CreateColumn(builder, builder.CreateString(c.Name), c.Type))
.ToArray();
columnsOffset = Column.CreateSortedVectorOfColumn(builder, columnsArray);
}
var layerOffsets = layers
.Select(l =>
{
Layer.StartLayer(builder);
if (columnsOffset.HasValue)
{
Layer.AddColumns(builder, columnsOffset.Value);
}
Layer.AddGeometryType(builder, l.GeometryType);
var layerOffset = Layer.EndLayer(builder);
return(layerOffset);
}).ToArray();
var layersOffset = Header.CreateLayersVector(builder, layerOffsets);
Header.StartHeader(builder);
Header.AddLayers(builder, layersOffset);
if (index != null)
{
Header.AddIndexNodesCount(builder, index.NumNodes);
}
Header.AddFeaturesCount(builder, count);
Header.AddFeaturesSize(builder, featuresSize);
var offset = Header.EndHeader(builder);
builder.FinishSizePrefixed(offset.Value);
return(builder.DataBuffer.ToSizedArray());
}