private void CreateTables()
{
if (_tables != null)
{
throw new BadMetadataException("Multiple table heaps.");
}
_reader.Advance(4); //reserved: 4, always 0
var header = new
{
MajorVersion = _reader.ReadUInt8(),
MinorVersion = _reader.ReadUInt8(),
HeapSizes = _reader.ReadUInt8(),
_ = _reader.Advance(1), //reserved: 1, always 1
Valid = _reader.ReadUInt64(),
Sorted = _reader.ReadUInt64(),
};
_heapSizes = header.HeapSizes;
// read table row nums
ulong present = header.Valid; // if bit is set table is presented, otherwise it is empty
_tables = new MetadataTable[MaxTableCount];
for (int i = 0; i < MaxTableCount; i++)
{
//NOTE: If flag set table is presented
if (((present >> i) & 1) != 0)
{
int rowCount = _reader.ReadInt32();
var id = (TableId)i;
var table = Schema.CreateTable(id);
table.RowCount = rowCount;
table.IsSorted = ((header.Sorted >> i) & 1) != 0;
_tables[i] = table;
}
}
//Setup tables (Offset, RowSize, Size)
long pos = _reader.Position;
for (int i = 0; i < MaxTableCount; i++)
{
var table = _tables[i];
if (table != null)
{
table.Offset = pos;
int rowSize = 0;
foreach (var column in table.Columns)
{
column.Size = GetColumnSize(column);
rowSize += column.Size;
}
table.RowSize = rowSize;
table.Size = table.RowCount * rowSize;
pos += table.Size;
}
}
}
private static object ReadValue(BufferedBinaryReader reader, SystemTypeCode type)
{
switch (type)
{
case SystemTypeCode.Boolean:
return(reader.ReadBoolean());
case SystemTypeCode.Int8:
return(reader.ReadInt8());
case SystemTypeCode.UInt8:
return(reader.ReadUInt8());
case SystemTypeCode.Int16:
return(reader.ReadInt16());
case SystemTypeCode.UInt16:
return(reader.ReadUInt16());
case SystemTypeCode.Int32:
return(reader.ReadInt32());
case SystemTypeCode.UInt32:
return(reader.ReadUInt32());
case SystemTypeCode.Int64:
return(reader.ReadInt64());
case SystemTypeCode.UInt64:
return(reader.ReadUInt64());
case SystemTypeCode.Single:
return(reader.ReadSingle());
case SystemTypeCode.Double:
return(reader.ReadDouble());
case SystemTypeCode.Decimal:
throw new NotImplementedException();
case SystemTypeCode.DateTime:
throw new NotImplementedException();
case SystemTypeCode.Char:
return(reader.ReadChar());
case SystemTypeCode.String:
return(reader.ReadCountedUtf8());
case SystemTypeCode.Enum:
throw new NotImplementedException();
default:
throw new ArgumentOutOfRangeException();
}
}
public void ReadFromStream(BufferedBinaryReader r)
{
ID = Utils.Latin1Encoding.GetString(r.ReadBytes(4));
StreamVersion = r.ReadByte();
TypeFlag = r.ReadByte();
AbsolutePosition = r.ReadUInt64();
Serial = r.ReadInt32();
PageNumber = r.ReadInt32();
Checksum = r.ReadUInt32();
Segments = r.ReadByte();
LacingValues = r.ReadBytes(Segments);
}
private static object ReadValue(ElementType type, BufferedBinaryReader reader)
{
switch (type)
{
case ElementType.Boolean:
return(reader.ReadBoolean());
case ElementType.Char:
return(reader.ReadChar());
case ElementType.Int8:
return(reader.ReadSByte());
case ElementType.UInt8:
return(reader.ReadByte());
case ElementType.Int16:
return(reader.ReadInt16());
case ElementType.UInt16:
return(reader.ReadUInt16());
case ElementType.Int32:
return(reader.ReadInt32());
case ElementType.UInt32:
return(reader.ReadUInt32());
case ElementType.Int64:
return(reader.ReadInt64());
case ElementType.UInt64:
return(reader.ReadUInt64());
case ElementType.Single:
return(reader.ReadSingle());
case ElementType.Double:
return(reader.ReadDouble());
case ElementType.String:
return(Encoding.Unicode.GetString(reader.ToArray()));
case ElementType.Class:
return(null);
default:
return(reader.ToArray());
}
}
internal static IndicatorSection BuildFrom(BufferedBinaryReader reader)
{
var fileStart = reader.Read(Constants.GribFileStart.Length);
if (!Constants.GribFileStart.SequenceEqual(fileStart))
{
throw new BadGribFormatException("Invalid file start.");
}
// Ignore the 2 reserved bytes
reader.Skip(2);
var disciplineNumber = reader.ReadUInt8();
var gribEdition = reader.ReadUInt8();
if (gribEdition != 2)
{
throw new NotSupportedException($"Only GRIB edition 2 is supported. GRIB edition {gribEdition} is not yet supported");
}
var totalLength = reader.ReadUInt64();
return(new IndicatorSection(disciplineNumber, 2, totalLength));
}
private bool getInfo(BufferedBinaryReader source, FileInfo info, ReadTagParams readTagParams)
{
// Get info from file
bool result = false;
bool isValidHeader = false;
// Check for ID3v2 (NB : this case should not even exist since OGG has its own native tagging system, and is not deemed compatible with ID3v2 according to the ID3 FAQ)
source.Seek(sizeInfo.ID3v2Size, SeekOrigin.Begin);
// Read global file header
info.IdentificationHeader.ReadFromStream(source);
if (info.IdentificationHeader.IsValid())
{
source.Seek(sizeInfo.ID3v2Size + info.IdentificationHeader.Segments + 27, SeekOrigin.Begin); // 27 being the size from 'ID' to 'Segments'
// Read Vorbis or Opus stream info
long position = source.Position;
String headerStart = Utils.Latin1Encoding.GetString(source.ReadBytes(3));
source.Seek(position, SeekOrigin.Begin);
if (VORBIS_HEADER_ID.StartsWith(headerStart))
{
contents = CONTENTS_VORBIS;
info.VorbisParameters.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(7));
isValidHeader = VORBIS_HEADER_ID.Equals(info.VorbisParameters.ID);
info.VorbisParameters.BitstreamVersion = source.ReadBytes(4);
info.VorbisParameters.ChannelMode = source.ReadByte();
info.VorbisParameters.SampleRate = source.ReadInt32();
info.VorbisParameters.BitRateMaximal = source.ReadInt32();
info.VorbisParameters.BitRateNominal = source.ReadInt32();
info.VorbisParameters.BitRateMinimal = source.ReadInt32();
info.VorbisParameters.BlockSize = source.ReadByte();
info.VorbisParameters.StopFlag = source.ReadByte();
}
else if (OPUS_HEADER_ID.StartsWith(headerStart))
{
contents = CONTENTS_OPUS;
info.OpusParameters.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(8));
isValidHeader = OPUS_HEADER_ID.Equals(info.OpusParameters.ID);
info.OpusParameters.Version = source.ReadByte();
info.OpusParameters.OutputChannelCount = source.ReadByte();
info.OpusParameters.PreSkip = source.ReadUInt16();
//info.OpusParameters.InputSampleRate = source.ReadUInt32();
info.OpusParameters.InputSampleRate = 48000; // Actual sample rate is hardware-dependent. Let's assume for now that the hardware ATL runs on supports 48KHz
source.Seek(4, SeekOrigin.Current);
info.OpusParameters.OutputGain = source.ReadInt16();
info.OpusParameters.ChannelMappingFamily = source.ReadByte();
if (info.OpusParameters.ChannelMappingFamily > 0)
{
info.OpusParameters.StreamCount = source.ReadByte();
info.OpusParameters.CoupledStreamCount = source.ReadByte();
info.OpusParameters.ChannelMapping = new byte[info.OpusParameters.OutputChannelCount];
for (int i = 0; i < info.OpusParameters.OutputChannelCount; i++)
{
info.OpusParameters.ChannelMapping[i] = source.ReadByte();
}
}
}
if (isValidHeader)
{
info.CommentHeaderStart = source.Position;
IList <long> pagePos = new List <long>();
// Reads all related Vorbis pages that describe Comment and Setup headers
// and concatenate their content into a single, continuous data stream
bool loop = true;
bool first = true;
using (MemoryStream s = new MemoryStream())
{
// Reconstruct the whole Comment header from OGG pages to a MemoryStream
while (loop)
{
info.SetupHeaderEnd = source.Position; // When the loop stops, cursor is starting to read a brand new page located after Comment _and_ Setup headers
info.CommentHeader.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(4));
info.CommentHeader.StreamVersion = source.ReadByte();
info.CommentHeader.TypeFlag = source.ReadByte();
// 0 marks a new page
if (0 == info.CommentHeader.TypeFlag)
{
loop = first;
}
if (loop)
{
info.CommentHeader.AbsolutePosition = source.ReadUInt64();
info.CommentHeader.Serial = source.ReadInt32();
info.CommentHeader.PageNumber = source.ReadInt32();
info.CommentHeader.Checksum = source.ReadUInt32();
info.CommentHeader.Segments = source.ReadByte();
info.CommentHeader.LacingValues = source.ReadBytes(info.CommentHeader.Segments);
s.Write(source.ReadBytes(info.CommentHeader.GetPageLength()), 0, info.CommentHeader.GetPageLength());
pagePos.Add(info.SetupHeaderEnd);
}
first = false;
}
if (readTagParams.PrepareForWriting) // Metrics to prepare writing
{
if (pagePos.Count > 1)
{
source.Position = pagePos[pagePos.Count - 2];
}
else
{
source.Position = pagePos[0];
}
// Determine the boundaries of 3rd header (Setup header) by searching from last-but-one page
if (StreamUtils.FindSequence(source, Utils.Latin1Encoding.GetBytes(VORBIS_SETUP_ID)))
{
info.SetupHeaderStart = source.Position - VORBIS_SETUP_ID.Length;
info.CommentHeaderEnd = info.SetupHeaderStart;
if (pagePos.Count > 1)
{
int firstSetupPage = -1;
for (int i = 1; i < pagePos.Count; i++)
{
if (info.CommentHeaderEnd < pagePos[i])
{
info.CommentHeaderSpanPages = i - 1;
firstSetupPage = i - 1;
}
if (info.SetupHeaderEnd < pagePos[i])
{
info.SetupHeaderSpanPages = i - firstSetupPage;
}
}
/// Not found yet => comment header takes up all pages, and setup header is on the end of the last page
if (-1 == firstSetupPage)
{
info.CommentHeaderSpanPages = pagePos.Count;
info.SetupHeaderSpanPages = 1;
}
}
else
{
info.CommentHeaderSpanPages = 1;
info.SetupHeaderSpanPages = 1;
}
}
}
// Get total number of samples
info.Samples = getSamples(source);
// Read metadata from the reconstructed Comment header inside the memoryStream
if (readTagParams.ReadTag)
{
BinaryReader msr = new BinaryReader(s);
s.Seek(0, SeekOrigin.Begin);
string tagId;
bool isValidTagHeader = false;
if (contents.Equals(CONTENTS_VORBIS))
{
tagId = Utils.Latin1Encoding.GetString(msr.ReadBytes(7));
isValidTagHeader = (VORBIS_TAG_ID.Equals(tagId));
}
else if (contents.Equals(CONTENTS_OPUS))
{
tagId = Utils.Latin1Encoding.GetString(msr.ReadBytes(8));
isValidTagHeader = (OPUS_TAG_ID.Equals(tagId));
}
if (isValidTagHeader)
{
vorbisTag.Clear();
vorbisTag.Read(msr, readTagParams);
}
}
} // using MemoryStream
result = true;
}
}
return(result);
}
// ---------------------------------------------------------------------------
// Read total samples of OGG file, which are located on the very last page of the file
private ulong getSamples(BufferedBinaryReader source)
{
OggHeader header = new OggHeader();
string headerId;
byte typeFlag;
byte[] lacingValues = new byte[255];
byte nbLacingValues = 0;
long nextPageOffset = 0;
// TODO - fine tune seekSize value
int seekSize = (int)Math.Round(MAX_PAGE_SIZE * 0.75);
if (seekSize > source.Length)
{
seekSize = (int)Math.Round(source.Length * 0.5);
}
source.Seek(-seekSize, SeekOrigin.End);
if (!StreamUtils.FindSequence(source, Utils.Latin1Encoding.GetBytes(OGG_PAGE_ID)))
{
LogDelegator.GetLogDelegate()(Log.LV_ERROR, "No OGG header found; aborting read operation"); // Throw exception ?
return(0);
}
source.Seek(-4, SeekOrigin.Current);
// Iterate until last page is encountered
do
{
if (source.Position + nextPageOffset + 27 > source.Length) // End of stream about to be reached => last OGG header did not have the proper type flag
{
break;
}
source.Seek(nextPageOffset, SeekOrigin.Current);
headerId = Utils.Latin1Encoding.GetString(source.ReadBytes(4));
if (headerId.Equals(OGG_PAGE_ID))
{
source.Seek(1, SeekOrigin.Current);
typeFlag = source.ReadByte();
source.Seek(20, SeekOrigin.Current);
nbLacingValues = source.ReadByte();
nextPageOffset = 0;
source.Read(lacingValues, 0, nbLacingValues);
for (int i = 0; i < nbLacingValues; i++)
{
nextPageOffset += lacingValues[i];
}
}
else
{
LogDelegator.GetLogDelegate()(Log.LV_ERROR, "Invalid OGG header found while looking for total samples; aborting read operation"); // Throw exception ?
return(0);
}
} while (0 == (typeFlag & 0x04)); // 0x04 marks the last page of the logical bitstream
// Stream is positioned at the end of the last page header; backtracking to read AbsolutePosition field
source.Seek(-nbLacingValues - 21, SeekOrigin.Current);
return(source.ReadUInt64());
}
private object ReadValue(BufferedBinaryReader reader, IType type)
{
var st = type.SystemType();
if (st != null)
{
switch (st.Code)
{
case SystemTypeCode.Boolean:
return reader.ReadBoolean();
case SystemTypeCode.Int8:
return reader.ReadInt8();
case SystemTypeCode.UInt8:
return reader.ReadUInt8();
case SystemTypeCode.Int16:
return reader.ReadInt16();
case SystemTypeCode.UInt16:
return reader.ReadUInt16();
case SystemTypeCode.Int32:
return reader.ReadInt32();
case SystemTypeCode.UInt32:
return reader.ReadUInt32();
case SystemTypeCode.Int64:
return reader.ReadInt64();
case SystemTypeCode.UInt64:
return reader.ReadUInt64();
case SystemTypeCode.Single:
return reader.ReadSingle();
case SystemTypeCode.Double:
return reader.ReadDouble();
case SystemTypeCode.Char:
return reader.ReadChar();
case SystemTypeCode.String:
return reader.ReadCountedUtf8();
case SystemTypeCode.Object:
//boxed value type
var e = (ElementType)reader.ReadInt8();
return ReadValue(reader, e);
case SystemTypeCode.Type:
return ReadType(reader);
}
}
if (type.TypeKind == TypeKind.Enum)
{
return ReadValue(reader, type.ValueType);
}
if (type.IsArray)
{
int numElem = reader.ReadInt32();
Array arr = null;
for (int i = 0; i < numElem; ++i)
{
var val = ReadValue(reader, type.ElementType);
if (arr == null)
arr = Array.CreateInstance(val.GetType(), numElem);
arr.SetValue(val, i);
}
return arr;
}
return null;
}
private object ReadValue(BufferedBinaryReader reader, ElementType e)
{
switch (e)
{
case ElementType.Boolean:
return reader.ReadBoolean();
case ElementType.Char:
return reader.ReadChar();
case ElementType.Int8:
return reader.ReadSByte();
case ElementType.UInt8:
return reader.ReadByte();
case ElementType.Int16:
return reader.ReadInt16();
case ElementType.UInt16:
return reader.ReadUInt16();
case ElementType.Int32:
return reader.ReadInt32();
case ElementType.UInt32:
return reader.ReadUInt32();
case ElementType.Int64:
return reader.ReadInt64();
case ElementType.UInt64:
return reader.ReadUInt64();
case ElementType.Single:
return reader.ReadSingle();
case ElementType.Double:
return reader.ReadDouble();
case ElementType.String:
return reader.ReadCountedUtf8();
case ElementType.Object:
case ElementType.CustomArgsBoxedObject:
{
var elem = (ElementType)reader.ReadInt8();
return ReadValue(reader, elem);
}
case ElementType.CustomArgsEnum:
{
string enumTypeName = reader.ReadCountedUtf8();
var enumType = FindType(enumTypeName);
if (enumType == null)
{
//TODO:
throw new BadMetadataException();
}
return ReadValue(reader, enumType);
}
case ElementType.CustomArgsType:
return ReadType(reader);
case ElementType.ArraySz:
{
var arrElemType = (ElementType)reader.ReadInt8();
return ReadArray(reader, arrElemType);
}
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Загрузка базы данных в память
/// </summary>
/// <returns></returns>
public void Load()
{
string filePath = Path.Combine(_environment.ContentRootPath, _dbFileName);
BinGeoModel binModel;
using (FileStream stream = new FileStream(filePath, FileMode.Open))
{
Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();
using (BufferedBinaryReader bufferedReader = new BufferedBinaryReader(stream, 65536))
{
stopwatch.Start();
bufferedReader.FillBuffer();
int version = bufferedReader.ReadInt32();
byte[] nameBytes = bufferedReader.Read(0, 32);
ulong timestamp = bufferedReader.ReadUInt64();
int records = bufferedReader.ReadInt32();
uint offsetRanges = bufferedReader.ReadUInt32();
uint offsetCities = bufferedReader.ReadUInt32();
uint offsetLocations = bufferedReader.ReadUInt32();
binModel = new BinGeoModel(version, nameBytes, timestamp, records, offsetRanges, offsetCities, offsetLocations);
int currentIndex = 0;
binModel.IpRangeCollection = new BinIpRange[binModel.RecordsCount];
while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount)
{
for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= IpRangeBytesCount; currentIndex++)
{
binModel.IpRangeCollection[currentIndex] = new BinIpRange()
{
IpFrom = bufferedReader.ReadUInt32(),
IpTo = bufferedReader.ReadUInt32(),
LocationIndex = bufferedReader.ReadUInt32()
};
}
}
currentIndex = 0;
binModel.LocationCollection = new BinLocation[binModel.RecordsCount];
while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount)
{
for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= LocationBytesCount; currentIndex++)
{
byte[] country = bufferedReader.Read(0, 8);
byte[] region = bufferedReader.Read(0, 12);
byte[] postal = bufferedReader.Read(0, 12);
byte[] city = bufferedReader.Read(0, 24);
byte[] organization = bufferedReader.Read(0, 32);
float latitude = bufferedReader.ReadSingle();
float longitude = bufferedReader.ReadSingle();
binModel.LocationCollection[currentIndex] = new BinLocation(country, region, postal, city, organization, latitude, longitude);
}
}
currentIndex = 0;
binModel.Indexes = new uint[binModel.RecordsCount];
while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount)
{
for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= IndexBytesCount; currentIndex++)
{
binModel.Indexes[currentIndex] = bufferedReader.ReadUInt32();
}
}
stopwatch.Stop();
DatabaseLoadedTimeMs = stopwatch.ElapsedMilliseconds;
_logger.LogInformation($"Database loading time: {stopwatch.ElapsedMilliseconds} ms");
}
// Отображение объектов сущностей двоичной базы в объекты бизнес сущностей
GeoModel = _mapper.Map <GeoModel>(binModel);
}
}
