private void CreateTables() { if (_tables != null) { throw new BadMetadataException("Multiple table heaps."); } _reader.Advance(4); //reserved: 4, always 0 var header = new { MajorVersion = _reader.ReadUInt8(), MinorVersion = _reader.ReadUInt8(), HeapSizes = _reader.ReadUInt8(), _ = _reader.Advance(1), //reserved: 1, always 1 Valid = _reader.ReadUInt64(), Sorted = _reader.ReadUInt64(), }; _heapSizes = header.HeapSizes; // read table row nums ulong present = header.Valid; // if bit is set table is presented, otherwise it is empty _tables = new MetadataTable[MaxTableCount]; for (int i = 0; i < MaxTableCount; i++) { //NOTE: If flag set table is presented if (((present >> i) & 1) != 0) { int rowCount = _reader.ReadInt32(); var id = (TableId)i; var table = Schema.CreateTable(id); table.RowCount = rowCount; table.IsSorted = ((header.Sorted >> i) & 1) != 0; _tables[i] = table; } } //Setup tables (Offset, RowSize, Size) long pos = _reader.Position; for (int i = 0; i < MaxTableCount; i++) { var table = _tables[i]; if (table != null) { table.Offset = pos; int rowSize = 0; foreach (var column in table.Columns) { column.Size = GetColumnSize(column); rowSize += column.Size; } table.RowSize = rowSize; table.Size = table.RowCount * rowSize; pos += table.Size; } } }
private static object ReadValue(BufferedBinaryReader reader, SystemTypeCode type) { switch (type) { case SystemTypeCode.Boolean: return(reader.ReadBoolean()); case SystemTypeCode.Int8: return(reader.ReadInt8()); case SystemTypeCode.UInt8: return(reader.ReadUInt8()); case SystemTypeCode.Int16: return(reader.ReadInt16()); case SystemTypeCode.UInt16: return(reader.ReadUInt16()); case SystemTypeCode.Int32: return(reader.ReadInt32()); case SystemTypeCode.UInt32: return(reader.ReadUInt32()); case SystemTypeCode.Int64: return(reader.ReadInt64()); case SystemTypeCode.UInt64: return(reader.ReadUInt64()); case SystemTypeCode.Single: return(reader.ReadSingle()); case SystemTypeCode.Double: return(reader.ReadDouble()); case SystemTypeCode.Decimal: throw new NotImplementedException(); case SystemTypeCode.DateTime: throw new NotImplementedException(); case SystemTypeCode.Char: return(reader.ReadChar()); case SystemTypeCode.String: return(reader.ReadCountedUtf8()); case SystemTypeCode.Enum: throw new NotImplementedException(); default: throw new ArgumentOutOfRangeException(); } }
public void ReadFromStream(BufferedBinaryReader r) { ID = Utils.Latin1Encoding.GetString(r.ReadBytes(4)); StreamVersion = r.ReadByte(); TypeFlag = r.ReadByte(); AbsolutePosition = r.ReadUInt64(); Serial = r.ReadInt32(); PageNumber = r.ReadInt32(); Checksum = r.ReadUInt32(); Segments = r.ReadByte(); LacingValues = r.ReadBytes(Segments); }
private static object ReadValue(ElementType type, BufferedBinaryReader reader) { switch (type) { case ElementType.Boolean: return(reader.ReadBoolean()); case ElementType.Char: return(reader.ReadChar()); case ElementType.Int8: return(reader.ReadSByte()); case ElementType.UInt8: return(reader.ReadByte()); case ElementType.Int16: return(reader.ReadInt16()); case ElementType.UInt16: return(reader.ReadUInt16()); case ElementType.Int32: return(reader.ReadInt32()); case ElementType.UInt32: return(reader.ReadUInt32()); case ElementType.Int64: return(reader.ReadInt64()); case ElementType.UInt64: return(reader.ReadUInt64()); case ElementType.Single: return(reader.ReadSingle()); case ElementType.Double: return(reader.ReadDouble()); case ElementType.String: return(Encoding.Unicode.GetString(reader.ToArray())); case ElementType.Class: return(null); default: return(reader.ToArray()); } }
internal static IndicatorSection BuildFrom(BufferedBinaryReader reader) { var fileStart = reader.Read(Constants.GribFileStart.Length); if (!Constants.GribFileStart.SequenceEqual(fileStart)) { throw new BadGribFormatException("Invalid file start."); } // Ignore the 2 reserved bytes reader.Skip(2); var disciplineNumber = reader.ReadUInt8(); var gribEdition = reader.ReadUInt8(); if (gribEdition != 2) { throw new NotSupportedException($"Only GRIB edition 2 is supported. GRIB edition {gribEdition} is not yet supported"); } var totalLength = reader.ReadUInt64(); return(new IndicatorSection(disciplineNumber, 2, totalLength)); }
private bool getInfo(BufferedBinaryReader source, FileInfo info, ReadTagParams readTagParams) { // Get info from file bool result = false; bool isValidHeader = false; // Check for ID3v2 (NB : this case should not even exist since OGG has its own native tagging system, and is not deemed compatible with ID3v2 according to the ID3 FAQ) source.Seek(sizeInfo.ID3v2Size, SeekOrigin.Begin); // Read global file header info.IdentificationHeader.ReadFromStream(source); if (info.IdentificationHeader.IsValid()) { source.Seek(sizeInfo.ID3v2Size + info.IdentificationHeader.Segments + 27, SeekOrigin.Begin); // 27 being the size from 'ID' to 'Segments' // Read Vorbis or Opus stream info long position = source.Position; String headerStart = Utils.Latin1Encoding.GetString(source.ReadBytes(3)); source.Seek(position, SeekOrigin.Begin); if (VORBIS_HEADER_ID.StartsWith(headerStart)) { contents = CONTENTS_VORBIS; info.VorbisParameters.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(7)); isValidHeader = VORBIS_HEADER_ID.Equals(info.VorbisParameters.ID); info.VorbisParameters.BitstreamVersion = source.ReadBytes(4); info.VorbisParameters.ChannelMode = source.ReadByte(); info.VorbisParameters.SampleRate = source.ReadInt32(); info.VorbisParameters.BitRateMaximal = source.ReadInt32(); info.VorbisParameters.BitRateNominal = source.ReadInt32(); info.VorbisParameters.BitRateMinimal = source.ReadInt32(); info.VorbisParameters.BlockSize = source.ReadByte(); info.VorbisParameters.StopFlag = source.ReadByte(); } else if (OPUS_HEADER_ID.StartsWith(headerStart)) { contents = CONTENTS_OPUS; info.OpusParameters.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(8)); isValidHeader = OPUS_HEADER_ID.Equals(info.OpusParameters.ID); info.OpusParameters.Version = source.ReadByte(); info.OpusParameters.OutputChannelCount = source.ReadByte(); info.OpusParameters.PreSkip = source.ReadUInt16(); //info.OpusParameters.InputSampleRate = source.ReadUInt32(); info.OpusParameters.InputSampleRate = 48000; // Actual sample rate is hardware-dependent. Let's assume for now that the hardware ATL runs on supports 48KHz source.Seek(4, SeekOrigin.Current); info.OpusParameters.OutputGain = source.ReadInt16(); info.OpusParameters.ChannelMappingFamily = source.ReadByte(); if (info.OpusParameters.ChannelMappingFamily > 0) { info.OpusParameters.StreamCount = source.ReadByte(); info.OpusParameters.CoupledStreamCount = source.ReadByte(); info.OpusParameters.ChannelMapping = new byte[info.OpusParameters.OutputChannelCount]; for (int i = 0; i < info.OpusParameters.OutputChannelCount; i++) { info.OpusParameters.ChannelMapping[i] = source.ReadByte(); } } } if (isValidHeader) { info.CommentHeaderStart = source.Position; IList <long> pagePos = new List <long>(); // Reads all related Vorbis pages that describe Comment and Setup headers // and concatenate their content into a single, continuous data stream bool loop = true; bool first = true; using (MemoryStream s = new MemoryStream()) { // Reconstruct the whole Comment header from OGG pages to a MemoryStream while (loop) { info.SetupHeaderEnd = source.Position; // When the loop stops, cursor is starting to read a brand new page located after Comment _and_ Setup headers info.CommentHeader.ID = Utils.Latin1Encoding.GetString(source.ReadBytes(4)); info.CommentHeader.StreamVersion = source.ReadByte(); info.CommentHeader.TypeFlag = source.ReadByte(); // 0 marks a new page if (0 == info.CommentHeader.TypeFlag) { loop = first; } if (loop) { info.CommentHeader.AbsolutePosition = source.ReadUInt64(); info.CommentHeader.Serial = source.ReadInt32(); info.CommentHeader.PageNumber = source.ReadInt32(); info.CommentHeader.Checksum = source.ReadUInt32(); info.CommentHeader.Segments = source.ReadByte(); info.CommentHeader.LacingValues = source.ReadBytes(info.CommentHeader.Segments); s.Write(source.ReadBytes(info.CommentHeader.GetPageLength()), 0, info.CommentHeader.GetPageLength()); pagePos.Add(info.SetupHeaderEnd); } first = false; } if (readTagParams.PrepareForWriting) // Metrics to prepare writing { if (pagePos.Count > 1) { source.Position = pagePos[pagePos.Count - 2]; } else { source.Position = pagePos[0]; } // Determine the boundaries of 3rd header (Setup header) by searching from last-but-one page if (StreamUtils.FindSequence(source, Utils.Latin1Encoding.GetBytes(VORBIS_SETUP_ID))) { info.SetupHeaderStart = source.Position - VORBIS_SETUP_ID.Length; info.CommentHeaderEnd = info.SetupHeaderStart; if (pagePos.Count > 1) { int firstSetupPage = -1; for (int i = 1; i < pagePos.Count; i++) { if (info.CommentHeaderEnd < pagePos[i]) { info.CommentHeaderSpanPages = i - 1; firstSetupPage = i - 1; } if (info.SetupHeaderEnd < pagePos[i]) { info.SetupHeaderSpanPages = i - firstSetupPage; } } /// Not found yet => comment header takes up all pages, and setup header is on the end of the last page if (-1 == firstSetupPage) { info.CommentHeaderSpanPages = pagePos.Count; info.SetupHeaderSpanPages = 1; } } else { info.CommentHeaderSpanPages = 1; info.SetupHeaderSpanPages = 1; } } } // Get total number of samples info.Samples = getSamples(source); // Read metadata from the reconstructed Comment header inside the memoryStream if (readTagParams.ReadTag) { BinaryReader msr = new BinaryReader(s); s.Seek(0, SeekOrigin.Begin); string tagId; bool isValidTagHeader = false; if (contents.Equals(CONTENTS_VORBIS)) { tagId = Utils.Latin1Encoding.GetString(msr.ReadBytes(7)); isValidTagHeader = (VORBIS_TAG_ID.Equals(tagId)); } else if (contents.Equals(CONTENTS_OPUS)) { tagId = Utils.Latin1Encoding.GetString(msr.ReadBytes(8)); isValidTagHeader = (OPUS_TAG_ID.Equals(tagId)); } if (isValidTagHeader) { vorbisTag.Clear(); vorbisTag.Read(msr, readTagParams); } } } // using MemoryStream result = true; } } return(result); }
// --------------------------------------------------------------------------- // Read total samples of OGG file, which are located on the very last page of the file private ulong getSamples(BufferedBinaryReader source) { OggHeader header = new OggHeader(); string headerId; byte typeFlag; byte[] lacingValues = new byte[255]; byte nbLacingValues = 0; long nextPageOffset = 0; // TODO - fine tune seekSize value int seekSize = (int)Math.Round(MAX_PAGE_SIZE * 0.75); if (seekSize > source.Length) { seekSize = (int)Math.Round(source.Length * 0.5); } source.Seek(-seekSize, SeekOrigin.End); if (!StreamUtils.FindSequence(source, Utils.Latin1Encoding.GetBytes(OGG_PAGE_ID))) { LogDelegator.GetLogDelegate()(Log.LV_ERROR, "No OGG header found; aborting read operation"); // Throw exception ? return(0); } source.Seek(-4, SeekOrigin.Current); // Iterate until last page is encountered do { if (source.Position + nextPageOffset + 27 > source.Length) // End of stream about to be reached => last OGG header did not have the proper type flag { break; } source.Seek(nextPageOffset, SeekOrigin.Current); headerId = Utils.Latin1Encoding.GetString(source.ReadBytes(4)); if (headerId.Equals(OGG_PAGE_ID)) { source.Seek(1, SeekOrigin.Current); typeFlag = source.ReadByte(); source.Seek(20, SeekOrigin.Current); nbLacingValues = source.ReadByte(); nextPageOffset = 0; source.Read(lacingValues, 0, nbLacingValues); for (int i = 0; i < nbLacingValues; i++) { nextPageOffset += lacingValues[i]; } } else { LogDelegator.GetLogDelegate()(Log.LV_ERROR, "Invalid OGG header found while looking for total samples; aborting read operation"); // Throw exception ? return(0); } } while (0 == (typeFlag & 0x04)); // 0x04 marks the last page of the logical bitstream // Stream is positioned at the end of the last page header; backtracking to read AbsolutePosition field source.Seek(-nbLacingValues - 21, SeekOrigin.Current); return(source.ReadUInt64()); }
private object ReadValue(BufferedBinaryReader reader, IType type) { var st = type.SystemType(); if (st != null) { switch (st.Code) { case SystemTypeCode.Boolean: return reader.ReadBoolean(); case SystemTypeCode.Int8: return reader.ReadInt8(); case SystemTypeCode.UInt8: return reader.ReadUInt8(); case SystemTypeCode.Int16: return reader.ReadInt16(); case SystemTypeCode.UInt16: return reader.ReadUInt16(); case SystemTypeCode.Int32: return reader.ReadInt32(); case SystemTypeCode.UInt32: return reader.ReadUInt32(); case SystemTypeCode.Int64: return reader.ReadInt64(); case SystemTypeCode.UInt64: return reader.ReadUInt64(); case SystemTypeCode.Single: return reader.ReadSingle(); case SystemTypeCode.Double: return reader.ReadDouble(); case SystemTypeCode.Char: return reader.ReadChar(); case SystemTypeCode.String: return reader.ReadCountedUtf8(); case SystemTypeCode.Object: //boxed value type var e = (ElementType)reader.ReadInt8(); return ReadValue(reader, e); case SystemTypeCode.Type: return ReadType(reader); } } if (type.TypeKind == TypeKind.Enum) { return ReadValue(reader, type.ValueType); } if (type.IsArray) { int numElem = reader.ReadInt32(); Array arr = null; for (int i = 0; i < numElem; ++i) { var val = ReadValue(reader, type.ElementType); if (arr == null) arr = Array.CreateInstance(val.GetType(), numElem); arr.SetValue(val, i); } return arr; } return null; }
private object ReadValue(BufferedBinaryReader reader, ElementType e) { switch (e) { case ElementType.Boolean: return reader.ReadBoolean(); case ElementType.Char: return reader.ReadChar(); case ElementType.Int8: return reader.ReadSByte(); case ElementType.UInt8: return reader.ReadByte(); case ElementType.Int16: return reader.ReadInt16(); case ElementType.UInt16: return reader.ReadUInt16(); case ElementType.Int32: return reader.ReadInt32(); case ElementType.UInt32: return reader.ReadUInt32(); case ElementType.Int64: return reader.ReadInt64(); case ElementType.UInt64: return reader.ReadUInt64(); case ElementType.Single: return reader.ReadSingle(); case ElementType.Double: return reader.ReadDouble(); case ElementType.String: return reader.ReadCountedUtf8(); case ElementType.Object: case ElementType.CustomArgsBoxedObject: { var elem = (ElementType)reader.ReadInt8(); return ReadValue(reader, elem); } case ElementType.CustomArgsEnum: { string enumTypeName = reader.ReadCountedUtf8(); var enumType = FindType(enumTypeName); if (enumType == null) { //TODO: throw new BadMetadataException(); } return ReadValue(reader, enumType); } case ElementType.CustomArgsType: return ReadType(reader); case ElementType.ArraySz: { var arrElemType = (ElementType)reader.ReadInt8(); return ReadArray(reader, arrElemType); } default: throw new ArgumentOutOfRangeException(); } }
/// <summary> /// Загрузка базы данных в память /// </summary> /// <returns></returns> public void Load() { string filePath = Path.Combine(_environment.ContentRootPath, _dbFileName); BinGeoModel binModel; using (FileStream stream = new FileStream(filePath, FileMode.Open)) { Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew(); using (BufferedBinaryReader bufferedReader = new BufferedBinaryReader(stream, 65536)) { stopwatch.Start(); bufferedReader.FillBuffer(); int version = bufferedReader.ReadInt32(); byte[] nameBytes = bufferedReader.Read(0, 32); ulong timestamp = bufferedReader.ReadUInt64(); int records = bufferedReader.ReadInt32(); uint offsetRanges = bufferedReader.ReadUInt32(); uint offsetCities = bufferedReader.ReadUInt32(); uint offsetLocations = bufferedReader.ReadUInt32(); binModel = new BinGeoModel(version, nameBytes, timestamp, records, offsetRanges, offsetCities, offsetLocations); int currentIndex = 0; binModel.IpRangeCollection = new BinIpRange[binModel.RecordsCount]; while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount) { for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= IpRangeBytesCount; currentIndex++) { binModel.IpRangeCollection[currentIndex] = new BinIpRange() { IpFrom = bufferedReader.ReadUInt32(), IpTo = bufferedReader.ReadUInt32(), LocationIndex = bufferedReader.ReadUInt32() }; } } currentIndex = 0; binModel.LocationCollection = new BinLocation[binModel.RecordsCount]; while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount) { for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= LocationBytesCount; currentIndex++) { byte[] country = bufferedReader.Read(0, 8); byte[] region = bufferedReader.Read(0, 12); byte[] postal = bufferedReader.Read(0, 12); byte[] city = bufferedReader.Read(0, 24); byte[] organization = bufferedReader.Read(0, 32); float latitude = bufferedReader.ReadSingle(); float longitude = bufferedReader.ReadSingle(); binModel.LocationCollection[currentIndex] = new BinLocation(country, region, postal, city, organization, latitude, longitude); } } currentIndex = 0; binModel.Indexes = new uint[binModel.RecordsCount]; while (bufferedReader.FillBuffer() && currentIndex < binModel.RecordsCount) { for (; currentIndex < binModel.RecordsCount && bufferedReader.NumBytesAvailable >= IndexBytesCount; currentIndex++) { binModel.Indexes[currentIndex] = bufferedReader.ReadUInt32(); } } stopwatch.Stop(); DatabaseLoadedTimeMs = stopwatch.ElapsedMilliseconds; _logger.LogInformation($"Database loading time: {stopwatch.ElapsedMilliseconds} ms"); } // Отображение объектов сущностей двоичной базы в объекты бизнес сущностей GeoModel = _mapper.Map <GeoModel>(binModel); } }