/// <summary> /// Reads the number of bytes using the specified reader. /// This handles copying across blocks if necessary. /// </summary> /// <param name="numBytes"></param> /// <param name="reader"></param> public void Read(int numBytes, BufferedBinaryReader reader) { if(Offset + numBytes < BlockSize) { if (numBytes > 0) { reader.Read(Blocks[CurrentBlock], Offset, numBytes); Offset += numBytes; } return; } int firstLen = BlockSize - Offset; int secondLen = numBytes - firstLen; reader.Read(Blocks[CurrentBlock], Offset, firstLen); Offset = 0; CurrentBlock += 1; if(Blocks.Count <= CurrentBlock)Blocks.Add(new byte[BlockSize]); reader.Read(Blocks[CurrentBlock], Offset, secondLen); Offset += secondLen; }
/// <summary> /// Obtains a typed list of ShapefilePoint structures with double values associated with the various coordinates. /// </summary> /// <param name="filename">A string filename</param> /// <param name="progressHandler">A progress indicator</param> private void FillPoints(string filename, IProgressHandler progressHandler) { // Check to ensure the filename is not null if (filename == null) { throw new NullReferenceException(MessageStrings.ArgumentNull_S.Replace("%S",filename)); } if (File.Exists(filename) == false) { throw new FileNotFoundException(MessageStrings.FileNotFound_S.Replace("%S", filename)); } // Reading the headers gives us an easier way to track the number of shapes and their overall length etc. List<ShapeHeader> shapeHeaders = ReadIndexFile(filename); // Get the basic header information. ShapefileHeader header = new ShapefileHeader(filename); Extent = new Extent(new[] { header.Xmin, header.Ymin, header.Xmax, header.Ymax }); Envelope = Extent.ToEnvelope(); // Check to ensure that the filename is the correct shape type if (header.ShapeType != ShapeTypes.Point && header.ShapeType != ShapeTypes.PointM && header.ShapeType != ShapeTypes.PointZ) { throw new ApplicationException(MessageStrings.FileNotPoints_S.Replace("%S", filename)); } // This will set up a reader so that we can read values in huge chunks, which is much faster than one value at a time. IO.BufferedBinaryReader bbReader = new IO.BufferedBinaryReader(filename, progressHandler); if (bbReader.FileLength == 100) { bbReader.Close(); // the file is empty so we are done reading return; } // Skip the shapefile header by skipping the first 100 bytes in the shapefile bbReader.Seek(100, SeekOrigin.Begin); int numShapes = shapeHeaders.Count; byte[] bigEndian = new byte[numShapes * 8]; byte[] allCoords = new byte[numShapes * 16]; bool isM = false; bool isZ = false; if(header.ShapeType == ShapeTypes.PointM || header.ShapeType == ShapeTypes.PointZ) { isM = true; } if (header.ShapeType == ShapeTypes.PointZ) { isZ = true; } byte[] allM = new byte[8]; if(isM) allM = new byte[numShapes * 8]; byte[] allZ = new byte[8]; if(isZ) allZ = new byte[numShapes * 8]; for (int shp = 0; shp < numShapes; shp++) { // Read from the index file because some deleted records // might still exist in the .shp file. long offset = (shapeHeaders[shp].ByteOffset); bbReader.Seek(offset, SeekOrigin.Begin); bbReader.Read(bigEndian, shp * 8, 8); ShapeTypes type = (ShapeTypes)bbReader.ReadInt32(); //bbReader.Seek(4, SeekOrigin.Current); bbReader.Read(allCoords, shp * 16, 16); if(isZ) { bbReader.Read(allZ, shp*8, 8); } if (isM) { bbReader.Read(allM, shp*8, 8); } ShapeRange shape = new ShapeRange(FeatureTypes.Point); shape.StartIndex = shp; shape.ContentLength = 8; shape.NumPoints = 1; shape.NumParts = 1; ShapeIndices.Add(shape); } double[] vert = new double[2 * numShapes]; Buffer.BlockCopy(allCoords, 0, vert, 0, numShapes*16); Vertex = vert; if(isM) { double[] m = new double[numShapes]; Buffer.BlockCopy(allM, 0, m, 0, numShapes * 8); M = m; } if(isZ) { double[] z = new double[numShapes]; Buffer.BlockCopy(allZ, 0, z, 0, numShapes * 8); Z = z; } for (int shp = 0; shp < numShapes; shp++) { PartRange part = new PartRange(vert, shp, 0, FeatureTypes.Point); part.NumVertices = 1; ShapeRange shape = ShapeIndices[shp]; shape.Parts.Add(part); shape.Extent = new Extent(new[] {vert[shp*2], vert[shp*2 + 1], vert[shp*2], vert[shp*2 + 1]}); } bbReader.Dispose(); }
/// <summary> /// Parses the first 100 bytes of a shapefile into the important values /// </summary> /// <param name="inFilename">The filename to read</param> public void Open(string inFilename) { _filename = inFilename; // Position Field Value Type ByteOrder // -------------------------------------------------------------- // Byte 0 File Code 9994 Integer Big // Byte 4 Unused 0 Integer Big // Byte 8 Unused 0 Integer Big // Byte 12 Unused 0 Integer Big // Byte 16 Unused 0 Integer Big // Byte 20 Unused 0 Integer Big // Byte 24 File Length File Length Integer Big // Byte 28 Version 1000 Integer Little // Byte 32 Shape Type Shape Type Integer Little // Byte 36 Bounding Box Xmin Double Little // Byte 44 Bounding Box Ymin Double Little // Byte 52 Bounding Box Xmax Double Little // Byte 60 Bounding Box Ymax Double Little // Byte 68 Bounding Box Zmin Double Little // Byte 76 Bounding Box Zmax Double Little // Byte 84 Bounding Box Mmin Double Little // Byte 92 Bounding Box Mmax Double Little BufferedBinaryReader bbReader = new BufferedBinaryReader(inFilename); bbReader.FillBuffer(100); // we only need to read 100 bytes from the header. bbReader.Close(); // Close the internal readers connected to the file, but don't close the file itself. // Reading BigEndian simply requires us to reverse the byte order. _fileCode = bbReader.ReadInt32(false); // Skip the next 20 bytes because they are unused bbReader.Seek(20, System.IO.SeekOrigin.Current); // Read the file length in reverse sequence _fileLength = bbReader.ReadInt32(false); // From this point on, all the header values are in little endian // Read the version _version = bbReader.ReadInt32(); // Read in the shapetype that should be the shapetype for the whole shapefile _shapeType = (ShapeTypes)bbReader.ReadInt32(); // Get the extents, each of which are double values. _xMin = bbReader.ReadDouble(); _yMin = bbReader.ReadDouble(); _xMax = bbReader.ReadDouble(); _yMax = bbReader.ReadDouble(); _zMin = bbReader.ReadDouble(); _zMax = bbReader.ReadDouble(); _mMin = bbReader.ReadDouble(); _mMax = bbReader.ReadDouble(); bbReader.Dispose(); }
/// <summary> /// Reads the entire index file in order to get a breakdown of how shapes are broken up. /// </summary> /// <param name="filename">A string filename of the .shx file to read.</param> /// <returns>A List of ShapeHeaders that give offsets and lengths so that reading can be optimized</returns> public List<ShapeHeader> ReadIndexFile(string filename) { string shxFilename = filename; string ext = System.IO.Path.GetExtension(filename); if (ext != ".shx") { shxFilename = System.IO.Path.ChangeExtension(filename, ".shx"); } if (shxFilename == null) { throw new NullReferenceException(MessageStrings.ArgumentNull_S.Replace("%S",filename)); } if (System.IO.File.Exists(shxFilename) == false) { throw new System.IO.FileNotFoundException(MessageStrings.FileNotFound_S.Replace("%S",filename)); } // This will store the header elements that we read from the file. List<ShapeHeader> result = new List<ShapeHeader>(); // Use a the length of the file to dimension the byte array IO.BufferedBinaryReader bbReader = new IO.BufferedBinaryReader(shxFilename); if (bbReader.FileLength == 100) { // the file is empty, so we are done bbReader.Close(); return result; } // Skip the header and begin reading from the first record bbReader.Seek(100, System.IO.SeekOrigin.Begin); _header.ShxLength = (int)bbReader.FileLength / 2; long length = bbReader.FileLength - 100; long numRecords = length / 8; // Each record consists of 2 Big-endian integers for a total of 8 bytes. for (long I = 0; I < numRecords; I++) { ShapeHeader sh = new ShapeHeader(); sh.Offset = bbReader.ReadInt32(false); sh.ContentLength = bbReader.ReadInt32(false); result.Add(sh); } bbReader.Close(); return result; }
// X Y MultiPoints: Total Length = 28 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 8 Integer 1 Little // Byte 12 Xmin Double 1 Little // Byte 20 Ymin Double 1 Little // Byte 28 Xmax Double 1 Little // Byte 36 Ymax Double 1 Little // Byte 48 NumPoints Integer 1 Little // Byte X Points Point NumPoints Little // X Y M MultiPoints: Total Length = 34 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 28 Integer 1 Little // Byte 12 Box Double 4 Little // Byte 44 NumPoints Integer 1 Little // Byte X Points Point NumPoints Little // Byte Y* Mmin Double 1 Little // Byte Y + 8* Mmax Double 1 Little // Byte Y + 16* Marray Double NumPoints Little // X Y Z M MultiPoints: Total Length = 44 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 18 Integer 1 Little // Byte 12 Box Double 4 Little // Byte 44 NumPoints Integer 1 Little // Byte X Points Point NumPoints Little // Byte Y Zmin Double 1 Little // Byte Y + 8 Zmax Double 1 Little // Byte Y + 16 Zarray Double NumPoints Little // Byte Z* Mmin Double 1 Little // Byte Z+8* Mmax Double 1 Little // Byte Z+16* Marray Double NumPoints Little private void FillPoints(string filename, IProgressHandler progressHandler) { // Check to ensure the filename is not null if (filename == null) { throw new NullReferenceException(MessageStrings.ArgumentNull_S.Replace("%S", filename)); } if (File.Exists(filename) == false) { throw new FileNotFoundException(MessageStrings.FileNotFound_S.Replace("%S", filename)); } // Get the basic header information. ShapefileHeader header = new ShapefileHeader(filename); Extent = new Extent(new[] { header.Xmin, header.Ymin, header.Xmax, header.Ymax }); // Check to ensure that the filename is the correct shape type if (header.ShapeType != ShapeTypes.MultiPoint && header.ShapeType != ShapeTypes.MultiPointM && header.ShapeType != ShapeTypes.MultiPointZ) { throw new ArgumentException(MessageStrings.FileNotLines_S.Replace("%S", filename)); } // Reading the headers gives us an easier way to track the number of shapes and their overall length etc. List<ShapeHeader> shapeHeaders = ReadIndexFile(filename); // This will set up a reader so that we can read values in huge chunks, which is much faster than one value at a time. IO.BufferedBinaryReader bbReader = new IO.BufferedBinaryReader(filename, progressHandler); if (bbReader.FileLength == 100) { // The shapefile is empty so we can simply return here bbReader.Close(); return; } // Skip the shapefile header by skipping the first 100 bytes in the shapefile bbReader.Seek(100, SeekOrigin.Begin); int numShapes = shapeHeaders.Count; byte[] bigEndians = new byte[numShapes * 8]; byte[] allBounds = new byte[numShapes * 32]; ByteBlock allCoords = new ByteBlock(BLOCKSIZE); bool isM = (header.ShapeType == ShapeTypes.MultiPointZ || header.ShapeType == ShapeTypes.MultiPointM); bool isZ = (header.ShapeType == ShapeTypes.PolyLineZ); ByteBlock allZ = null; ByteBlock allM = null; if (isZ) { allZ = new ByteBlock(BLOCKSIZE); } if (isM) { allM = new ByteBlock(BLOCKSIZE); } int pointOffset = 0; for (int shp = 0; shp < numShapes; shp++) { // Read from the index file because some deleted records // might still exist in the .shp file. long offset = (shapeHeaders[shp].ByteOffset); bbReader.Seek(offset, SeekOrigin.Begin); // time: 200 ms ShapeRange shape = new ShapeRange(FeatureTypes.MultiPoint); shape.RecordNumber = bbReader.ReadInt32(false); // Byte 0 Record Number Integer 1 Big shape.ContentLength = bbReader.ReadInt32(false); // Byte 4 Content Length Integer 1 Big //bbReader.Read(bigEndians, shp * 8, 8); shape.ShapeType = (ShapeTypes)bbReader.ReadInt32(); shape.StartIndex = pointOffset; if (shape.ShapeType == ShapeTypes.NullShape) { continue; } bbReader.Read(allBounds, shp * 32, 32); shape.NumParts = 1; shape.NumPoints = bbReader.ReadInt32(); allCoords.Read(shape.NumPoints * 16, bbReader); pointOffset += shape.NumPoints; if (header.ShapeType == ShapeTypes.MultiPointM) { // These are listed as "optional" but there isn't a good indicator of how to determine if they were added. // To handle the "optional" M values, check the contentLength for the feature. // The content length does not include the 8-byte record header and is listed in 16-bit words. if (shape.ContentLength * 2 > 44 + 4 * shape.NumParts + 16 * shape.NumPoints) { double mMin = bbReader.ReadDouble(); double mMax = bbReader.ReadDouble(); //bbReader.Seek(16, SeekOrigin.Current); if (allM != null) allM.Read(shape.NumPoints * 8, bbReader); } } if (header.ShapeType == ShapeTypes.MultiPointZ) { bool hasM = shape.ContentLength * 2 > 60 + 4 * shape.NumParts + 24 * shape.NumPoints; double zMin = bbReader.ReadDouble(); double zMax = bbReader.ReadDouble(); // For Z shapefiles, the Z part is not optional. if (allZ != null) allZ.Read(shape.NumPoints * 8, bbReader); // These are listed as "optional" but there isn't a good indicator of how to determine if they were added. // To handle the "optional" M values, check the contentLength for the feature. // The content length does not include the 8-byte record header and is listed in 16-bit words. if (hasM) { double mMin = bbReader.ReadDouble(); double mMax = bbReader.ReadDouble(); if (allM != null) allM.Read(shape.NumPoints * 8, bbReader); } } // Now that we have read all the values, create the geometries from the points and parts arrays. ShapeIndices.Add(shape); } double[] vert = allCoords.ToDoubleArray(); Vertex = vert; if (isM) M = allM.ToDoubleArray(); if (isZ) Z = allZ.ToDoubleArray(); Array.Reverse(bigEndians); List<ShapeRange> shapes = ShapeIndices; double[] bounds = new double[numShapes * 4]; Buffer.BlockCopy(allBounds, 0, bounds, 0, allBounds.Length); for (int shp = 0; shp < numShapes; shp++) { ShapeRange shape = shapes[shp]; shape.Extent = new Extent(bounds, shp * 4); int endIndex = shape.NumPoints + shape.StartIndex; int startIndex = shape.StartIndex; int count = endIndex - startIndex; PartRange partR = new PartRange(vert, shape.StartIndex, 0, FeatureTypes.MultiPoint); partR.NumVertices = count; shape.Parts.Add(partR); } GC.Collect(); bbReader.Dispose(); }
// X Y Poly Lines: Total Length = 28 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 3 Integer 1 Little // Byte 12 Xmin Double 1 Little // Byte 20 Ymin Double 1 Little // Byte 28 Xmax Double 1 Little // Byte 36 Ymax Double 1 Little // Byte 44 NumParts Integer 1 Little // Byte 48 NumPoints Integer 1 Little // Byte 52 Parts Integer NumParts Little // Byte X Points Point NumPoints Little // X Y M Poly Lines: Total Length = 34 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 23 Integer 1 Little // Byte 12 Box Double 4 Little // Byte 44 NumParts Integer 1 Little // Byte 48 NumPoints Integer 1 Little // Byte 52 Parts Integer NumParts Little // Byte X Points Point NumPoints Little // Byte Y* Mmin Double 1 Little // Byte Y + 8* Mmax Double 1 Little // Byte Y + 16* Marray Double NumPoints Little // X Y Z M Poly Lines: Total Length = 44 Bytes // --------------------------------------------------------- // Position Value Type Number Byte Order // --------------------------------------------------------- // Byte 0 Record Number Integer 1 Big // Byte 4 Content Length Integer 1 Big // Byte 8 Shape Type 13 Integer 1 Little // Byte 12 Box Double 4 Little // Byte 44 NumParts Integer 1 Little // Byte 48 NumPoints Integer 1 Little // Byte 52 Parts Integer NumParts Little // Byte X Points Point NumPoints Little // Byte Y Zmin Double 1 Little // Byte Y + 8 Zmax Double 1 Little // Byte Y + 16 Zarray Double NumPoints Little // Byte Z* Mmin Double 1 Little // Byte Z+8* Mmax Double 1 Little // Byte Z+16* Marray Double NumPoints Little private void FillPolygons(string filename, IProgressHandler progressHandler) { // Check to ensure the filename is not null if (filename == null) { throw new NullReferenceException(MessageStrings.ArgumentNull_S.Replace("%S", filename)); } if (File.Exists(filename) == false) { throw new FileNotFoundException(MessageStrings.FileNotFound_S.Replace("%S", filename)); } // Get the basic header information. ShapefileHeader header = new ShapefileHeader(filename); Extent = new Extent(new[]{header.Xmin, header.Ymin, header.Xmax, header.Ymax}); // Check to ensure that the filename is the correct shape type if (header.ShapeType != ShapeTypes.Polygon && header.ShapeType != ShapeTypes.PolygonM && header.ShapeType != ShapeTypes.PolygonZ) { throw new ArgumentException(MessageStrings.FileNotLines_S.Replace("%S", filename)); } // Reading the headers gives us an easier way to track the number of shapes and their overall length etc. List<ShapeHeader> shapeHeaders = ReadIndexFile(filename); // This will set up a reader so that we can read values in huge chunks, which is much faster than one value at a time. IO.BufferedBinaryReader bbReader = new IO.BufferedBinaryReader(filename, progressHandler); if (bbReader.FileLength == 100) { // The shapefile is empty so we can simply return here bbReader.Close(); return; } // Skip the shapefile header by skipping the first 100 bytes in the shapefile bbReader.Seek(100, SeekOrigin.Begin); int numShapes = shapeHeaders.Count; int [] partOffsets = new int[numShapes]; byte[] bigEndians = new byte[numShapes * 8]; byte[] allBounds = new byte[numShapes * 32]; ByteBlock allParts = new ByteBlock(BLOCKSIZE); // probably all will be in one block, but use a byteBlock just in case. ByteBlock allCoords = new ByteBlock(BLOCKSIZE); bool isM = (header.ShapeType == ShapeTypes.PolyLineM || header.ShapeType == ShapeTypes.PolyLineZ); bool isZ = (header.ShapeType == ShapeTypes.PolyLineZ); ByteBlock allZ = null; ByteBlock allM = null; if (isZ) { allZ = new ByteBlock(BLOCKSIZE); } if (isM) { allM = new ByteBlock(BLOCKSIZE); } int pointOffset = 0; for (int shp = 0; shp < numShapes; shp++) { // Read from the index file because some deleted records // might still exist in the .shp file. long offset = (shapeHeaders[shp].ByteOffset); bbReader.Seek(offset, SeekOrigin.Begin); // Position Value Type Number Byte Order ShapeRange shape = new ShapeRange(FeatureTypes.Polygon); //-------------------------------------------------------------------- shape.RecordNumber = bbReader.ReadInt32(false); // Byte 0 Record Number Integer 1 Big shape.ContentLength = bbReader.ReadInt32(false); // Byte 4 Content Length Integer 1 Big shape.ShapeType = (ShapeTypes)bbReader.ReadInt32(); // Byte 8 Shape Type Integer 1 Little shape.StartIndex = pointOffset; if (shape.ShapeType == ShapeTypes.NullShape) { continue; } bbReader.Read(allBounds, shp*32, 32); //double xMin = bbReader.ReadDouble(); // Byte 12 Xmin Double 1 Little // double yMin = bbReader.ReadDouble(); // Byte 20 Ymin Double 1 Little //double xMax = bbReader.ReadDouble(); // Byte 28 Xmax Double 1 Little //double yMax = bbReader.ReadDouble(); // Byte 36 Ymax Double 1 Little shape.NumParts = bbReader.ReadInt32(); // Byte 44 NumParts Integer 1 Little //feature.NumPoints = bbReader.ReadInt32(); // Byte 48 NumPoints Integer 1 Little shape.NumPoints = bbReader.ReadInt32(); // Create an envelope from the extents box in the file. //feature.Envelope = new Envelope(xMin, xMax, yMin, yMax); partOffsets[shp] = allParts.IntOffset(); allParts.Read(shape.NumParts * 4, bbReader); allCoords.Read(shape.NumPoints * 16, bbReader); pointOffset += shape.NumPoints; if (header.ShapeType == ShapeTypes.PolygonM) { // These are listed as "optional" but there isn't a good indicator of how to determine if they were added. // To handle the "optional" M values, check the contentLength for the feature. // The content length does not include the 8-byte record header and is listed in 16-bit words. if (shape.ContentLength * 2 > 44 + 4 * shape.NumParts + 16 * shape.NumPoints) { double mMin = bbReader.ReadDouble(); double mMax = bbReader.ReadDouble(); if(allM != null)allM.Read(shape.NumPoints * 8, bbReader); } } if (header.ShapeType == ShapeTypes.PolygonZ) { bool hasM = shape.ContentLength * 2 > 60 + 4 * shape.NumParts + 24 * shape.NumPoints; double zMin = bbReader.ReadDouble(); double zMax = bbReader.ReadDouble(); // For Z shapefiles, the Z part is not optional. if (allZ != null) allZ.Read(shape.NumPoints * 8, bbReader); // These are listed as "optional" but there isn't a good indicator of how to determine if they were added. // To handle the "optional" M values, check the contentLength for the feature. // The content length does not include the 8-byte record header and is listed in 16-bit words. if (hasM) { double mMin = bbReader.ReadDouble(); double mMax = bbReader.ReadDouble(); if (allM != null) allM.Read(shape.NumPoints * 8, bbReader); } } ShapeIndices.Add(shape); } double[] vert = allCoords.ToDoubleArray(); Vertex = vert; if (isM) M = allM.ToDoubleArray(); if (isZ) Z = allZ.ToDoubleArray(); List<ShapeRange> shapes = ShapeIndices; double[] bounds = new double[numShapes * 4]; Buffer.BlockCopy(allBounds, 0, bounds, 0, allBounds.Length); int[] parts = allParts.ToIntArray(); ProgressMeter = new ProgressMeter(ProgressHandler, "Testing Parts and Holes", numShapes); for (int shp = 0; shp < numShapes; shp++) { ShapeRange shape = shapes[shp]; shape.Extent = new Extent(bounds, shp * 4); for (int part = 0; part < shape.NumParts; part++) { int offset = partOffsets[shp]; int endIndex = shape.NumPoints + shape.StartIndex; int startIndex = parts[offset + part] + shape.StartIndex; if (part < shape.NumParts - 1) endIndex = parts[offset + part + 1] + shape.StartIndex; int count = endIndex - startIndex; PartRange partR = new PartRange(vert, shape.StartIndex, parts[offset + part], FeatureTypes.Polygon); partR.NumVertices = count; shape.Parts.Add(partR); } ProgressMeter.CurrentValue = shp; } ProgressMeter.Reset(); GC.Collect(); }