// public methods -------------------------------------------------------- /// <summary> /// <p> /// ICU data header reader method. Takes a ICU generated big-endian input /// stream, parse the ICU standard file header and authenticates them. /// </p> /// <p> /// Header format: /// <ul> /// <li>Header size (char) /// <li>Magic number 1 (byte) /// <li>Magic number 2 (byte) /// <li>Rest of the header size (char) /// <li>Reserved word (char) /// <li>Big endian indicator (byte) /// <li>Character set family indicator (byte) /// <li>Size of a char (byte) for c++ and c use /// <li>Reserved byte (byte) /// <li>Data format identifier (4 bytes), each ICU data has its own /// identifier to distinguish them. [0] major [1] minor [2] milli [3] micro /// <li>Data version (4 bytes), the change version of the ICU data [0] major /// [1] minor [2] milli [3] micro /// <li>Unicode version (4 bytes) this ICU is based on. /// </ul> /// </p> /// <p> /// Example of use:<br> /// <pre> /// try { /// FileInputStream input = new FileInputStream(filename); /// If (Utility.readICUDataHeader(input, dataformat, dataversion, /// unicode) { /// System.out.println("Verified file header, this is a ICU data file"); /// } /// } catch (IOException e) { /// System.out.println("This is not a ICU data file"); /// } /// </pre> /// </p> /// </summary> /// /// <param name="inputStream">input stream that contains the ICU data header</param> /// <param name="dataFormatIDExpected">Data format expected. An array of 4 bytes information aboutthe data format. E.g. data format ID 1.2.3.4. will became anarray of {1, 2, 3, 4}</param> /// <param name="authenticate">user defined extra data authentication. This value can benull, if no extra authentication is needed.</param> /// <exception cref="IOException">thrown if there is a read error or when headerauthentication fails.</exception> /// @draft 2.1 public static byte[] ReadHeader(Stream inputStream, byte[] dataFormatIDExpected, ICUBinary.Authenticate authenticate) { DataInputStream input = new DataInputStream(inputStream); char headersize = input.ReadChar(); int readcount = 2; // reading the header format byte magic1 = (byte)input.ReadByte(); readcount++; byte magic2 = (byte)input.ReadByte(); readcount++; if (magic1 != MAGIC1 || magic2 != MAGIC2) { throw new IOException(MAGIC_NUMBER_AUTHENTICATION_FAILED_); } input.ReadChar(); // reading size readcount += 2; input.ReadChar(); // reading reserved word readcount += 2; sbyte bigendian = input.ReadByte(); readcount++; sbyte charset = input.ReadByte(); readcount++; sbyte charsize = input.ReadByte(); readcount++; input.ReadByte(); // reading reserved byte readcount++; byte[] dataFormatID = new byte[4]; input.ReadFully(dataFormatID); readcount += 4; byte[] dataVersion = new byte[4]; input.ReadFully(dataVersion); readcount += 4; byte[] unicodeVersion = new byte[4]; input.ReadFully(unicodeVersion); readcount += 4; if (headersize < readcount) { throw new IOException("Internal Error: Header size error"); } input.SkipBytes(headersize - readcount); if (bigendian != BIG_ENDIAN_ || charset != CHAR_SET_ || charsize != CHAR_SIZE_ || !ILOG.J2CsMapping.Collections.Arrays.Equals(dataFormatIDExpected, dataFormatID) || (authenticate != null && !authenticate .IsDataVersionAcceptable(dataVersion))) { throw new IOException(HEADER_AUTHENTICATION_FAILED_); } return(unicodeVersion); }
/// <exception cref="System.IO.IOException"/> protected internal virtual void ReadObject(Writable obj, DataInputStream inStream ) { int numBytes = WritableUtils.ReadVInt(inStream); byte[] buffer; // For BytesWritable and Text, use the specified length to set the length // this causes the "obvious" translations to work. So that if you emit // a string "abc" from C++, it shows up as "abc". if (obj is BytesWritable) { buffer = new byte[numBytes]; inStream.ReadFully(buffer); ((BytesWritable)obj).Set(buffer, 0, numBytes); } else { if (obj is Text) { buffer = new byte[numBytes]; inStream.ReadFully(buffer); ((Text)obj).Set(buffer); } else { obj.ReadFields(inStream); } } }
/// <summary> /// Read more data and get them processed <br /> /// Entry condition: ostart = ofinish <br /> /// Exit condition: ostart <= ofinish <br> /// return (ofinish-ostart) (we have this many bytes for you), 0 (no data now, /// but could have more later), or -1 (absolutely no more data) /// </summary> /// <exception cref="System.IO.IOException"/> private int ReadMoreData() { try { inStream.ReadFully(lengthBuf); int length = UnsignedBytesToInt(lengthBuf); if (Log.IsDebugEnabled()) { Log.Debug("Actual length is " + length); } saslToken = new byte[length]; inStream.ReadFully(saslToken); } catch (EOFException) { return(-1); } try { if (saslServer != null) { // using saslServer obuffer = saslServer.Unwrap(saslToken, 0, saslToken.Length); } else { // using saslClient obuffer = saslClient.Unwrap(saslToken, 0, saslToken.Length); } } catch (SaslException se) { try { DisposeSasl(); } catch (SaslException) { } throw; } ostart = 0; if (obuffer == null) { ofinish = 0; } else { ofinish = obuffer.Length; } return(ofinish); }
/// <summary> /// Reads an automaton from a byte stream. /// </summary> /// <param name="stream">The stream to read.</param> internal CFSA2(Stream stream) { using (DataInputStream input = new DataInputStream(stream, true)) { // Read flags. ushort flagBits = (ushort)input.ReadInt16(); flags = 0; flags = (FSAFlags)flagBits; if (flagBits != (ushort)flags) { throw new IOException($"Unrecognized flags: 0x{((int)flagBits).ToHexString()}"); } this.hasNumbers = (flags & FSAFlags.Numbers) != 0; /* * Read mapping dictionary. */ int labelMappingSize = input.ReadByte() & 0xff; LabelMapping = new byte[labelMappingSize]; input.ReadFully(LabelMapping); /* * Read arcs' data. */ Arcs = ReadRemaining(input); } }
/// <summary>Read checksum into given buffer</summary> /// <param name="buf">buffer to read the checksum into</param> /// <param name="checksumOffset">offset at which to write the checksum into buf</param> /// <param name="checksumLen">length of checksum to write</param> /// <exception cref="System.IO.IOException">on error</exception> private void ReadChecksum(byte[] buf, int checksumOffset, int checksumLen) { if (checksumSize <= 0 && checksumIn == null) { return; } try { checksumIn.ReadFully(buf, checksumOffset, checksumLen); } catch (IOException e) { Log.Warn(" Could not read or failed to veirfy checksum for data" + " at offset " + offset + " for block " + block, e); IOUtils.CloseStream(checksumIn); checksumIn = null; if (corruptChecksumOk) { if (checksumOffset < checksumLen) { // Just fill the array with zeros. Arrays.Fill(buf, checksumOffset, checksumLen, unchecked ((byte)0)); } } else { throw; } } }
private void ReadData(Stream stream) { DataInputStream ds = new DataInputStream(stream); /* if (DEBUG) * System.Console.Out.WriteLine("The DataInputStream class is: " + ds.GetType().FullName); + if (DEBUG) + System.Console.Out + .WriteLine("The available bytes in the stream before reading the data: " + ds.Available()); */ /* * The following will read two integers before ds.mark(). Later, the two * integers need to be placed into data[], then ds.reset(), then * ds.readFully(into rest of data[]). * * This is necessary because we don't know the readLimit for ds.mark() * until we have read the second integer (indexLength). */ rootRes = ds.ReadInt(); // read the variable-length indexes[] array int indexLength = ds.ReadInt(); ds.Mark((indexLength - 1) * 4); indexes = new int[indexLength]; indexes[URES_INDEX_LENGTH] = indexLength; for (int i = 1; i < indexLength; i++) { indexes[i] = ds.ReadInt(); } // determine if this resource bundle falls back to a parent bundle // along normal locale ID fallback noFallback = indexLength > URES_INDEX_ATTRIBUTES && (indexes[URES_INDEX_ATTRIBUTES] & URES_ATT_NO_FALLBACK) != 0; // read the entire bundle (after the header) into data[] // put rootRes and indexLength into data[0..7] // and the rest of the data into data[8..length-1] int length = indexes[URES_INDEX_BUNDLE_TOP] * 4; if (DEBUG) { System.Console.Out.WriteLine("The number of bytes in the bundle: " + length); } data = new sbyte[length]; WriteInt(rootRes, data, 0); WriteInt(indexLength, data, 4); // now reset to the mark, which was set after reading rootRes and // indexLength ds.Reset(); ds.ReadFully(data, 8, length - 8); }
// read a long value from the scanner /// <exception cref="System.IO.IOException"/> public virtual byte[] ReadLongValue(TFile.Reader.Scanner scanner, int len) { DataInputStream din = scanner.Entry().GetValueStream(); byte[] b = new byte[len]; din.ReadFully(b); din.Close(); return(b); }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInputStream @in) { this.packetLen = @in.ReadInt(); short protoLen = @in.ReadShort(); byte[] data = new byte[protoLen]; @in.ReadFully(data); proto = DataTransferProtos.PacketHeaderProto.ParseFrom(data); }
/// <exception cref="System.IO.IOException"/> private void SendRecvData(string testDescription, bool eofExpected) { /* Opens a socket to datanode * sends the data in sendBuf. * If there is data in expectedBuf, expects to receive the data * from datanode that matches expectedBuf. * If there is an exception while recieving, throws it * only if exceptionExcepted is false. */ Socket sock = null; try { if (testDescription != null) { Log.Info("Testing : " + testDescription); } Log.Info("Going to write:" + StringUtils.ByteToHexString(sendBuf.ToByteArray())); sock = new Socket(); sock.Connect(dnAddr, HdfsServerConstants.ReadTimeout); sock.ReceiveTimeout = HdfsServerConstants.ReadTimeout; OutputStream @out = sock.GetOutputStream(); // Should we excuse byte[] retBuf = new byte[recvBuf.Size()]; DataInputStream @in = new DataInputStream(sock.GetInputStream()); @out.Write(sendBuf.ToByteArray()); @out.Flush(); try { @in.ReadFully(retBuf); } catch (EOFException eof) { if (eofExpected) { Log.Info("Got EOF as expected."); return; } throw; } string received = StringUtils.ByteToHexString(retBuf); string expected = StringUtils.ByteToHexString(recvBuf.ToByteArray()); Log.Info("Received: " + received); Log.Info("Expected: " + expected); if (eofExpected) { throw new IOException("Did not recieve IOException when an exception " + "is expected while reading from " + datanode); } NUnit.Framework.Assert.AreEqual(expected, received); } finally { IOUtils.CloseSocket(sock); } }
/// <summary> /// Loads the rules from a DateInputStream, often in a jar file. /// </summary> /// <param name="dis"> the DateInputStream to load, not null </param> /// <exception cref="Exception"> if an error occurs </exception> //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private void load(java.io.DataInputStream dis) throws Exception private void Load(DataInputStream dis) { if (dis.ReadByte() != 1) { throw new StreamCorruptedException("File format not recognised"); } // group String groupId = dis.ReadUTF(); if ("TZDB".Equals(groupId) == false) { throw new StreamCorruptedException("File format not recognised"); } // versions int versionCount = dis.ReadShort(); for (int i = 0; i < versionCount; i++) { VersionId = dis.ReadUTF(); } // regions int regionCount = dis.ReadShort(); String[] regionArray = new String[regionCount]; for (int i = 0; i < regionCount; i++) { regionArray[i] = dis.ReadUTF(); } RegionIds = Arrays.AsList(regionArray); // rules int ruleCount = dis.ReadShort(); Object[] ruleArray = new Object[ruleCount]; for (int i = 0; i < ruleCount; i++) { sbyte[] bytes = new sbyte[dis.ReadShort()]; dis.ReadFully(bytes); ruleArray[i] = bytes; } // link version-region-rules for (int i = 0; i < versionCount; i++) { int versionRegionCount = dis.ReadShort(); RegionToRules.Clear(); for (int j = 0; j < versionRegionCount; j++) { String region = regionArray[dis.ReadShort()]; Object rule = ruleArray[dis.ReadShort() & 0xffff]; RegionToRules[region] = rule; } } }
/// <exception cref="System.IO.IOException"/> private void ReadNumMetablocks(TFile.Reader reader, int n) { int len = Runtime.GetBytesForString(("something to test" + 0)).Length; for (int i = 0; i < n; i++) { DataInputStream din = reader.GetMetaBlock("TfileMeta" + i); byte[] b = new byte[len]; din.ReadFully(b); Assert.True("faield to match metadata", Arrays.Equals(Runtime.GetBytesForString (("something to test" + i)), b)); din.Close(); } }
// private methods --------------------------------------------------- /// <summary> /// Reads an individual record of AlgorithmNames /// </summary> /// /// <returns>an instance of AlgorithNames if read is successful otherwise null</returns> /// <exception cref="IOException">thrown when file read error occurs or data is corrupted</exception> private UCharacterName.AlgorithmName ReadAlg() { UCharacterName.AlgorithmName result = new UCharacterName.AlgorithmName(); int rangestart = m_dataInputStream_.ReadInt(); int rangeend = m_dataInputStream_.ReadInt(); sbyte type = m_dataInputStream_.ReadByte(); sbyte variant = m_dataInputStream_.ReadByte(); if (!result.SetInfo(rangestart, rangeend, type, variant)) { return(null); } int size = m_dataInputStream_.ReadChar(); if (type == IBM.ICU.Impl.UCharacterName.AlgorithmName.TYPE_1_) { char[] factor = new char[variant]; for (int j = 0; j < variant; j++) { factor[j] = m_dataInputStream_.ReadChar(); } result.SetFactor(factor); size -= (variant << 1); } StringBuilder prefix = new StringBuilder(); char c = (char)(m_dataInputStream_.ReadByte() & 0x00FF); while (c != 0) { prefix.Append(c); c = (char)(m_dataInputStream_.ReadByte() & 0x00FF); } result.SetPrefix(prefix.ToString()); size -= (ALG_INFO_SIZE_ + prefix.Length + 1); if (size > 0) { byte[] str0 = new byte[size]; m_dataInputStream_.ReadFully(str0); result.SetFactorString(str0); } return(result); }
/// <summary> /// Convenience method for reading a token storage file directly from a /// datainputstream /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void ReadTokenStorageStream(DataInputStream @in) { byte[] magic = new byte[TokenStorageMagic.Length]; @in.ReadFully(magic); if (!Arrays.Equals(magic, TokenStorageMagic)) { throw new IOException("Bad header found in token storage."); } byte version = @in.ReadByte(); if (version != TokenStorageVersion) { throw new IOException("Unknown version " + version + " in token storage."); } ReadFields(@in); }
/// <exception cref="System.IO.IOException"/> public virtual void Map(Text key, LongWritable value, OutputCollector <Text, LongWritable > collector, Reporter reporter) { string name = key.ToString(); long size = value.Get(); long seed = long.Parse(name); random.SetSeed(seed); reporter.SetStatus("opening " + name); DataInputStream @in = new DataInputStream(fs.Open(new Path(DataDir, name))); long read = 0; try { while (read < size) { long remains = size - read; int n = (remains <= buffer.Length) ? (int)remains : buffer.Length; @in.ReadFully(buffer, 0, n); read += n; if (fastCheck) { Arrays.Fill(check, unchecked ((byte)random.Next(byte.MaxValue))); } else { random.NextBytes(check); } if (n != buffer.Length) { Arrays.Fill(buffer, n, buffer.Length, unchecked ((byte)0)); Arrays.Fill(check, n, check.Length, unchecked ((byte)0)); } NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, check)); reporter.SetStatus("reading " + name + "@" + read + "/" + size); } } finally { @in.Close(); } collector.Collect(new Text("bytes"), new LongWritable(read)); reporter.SetStatus("read " + name); }
// all messages must be RPC SASL wrapped, else an exception is thrown /// <exception cref="System.IO.IOException"/> private void ReadNextRpcPacket() { SaslRpcClient.Log.Debug("reading next wrapped RPC packet"); DataInputStream dis = new DataInputStream(this.@in); int rpcLen = dis.ReadInt(); byte[] rpcBuf = new byte[rpcLen]; dis.ReadFully(rpcBuf); // decode the RPC header ByteArrayInputStream bis = new ByteArrayInputStream(rpcBuf); RpcHeaderProtos.RpcResponseHeaderProto.Builder headerBuilder = RpcHeaderProtos.RpcResponseHeaderProto .NewBuilder(); headerBuilder.MergeDelimitedFrom(bis); bool isWrapped = false; // Must be SASL wrapped, verify and decode. if (headerBuilder.GetCallId() == Server.AuthProtocol.Sasl.callId) { RpcHeaderProtos.RpcSaslProto.Builder saslMessage = RpcHeaderProtos.RpcSaslProto.NewBuilder (); saslMessage.MergeDelimitedFrom(bis); if (saslMessage.GetState() == RpcHeaderProtos.RpcSaslProto.SaslState.Wrap) { isWrapped = true; byte[] token = saslMessage.GetToken().ToByteArray(); if (SaslRpcClient.Log.IsDebugEnabled()) { SaslRpcClient.Log.Debug("unwrapping token of length:" + token.Length); } token = this._enclosing.saslClient.Unwrap(token, 0, token.Length); this.unwrappedRpcBuffer = ByteBuffer.Wrap(token); } } if (!isWrapped) { throw new SaslException("Server sent non-wrapped response"); } }
/// <summary>Reads a header from the given input stream</summary> /// <param name="in">input stream to read from</param> /// <returns>ReadInfo</returns> /// <exception cref="System.IO.IOException">if a read error occurs or EOF occurs</exception> /// <exception cref="BadFileException">if end of file occurs or the byte amount read is invalid /// </exception> /// <exception cref="Org.Apache.Hadoop.FS.Slive.BadFileException"/> internal virtual DataVerifier.ReadInfo ReadHeader(DataInputStream @in) { int headerLen = DataWriter.GetHeaderLength(); ByteBuffer headerBuf = ByteBuffer.Wrap(new byte[headerLen]); long elapsed = 0; { long startTime = Timer.Now(); @in.ReadFully(((byte[])headerBuf.Array())); elapsed += Timer.Elapsed(startTime); } headerBuf.Rewind(); long hashValue = headerBuf.GetLong(); long byteAvailable = headerBuf.GetLong(); if (byteAvailable < 0) { throw new BadFileException("Invalid negative amount " + byteAvailable + " determined for header data amount" ); } return(new DataVerifier.ReadInfo(byteAvailable, hashValue, elapsed, headerLen)); }
/// <summary> /// Creates a new automaton, reading it from a file in FSA format, version 5. /// </summary> internal CFSA(Stream stream) { using (DataInputStream input = new DataInputStream(stream, true)) { // Skip legacy header fields. input.ReadByte(); // filler input.ReadByte(); // annotation byte hgtl = (byte)input.ReadByte(); /* * Determine if the automaton was compiled with NUMBERS. If so, modify * ctl and goto fields accordingly. */ flags = FSAFlags.Flexible | FSAFlags.StopBit | FSAFlags.NextBit; if ((hgtl & 0xf0) != 0) { this.NodeDataLength = (hgtl.TripleShift(4)) & 0x0f; this.GoToLength = hgtl & 0x0f; flags |= FSAFlags.Numbers; } else { this.NodeDataLength = 0; this.GoToLength = hgtl & 0x0f; } /* * Read mapping dictionary. */ LabelMapping = new byte[1 << 5]; input.ReadFully(LabelMapping); /* * Read arcs' data. */ Arcs = ReadRemaining(input); } }
/// <summary> /// Verifies a given number of bytes from a file - less number of bytes may be /// read if a header can not be read in due to the byte limit /// </summary> /// <param name="byteAm"> /// the byte amount to limit to (should be less than or equal to file /// size) /// </param> /// <param name="bytesRead">the starting byte location</param> /// <param name="in">the input stream to read from</param> /// <returns>VerifyOutput with data about reads</returns> /// <exception cref="System.IO.IOException">if a read failure occurs</exception> /// <exception cref="BadFileException"> /// if a header can not be read or end of file is reached /// unexpectedly /// </exception> /// <exception cref="Org.Apache.Hadoop.FS.Slive.BadFileException"/> private DataVerifier.VerifyOutput VerifyBytes(long byteAm, long bytesRead, DataInputStream @in) { if (byteAm <= 0) { return(new DataVerifier.VerifyOutput(0, 0, 0, 0)); } long chunksSame = 0; long chunksDifferent = 0; long readTime = 0; long bytesLeft = byteAm; long bufLeft = 0; long bufRead = 0; long seqNum = 0; DataHasher hasher = null; ByteBuffer readBuf = ByteBuffer.Wrap(new byte[bufferSize]); while (bytesLeft > 0) { if (bufLeft <= 0) { if (bytesLeft < DataWriter.GetHeaderLength()) { // no bytes left to read a header break; } // time to read a new header DataVerifier.ReadInfo header = null; try { header = ReadHeader(@in); } catch (EOFException) { // eof ok on header reads // but not on data readers break; } ++seqNum; hasher = new DataHasher(header.GetHashValue()); bufLeft = header.GetByteAm(); readTime += header.GetTimeTaken(); bytesRead += header.GetBytesRead(); bytesLeft -= header.GetBytesRead(); bufRead = 0; // number of bytes to read greater than how many we want to read if (bufLeft > bytesLeft) { bufLeft = bytesLeft; } // does the buffer amount have anything?? if (bufLeft <= 0) { continue; } } // figure out the buffer size to read int bufSize = bufferSize; if (bytesLeft < bufSize) { bufSize = (int)bytesLeft; } if (bufLeft < bufSize) { bufSize = (int)bufLeft; } // read it in try { readBuf.Rewind(); long startTime = Timer.Now(); @in.ReadFully(((byte[])readBuf.Array()), 0, bufSize); readTime += Timer.Elapsed(startTime); } catch (EOFException e) { throw new BadFileException("Could not read the number of expected data bytes " + bufSize + " due to unexpected end of file during sequence " + seqNum, e); } // update the counters bytesRead += bufSize; bytesLeft -= bufSize; bufLeft -= bufSize; // verify what we read readBuf.Rewind(); // figure out the expected hash offset start point long vOffset = DetermineOffset(bufRead); // now update for new position bufRead += bufSize; // verify DataVerifier.VerifyInfo verifyRes = VerifyBuffer(readBuf, bufSize, vOffset, hasher ); // update the verification counters chunksSame += verifyRes.GetSame(); chunksDifferent += verifyRes.GetDifferent(); } return(new DataVerifier.VerifyOutput(chunksSame, chunksDifferent, bytesRead, readTime )); }
private static sbyte[] InitNamePool() { lock (typeof(CharacterName)) { sbyte[] strPool = null; if (RefStrPool != null && (strPool = RefStrPool.get()) != null) { return(strPool); } DataInputStream dis = null; try { dis = new DataInputStream(new InflaterInputStream(AccessController.doPrivileged(new PrivilegedActionAnonymousInnerClassHelper()))); Lookup = new int[(Character.MAX_CODE_POINT + 1) >> 8][]; int total = dis.ReadInt(); int cpEnd = dis.ReadInt(); sbyte[] ba = new sbyte[cpEnd]; dis.ReadFully(ba); int nameOff = 0; int cpOff = 0; int cp = 0; do { int len = ba[cpOff++] & 0xff; if (len == 0) { len = ba[cpOff++] & 0xff; // always big-endian cp = ((ba[cpOff++] & 0xff) << 16) | ((ba[cpOff++] & 0xff) << 8) | ((ba[cpOff++] & 0xff)); } else { cp++; } int hi = cp >> 8; if (Lookup[hi] == null) { Lookup[hi] = new int[0x100]; } Lookup[hi][cp & 0xff] = (nameOff << 8) | len; nameOff += len; } while (cpOff < cpEnd); strPool = new sbyte[total - cpEnd]; dis.ReadFully(strPool); RefStrPool = new SoftReference <>(strPool); } catch (Exception x) { throw new InternalError(x.Message, x); } finally { try { if (dis != null) { dis.Close(); } } catch (Exception) { } } return(strPool); } }
/// <summary>Find out the number of bytes in the block that match its crc.</summary> /// <remarks> /// Find out the number of bytes in the block that match its crc. /// This algorithm assumes that data corruption caused by unexpected /// datanode shutdown occurs only in the last crc chunk. So it checks /// only the last chunk. /// </remarks> /// <param name="blockFile">the block file</param> /// <param name="genStamp">generation stamp of the block</param> /// <returns>the number of valid bytes</returns> private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp) { DataInputStream checksumIn = null; InputStream blockIn = null; try { FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp); long blockFileLen = blockFile.Length(); long metaFileLen = metaFile.Length(); int crcHeaderLen = DataChecksum.GetChecksumHeaderSize(); if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen < crcHeaderLen) { return(0); } checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile ), HdfsConstants.IoFileBufferSize)); // read and handle the common header here. For now just a version DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile ); int bytesPerChecksum = checksum.GetBytesPerChecksum(); int checksumSize = checksum.GetChecksumSize(); long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum , (metaFileLen - crcHeaderLen) / checksumSize); if (numChunks == 0) { return(0); } IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize); blockIn = new FileInputStream(blockFile); long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum; IOUtils.SkipFully(blockIn, lastChunkStartPos); int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos ); byte[] buf = new byte[lastChunkSize + checksumSize]; checksumIn.ReadFully(buf, lastChunkSize, checksumSize); IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize); checksum.Update(buf, 0, lastChunkSize); long validFileLength; if (checksum.Compare(buf, lastChunkSize)) { // last chunk matches crc validFileLength = lastChunkStartPos + lastChunkSize; } else { // last chunck is corrupt validFileLength = lastChunkStartPos; } // truncate if extra bytes are present without CRC if (blockFile.Length() > validFileLength) { RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw"); try { // truncate blockFile blockRAF.SetLength(validFileLength); } finally { blockRAF.Close(); } } return(validFileLength); } catch (IOException e) { FsDatasetImpl.Log.Warn(e); return(0); } finally { IOUtils.CloseStream(checksumIn); IOUtils.CloseStream(blockIn); } }
// ---------------------------------------------------------------- // Constructor /// <summary> /// Constructs a UPropertyAliases object. The binary file DATA_FILE_NAME is /// read from the jar/classpath and unflattened into member variables of this /// object. /// </summary> /// public UPropertyAliases() { // Open the .icu file from the jar/classpath Stream mask0 = IBM.ICU.Impl.ICUData.GetRequiredStream(DATA_FILE_NAME); BufferedStream b = new BufferedStream(mask0, DATA_BUFFER_SIZE); // Read and discard Unicode version... /* byte unicodeVersion[] = */ IBM.ICU.Impl.ICUBinary.ReadHeader(b, DATA_FORMAT_ID, this); DataInputStream d = new DataInputStream(b); // Record the origin position of the file. Keep enough around // to seek back to the start of the header. d.Mark(256); short enumToName_offset = d.ReadShort(); short nameToEnum_offset = d.ReadShort(); short enumToValue_offset = d.ReadShort(); short total_size = d.ReadShort(); short valueMap_offset = d.ReadShort(); short valueMap_count = d.ReadShort(); short nameGroupPool_offset = d.ReadShort(); short nameGroupPool_count = d.ReadShort(); short stringPool_offset = d.ReadShort(); short stringPool_count = d.ReadShort(); if (DEBUG) { System.Console.Out.WriteLine("enumToName_offset=" + enumToName_offset + "\n" + "nameToEnum_offset=" + nameToEnum_offset + "\n" + "enumToValue_offset=" + enumToValue_offset + "\n" + "total_size=" + total_size + "\n" + "valueMap_offset=" + valueMap_offset + "\n" + "valueMap_count=" + valueMap_count + "\n" + "nameGroupPool_offset=" + nameGroupPool_offset + "\n" + "nameGroupPool_count=" + nameGroupPool_count + "\n" + "stringPool_offset=" + stringPool_offset + "\n" + "stringPool_count=" + stringPool_count); } byte[] raw = new byte[total_size]; d.Reset(); d.ReadFully(raw); d.Close(); UPropertyAliases.Builder builder = new UPropertyAliases.Builder(raw); stringPool = builder .ReadStringPool(stringPool_offset, stringPool_count); nameGroupPool = builder.ReadNameGroupPool(nameGroupPool_offset, nameGroupPool_count); builder.SetupValueMap_map(valueMap_offset, valueMap_count); // Some of the following data structures have to be set up // here, _not_ in Builder. That's because they are instances // of non-static inner classes, and they contain implicit // references to this. builder.Seek(enumToName_offset); enumToName = new UPropertyAliases.NonContiguousEnumToShort(builder); builder.NameGroupOffsetToIndex(enumToName.offsetArray); builder.Seek(nameToEnum_offset); nameToEnum = new UPropertyAliases.NameToEnum(this, builder); builder.Seek(enumToValue_offset); enumToValue = new UPropertyAliases.NonContiguousEnumToShort(builder); builder.ValueMapOffsetToIndex(enumToValue.offsetArray); valueMapArray = new UPropertyAliases.ValueMap [valueMap_count]; for (int i = 0; i < valueMap_count; ++i) { // Must seek to the start of each entry. builder.Seek(builder.valueMap_map[i]); valueMapArray[i] = new UPropertyAliases.ValueMap(this, builder); } builder.Close(); }
/// <exception cref="IOException"/> private static TypeAnnotation Parse(DataInputStream data, ConstantPool pool) { int targetType = data.ReadUnsignedByte(); int target = targetType << 24; switch (targetType) { case TypeAnnotation.Class_Type_Parameter: case TypeAnnotation.Method_Type_Parameter: case TypeAnnotation.Method_Parameter: { target |= data.ReadUnsignedByte(); break; } case TypeAnnotation.Super_Type_Reference: case TypeAnnotation.Class_Type_Parameter_Bound: case TypeAnnotation.Method_Type_Parameter_Bound: case TypeAnnotation.Throws_Reference: case TypeAnnotation.Catch_Clause: case TypeAnnotation.Expr_Instanceof: case TypeAnnotation.Expr_New: case TypeAnnotation.Expr_Constructor_Ref: case TypeAnnotation.Expr_Method_Ref: { target |= data.ReadUnsignedShort(); break; } case TypeAnnotation.Type_Arg_Cast: case TypeAnnotation.Type_Arg_Constructor_Call: case TypeAnnotation.Type_Arg_Method_Call: case TypeAnnotation.Type_Arg_Constructor_Ref: case TypeAnnotation.Type_Arg_Method_Ref: { data.SkipBytes(3); break; } case TypeAnnotation.Local_Variable: case TypeAnnotation.Resource_Variable: { data.SkipBytes(data.ReadUnsignedShort() * 6); break; } case TypeAnnotation.Field: case TypeAnnotation.Method_Return_Type: case TypeAnnotation.Method_Receiver: { break; } default: { throw new Exception("unknown target type: " + targetType); } } int pathLength = data.ReadUnsignedByte(); byte[] path = null; if (pathLength > 0) { path = new byte[2 * pathLength]; data.ReadFully(path); } AnnotationExprent annotation = StructAnnotationAttribute.ParseAnnotation(data, pool ); return(new TypeAnnotation(target, path, annotation)); }
public static void Main(String[] args) { int ident; int dirofs; int dirlen; int i; int numLumps; byte[] name = new byte[56]; string nameString; int filepos; int filelen; FileStream readLump; DataInputStream directory; string pakName; string pattern; if (args.length == 2) { if (!args[0].Equals("-list")) { Usage(); } pakName = args[1]; pattern = null; } else if (args.length == 3) { pakName = args[0]; pattern = args[1]; } else { pakName = null; pattern = null; Usage(); } try { directory = new DataInputStream(new FileInputStream(pakName)); readLump = new FileStream(pakName, "r"); ident = IntSwap(directory.ReadInt32()); dirofs = IntSwap(directory.ReadInt32()); dirlen = IntSwap(directory.ReadInt32()); if (ident != IDPAKHEADER) { System.Diagnostics.Debug.WriteLine(pakName + " is not a pakfile."); System.Exit(1); } directory.SkipBytes(dirofs - 12); numLumps = dirlen / 64; System.Diagnostics.Debug.WriteLine(numLumps + " lumps in " + pakName); for (i = 0; i < numLumps; i++) { directory.ReadFully(name); filepos = IntSwap(directory.ReadInt32()); filelen = IntSwap(directory.ReadInt32()); nameString = new string (name); nameString = nameString.Substring(0, nameString.IndexOf(0)); if (pattern == null) { System.Diagnostics.Debug.WriteLine(nameString + " : " + filelen + "bytes"); } else if (PatternMatch(pattern, nameString)) { File writeFile; DataOutputStream writeLump; byte[] buffer = new byte[filelen]; StringBuffer fixedString; string finalName; int index; System.Diagnostics.Debug.WriteLine("Unpaking " + nameString + " " + filelen + " bytes"); readLump.Seek(filepos); readLump.ReadFully(buffer); fixedString = new StringBuffer(args[2] + File.separator + nameString); for (index = 0; index < fixedString.Length; index++) { if (fixedString[index] == '/') { fixedString.Se[index, File.separatorChar]; } } finalName = fixedString.ToString(); index = finalName.LastIndexOf(File.separatorChar); if (index != -1) { string finalPath; File writePath; finalPath = finalName.Substring(0, index); writePath = new File(finalPath); writePath.Mkdirs(); } writeFile = new File(finalName); writeLump = new DataOutputStream(new FileOutputStream(writeFile)); writeLump.Write(buffer); writeLump.Close(); } } readLump.Close(); directory.Close(); } catch (IOException e) { System.Diagnostics.Debug.WriteLine(e.ToString()); } }