// public methods --------------------------------------------------------
/// <summary>
/// <p>
/// ICU data header reader method. Takes a ICU generated big-endian input
/// stream, parse the ICU standard file header and authenticates them.
/// </p>
/// <p>
/// Header format:
/// <ul>
/// <li>Header size (char)
/// <li>Magic number 1 (byte)
/// <li>Magic number 2 (byte)
/// <li>Rest of the header size (char)
/// <li>Reserved word (char)
/// <li>Big endian indicator (byte)
/// <li>Character set family indicator (byte)
/// <li>Size of a char (byte) for c++ and c use
/// <li>Reserved byte (byte)
/// <li>Data format identifier (4 bytes), each ICU data has its own
/// identifier to distinguish them. [0] major [1] minor [2] milli [3] micro
/// <li>Data version (4 bytes), the change version of the ICU data [0] major
/// [1] minor [2] milli [3] micro
/// <li>Unicode version (4 bytes) this ICU is based on.
/// </ul>
/// </p>
/// <p>
/// Example of use:<br>
/// <pre>
/// try {
/// FileInputStream input = new FileInputStream(filename);
/// If (Utility.readICUDataHeader(input, dataformat, dataversion,
/// unicode) {
/// System.out.println("Verified file header, this is a ICU data file");
/// }
/// } catch (IOException e) {
/// System.out.println("This is not a ICU data file");
/// }
/// </pre>
/// </p>
/// </summary>
///
/// <param name="inputStream">input stream that contains the ICU data header</param>
/// <param name="dataFormatIDExpected">Data format expected. An array of 4 bytes information aboutthe data format. E.g. data format ID 1.2.3.4. will became anarray of {1, 2, 3, 4}</param>
/// <param name="authenticate">user defined extra data authentication. This value can benull, if no extra authentication is needed.</param>
/// <exception cref="IOException">thrown if there is a read error or when headerauthentication fails.</exception>
/// @draft 2.1
public static byte[] ReadHeader(Stream inputStream,
byte[] dataFormatIDExpected, ICUBinary.Authenticate authenticate)
{
DataInputStream input = new DataInputStream(inputStream);
char headersize = input.ReadChar();
int readcount = 2;
// reading the header format
byte magic1 = (byte)input.ReadByte();
readcount++;
byte magic2 = (byte)input.ReadByte();
readcount++;
if (magic1 != MAGIC1 || magic2 != MAGIC2)
{
throw new IOException(MAGIC_NUMBER_AUTHENTICATION_FAILED_);
}
input.ReadChar(); // reading size
readcount += 2;
input.ReadChar(); // reading reserved word
readcount += 2;
sbyte bigendian = input.ReadByte();
readcount++;
sbyte charset = input.ReadByte();
readcount++;
sbyte charsize = input.ReadByte();
readcount++;
input.ReadByte(); // reading reserved byte
readcount++;
byte[] dataFormatID = new byte[4];
input.ReadFully(dataFormatID);
readcount += 4;
byte[] dataVersion = new byte[4];
input.ReadFully(dataVersion);
readcount += 4;
byte[] unicodeVersion = new byte[4];
input.ReadFully(unicodeVersion);
readcount += 4;
if (headersize < readcount)
{
throw new IOException("Internal Error: Header size error");
}
input.SkipBytes(headersize - readcount);
if (bigendian != BIG_ENDIAN_ ||
charset != CHAR_SET_ ||
charsize != CHAR_SIZE_ ||
!ILOG.J2CsMapping.Collections.Arrays.Equals(dataFormatIDExpected, dataFormatID) ||
(authenticate != null && !authenticate
.IsDataVersionAcceptable(dataVersion)))
{
throw new IOException(HEADER_AUTHENTICATION_FAILED_);
}
return(unicodeVersion);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void ReadObject(Writable obj, DataInputStream inStream
)
{
int numBytes = WritableUtils.ReadVInt(inStream);
byte[] buffer;
// For BytesWritable and Text, use the specified length to set the length
// this causes the "obvious" translations to work. So that if you emit
// a string "abc" from C++, it shows up as "abc".
if (obj is BytesWritable)
{
buffer = new byte[numBytes];
inStream.ReadFully(buffer);
((BytesWritable)obj).Set(buffer, 0, numBytes);
}
else
{
if (obj is Text)
{
buffer = new byte[numBytes];
inStream.ReadFully(buffer);
((Text)obj).Set(buffer);
}
else
{
obj.ReadFields(inStream);
}
}
}
/// <summary>
/// Read more data and get them processed <br />
/// Entry condition: ostart = ofinish <br />
/// Exit condition: ostart <= ofinish <br>
/// return (ofinish-ostart) (we have this many bytes for you), 0 (no data now,
/// but could have more later), or -1 (absolutely no more data)
/// </summary>
/// <exception cref="System.IO.IOException"/>
private int ReadMoreData()
{
try
{
inStream.ReadFully(lengthBuf);
int length = UnsignedBytesToInt(lengthBuf);
if (Log.IsDebugEnabled())
{
Log.Debug("Actual length is " + length);
}
saslToken = new byte[length];
inStream.ReadFully(saslToken);
}
catch (EOFException)
{
return(-1);
}
try
{
if (saslServer != null)
{
// using saslServer
obuffer = saslServer.Unwrap(saslToken, 0, saslToken.Length);
}
else
{
// using saslClient
obuffer = saslClient.Unwrap(saslToken, 0, saslToken.Length);
}
}
catch (SaslException se)
{
try
{
DisposeSasl();
}
catch (SaslException)
{
}
throw;
}
ostart = 0;
if (obuffer == null)
{
ofinish = 0;
}
else
{
ofinish = obuffer.Length;
}
return(ofinish);
}
/// <summary>
/// Reads an automaton from a byte stream.
/// </summary>
/// <param name="stream">The stream to read.</param>
internal CFSA2(Stream stream)
{
using (DataInputStream input = new DataInputStream(stream, true))
{
// Read flags.
ushort flagBits = (ushort)input.ReadInt16();
flags = 0;
flags = (FSAFlags)flagBits;
if (flagBits != (ushort)flags)
{
throw new IOException($"Unrecognized flags: 0x{((int)flagBits).ToHexString()}");
}
this.hasNumbers = (flags & FSAFlags.Numbers) != 0;
/*
* Read mapping dictionary.
*/
int labelMappingSize = input.ReadByte() & 0xff;
LabelMapping = new byte[labelMappingSize];
input.ReadFully(LabelMapping);
/*
* Read arcs' data.
*/
Arcs = ReadRemaining(input);
}
}
/// <summary>Read checksum into given buffer</summary>
/// <param name="buf">buffer to read the checksum into</param>
/// <param name="checksumOffset">offset at which to write the checksum into buf</param>
/// <param name="checksumLen">length of checksum to write</param>
/// <exception cref="System.IO.IOException">on error</exception>
private void ReadChecksum(byte[] buf, int checksumOffset, int checksumLen)
{
if (checksumSize <= 0 && checksumIn == null)
{
return;
}
try
{
checksumIn.ReadFully(buf, checksumOffset, checksumLen);
}
catch (IOException e)
{
Log.Warn(" Could not read or failed to veirfy checksum for data" + " at offset "
+ offset + " for block " + block, e);
IOUtils.CloseStream(checksumIn);
checksumIn = null;
if (corruptChecksumOk)
{
if (checksumOffset < checksumLen)
{
// Just fill the array with zeros.
Arrays.Fill(buf, checksumOffset, checksumLen, unchecked ((byte)0));
}
}
else
{
throw;
}
}
}
private void ReadData(Stream stream)
{
DataInputStream ds = new DataInputStream(stream);
/* if (DEBUG)
* System.Console.Out.WriteLine("The DataInputStream class is: "
+ ds.GetType().FullName);
+ if (DEBUG)
+ System.Console.Out
+ .WriteLine("The available bytes in the stream before reading the data: "
+ ds.Available());
*/
/*
* The following will read two integers before ds.mark(). Later, the two
* integers need to be placed into data[], then ds.reset(), then
* ds.readFully(into rest of data[]).
*
* This is necessary because we don't know the readLimit for ds.mark()
* until we have read the second integer (indexLength).
*/
rootRes = ds.ReadInt();
// read the variable-length indexes[] array
int indexLength = ds.ReadInt();
ds.Mark((indexLength - 1) * 4);
indexes = new int[indexLength];
indexes[URES_INDEX_LENGTH] = indexLength;
for (int i = 1; i < indexLength; i++)
{
indexes[i] = ds.ReadInt();
}
// determine if this resource bundle falls back to a parent bundle
// along normal locale ID fallback
noFallback = indexLength > URES_INDEX_ATTRIBUTES &&
(indexes[URES_INDEX_ATTRIBUTES] & URES_ATT_NO_FALLBACK) != 0;
// read the entire bundle (after the header) into data[]
// put rootRes and indexLength into data[0..7]
// and the rest of the data into data[8..length-1]
int length = indexes[URES_INDEX_BUNDLE_TOP] * 4;
if (DEBUG)
{
System.Console.Out.WriteLine("The number of bytes in the bundle: " + length);
}
data = new sbyte[length];
WriteInt(rootRes, data, 0);
WriteInt(indexLength, data, 4);
// now reset to the mark, which was set after reading rootRes and
// indexLength
ds.Reset();
ds.ReadFully(data, 8, length - 8);
}
// read a long value from the scanner
/// <exception cref="System.IO.IOException"/>
public virtual byte[] ReadLongValue(TFile.Reader.Scanner scanner, int len)
{
DataInputStream din = scanner.Entry().GetValueStream();
byte[] b = new byte[len];
din.ReadFully(b);
din.Close();
return(b);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInputStream @in)
{
this.packetLen = @in.ReadInt();
short protoLen = @in.ReadShort();
byte[] data = new byte[protoLen];
@in.ReadFully(data);
proto = DataTransferProtos.PacketHeaderProto.ParseFrom(data);
}
/// <exception cref="System.IO.IOException"/>
private void SendRecvData(string testDescription, bool eofExpected)
{
/* Opens a socket to datanode
* sends the data in sendBuf.
* If there is data in expectedBuf, expects to receive the data
* from datanode that matches expectedBuf.
* If there is an exception while recieving, throws it
* only if exceptionExcepted is false.
*/
Socket sock = null;
try
{
if (testDescription != null)
{
Log.Info("Testing : " + testDescription);
}
Log.Info("Going to write:" + StringUtils.ByteToHexString(sendBuf.ToByteArray()));
sock = new Socket();
sock.Connect(dnAddr, HdfsServerConstants.ReadTimeout);
sock.ReceiveTimeout = HdfsServerConstants.ReadTimeout;
OutputStream @out = sock.GetOutputStream();
// Should we excuse
byte[] retBuf = new byte[recvBuf.Size()];
DataInputStream @in = new DataInputStream(sock.GetInputStream());
@out.Write(sendBuf.ToByteArray());
@out.Flush();
try
{
@in.ReadFully(retBuf);
}
catch (EOFException eof)
{
if (eofExpected)
{
Log.Info("Got EOF as expected.");
return;
}
throw;
}
string received = StringUtils.ByteToHexString(retBuf);
string expected = StringUtils.ByteToHexString(recvBuf.ToByteArray());
Log.Info("Received: " + received);
Log.Info("Expected: " + expected);
if (eofExpected)
{
throw new IOException("Did not recieve IOException when an exception " + "is expected while reading from "
+ datanode);
}
NUnit.Framework.Assert.AreEqual(expected, received);
}
finally
{
IOUtils.CloseSocket(sock);
}
}
/// <summary>
/// Loads the rules from a DateInputStream, often in a jar file.
/// </summary>
/// <param name="dis"> the DateInputStream to load, not null </param>
/// <exception cref="Exception"> if an error occurs </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private void load(java.io.DataInputStream dis) throws Exception
private void Load(DataInputStream dis)
{
if (dis.ReadByte() != 1)
{
throw new StreamCorruptedException("File format not recognised");
}
// group
String groupId = dis.ReadUTF();
if ("TZDB".Equals(groupId) == false)
{
throw new StreamCorruptedException("File format not recognised");
}
// versions
int versionCount = dis.ReadShort();
for (int i = 0; i < versionCount; i++)
{
VersionId = dis.ReadUTF();
}
// regions
int regionCount = dis.ReadShort();
String[] regionArray = new String[regionCount];
for (int i = 0; i < regionCount; i++)
{
regionArray[i] = dis.ReadUTF();
}
RegionIds = Arrays.AsList(regionArray);
// rules
int ruleCount = dis.ReadShort();
Object[] ruleArray = new Object[ruleCount];
for (int i = 0; i < ruleCount; i++)
{
sbyte[] bytes = new sbyte[dis.ReadShort()];
dis.ReadFully(bytes);
ruleArray[i] = bytes;
}
// link version-region-rules
for (int i = 0; i < versionCount; i++)
{
int versionRegionCount = dis.ReadShort();
RegionToRules.Clear();
for (int j = 0; j < versionRegionCount; j++)
{
String region = regionArray[dis.ReadShort()];
Object rule = ruleArray[dis.ReadShort() & 0xffff];
RegionToRules[region] = rule;
}
}
}
/// <exception cref="System.IO.IOException"/>
private void ReadNumMetablocks(TFile.Reader reader, int n)
{
int len = Runtime.GetBytesForString(("something to test" + 0)).Length;
for (int i = 0; i < n; i++)
{
DataInputStream din = reader.GetMetaBlock("TfileMeta" + i);
byte[] b = new byte[len];
din.ReadFully(b);
Assert.True("faield to match metadata", Arrays.Equals(Runtime.GetBytesForString
(("something to test" + i)), b));
din.Close();
}
}
// private methods ---------------------------------------------------
/// <summary>
/// Reads an individual record of AlgorithmNames
/// </summary>
///
/// <returns>an instance of AlgorithNames if read is successful otherwise null</returns>
/// <exception cref="IOException">thrown when file read error occurs or data is corrupted</exception>
private UCharacterName.AlgorithmName ReadAlg()
{
UCharacterName.AlgorithmName result = new UCharacterName.AlgorithmName();
int rangestart = m_dataInputStream_.ReadInt();
int rangeend = m_dataInputStream_.ReadInt();
sbyte type = m_dataInputStream_.ReadByte();
sbyte variant = m_dataInputStream_.ReadByte();
if (!result.SetInfo(rangestart, rangeend, type, variant))
{
return(null);
}
int size = m_dataInputStream_.ReadChar();
if (type == IBM.ICU.Impl.UCharacterName.AlgorithmName.TYPE_1_)
{
char[] factor = new char[variant];
for (int j = 0; j < variant; j++)
{
factor[j] = m_dataInputStream_.ReadChar();
}
result.SetFactor(factor);
size -= (variant << 1);
}
StringBuilder prefix = new StringBuilder();
char c = (char)(m_dataInputStream_.ReadByte() & 0x00FF);
while (c != 0)
{
prefix.Append(c);
c = (char)(m_dataInputStream_.ReadByte() & 0x00FF);
}
result.SetPrefix(prefix.ToString());
size -= (ALG_INFO_SIZE_ + prefix.Length + 1);
if (size > 0)
{
byte[] str0 = new byte[size];
m_dataInputStream_.ReadFully(str0);
result.SetFactorString(str0);
}
return(result);
}
/// <summary>
/// Convenience method for reading a token storage file directly from a
/// datainputstream
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void ReadTokenStorageStream(DataInputStream @in)
{
byte[] magic = new byte[TokenStorageMagic.Length];
@in.ReadFully(magic);
if (!Arrays.Equals(magic, TokenStorageMagic))
{
throw new IOException("Bad header found in token storage.");
}
byte version = @in.ReadByte();
if (version != TokenStorageVersion)
{
throw new IOException("Unknown version " + version + " in token storage.");
}
ReadFields(@in);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(Text key, LongWritable value, OutputCollector <Text, LongWritable
> collector, Reporter reporter)
{
string name = key.ToString();
long size = value.Get();
long seed = long.Parse(name);
random.SetSeed(seed);
reporter.SetStatus("opening " + name);
DataInputStream @in = new DataInputStream(fs.Open(new Path(DataDir, name)));
long read = 0;
try
{
while (read < size)
{
long remains = size - read;
int n = (remains <= buffer.Length) ? (int)remains : buffer.Length;
@in.ReadFully(buffer, 0, n);
read += n;
if (fastCheck)
{
Arrays.Fill(check, unchecked ((byte)random.Next(byte.MaxValue)));
}
else
{
random.NextBytes(check);
}
if (n != buffer.Length)
{
Arrays.Fill(buffer, n, buffer.Length, unchecked ((byte)0));
Arrays.Fill(check, n, check.Length, unchecked ((byte)0));
}
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, check));
reporter.SetStatus("reading " + name + "@" + read + "/" + size);
}
}
finally
{
@in.Close();
}
collector.Collect(new Text("bytes"), new LongWritable(read));
reporter.SetStatus("read " + name);
}
// all messages must be RPC SASL wrapped, else an exception is thrown
/// <exception cref="System.IO.IOException"/>
private void ReadNextRpcPacket()
{
SaslRpcClient.Log.Debug("reading next wrapped RPC packet");
DataInputStream dis = new DataInputStream(this.@in);
int rpcLen = dis.ReadInt();
byte[] rpcBuf = new byte[rpcLen];
dis.ReadFully(rpcBuf);
// decode the RPC header
ByteArrayInputStream bis = new ByteArrayInputStream(rpcBuf);
RpcHeaderProtos.RpcResponseHeaderProto.Builder headerBuilder = RpcHeaderProtos.RpcResponseHeaderProto
.NewBuilder();
headerBuilder.MergeDelimitedFrom(bis);
bool isWrapped = false;
// Must be SASL wrapped, verify and decode.
if (headerBuilder.GetCallId() == Server.AuthProtocol.Sasl.callId)
{
RpcHeaderProtos.RpcSaslProto.Builder saslMessage = RpcHeaderProtos.RpcSaslProto.NewBuilder
();
saslMessage.MergeDelimitedFrom(bis);
if (saslMessage.GetState() == RpcHeaderProtos.RpcSaslProto.SaslState.Wrap)
{
isWrapped = true;
byte[] token = saslMessage.GetToken().ToByteArray();
if (SaslRpcClient.Log.IsDebugEnabled())
{
SaslRpcClient.Log.Debug("unwrapping token of length:" + token.Length);
}
token = this._enclosing.saslClient.Unwrap(token, 0, token.Length);
this.unwrappedRpcBuffer = ByteBuffer.Wrap(token);
}
}
if (!isWrapped)
{
throw new SaslException("Server sent non-wrapped response");
}
}
/// <summary>Reads a header from the given input stream</summary>
/// <param name="in">input stream to read from</param>
/// <returns>ReadInfo</returns>
/// <exception cref="System.IO.IOException">if a read error occurs or EOF occurs</exception>
/// <exception cref="BadFileException">if end of file occurs or the byte amount read is invalid
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.BadFileException"/>
internal virtual DataVerifier.ReadInfo ReadHeader(DataInputStream @in)
{
int headerLen = DataWriter.GetHeaderLength();
ByteBuffer headerBuf = ByteBuffer.Wrap(new byte[headerLen]);
long elapsed = 0;
{
long startTime = Timer.Now();
@in.ReadFully(((byte[])headerBuf.Array()));
elapsed += Timer.Elapsed(startTime);
}
headerBuf.Rewind();
long hashValue = headerBuf.GetLong();
long byteAvailable = headerBuf.GetLong();
if (byteAvailable < 0)
{
throw new BadFileException("Invalid negative amount " + byteAvailable + " determined for header data amount"
);
}
return(new DataVerifier.ReadInfo(byteAvailable, hashValue, elapsed, headerLen));
}
/// <summary>
/// Creates a new automaton, reading it from a file in FSA format, version 5.
/// </summary>
internal CFSA(Stream stream)
{
using (DataInputStream input = new DataInputStream(stream, true))
{
// Skip legacy header fields.
input.ReadByte(); // filler
input.ReadByte(); // annotation
byte hgtl = (byte)input.ReadByte();
/*
* Determine if the automaton was compiled with NUMBERS. If so, modify
* ctl and goto fields accordingly.
*/
flags = FSAFlags.Flexible | FSAFlags.StopBit | FSAFlags.NextBit;
if ((hgtl & 0xf0) != 0)
{
this.NodeDataLength = (hgtl.TripleShift(4)) & 0x0f;
this.GoToLength = hgtl & 0x0f;
flags |= FSAFlags.Numbers;
}
else
{
this.NodeDataLength = 0;
this.GoToLength = hgtl & 0x0f;
}
/*
* Read mapping dictionary.
*/
LabelMapping = new byte[1 << 5];
input.ReadFully(LabelMapping);
/*
* Read arcs' data.
*/
Arcs = ReadRemaining(input);
}
}
/// <summary>
/// Verifies a given number of bytes from a file - less number of bytes may be
/// read if a header can not be read in due to the byte limit
/// </summary>
/// <param name="byteAm">
/// the byte amount to limit to (should be less than or equal to file
/// size)
/// </param>
/// <param name="bytesRead">the starting byte location</param>
/// <param name="in">the input stream to read from</param>
/// <returns>VerifyOutput with data about reads</returns>
/// <exception cref="System.IO.IOException">if a read failure occurs</exception>
/// <exception cref="BadFileException">
/// if a header can not be read or end of file is reached
/// unexpectedly
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.BadFileException"/>
private DataVerifier.VerifyOutput VerifyBytes(long byteAm, long bytesRead, DataInputStream
@in)
{
if (byteAm <= 0)
{
return(new DataVerifier.VerifyOutput(0, 0, 0, 0));
}
long chunksSame = 0;
long chunksDifferent = 0;
long readTime = 0;
long bytesLeft = byteAm;
long bufLeft = 0;
long bufRead = 0;
long seqNum = 0;
DataHasher hasher = null;
ByteBuffer readBuf = ByteBuffer.Wrap(new byte[bufferSize]);
while (bytesLeft > 0)
{
if (bufLeft <= 0)
{
if (bytesLeft < DataWriter.GetHeaderLength())
{
// no bytes left to read a header
break;
}
// time to read a new header
DataVerifier.ReadInfo header = null;
try
{
header = ReadHeader(@in);
}
catch (EOFException)
{
// eof ok on header reads
// but not on data readers
break;
}
++seqNum;
hasher = new DataHasher(header.GetHashValue());
bufLeft = header.GetByteAm();
readTime += header.GetTimeTaken();
bytesRead += header.GetBytesRead();
bytesLeft -= header.GetBytesRead();
bufRead = 0;
// number of bytes to read greater than how many we want to read
if (bufLeft > bytesLeft)
{
bufLeft = bytesLeft;
}
// does the buffer amount have anything??
if (bufLeft <= 0)
{
continue;
}
}
// figure out the buffer size to read
int bufSize = bufferSize;
if (bytesLeft < bufSize)
{
bufSize = (int)bytesLeft;
}
if (bufLeft < bufSize)
{
bufSize = (int)bufLeft;
}
// read it in
try
{
readBuf.Rewind();
long startTime = Timer.Now();
@in.ReadFully(((byte[])readBuf.Array()), 0, bufSize);
readTime += Timer.Elapsed(startTime);
}
catch (EOFException e)
{
throw new BadFileException("Could not read the number of expected data bytes " +
bufSize + " due to unexpected end of file during sequence " + seqNum, e);
}
// update the counters
bytesRead += bufSize;
bytesLeft -= bufSize;
bufLeft -= bufSize;
// verify what we read
readBuf.Rewind();
// figure out the expected hash offset start point
long vOffset = DetermineOffset(bufRead);
// now update for new position
bufRead += bufSize;
// verify
DataVerifier.VerifyInfo verifyRes = VerifyBuffer(readBuf, bufSize, vOffset, hasher
);
// update the verification counters
chunksSame += verifyRes.GetSame();
chunksDifferent += verifyRes.GetDifferent();
}
return(new DataVerifier.VerifyOutput(chunksSame, chunksDifferent, bytesRead, readTime
));
}
private static sbyte[] InitNamePool()
{
lock (typeof(CharacterName))
{
sbyte[] strPool = null;
if (RefStrPool != null && (strPool = RefStrPool.get()) != null)
{
return(strPool);
}
DataInputStream dis = null;
try
{
dis = new DataInputStream(new InflaterInputStream(AccessController.doPrivileged(new PrivilegedActionAnonymousInnerClassHelper())));
Lookup = new int[(Character.MAX_CODE_POINT + 1) >> 8][];
int total = dis.ReadInt();
int cpEnd = dis.ReadInt();
sbyte[] ba = new sbyte[cpEnd];
dis.ReadFully(ba);
int nameOff = 0;
int cpOff = 0;
int cp = 0;
do
{
int len = ba[cpOff++] & 0xff;
if (len == 0)
{
len = ba[cpOff++] & 0xff;
// always big-endian
cp = ((ba[cpOff++] & 0xff) << 16) | ((ba[cpOff++] & 0xff) << 8) | ((ba[cpOff++] & 0xff));
}
else
{
cp++;
}
int hi = cp >> 8;
if (Lookup[hi] == null)
{
Lookup[hi] = new int[0x100];
}
Lookup[hi][cp & 0xff] = (nameOff << 8) | len;
nameOff += len;
} while (cpOff < cpEnd);
strPool = new sbyte[total - cpEnd];
dis.ReadFully(strPool);
RefStrPool = new SoftReference <>(strPool);
}
catch (Exception x)
{
throw new InternalError(x.Message, x);
}
finally
{
try
{
if (dis != null)
{
dis.Close();
}
}
catch (Exception)
{
}
}
return(strPool);
}
}
/// <summary>Find out the number of bytes in the block that match its crc.</summary>
/// <remarks>
/// Find out the number of bytes in the block that match its crc.
/// This algorithm assumes that data corruption caused by unexpected
/// datanode shutdown occurs only in the last crc chunk. So it checks
/// only the last chunk.
/// </remarks>
/// <param name="blockFile">the block file</param>
/// <param name="genStamp">generation stamp of the block</param>
/// <returns>the number of valid bytes</returns>
private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp)
{
DataInputStream checksumIn = null;
InputStream blockIn = null;
try
{
FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp);
long blockFileLen = blockFile.Length();
long metaFileLen = metaFile.Length();
int crcHeaderLen = DataChecksum.GetChecksumHeaderSize();
if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen
< crcHeaderLen)
{
return(0);
}
checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile
), HdfsConstants.IoFileBufferSize));
// read and handle the common header here. For now just a version
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile
);
int bytesPerChecksum = checksum.GetBytesPerChecksum();
int checksumSize = checksum.GetChecksumSize();
long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum
, (metaFileLen - crcHeaderLen) / checksumSize);
if (numChunks == 0)
{
return(0);
}
IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize);
blockIn = new FileInputStream(blockFile);
long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
IOUtils.SkipFully(blockIn, lastChunkStartPos);
int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos
);
byte[] buf = new byte[lastChunkSize + checksumSize];
checksumIn.ReadFully(buf, lastChunkSize, checksumSize);
IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize);
checksum.Update(buf, 0, lastChunkSize);
long validFileLength;
if (checksum.Compare(buf, lastChunkSize))
{
// last chunk matches crc
validFileLength = lastChunkStartPos + lastChunkSize;
}
else
{
// last chunck is corrupt
validFileLength = lastChunkStartPos;
}
// truncate if extra bytes are present without CRC
if (blockFile.Length() > validFileLength)
{
RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw");
try
{
// truncate blockFile
blockRAF.SetLength(validFileLength);
}
finally
{
blockRAF.Close();
}
}
return(validFileLength);
}
catch (IOException e)
{
FsDatasetImpl.Log.Warn(e);
return(0);
}
finally
{
IOUtils.CloseStream(checksumIn);
IOUtils.CloseStream(blockIn);
}
}
// ----------------------------------------------------------------
// Constructor
/// <summary>
/// Constructs a UPropertyAliases object. The binary file DATA_FILE_NAME is
/// read from the jar/classpath and unflattened into member variables of this
/// object.
/// </summary>
///
public UPropertyAliases()
{
// Open the .icu file from the jar/classpath
Stream
mask0 = IBM.ICU.Impl.ICUData.GetRequiredStream(DATA_FILE_NAME);
BufferedStream b = new BufferedStream(mask0, DATA_BUFFER_SIZE);
// Read and discard Unicode version...
/* byte unicodeVersion[] = */
IBM.ICU.Impl.ICUBinary.ReadHeader(b, DATA_FORMAT_ID,
this);
DataInputStream d = new DataInputStream(b);
// Record the origin position of the file. Keep enough around
// to seek back to the start of the header.
d.Mark(256);
short enumToName_offset = d.ReadShort();
short nameToEnum_offset = d.ReadShort();
short enumToValue_offset = d.ReadShort();
short total_size = d.ReadShort();
short valueMap_offset = d.ReadShort();
short valueMap_count = d.ReadShort();
short nameGroupPool_offset = d.ReadShort();
short nameGroupPool_count = d.ReadShort();
short stringPool_offset = d.ReadShort();
short stringPool_count = d.ReadShort();
if (DEBUG)
{
System.Console.Out.WriteLine("enumToName_offset=" + enumToName_offset + "\n"
+ "nameToEnum_offset=" + nameToEnum_offset + "\n"
+ "enumToValue_offset=" + enumToValue_offset + "\n"
+ "total_size=" + total_size + "\n" + "valueMap_offset="
+ valueMap_offset + "\n" + "valueMap_count="
+ valueMap_count + "\n" + "nameGroupPool_offset="
+ nameGroupPool_offset + "\n" + "nameGroupPool_count="
+ nameGroupPool_count + "\n" + "stringPool_offset="
+ stringPool_offset + "\n" + "stringPool_count="
+ stringPool_count);
}
byte[] raw = new byte[total_size];
d.Reset();
d.ReadFully(raw);
d.Close();
UPropertyAliases.Builder builder = new UPropertyAliases.Builder(raw);
stringPool = builder
.ReadStringPool(stringPool_offset, stringPool_count);
nameGroupPool = builder.ReadNameGroupPool(nameGroupPool_offset,
nameGroupPool_count);
builder.SetupValueMap_map(valueMap_offset, valueMap_count);
// Some of the following data structures have to be set up
// here, _not_ in Builder. That's because they are instances
// of non-static inner classes, and they contain implicit
// references to this.
builder.Seek(enumToName_offset);
enumToName = new UPropertyAliases.NonContiguousEnumToShort(builder);
builder.NameGroupOffsetToIndex(enumToName.offsetArray);
builder.Seek(nameToEnum_offset);
nameToEnum = new UPropertyAliases.NameToEnum(this, builder);
builder.Seek(enumToValue_offset);
enumToValue = new UPropertyAliases.NonContiguousEnumToShort(builder);
builder.ValueMapOffsetToIndex(enumToValue.offsetArray);
valueMapArray = new UPropertyAliases.ValueMap [valueMap_count];
for (int i = 0; i < valueMap_count; ++i)
{
// Must seek to the start of each entry.
builder.Seek(builder.valueMap_map[i]);
valueMapArray[i] = new UPropertyAliases.ValueMap(this, builder);
}
builder.Close();
}
/// <exception cref="IOException"/>
private static TypeAnnotation Parse(DataInputStream data, ConstantPool pool)
{
int targetType = data.ReadUnsignedByte();
int target = targetType << 24;
switch (targetType)
{
case TypeAnnotation.Class_Type_Parameter:
case TypeAnnotation.Method_Type_Parameter:
case TypeAnnotation.Method_Parameter:
{
target |= data.ReadUnsignedByte();
break;
}
case TypeAnnotation.Super_Type_Reference:
case TypeAnnotation.Class_Type_Parameter_Bound:
case TypeAnnotation.Method_Type_Parameter_Bound:
case TypeAnnotation.Throws_Reference:
case TypeAnnotation.Catch_Clause:
case TypeAnnotation.Expr_Instanceof:
case TypeAnnotation.Expr_New:
case TypeAnnotation.Expr_Constructor_Ref:
case TypeAnnotation.Expr_Method_Ref:
{
target |= data.ReadUnsignedShort();
break;
}
case TypeAnnotation.Type_Arg_Cast:
case TypeAnnotation.Type_Arg_Constructor_Call:
case TypeAnnotation.Type_Arg_Method_Call:
case TypeAnnotation.Type_Arg_Constructor_Ref:
case TypeAnnotation.Type_Arg_Method_Ref:
{
data.SkipBytes(3);
break;
}
case TypeAnnotation.Local_Variable:
case TypeAnnotation.Resource_Variable:
{
data.SkipBytes(data.ReadUnsignedShort() * 6);
break;
}
case TypeAnnotation.Field:
case TypeAnnotation.Method_Return_Type:
case TypeAnnotation.Method_Receiver:
{
break;
}
default:
{
throw new Exception("unknown target type: " + targetType);
}
}
int pathLength = data.ReadUnsignedByte();
byte[] path = null;
if (pathLength > 0)
{
path = new byte[2 * pathLength];
data.ReadFully(path);
}
AnnotationExprent annotation = StructAnnotationAttribute.ParseAnnotation(data, pool
);
return(new TypeAnnotation(target, path, annotation));
}
public static void Main(String[] args)
{
int ident;
int dirofs;
int dirlen;
int i;
int numLumps;
byte[] name = new byte[56];
string nameString;
int filepos;
int filelen;
FileStream readLump;
DataInputStream directory;
string pakName;
string pattern;
if (args.length == 2)
{
if (!args[0].Equals("-list"))
{
Usage();
}
pakName = args[1];
pattern = null;
}
else if (args.length == 3)
{
pakName = args[0];
pattern = args[1];
}
else
{
pakName = null;
pattern = null;
Usage();
}
try
{
directory = new DataInputStream(new FileInputStream(pakName));
readLump = new FileStream(pakName, "r");
ident = IntSwap(directory.ReadInt32());
dirofs = IntSwap(directory.ReadInt32());
dirlen = IntSwap(directory.ReadInt32());
if (ident != IDPAKHEADER)
{
System.Diagnostics.Debug.WriteLine(pakName + " is not a pakfile.");
System.Exit(1);
}
directory.SkipBytes(dirofs - 12);
numLumps = dirlen / 64;
System.Diagnostics.Debug.WriteLine(numLumps + " lumps in " + pakName);
for (i = 0; i < numLumps; i++)
{
directory.ReadFully(name);
filepos = IntSwap(directory.ReadInt32());
filelen = IntSwap(directory.ReadInt32());
nameString = new string (name);
nameString = nameString.Substring(0, nameString.IndexOf(0));
if (pattern == null)
{
System.Diagnostics.Debug.WriteLine(nameString + " : " + filelen + "bytes");
}
else if (PatternMatch(pattern, nameString))
{
File writeFile;
DataOutputStream writeLump;
byte[] buffer = new byte[filelen];
StringBuffer fixedString;
string finalName;
int index;
System.Diagnostics.Debug.WriteLine("Unpaking " + nameString + " " + filelen + " bytes");
readLump.Seek(filepos);
readLump.ReadFully(buffer);
fixedString = new StringBuffer(args[2] + File.separator + nameString);
for (index = 0; index < fixedString.Length; index++)
{
if (fixedString[index] == '/')
{
fixedString.Se[index, File.separatorChar];
}
}
finalName = fixedString.ToString();
index = finalName.LastIndexOf(File.separatorChar);
if (index != -1)
{
string finalPath;
File writePath;
finalPath = finalName.Substring(0, index);
writePath = new File(finalPath);
writePath.Mkdirs();
}
writeFile = new File(finalName);
writeLump = new DataOutputStream(new FileOutputStream(writeFile));
writeLump.Write(buffer);
writeLump.Close();
}
}
readLump.Close();
directory.Close();
}
catch (IOException e)
{
System.Diagnostics.Debug.WriteLine(e.ToString());
}
}
