public void BasicRoundTrip()
{
MemoryStream ms = new MemoryStream();
BZip2OutputStream outStream = new BZip2OutputStream(ms);
byte[] buf = new byte[10000];
System.Random rnd = new Random();
rnd.NextBytes(buf);
outStream.Write(buf, 0, buf.Length);
outStream.Close();
ms = new MemoryStream(ms.GetBuffer());
ms.Seek(0, SeekOrigin.Begin);
using (BZip2InputStream inStream = new BZip2InputStream(ms))
{
byte[] buf2 = new byte[buf.Length];
int pos = 0;
while (true)
{
int numRead = inStream.Read(buf2, pos, 4096);
if (numRead <= 0)
{
break;
}
pos += numRead;
}
for (int i = 0; i < buf.Length; ++i)
{
Assert.AreEqual(buf2[i], buf[i]);
}
}
}
public static void Decompress(Stream inStream, Stream outStream)
{
if (inStream == null)
{
throw new ArgumentNullException("inStream");
}
if (outStream == null)
{
throw new ArgumentNullException("outStream");
}
try
{
using (BZip2InputStream bzip2InputStream = new BZip2InputStream(inStream))
{
for (int num = bzip2InputStream.ReadByte(); num != -1; num = bzip2InputStream.ReadByte())
{
outStream.WriteByte((byte)num);
}
}
}
finally
{
if (outStream != null)
{
((IDisposable)outStream).Dispose();
}
}
}
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner, Action <int> progressPercent = null)
{
if (inStream == null || outStream == null)
{
throw new Exception("Null Stream");
}
try
{
using (BZip2InputStream bzipInput = new BZip2InputStream(inStream))
{
bzipInput.IsStreamOwner = isStreamOwner;
Core.StreamUtils.Copy(bzipInput, outStream, new byte[4096], progressBytes =>
{
var percent = (int)(progressBytes * 100.0 / inStream.Length);
progressPercent?.Invoke(percent);
});
}
}
finally
{
if (isStreamOwner)
{
// inStream is closed by the BZip2InputStream if stream owner
outStream.Dispose();
}
}
}
public void BZip2_Compress_Extract_Test() {
var plainStream = PlainText.ToStream();
plainStream.Seek(0, SeekOrigin.Begin);
var plainData = Encoding.UTF8.GetBytes(PlainText);
byte[] compressedData;
byte[] extractedData;
// Compress
using(var compressedStream = new MemoryStream())
using(var bz2 = new BZip2OutputStream(compressedStream)) {
bz2.Write(plainData, 0, plainData.Length);
bz2.Close();
compressedData = compressedStream.ToArray();
}
Assert.IsNotNull(compressedData);
// Array.Resize(ref compressedData, compressedData.Length+1);
// compressedData[compressedData.Length - 1] = (byte)0;
// Extract
using(var compressedStream = new MemoryStream(compressedData))
using(var bz2 = new BZip2InputStream(compressedStream))
using(var extractedStream = new MemoryStream()) {
StreamTool.CopyStreamToStream(bz2, extractedStream);
extractedData = extractedStream.ToArray();
}
Assert.IsNotNull(extractedData);
string extractedText = Encoding.UTF8.GetString(extractedData).TrimEnd('\0');
Assert.AreEqual(PlainText, extractedText);
}
public void CreateEmptyArchive()
{
MemoryStream ms = new MemoryStream();
BZip2OutputStream outStream = new BZip2OutputStream(ms);
outStream.Close();
ms = new MemoryStream(ms.GetBuffer());
ms.Seek(0, SeekOrigin.Begin);
BZip2InputStream inStream = new BZip2InputStream(ms);
byte[] buffer = new byte[1024];
int pos = 0;
while (true)
{
int numRead = inStream.Read(buffer, 0, buffer.Length);
if (numRead <= 0)
{
break;
}
pos += numRead;
}
Assertion.AssertEquals(pos, 0);
}
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner)
{
if (inStream == null)
{
throw new ArgumentNullException(nameof(inStream));
}
if (outStream == null)
{
throw new ArgumentNullException(nameof(outStream));
}
try
{
using (BZip2InputStream bzipInput = new BZip2InputStream(inStream))
{
bzipInput.IsStreamOwner = isStreamOwner;
Core.StreamUtils.Copy(bzipInput, outStream, new byte[4096]);
}
}
finally
{
if (isStreamOwner)
{
// inStream is closed by the BZip2InputStream if stream owner
outStream.Dispose();
}
}
}
/// <summary>
/// 압축된 데이타를 복원한다.
/// </summary>
/// <param name="input">복원할 Data</param>
/// <returns>복원된 Data</returns>
public override byte[] Decompress(byte[] input) {
if(IsDebugEnabled)
log.Debug(CompressorTool.SR.DecompressStartMsg);
// check input data
if(input.IsZeroLength()) {
if(IsDebugEnabled)
log.Debug(CompressorTool.SR.InvalidInputDataMsg);
return CompressorTool.EmptyBytes;
}
byte[] output;
using(var inStream = new MemoryStream(input)) {
using(var bz2 = new BZip2InputStream(inStream))
using(var outStream = new MemoryStream(input.Length * 2)) {
StreamTool.CopyStreamToStream(bz2, outStream, CompressorTool.BUFFER_SIZE);
output = outStream.ToArray();
}
}
if(IsDebugEnabled)
log.Debug(CompressorTool.SR.DecompressResultMsg, input.Length, output.Length, output.Length / (double)input.Length);
return output;
}
public void CreateEmptyArchive()
{
MemoryStream ms = new MemoryStream();
BZip2OutputStream outStream = new BZip2OutputStream(ms);
outStream.Close();
ms = new MemoryStream(ms.GetBuffer());
ms.Seek(0, SeekOrigin.Begin);
using (BZip2InputStream inStream = new BZip2InputStream(ms))
{
byte[] buffer = new byte[1024];
int pos = 0;
while (true)
{
int numRead = inStream.Read(buffer, 0, buffer.Length);
if (numRead <= 0)
{
break;
}
pos += numRead;
}
Assert.AreEqual(pos, 0);
}
}
public void BasicRoundTrip()
{
MemoryStream ms = new MemoryStream();
BZip2OutputStream outStream = new BZip2OutputStream(ms);
byte[] buf = new byte[10000];
System.Random rnd = new Random();
rnd.NextBytes(buf);
outStream.Write(buf, 0, buf.Length);
outStream.Close();
ms = new MemoryStream(ms.GetBuffer());
ms.Seek(0, SeekOrigin.Begin);
BZip2InputStream inStream = new BZip2InputStream(ms);
byte[] buf2 = new byte[buf.Length];
int pos = 0;
while (true)
{
int numRead = inStream.Read(buf2, pos, 4096);
if (numRead <= 0)
{
break;
}
pos += numRead;
}
for (int i = 0; i < buf.Length; ++i)
{
Assertion.AssertEquals(buf2[i], buf[i]);
}
}
public void Extract(string path, string dest_dir)
{
BZip2InputStream bz2stream = new BZip2InputStream(new FileStream( path, FileMode.Open));
TarExtracter untar = new TarExtracter();
untar.Extract(bz2stream, dest_dir);
bz2stream.Close();
}
private void InitBlock()
{
char c = this.BsGetUChar();
char c2 = this.BsGetUChar();
char c3 = this.BsGetUChar();
char c4 = this.BsGetUChar();
char c5 = this.BsGetUChar();
char c6 = this.BsGetUChar();
if (c == '\u0017' && c2 == 'r' && c3 == 'E' && c4 == '8' && c5 == 'P' && c6 == '\u0090')
{
this.Complete();
return;
}
if (c != '1' || c2 != 'A' || c3 != 'Y' || c4 != '&' || c5 != 'S' || c6 != 'Y')
{
BZip2InputStream.BadBlockHeader();
this.streamEnd = true;
return;
}
this.storedBlockCRC = this.BsGetInt32();
this.blockRandomised = (this.BsR(1) == 1);
this.GetAndMoveToFrontDecode();
this.mCrc.Reset();
this.currentState = 1;
}
/// <summary>
/// Decompress <paramref name="inStream">input</paramref> writing
/// decompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The stream containing data to decompress.</param>
/// <param name="outStream">The stream to write decompressed data to.</param>
/// <remarks>Both streams are closed on completion</remarks>
public static void Decompress(Stream inStream, Stream outStream)
{
if (inStream == null)
{
throw new ArgumentNullException("inStream");
}
if (outStream == null)
{
throw new ArgumentNullException("outStream");
}
using (outStream)
{
using (BZip2InputStream bzis = new BZip2InputStream(inStream))
{
int ch = bzis.ReadByte();
while (ch != -1)
{
outStream.WriteByte((byte)ch);
ch = bzis.ReadByte();
}
}
}
}
public static void Untbz(Stream stream, string outDirPath)
{
using (var inputStream = new BZip2InputStream(stream)) {
var archive = TarArchive.CreateInputTarArchive(inputStream);
archive.AsciiTranslate = false;
archive.ExtractContents(outDirPath);
}
}
/// <summary> Loads a TrigradCompressed image from a stream. </summary>
public TrigradCompressed(Stream s)
{
using (BZip2InputStream dezipper = new BZip2InputStream(s))
using (BinaryReader reader = new BinaryReader(dezipper))
{
Width = reader.ReadUInt16();
Height = reader.ReadUInt16();
uint count = reader.ReadUInt32();
Point[] points = new Point[count];
for (int i = 0; i < count; i++)
points[i].X = reader.ReadUInt16();
for (int i = 0; i < count; i++)
points[i].Y = reader.ReadUInt16();
ColorStruct[] colors = new ColorStruct[count];
for (int i = 0; i < count; i++)
colors[i].R = reader.ReadByte();
for (int i = 0; i < count; i++)
colors[i].G = reader.ReadByte();
for (int i = 0; i < count; i++)
colors[i].B = reader.ReadByte();
for (int i = 0; i < count; i++)
{
SampleTable.Add(points[i],colors[i].Color);
}
uint meshCount = reader.ReadUInt32();
SampleTri[] tris = new SampleTri[meshCount];
for (int i = 0; i < meshCount; i++)
tris[i] = new SampleTri();
for (int i = 0; i < meshCount; i++)
tris[i].U = new Sample(points[reader.ReadInt32()], new Pixel(Color.Black));
for (int i = 0; i < meshCount; i++)
tris[i].V = new Sample(points[reader.ReadInt32()], new Pixel(Color.Black));
for (int i = 0; i < meshCount; i++)
tris[i].W = new Sample(points[reader.ReadInt32()], new Pixel(Color.Black));
foreach (var tri in tris)
{
tri.U.Color = SampleTable[tri.U.Point];
tri.V.Color = SampleTable[tri.V.Point];
tri.W.Color = SampleTable[tri.W.Point];
}
Mesh = tris.ToList();
}
}
public static void Untbz(string inPath, string outDirPath)
{
using (var fs = File.OpenRead(inPath)) {
using (var stream = new BZip2InputStream(fs)) {
var archive = TarArchive.CreateInputTarArchive(stream);
archive.ExtractContents(outDirPath);
}
}
}
private void Complete()
{
this.storedCombinedCRC = this.BsGetInt32();
if (this.storedCombinedCRC != (int)this.computedCombinedCRC)
{
BZip2InputStream.CrcError();
}
this.streamEnd = true;
}
private void EndBlock()
{
this.computedBlockCRC = (int)this.mCrc.Value;
if (this.storedBlockCRC != this.computedBlockCRC)
{
BZip2InputStream.CrcError();
}
this.computedCombinedCRC = ((this.computedCombinedCRC << 1 & uint.MaxValue) | this.computedCombinedCRC >> 31);
this.computedCombinedCRC ^= (uint)this.computedBlockCRC;
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Usage : MtUnZip {NameOfOsm.Bz2} {MtIsEnabled}");
return;
}
if (!File.Exists(args[0]))
{
Console.WriteLine("File {0} does not exist", args[0]);
return;
}
bool mtIsEnabled = bool.Parse(args[1]);
var timer = Stopwatch.StartNew();
var unzipper = new BZip2InputStream(File.OpenRead(args[0]));
Stream bufferedReader = unzipper;
if (mtIsEnabled)
bufferedReader = new MultiThreadBufferedReader(unzipper);
var rd = XmlReader.Create(bufferedReader);
var nodeNameCounts = new List<NodeStats>();
while (rd.Read())
{
if (rd.NodeType == XmlNodeType.Element)
{
var name = rd.Name;
var stats = nodeNameCounts.FirstOrDefault(nodeStats => nodeStats.Name == name);
if (stats == null)
{
stats = new NodeStats(name);
nodeNameCounts.Add(stats);
}
stats.Update(rd);
}
}
nodeNameCounts.Sort((lhs, rhs) => lhs.Name.CompareTo(rhs.Name));
foreach (var stats in nodeNameCounts)
{
Console.WriteLine("{0,20} : {1,10:N0}", stats.Name, stats.Count);
}
Console.WriteLine("Took {0:N0} [ms]", timer.ElapsedMilliseconds);
Console.WriteLine("Total Processor time {0:N0} [ms]", Process.GetCurrentProcess().TotalProcessorTime.TotalMilliseconds);
}
// about 800ms here, not that slow.
public static void ExtractTemplate(string packedTemplatePath)
{
var appRootDir = Path.GetDirectoryName(Application.ExecutablePath);
using (var packedTemplate = File.OpenRead(packedTemplatePath))
using (var bz2 = new BZip2InputStream(packedTemplate))
using (var tar = TarArchive.CreateInputTarArchive(bz2))
{
tar.ExtractContents(appRootDir);
}
}
public static void Decompress(Stream instream, Stream outstream)
{
Stream stream = outstream;
Stream stream2 = instream;
BZip2InputStream stream3 = new BZip2InputStream(stream2);
for (int i = stream3.ReadByte(); i != -1; i = stream3.ReadByte())
{
stream.WriteByte((byte)i);
}
stream.Flush();
}
public static void Decompress(Stream instream, Stream outstream)
{
System.IO.Stream bos = outstream;
System.IO.Stream bis = instream;
BZip2InputStream bzis = new BZip2InputStream(bis);
int ch = bzis.ReadByte();
while (ch != -1) {
bos.WriteByte((byte)ch);
ch = bzis.ReadByte();
}
bos.Flush();
}
private static void RemoveLineBreaks(string source, string destination)
{
Parallel.ForEach(Directory.GetFiles(source, "*.bz2"), file =>
{
using(var fileStream = new FileStream(file, FileMode.Open))
using (var bz2Stream = new BZip2InputStream(fileStream))
using(var reader = new StreamReader(bz2Stream))
{
var content = reader.ReadToEnd().Replace(Environment.NewLine, string.Empty);
File.WriteAllText(Path.Combine(destination, Path.GetFileNameWithoutExtension(file)), content);
}
});
}
/// <summary>
/// Decompress <paramref name="instream">input</paramref> writing
/// decompressed data to <paramref name="outstream">output stream</paramref>
/// </summary>
public static void Decompress(Stream instream, Stream outstream)
{
var bos = outstream;
var bis = instream;
var bzis = new BZip2InputStream(bis);
var ch = bzis.ReadByte();
while (ch != -1)
{
bos.WriteByte((byte)ch);
ch = bzis.ReadByte();
}
bos.Flush();
}
/// <summary>
/// Decompress <paramref name="instream">input</paramref> writing
/// decompressed data to <paramref name="outstream">output stream</paramref>
/// </summary>
public static void Decompress(Stream instream, Stream outstream)
{
System.IO.Stream bos = outstream;
System.IO.Stream bis = instream;
BZip2InputStream bzis = new BZip2InputStream(bis);
int ch = bzis.ReadByte();
while (ch != -1)
{
bos.WriteByte((byte)ch);
ch = bzis.ReadByte();
}
bos.Flush();
}
public static byte[] Decompress(byte[] input, int uncompressedLength)
{
Contract.Requires(input != null);
Contract.Requires(uncompressedLength >= 0);
Contract.Ensures(Contract.Result<byte[]>() != null);
Contract.Ensures(Contract.Result<byte[]>().Length == uncompressedLength);
var output = new byte[uncompressedLength];
using (var stream = new BZip2InputStream(new MemoryStream(input, false)))
stream.Read(output, 0, uncompressedLength);
return output;
}
/// <summary>
/// 解压缩字符串(ICSharpCode.SharpZipLib版)
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public static string Decompress(string input)
{
string result = string.Empty;
byte[] buffer = Convert.FromBase64String(input);
using (Stream inputStream = new MemoryStream(buffer))
{
BZip2InputStream zipStream = new BZip2InputStream(inputStream);
using (StreamReader reader = new StreamReader(zipStream, Encoding.UTF8))
{
//输出
result = reader.ReadToEnd();
}
}
return result;
}
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner) {
if (inStream==null||outStream==null) {
throw new Exception("Null Stream");
}
try {
using (var bzipInput=new BZip2InputStream(inStream)) {
bzipInput.IsStreamOwner=isStreamOwner;
StreamUtils.Copy(bzipInput, outStream, new byte[4096]);
}
} finally {
if (isStreamOwner) {
// inStream is closed by the BZip2InputStream if stream owner
outStream.Dispose();
}
}
}
private static Stopwatch ParseDisks(Action<Disk> addToBatch)
{
int i = 0;
var parser = new Parser();
var buffer = new byte[1024*1024];// more than big enough for all files
var sp = Stopwatch.StartNew();
using (var bz2 = new BZip2InputStream(File.Open(@"D:\Data\freedb-complete-20120101.tar.bz2", FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while((entry=tar.GetNextEntry()) != null)
{
if(entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
continue;
var readSoFar = 0;
while(true)
{
var read = tar.Read(buffer, readSoFar, ((int) entry.Size) - readSoFar);
if (read == 0)
break;
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer,0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
addToBatch(disk);
if (i++ % BatchSize == 0)
Console.Write("\r{0} {1:#,#} {2} ", entry.Name, i, sp.Elapsed);
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(entry.Name);
Console.WriteLine(e);
return sp;
}
}
}
return sp;
}
/// <summary>
/// 解压缩字符串
/// </summary>
/// <param name="str">需要解压的base64位字符串</param>
/// <returns>字符串</returns>
public string UnCompressStr(string str)
{
//判断需解压字符串是否为空
if (str == null)
{
//如果为空，返回空
return("");
}
//如果不为空则执行以下语句
try
{
//定义字符串数组，获取需解压字符串数据包
string[] temp = str.Split(' ');
//如果字符小于2返回空
if (temp.Length < 2)
{
return("");
}
//调用文本可变字符串类
System.Text.StringBuilder uncompressedString = new System.Text.StringBuilder();
int totalLength = 0;
//base64位字符串数字转化为8位字符数组
byte[] bytInput = System.Convert.FromBase64String(temp[0]);
//转化为等效32位字符数组
byte[] writeData = new byte[Convert.ToInt32(temp[1])];
//构建读取流
Stream s2 = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(new MemoryStream(bytInput));
//循环解压文本
while (true)
{
int size = s2.Read(writeData, 0, writeData.Length);
if (size > 0)
{
totalLength += size;
uncompressedString.Append(System.Text.Encoding.Unicode.GetString(writeData, 0, size));
//返回解压后文本
return(uncompressedString.ToString());
}
return("");
}
}
catch (Exception ex)
{
throw ex;
}
}
private void FillBuffer()
{
int num = 0;
try
{
num = this.baseStream.ReadByte();
}
catch (Exception)
{
BZip2InputStream.CompressedStreamEOF();
}
if (num == -1)
{
BZip2InputStream.CompressedStreamEOF();
}
this.bsBuff = (this.bsBuff << 8 | (num & 255));
this.bsLive += 8;
}
public DebArchiveInputStream(Stream inputStream)
: base(inputStream)
{
this.arIn = new ArArchiveInputStream (inputStream);
ArArchiveEntry binary = arIn.GetNextArEntry ();
if (!binary.Name.Equals ("debian-binary"))
{
// TODO Customize
throw new IOException ("Invalid name, expected debian-binary, readed:" + binary.Name);
}
ArArchiveEntry control = arIn.GetNextArEntry ();
if (!control.Name.Equals ("control.tar.gz"))
{
throw new IOException ("Invalid name, expected control.tar.gz, readed:" + control.Name);
}
ArArchiveEntry data = arIn.GetNextArEntry ();
Stream compressedStream = null;
if (data.Name.Equals ("data.tar.gz"))
{
compressedStream = new GZipStream (arIn, CompressionMode.Decompress);
}
else if (data.Name.Equals ("data.tar.bz2"))
{
compressedStream = new BZip2InputStream (arIn);
}
else if (data.Name.Equals ("data.tar.bz2"))
{
compressedStream = new LZMAInputStream (arIn);
}
else if (data.Name.Equals ("data.tar"))
{
compressedStream = arIn;
}
else
{
throw new IOException ("Unsupported compressed data:" + data.Name);
}
this.tarStream = new TarInputStream (compressedStream);
}
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner)
{
if (inStream == null || outStream == null)
{
throw new Exception("Null Stream");
}
try {
using (var bzipInput = new BZip2InputStream(inStream)) {
bzipInput.IsStreamOwner = isStreamOwner;
StreamUtils.Copy(bzipInput, outStream, new byte[4096]);
}
} finally {
if (isStreamOwner)
{
// inStream is closed by the BZip2InputStream if stream owner
outStream.Dispose();
}
}
}
public static void Decompress(Stream inStream, Stream outStream)
{
if ( inStream == null ) {
throw new ArgumentNullException("inStream");
}
if ( outStream == null ) {
throw new ArgumentNullException("outStream");
}
using ( outStream ) {
using ( BZip2InputStream bzis = new BZip2InputStream(inStream) ) {
int ch = bzis.ReadByte();
while (ch != -1) {
outStream.WriteByte((byte)ch);
ch = bzis.ReadByte();
}
}
}
}
private static void ParseDisks(BulkInsertOperation insert)
{
int i = 0;
var parser = new Parser();
var buffer = new byte[1024*1024];// more than big enough for all files
using (var bz2 = new BZip2InputStream(File.Open(@"D:\Scratch\freedb-complete-20150101.tar.bz2", FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while((entry=tar.GetNextEntry()) != null)
{
if(entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
continue;
var readSoFar = 0;
while(true)
{
var read = tar.Read(buffer, readSoFar, ((int) entry.Size) - readSoFar);
if (read == 0)
break;
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer,0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
insert.Store(disk);
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(entry.Name);
Console.WriteLine(e);
}
}
}
}
public static void Decompress(Stream inStream, Stream outStream)
{
if (inStream == null)
{
throw new ArgumentNullException("inStream");
}
if (outStream == null)
{
throw new ArgumentNullException("outStream");
}
using (outStream)
{
using (BZip2InputStream stream = new BZip2InputStream(inStream))
{
for (int i = stream.ReadByte(); i != -1; i = stream.ReadByte())
{
outStream.WriteByte((byte)i);
}
}
}
}
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner)
{
if (inStream == null || outStream == null)
{
throw new Exception("Null Stream");
}
try
{
using (BZip2InputStream bZip2InputStream = new BZip2InputStream(inStream))
{
bZip2InputStream.IsStreamOwner = isStreamOwner;
StreamUtils.Copy(bZip2InputStream, outStream, new byte[4096]);
}
}
finally
{
if (isStreamOwner)
{
outStream.Close();
}
}
}
public static string UnzipString(string compbytes)
{
string result;
MemoryStream m_msBZip2 = null;
BZip2InputStream m_isBZip2 = null;
try
{
m_msBZip2 = new MemoryStream(Convert.FromBase64String(compbytes));
// read final uncompressed string size stored in first 4 bytes
//
using (BinaryReader reader = new BinaryReader(m_msBZip2, System.Text.Encoding.ASCII))
{
Int32 size = reader.ReadInt32();
m_isBZip2 = new BZip2InputStream(m_msBZip2);
byte[] bytesUncompressed = new byte[size];
m_isBZip2.Read(bytesUncompressed, 0, bytesUncompressed.Length);
m_isBZip2.Close();
m_msBZip2.Close();
result = Encoding.ASCII.GetString(bytesUncompressed);
reader.Close();
}
}
finally
{
if (m_isBZip2 != null)
{
m_isBZip2.Dispose();
}
if (m_msBZip2 != null)
{
m_msBZip2.Dispose();
}
}
return result;
}
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner)
{
if (inStream == null || outStream == null)
{
throw new Exception("Null Stream");
}
try {
using (BZip2InputStream bzipInput = new BZip2InputStream(inStream)) {
bzipInput.IsStreamOwner = isStreamOwner;
Core.StreamUtils.Copy(bzipInput, outStream, new byte[4096]);
}
} finally {
if (isStreamOwner)
{
// inStream is closed by the BZip2InputStream if stream owner
#if NET45
outStream.Close();
#elif NETSTANDARD1_3
outStream.Dispose();
#endif
}
}
}
// TODO: Create a class Archive.cs and do all the archiving stuff there!
public static bool UncompressBzip2File(string Filename, string To, Gtk.ProgressBar bar)
{
try{
Console.WriteLine(Filename);
BZip2InputStream bzipIn = new BZip2InputStream(File.OpenRead(Filename));
FileStream streamWriter = File.Create(To+Path.GetFileNameWithoutExtension(Filename));
long size=0;
byte[] data = new byte[1024];
while (true)
{
size = bzipIn.Read(data, 0, data.Length);
if (size > 0) streamWriter.Write(data, 0, (int) size);
else break;
}
streamWriter.Close();
Console.WriteLine("Deflating the gzip file done!");
return true;
}
catch(Exception e) {
Console.WriteLine("An exception occured while deflating the bzip2 file: "+e.Message);
return false;
}
}
/// <summary>
/// This is the "real" main. The class main() instantiates a tar object
/// for the application and then calls this method. Process the arguments
/// and perform the requested operation.
/// </summary>
public void InstanceMain(string[] argv)
{
TarArchive archive = null;
int argIdx = this.ProcessArguments(argv);
if (this.archiveName != null && ! this.archiveName.Equals("-")) {
if (operation == Operation.Create) {
if (!Directory.Exists(Path.GetDirectoryName(archiveName))) {
Console.Error.WriteLine("Directory for archive doesnt exist");
return;
}
}
else {
if (File.Exists(this.archiveName) == false) {
Console.Error.WriteLine("File does not exist " + this.archiveName);
return;
}
}
}
if (operation == Operation.Create) { // WRITING
Stream outStream = Console.OpenStandardOutput();
if (this.archiveName != null && ! this.archiveName.Equals("-")) {
outStream = File.Create(archiveName);
}
if (outStream != null) {
switch (this.compression) {
case Compression.Compress:
outStream = new DeflaterOutputStream(outStream);
break;
case Compression.Gzip:
outStream = new GZipOutputStream(outStream);
break;
case Compression.Bzip2:
outStream = new BZip2OutputStream(outStream, 9);
break;
}
archive = TarArchive.CreateOutputTarArchive(outStream, this.blockingFactor);
}
} else { // EXTRACTING OR LISTING
Stream inStream = Console.OpenStandardInput();
if (this.archiveName != null && ! this.archiveName.Equals( "-" )) {
inStream = File.OpenRead(archiveName);
}
if (inStream != null) {
switch (this.compression) {
case Compression.Compress:
inStream = new InflaterInputStream(inStream);
break;
case Compression.Gzip:
inStream = new GZipInputStream(inStream);
break;
case Compression.Bzip2:
inStream = new BZip2InputStream(inStream);
break;
}
archive = TarArchive.CreateInputTarArchive(inStream, this.blockingFactor);
}
}
if (archive != null) { // SET ARCHIVE OPTIONS
archive.SetKeepOldFiles(this.keepOldFiles);
archive.AsciiTranslate = this.asciiTranslate;
archive.SetUserInfo(this.userId, this.userName, this.groupId, this.groupName);
}
if (archive == null) {
Console.Error.WriteLine( "no processing due to errors" );
} else if (operation == Operation.Create) { // WRITING
if (verbose) {
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
}
for ( ; argIdx < argv.Length ; ++argIdx ) {
string[] fileNames = GetFilesForSpec(argv[argIdx]);
if (fileNames.Length > 0) {
foreach (string name in fileNames) {
TarEntry entry = TarEntry.CreateEntryFromFile(name);
archive.WriteEntry(entry, true);
}
} else {
Console.Error.Write("No files for " + argv[argIdx]);
}
}
} else if (operation == Operation.List) { // LISTING
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
archive.ListContents();
} else { // EXTRACTING
string userDir = Environment.CurrentDirectory;
if (verbose) {
archive.ProgressMessageEvent += new ProgressMessageHandler(ShowTarProgressMessage);
}
if (userDir != null) {
archive.ExtractContents(userDir);
}
}
if (archive != null) { // CLOSE ARCHIVE
archive.Close();
}
}
/// <summary>
/// required for applying a binary patch
/// </summary>
private FileStream bz2read(ref BZip2InputStream bz, Int32 offset, string fname)
{
FileStream ReturnValue;
FileStream fs;
try
{
fs = new FileStream(fname, FileMode.Open, FileAccess.Read, FileShare.Read);
fs.Seek(offset, SeekOrigin.Begin);
bz = new BZip2InputStream(fs);
ReturnValue = fs;
}
catch (Exception e)
{
throw new Exception("Cannot open file " + fname + " for reading; " + e.Message);
}
return ReturnValue;
}
/// <summary>
/// required for applying a binary patch
/// </summary>
private Int32 loopread(ref BZip2InputStream zipStream, ref byte[] buf, Int32 offset, Int32 nbytes)
{
Int32 ptr;
Int32 lenread;
ptr = 0;
while (ptr < nbytes)
{
lenread = zipStream.Read(buf, offset + ptr, nbytes);
if (lenread == 0)
{
return ptr;
}
if (lenread == -1)
{
return -1;
}
ptr = ptr + lenread;
}
return ptr;
}
private void GetAndMoveToFrontDecode()
{
byte[] array = new byte[256];
int num = 100000 * this.blockSize100k;
this.origPtr = this.BsGetIntVS(24);
this.RecvDecodingTables();
int num2 = this.nInUse + 1;
int num3 = -1;
int num4 = 0;
for (int i = 0; i <= 255; i++)
{
this.unzftab[i] = 0;
}
for (int j = 0; j <= 255; j++)
{
array[j] = (byte)j;
}
this.last = -1;
if (num4 == 0)
{
num3++;
num4 = 50;
}
num4--;
int num5 = (int)this.selector[num3];
int num6 = this.minLens[num5];
int k;
int num7;
for (k = this.BsR(num6); k > this.limit[num5][num6]; k = (k << 1 | num7))
{
if (num6 > 20)
{
throw new BZip2Exception("Bzip data error");
}
num6++;
while (this.bsLive < 1)
{
this.FillBuffer();
}
num7 = (this.bsBuff >> this.bsLive - 1 & 1);
this.bsLive--;
}
if (k - this.baseArray[num5][num6] < 0 || k - this.baseArray[num5][num6] >= 258)
{
throw new BZip2Exception("Bzip data error");
}
int num8 = this.perm[num5][k - this.baseArray[num5][num6]];
while (num8 != num2)
{
if (num8 == 0 || num8 == 1)
{
int l = -1;
int num9 = 1;
do
{
if (num8 == 0)
{
l += num9;
}
else if (num8 == 1)
{
l += 2 * num9;
}
num9 <<= 1;
if (num4 == 0)
{
num3++;
num4 = 50;
}
num4--;
num5 = (int)this.selector[num3];
num6 = this.minLens[num5];
for (k = this.BsR(num6); k > this.limit[num5][num6]; k = (k << 1 | num7))
{
num6++;
while (this.bsLive < 1)
{
this.FillBuffer();
}
num7 = (this.bsBuff >> this.bsLive - 1 & 1);
this.bsLive--;
}
num8 = this.perm[num5][k - this.baseArray[num5][num6]];
}while (num8 == 0 || num8 == 1);
l++;
byte b = this.seqToUnseq[(int)array[0]];
this.unzftab[(int)b] += l;
while (l > 0)
{
this.last++;
this.ll8[this.last] = b;
l--;
}
if (this.last >= num)
{
BZip2InputStream.BlockOverrun();
}
}
else
{
this.last++;
if (this.last >= num)
{
BZip2InputStream.BlockOverrun();
}
byte b2 = array[num8 - 1];
this.unzftab[(int)this.seqToUnseq[(int)b2]]++;
this.ll8[this.last] = this.seqToUnseq[(int)b2];
for (int m = num8 - 1; m > 0; m--)
{
array[m] = array[m - 1];
}
array[0] = b2;
if (num4 == 0)
{
num3++;
num4 = 50;
}
num4--;
num5 = (int)this.selector[num3];
num6 = this.minLens[num5];
for (k = this.BsR(num6); k > this.limit[num5][num6]; k = (k << 1 | num7))
{
num6++;
while (this.bsLive < 1)
{
this.FillBuffer();
}
num7 = (this.bsBuff >> this.bsLive - 1 & 1);
this.bsLive--;
}
num8 = this.perm[num5][k - this.baseArray[num5][num6]];
}
}
}
private void RecvDecodingTables()
{
char[][] array = new char[6][];
for (int i = 0; i < 6; i++)
{
array[i] = new char[258];
}
bool[] array2 = new bool[16];
for (int j = 0; j < 16; j++)
{
array2[j] = (this.BsR(1) == 1);
}
for (int k = 0; k < 16; k++)
{
if (array2[k])
{
for (int l = 0; l < 16; l++)
{
this.inUse[k * 16 + l] = (this.BsR(1) == 1);
}
}
else
{
for (int m = 0; m < 16; m++)
{
this.inUse[k * 16 + m] = false;
}
}
}
this.MakeMaps();
int num = this.nInUse + 2;
int num2 = this.BsR(3);
int num3 = this.BsR(15);
for (int n = 0; n < num3; n++)
{
int num4 = 0;
while (this.BsR(1) == 1)
{
num4++;
}
this.selectorMtf[n] = (byte)num4;
}
byte[] array3 = new byte[6];
for (int num5 = 0; num5 < num2; num5++)
{
array3[num5] = (byte)num5;
}
for (int num6 = 0; num6 < num3; num6++)
{
int num7 = (int)this.selectorMtf[num6];
byte b = array3[num7];
while (num7 > 0)
{
array3[num7] = array3[num7 - 1];
num7--;
}
array3[0] = b;
this.selector[num6] = b;
}
for (int num8 = 0; num8 < num2; num8++)
{
int num9 = this.BsR(5);
for (int num10 = 0; num10 < num; num10++)
{
while (this.BsR(1) == 1)
{
if (this.BsR(1) == 0)
{
num9++;
}
else
{
num9--;
}
}
array[num8][num10] = (char)num9;
}
}
for (int num11 = 0; num11 < num2; num11++)
{
int num12 = 32;
int num13 = 0;
for (int num14 = 0; num14 < num; num14++)
{
num13 = Math.Max(num13, (int)array[num11][num14]);
num12 = Math.Min(num12, (int)array[num11][num14]);
}
BZip2InputStream.HbCreateDecodeTables(this.limit[num11], this.baseArray[num11], this.perm[num11], array[num11], num12, num13, num);
this.minLens[num11] = num12;
}
}
public void Performance()
{
window_ = new WindowedStream(0x150000);
outStream_ = new BZip2OutputStream(window_, 1);
const long Target = 0x10000000;
readTarget_ = writeTarget_ = Target;
Thread reader = new Thread(Reader);
reader.Name = "Reader";
Thread writer = new Thread(Writer);
writer.Name = "Writer";
DateTime startTime = DateTime.Now;
writer.Start();
inStream_ = new BZip2InputStream(window_);
reader.Start();
Assert.IsTrue(writer.Join(TimeSpan.FromMinutes(5.0D)));
Assert.IsTrue(reader.Join(TimeSpan.FromMinutes(5.0D)));
DateTime endTime = DateTime.Now;
TimeSpan span = endTime - startTime;
Console.WriteLine("Time {0} throughput {1} KB/Sec", span, (Target / 1024) / span.TotalSeconds);
}
/// <summary>
/// Extracts the files from the archive.
/// </summary>
protected override void ExecuteTask()
{
Stream fs = null;
Stream instream = null;
try {
// ensure archive exists
if (!SrcFile.Exists)
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Tar file '{0}' does not exist.", SrcFile.FullName),
Location);
fs = SrcFile.OpenRead();
// wrap inputstream with corresponding compression method
switch (CompressionMethod) {
case TarCompressionMethod.GZip:
instream = new GZipInputStream(fs);
break;
case TarCompressionMethod.BZip2:
instream = new BZip2InputStream(fs);
break;
default:
instream = fs;
break;
}
using (TarInputStream s = new TarInputStream(instream)) {
Log(Level.Info, "Expanding '{0}' to '{1}'.",
SrcFile.FullName, DestinationDirectory.FullName);
TarEntry entry;
// extract the file or directory entry
while ((entry = s.GetNextEntry()) != null) {
if (entry.IsDirectory) {
ExtractDirectory(s, DestinationDirectory.FullName,
entry.Name, entry.ModTime);
} else {
ExtractFile(s, DestinationDirectory.FullName,
entry.Name, entry.ModTime, entry.Size);
}
}
}
} catch (IOException ex) {
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Failed to expand '{0}' to '{1}'.", SrcFile.FullName,
DestinationDirectory.FullName), Location, ex);
} catch (TarException ex) {
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Invalid tar file '{0}'.", SrcFile.FullName), Location, ex);
} catch (BZip2Exception ex) {
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Invalid bzip2'd tar file '{0}'.", SrcFile.FullName), Location, ex);
} catch (GZipException ex) {
throw new BuildException(string.Format(CultureInfo.InvariantCulture,
"Invalid gzipped tar file '{0}'.", SrcFile.FullName), Location, ex);
} finally {
// close the filestream
if (fs != null)
fs.Close ();
}
}
private Stopwatch ParseDisks(Action<Disk> addToBatch)
{
int i = 0;
var parser = new Parser();
var buffer = new byte[1024 * 1024];// more than big enough for all files
var sp = Stopwatch.StartNew();
using (var bz2 = new BZip2InputStream(File.Open(dataLocation, FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
continue;
var readSoFar = 0;
while (true)
{
var read = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar);
if (read == 0)
break;
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
addToBatch(disk);
if (i++ % BatchSize == 0)
{
process.Refresh();
MemoryUsage.Add(process.WorkingSet64);
logger.Info("\r{0} {1:#,#} {2} ", entry.Name, i, sp.Elapsed);
}
}
catch (Exception e)
{
logger.Error("");
logger.Error(entry.Name);
logger.Error(e);
return sp;
}
}
}
return sp;
}
private void OpenFile(object param)
{
IProgressFeedback progress = param as IProgressFeedback;
FileStream fs = new FileStream(openFileDialog.FileName, FileMode.Open);
BZip2InputStream stream = new BZip2InputStream(fs);
dataSet.ReadXml(stream);
stream.Close();
fs.Close();
dataSet.Tables[0].EndLoadData();
progress.ProgressUpdate("Opened", 100);
AnalyzePackets(progress);
progress.ProgressUpdate("Finishing", 100);
Invoke(new ThreadStart(RestoreDataSource));
progress.OperationComplete();
}
private void ParseComressedManFile ()
{
try {
Stream stream = null;
if (Extension == ".gz")
stream = new GZipInputStream (Stream);
else if (Extension == ".bz2")
stream = new BZip2InputStream (Stream);
else if (Extension == ".lzma")
stream = GetLzmaStream (Stream);
compressed_reader = new StreamReader (stream);
} catch (Exception e) {
Log.Error (e, "Error in opening compressed man page");
if (compressed_reader != null)
compressed_reader.Close ();
Error ();
return;
}
ParseManFile (compressed_reader);
}
public static JSONNode LoadFromCompressedStream(System.IO.Stream aData)
{
var zin = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(aData);
return(LoadFromStream(zin));
}
private void ReadFile()
{
var buffer = new byte[1024 * 1024];// more than big enough for all files
using (var bz2 = new BZip2InputStream(File.Open(_path, FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
continue;
var readSoFar = 0;
while (true)
{
var bytes = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar);
if (bytes == 0)
break;
readSoFar += bytes;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can reads the values properly
var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd();
_entries.Add(fileText);
Interlocked.Increment(ref reads);
}
}
_entries.Add(null);
}
