private void ReadComplete(IAsyncResult res)
{
ReadState rs = (ReadState)res.AsyncState;
using (rs.file)
{
rs.file.EndRead(res);
if (rs.compressLevel > 0)
{
byte[] data = rs.data;
// don't compress things which are already compressed
if ((data.Length > 2) && ((data[0] != 0x1f) || (data[1] != 0x8b)))
{
MemoryStream compStream = new MemoryStream(rs.data.Length);
using (GZipOutputStream gzOut = new GZipOutputStream(compStream))
{
gzOut.SetLevel(rs.compressLevel);
gzOut.Write(rs.data, 0, rs.data.Length);
gzOut.Finish();
rs.data = compStream.ToArray();
}
}
}
lock (writeData)
{
writeData[rs.writeDataIndex] = rs.data;
newWriteSignal.Release(1);
}
}
outstandingReads.Increment();
}
// write manifest & checksum of manifest
private static void WriteManifest(PackageBuildInfo buildInfo, Manifest manifest)
{
var json = JsonUtility.ToJson(manifest);
var bytes = Encoding.UTF8.GetBytes(json);
var manifestRawPath = Path.Combine(buildInfo.packagePath, Manifest.ManifestFileName + ".json");
var manifestChecksumPath = Path.Combine(buildInfo.packagePath, Manifest.ChecksumFileName);
byte[] zData;
using (var zStream = new MemoryStream())
{
using (var outputStream = new GZipOutputStream(zStream))
{
outputStream.SetLevel(Deflater.BEST_COMPRESSION);
outputStream.Write(bytes, 0, bytes.Length);
outputStream.Flush();
}
zStream.Flush();
zData = zStream.ToArray();
}
buildInfo.filelist.Add(Manifest.ManifestFileName);
buildInfo.filelist.Add(Manifest.ManifestFileName + ".json");
var fileEntry = AsManifestEntry(EncryptData(buildInfo, Manifest.ManifestFileName, zData),
buildInfo.data.chunkSize);
var fileEntryJson = JsonUtility.ToJson(fileEntry);
Debug.LogFormat("write manifest: {0}", fileEntryJson);
File.WriteAllBytes(manifestRawPath, bytes);
File.WriteAllText(manifestChecksumPath, fileEntryJson);
}
public override void Initialize()
{
var tasks = new Task[options.Threads];
for (var i = 0; i < options.Threads; i++)
{
var i1 = i;
tasks[i1] = Task.Run(() =>
{
var data = DataGenerator.GenerateString((int)(volume / options.Threads));
using var s = new MemoryStream();
using (var stream = new GZipOutputStream(s))
{
stream.SetLevel(9);
using var sw = new StreamWriter(stream);
sw.Write(data);
sw.Flush();
stream.Finish();
stream.IsStreamOwner = false;
}
s.Seek(0, SeekOrigin.Begin);
datas[i1] = s.ToArray();
});
}
Task.WaitAll(tasks);
}
public byte[] Compress(string message)
{
if (message == null)
{
return(null);
}
using (var dataStream = new MemoryStream())
using (var zipStream = new GZipOutputStream(dataStream))
{
zipStream.SetLevel((int)CompressionLevel);
var rawBytes = Encoding.UTF8.GetBytes(message);
zipStream.Write(rawBytes, 0, rawBytes.Length);
zipStream.Flush();
zipStream.Finish();
var compressedBytes = new byte[dataStream.Length];
dataStream.Seek(0, SeekOrigin.Begin);
dataStream.Read(compressedBytes, 0, compressedBytes.Length);
return(compressedBytes);
}
}
/// <summary>
/// When overridden in a derived class, executes the task.
/// </summary>
/// <returns>
/// true if the task successfully executed; otherwise, false.
/// </returns>
public override bool Execute()
{
var buffer = new byte[32 * 1024];
var rootName = string.IsNullOrWhiteSpace(RootName) ? "" : RootName + "/";
using (var gz = new GZipOutputStream(File.Create(Out)))
using (var outStream = new TarOutputStream(gz))
{
gz.SetLevel(3);
foreach (var file in Include.Select(x => x.ItemSpec.ToNPath()).Where(x => x.FileExists()))
{
AddToArchive(rootName, file, outStream, ref buffer);
}
foreach (var dir in Include.Select(x => x.ItemSpec.ToNPath()).Where(x => x.DirectoryExists()))
{
foreach (var file in dir.Contents(true))
{
var relativePath = file.DirectoryExists() ? file.RelativeTo(dir) : file.Parent.RelativeTo(dir);
var baseName = rootName;
if (!relativePath.IsEmpty)
{
baseName += relativePath.ToString(SlashMode.Forward) + "/";
}
AddToArchive(baseName, file, outStream, ref buffer);
}
}
}
return(true);
}
public override void Run()
{
var tasks = new Task[options.Threads];
for (var i = 0; i < options.Threads; i++)
{
var i1 = i;
tasks[i] = ThreadAffinity.RunAffinity(1uL << i, () =>
{
using (Stream s = new MemoryStream())
{
using var stream = new GZipOutputStream(s);
stream.SetLevel(9);
using var sw = new StreamWriter(stream);
sw.Write(datas[i1]);
sw.Flush();
stream.Finish();
}
BenchmarkRunner.ReportProgress();
});
}
Task.WaitAll(tasks);
}
/// <summary>
/// 지정된 데이타를 압축한다.
/// </summary>
/// <param name="input">압축할 Data</param>
/// <returns>압축된 Data</returns>
public override byte[] Compress(byte[] input)
{
if (IsDebugEnabled)
{
log.Debug(CompressorTool.SR.CompressStartMsg);
}
// check input data
if (input.IsZeroLength())
{
if (IsDebugEnabled)
{
log.Debug(CompressorTool.SR.InvalidInputDataMsg);
}
return(CompressorTool.EmptyBytes);
}
byte[] output;
using (var compressedStream = new MemoryStream(input.Length)) {
using (var gzs = new GZipOutputStream(compressedStream)) {
gzs.SetLevel(ZipLevel);
gzs.Write(input, 0, input.Length);
gzs.Finish();
}
output = compressedStream.ToArray();
}
if (IsDebugEnabled)
{
log.Debug(CompressorTool.SR.CompressResultMsg, input.Length, output.Length, output.Length / (double)input.Length);
}
return(output);
}
public TcpRawMessage Compress(CompressionLevel compressionLevel)
{
CheckDisposed();
TcpRawMessage compressedMessage = new TcpRawMessage(this.memoryStreamPool, (int)this.stream.Length);
compressedMessage.Flags = this.Flags;
this.stream.Position = 0;
byte[] buffer = null;
try
{
buffer = ArrayPool <byte> .Shared.Rent(BUFFER_SIZE);
using (GZipOutputStream gzip = new GZipOutputStream(compressedMessage.stream, BUFFER_SIZE))
{
gzip.SetLevel(6);
gzip.IsStreamOwner = false;
int readBytes;
while ((readBytes = this.stream.Read(buffer, 0, buffer.Length)) > 0)
{
gzip.Write(buffer, 0, readBytes);
}
}
}
finally
{
if (buffer != null)
{
ArrayPool <byte> .Shared.Return(buffer);
}
}
compressedMessage.Position = 0;
return(compressedMessage);
}
private List <FSEntry> RecompressDir(string dir)
{
string[] files = Directory.GetFiles(dir);
List <FSEntry> entries = new List <FSEntry>(files.Length);
byte[] compBuffer = new byte[8192];
foreach (string f in files)
{
FSEntry fe = new FSEntry();
fe.tocEntry = new TocEntry();
fe.tocEntry.dateTime = GetFileCreationTimeInSeconds(f);
fe.tocEntry.flags = TOCFlags_File;
fe.tocEntry.offsetIndex = 0;
fe.tocEntry.name = new byte[25];
fe.diskFile = f;
using (FileStream fs = new FileStream(f, FileMode.Open, FileAccess.Read))
{
MemoryStream ms = new MemoryStream((int)fs.Length);
GZipOutputStream gzOut = new GZipOutputStream(ms);
gzOut.SetLevel(2);
StreamUtils.Copy(fs, gzOut, compBuffer);
byte[] compData = ms.GetBuffer();
Array.Resize(ref compData, (int)ms.Position);
fe.compressedFile = compData;
}
string gzName = Path.GetFileName(f) + ".gz";
fe.name = gzName;
entries.Add(fe);
}
return(entries);
}
/// <summary>
/// Compress a given file into Tar archive.
/// </summary>
public void Compress()
{
Common.ValidateOverwrite(this.overwriteTarget, this.target);
DirectoryInfo dir = new DirectoryInfo(Path.GetDirectoryName(this.source));
this.Log(String.Format("Archiving: [{0}] -> [{1}].", this.source, this.target), LogLevel.Minimal);
if (tarLevel == TarCompressionLevel.None)
{
using (Stream outStream = File.Create(this.target))
{
ArchiveFile(outStream, dir);
}
}
else if (tarLevel == TarCompressionLevel.BZip2)
{
using (BZip2OutputStream bz2Stream = new BZip2OutputStream(File.Create(this.target), 9))
{
ArchiveFile(bz2Stream, dir);
}
}
else if (tarLevel == TarCompressionLevel.GZip)
{
using (GZipOutputStream gzoStream = new GZipOutputStream(File.Create(this.target)))
{
gzoStream.SetLevel(9);
ArchiveFile(gzoStream, dir);
}
}
this.Log(String.Format("Successfully Archived: [{0}] -> [{1}].", this.source, this.target), LogLevel.Minimal);
Common.RemoveFile(this.removeSource, this.source);
}
public static async Task CreateGzipAsync(FileSystemStorageFile Source, string NewZipPath, CompressionLevel Level, ProgressChangedEventHandler ProgressHandler = null)
{
if (Level == CompressionLevel.Undefine)
{
throw new ArgumentException("Undefine is not allowed in this function", nameof(Level));
}
if (await FileSystemStorageItemBase.CreateAsync(NewZipPath, StorageItemTypes.File, CreateOption.GenerateUniqueName).ConfigureAwait(false) is FileSystemStorageFile NewFile)
{
using (FileStream SourceFileStream = await Source.GetFileStreamFromFileAsync(AccessMode.Read))
using (FileStream NewFileStream = await NewFile.GetFileStreamFromFileAsync(AccessMode.Exclusive))
using (GZipOutputStream GZipStream = new GZipOutputStream(NewFileStream))
{
GZipStream.SetLevel((int)Level);
GZipStream.IsStreamOwner = false;
GZipStream.FileName = Source.Name;
await SourceFileStream.CopyToAsync(GZipStream, ProgressHandler : ProgressHandler).ConfigureAwait(false);
}
}
else
{
throw new UnauthorizedAccessException();
}
}
public byte[] Compress <ObjectType>(ObjectType ObjToCompress)
{
if (ObjToCompress == null)
{
return(null);
}
string toCompress = JsonConvert.SerializeObject(ObjToCompress, Formatting.None, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
DefaultValueHandling = DefaultValueHandling.Ignore
});
using (Stream memOutput = new MemoryStream())
{
GZipOutputStream zipOut = new GZipOutputStream(memOutput);
zipOut.SetLevel(9);
StreamWriter writer = new StreamWriter(zipOut);
writer.Write(toCompress);
writer.Flush();
zipOut.Finish();
byte[] bytes = new byte[memOutput.Length];
memOutput.Seek(0, SeekOrigin.Begin);
memOutput.Read(bytes, 0, bytes.Length);
return(bytes);
}
}
/// <summary>
/// 压缩字节流数据
/// <para>注意:</para>
/// <para>要压缩的字节流数据如果过小则有可能会起到反效果</para>
/// </summary>
/// <param name="fromBytes">要压缩的字节流数据</param>
/// <returns></returns>
public static byte[] Compress(this byte[] fromBytes)
{
//初始化返回值
byte[] result = fromBytes;
try
{
//声明一个内存流，在内存中压缩
using (MemoryStream resultStream = new MemoryStream())
{
//声明一个GZip输出压缩流
using (GZipOutputStream gZip = new GZipOutputStream(resultStream))
{
//设置压缩等级为6级（等级越高效果越好）
gZip.SetLevel(6);
//将压缩好的数据输出到内存流中
gZip.Write(fromBytes, 0, fromBytes.Length);
}
result = resultStream.ToArray();
}
}catch (Exception ex)
{
//如果发生异常就认为压缩失败，返回原字节流数据
LogUtil.WriteException(ex.ToString());
}
return(result);
}
public Stream GetBackupWriter(Dictionary <string, List <string> > config, Stream writeToStream)
{
ParameterInfo.ValidateParams(mBackupParamSchema, config);
int level = 9;
List <string> sLevel;
if (config.TryGetValue("level", out sLevel))
{
if (!int.TryParse(sLevel[0], out level))
{
throw new ArgumentException(string.Format("gzip: Unable to parse the integer: {0}", sLevel));
}
}
if (level < 1 || level > 9)
{
throw new ArgumentException(string.Format("gzip: Level must be between 1 and 9: {0}", level));
}
Console.WriteLine(string.Format("gzip: level = {0}", level));
GZipOutputStream gzos = new GZipOutputStream(writeToStream, 4 * 1024 * 1024);
gzos.SetLevel(level);
return(gzos);
}
static void Build()
{
using (FileStream file = new FileStream("solodata.dat", FileMode.Create, FileAccess.ReadWrite))
{
using (TextReader input = new StreamReader(File.OpenRead("Menus.csv"), Encoding.UTF8))
{
using (CsvReader csv = new CsvReader(input))
{
file.WriteUInt(0);
uint rowCount = 0;
csv.Read();
while (csv.Read())
{
string value = csv.GetField(1);
file.WriteUShort(ushort.Parse(value, System.Globalization.NumberStyles.HexNumber));
rowCount++;
}
long filePosition = file.Position;
file.Position = 0;
file.WriteUInt(rowCount);
file.Position = filePosition;
}
}
ImportCSV("Cars\\Cars.csv");
var filenames = Directory.EnumerateFiles("Cars", "*.csv").Where(csv => Path.GetFileName(csv) != "Cars.csv");
foreach (string filename in filenames)
{
ImportCSV(filename);
}
long startPosition = file.Position;
file.WriteUInt(0);
foreach (var car in Cars)
{
file.WriteUInt(CarNameConversion.ToCarID(car.Key));
file.WriteUInt(car.Value);
}
file.Position = startPosition;
file.WriteUInt((uint)Cars.Count);
file.Position = 0;
using (var gzip = new FileStream("solodata.dat.gz", FileMode.Create, FileAccess.Write))
{
using (var compression = new GZipOutputStream(gzip))
{
compression.SetLevel(8);
compression.IsStreamOwner = false;
file.CopyTo(compression);
}
}
}
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("Expected arguments: fileFrom fileTo [-b64] [-unzip]");
return;
}
bool useBase64 = false;
bool unzip = false;
for (int i = 2; i < args.Length; i++)
{
if (args[i].Equals("-b64"))
{
useBase64 = true;
}
if (args[i].Equals("unzip"))
{
unzip = true;
}
}
string fileFrom = args[0];
string fileTo = args[1];
if (unzip)
{
byte[] data;
if (useBase64)
{
string text = File.ReadAllLines(fileFrom)[0];
data = Convert.FromBase64String(text);
}
else
{
data = File.ReadAllBytes(fileFrom);
}
Stream stream = new FileStream(fileTo, FileMode.Create, FileAccess.Write);
BinaryWriter writer = new BinaryWriter(stream);
writer.Write(data.Length);
//stream.Close();
GZipOutputStream gZipOutputStream = new GZipOutputStream(stream);
gZipOutputStream.SetLevel(5);
gZipOutputStream.Write(data, 0, data.Length);
gZipOutputStream.Close();
}
else
{
Stream reader = new FileStream(fileFrom, FileMode.Open, FileAccess.Read);
reader.Position = 1;
GZipInputStream gZipInputStream = new GZipInputStream(reader);
gZipInputStream.CopyTo(new FileStream(fileTo, FileMode.Create, FileAccess.Write));
}
}
private TarOutputStream newTarOutputStream(string filename)
{
Stream oStream = File.Create(filename);
//gzipStream = new BZip2OutputStream(oStream);
GZipOutputStream gzipStream = new GZipOutputStream(oStream);
gzipStream.SetLevel(3);
return(new TarOutputStream(gzipStream));
}
public static long GZipStream(Stream inputStream, Stream outputStream)
{
using (GZipOutputStream compressionStream = new GZipOutputStream(outputStream))
{
compressionStream.SetLevel(9);
inputStream.CopyTo(compressionStream);
compressionStream.Finish();
return(compressionStream.Length);
}
}
public byte[] Compress(byte[] bytes, int level)
{
using (MemoryStream ms = new MemoryStream()) {
GZipOutputStream gzip = new GZipOutputStream(ms);
gzip.Write(bytes, 0, bytes.Length);
gzip.SetLevel(level);
gzip.Close();
return(ms.ToArray());
}
}
static void PackGTMenu()
{
using (var output = new FileStream("gtmenudat.dat", FileMode.Create, FileAccess.Write))
{
using (var index = new FileStream("gtmenudat.idx", FileMode.Create, FileAccess.Write))
{
index.WriteUInt(0);
uint fileCount = 0;
foreach (string filename in Directory.EnumerateFiles("gtmenudat\\"))
{
if (filename.EndsWith(".gz") && File.Exists(filename.Substring(0, filename.Length - 3)))
{
continue;
}
fileCount++;
index.WriteUInt((uint)output.Position);
Console.WriteLine($"Adding {filename}");
using (var input = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
if (filename.EndsWith(".gz"))
{
input.CopyTo(output);
}
else
{
using (var memory = new MemoryStream())
{
using (var compression = new GZipOutputStream(memory))
{
compression.SetLevel(8);
compression.IsStreamOwner = false;
input.CopyTo(compression);
}
memory.Position = 0;
memory.CopyTo(output);
}
}
}
long misalignedBytes = output.Length % 4;
if (misalignedBytes != 0)
{
output.Position += 4 - misalignedBytes;
}
}
index.Position = 0;
index.WriteUInt(fileCount);
}
}
}
public static MemoryStream GZipCompressStream(Stream decompStream)
{
MemoryStream ms = new MemoryStream((int)decompStream.Length);
using (GZipOutputStream gzos = new GZipOutputStream(ms))
{
gzos.SetLevel(Globals.App.CompressionLevel);
byte[] buffer = new byte[65536];
StreamUtils.Copy(decompStream, gzos, buffer);
}
return(ms);
}
public Packer(string fileName)
{
_fileName = fileName;
var fileStream = File.Create(fileName);
var gzipStream = new GZipOutputStream(fileStream);
gzipStream.SetLevel(9);
_stream = new TarOutputStream(gzipStream);
}
public static long GZipStream(Stream stream)
{
using (var compressedFileStream = new MemoryStream())
{
using (GZipOutputStream compressionStream = new GZipOutputStream(compressedFileStream))
{
compressionStream.SetLevel(9);
stream.CopyTo(compressionStream);
compressionStream.Finish();
return(compressionStream.Length);
}
}
}
public static byte[] CompressData(byte[] source, int offset, int size)
{
MemoryStream output = new MemoryStream();
// BZip2OutputStream gzo = new BZip2OutputStream (output);
GZipOutputStream gzo = new GZipOutputStream(output);
gzo.SetLevel(9);
gzo.Write(source, offset, size);
gzo.Close();
byte [] result = output.ToArray();
output.Close();
return(result);
}
static void Compress(Stream inputFs, Stream outputFs)
{
using (GZipOutputStream stream = new GZipOutputStream(outputFs))
{
stream.SetLevel(CompressLevel);
int size;
while ((size = inputFs.Read(MBytesCache, 0, MBytesCache.Length)) > 0)
{
stream.Write(MBytesCache, 0, size);
}
stream.Flush();
}
}
/// <summary>
/// 创建压缩包文件
/// </summary>
/// <param name="filePath">要创建的文件路径</param>
/// <param name="dirPath">要打包的目录</param>
/// <param name="level">压缩等级，取值范围在 <see cref="Deflater.NO_COMPRESSION"/> ~ <see cref="Deflater.BEST_COMPRESSION"/></param>
/// <param name="progress">进度值监听</param>
/// <returns></returns>
public static bool Create(string filePath, string dirPath, int level = Deflater.BEST_COMPRESSION, ProgressMessageHandler?progress = null)
{
if (!filePath.EndsWith(FileEx.TAR_GZ))
{
filePath += FileEx.TAR_GZ;
}
if (File.Exists(filePath))
{
return(false);
}
if (!Directory.Exists(dirPath))
{
return(false);
}
using var fs = File.Create(filePath);
using var s = new GZipOutputStream(fs);
s.SetLevel(level);
using var archive = TarArchive.CreateOutputTarArchive(s, TarBuffer.DefaultBlockFactor, EncodingCache.UTF8NoBOM);
if (progress != null)
{
archive.ProgressMessageEvent += progress;
}
HandleFiles(dirPath);
HandleDirs(dirPath);
void HandleFiles(string dirPath_)
{
foreach (var file in Directory.GetFiles(dirPath_))
{
var entry = TarEntry.CreateEntryFromFile(file);
entry.Name = file.TrimStart(dirPath).Trim(Path.DirectorySeparatorChar);
if (Path.DirectorySeparatorChar != IOPath.UnixDirectorySeparatorChar)
{
entry.Name = entry.Name.Replace(Path.DirectorySeparatorChar, IOPath.UnixDirectorySeparatorChar);
}
archive.WriteEntry(entry, false);
}
}
void HandleDirs(string dirPath_)
{
foreach (var dir in Directory.GetDirectories(dirPath_))
{
HandleFiles(dir);
}
}
return(true);
}
/// <summary>
/// Wraps a file into the target gz file
/// </summary>
/// <param name="inputFile">the original file</param>
/// <param name="outputFile">the output file</param>
private void Wrap(string inputFile, string outputFile)
{
using (FileStream fsi = new FileStream(inputFile, FileMode.Open, FileAccess.Read))
{
using (FileStream fso = new FileStream(outputFile, FileMode.Create, FileAccess.ReadWrite))
{
using (GZipOutputStream gzo = new GZipOutputStream(fso))
{
gzo.SetLevel(compressionLevel);
fsi.CopyTo(gzo);
}
}
}
}
public static byte[] Compress(string content)
{
MemoryStream memoryStream = new MemoryStream();
using (GZipOutputStream outStream = new GZipOutputStream(memoryStream))
{
var data = Encoding.UTF8.GetBytes(content);
outStream.IsStreamOwner = false;
outStream.SetLevel(4);
outStream.Write(data, 0, data.Length);
outStream.Flush();
outStream.Finish();
}
return(memoryStream.GetBuffer());
}
public static long GZipFile(string name)
{
using (var fileStream = File.OpenRead(name))
{
using (var compressedFileStream = File.Create(name + ".gz"))
{
using (GZipOutputStream compressionStream = new GZipOutputStream(compressedFileStream))
{
compressionStream.SetLevel(9);
fileStream.CopyTo(compressionStream);
compressionStream.Finish();
return(compressionStream.Length);
}
}
}
}
private static void CompressResponse(Response response)
{
response.Headers.Add("Expires", DateTime.Now.Add(TimeSpan.FromDays(365)).ToString("R"));
response.Headers["Transfer-Encoding"] = "chunked";
response.Headers["Content-Encoding"] = "gzip";
response.Headers["Cache-Control"] = "private, max-age=0";
response.ContentType = "text/html; charset=utf-8";
var contents = response.Contents;
response.Contents = responseStream => {
using (var compression = new GZipOutputStream(responseStream)) {
compression.SetLevel(9);
contents(compression);
}
};
}