private Stream CreateCompressionStream(Stream stream, WaveCompression type, CompressionMode mode) { switch (type) { case WaveCompression.Uncompressed: return(stream); case WaveCompression.GZip: return(new GZipStream(stream, mode, true)); case WaveCompression.DEFLATE: return(new DeflateStream(stream, mode, true)); case WaveCompression.LZMA: return(mode == CompressionMode.Compress ? LzmaStream.Encode(stream) : LzmaStream.Decode(stream)); case WaveCompression.Bzip2: return(mode == CompressionMode.Compress ? Bzip2Stream.Output(new SkipStream(stream, 2), false, 9) : Bzip2Stream.Input(stream, true)); case WaveCompression.Flac: return(new FlacStream(stream, true, BitsPerSample)); default: throw new CustomException("This compression method is not supported."); } }
private Stream GetWriteStream(Stream writeStream) { this.counting = new CountingWritableSubStream(writeStream); Stream counting = this.counting; switch (this.writer.zipCompressionInfo.Compression) { case ZipCompressionMethod.BZip2: return(new BZip2Stream(this.counting, CompressionMode.Compress, true, false)); case ZipCompressionMethod.LZMA: { this.counting.WriteByte(9); this.counting.WriteByte(20); this.counting.WriteByte(5); this.counting.WriteByte(0); LzmaStream stream2 = new LzmaStream(new LzmaEncoderProperties(!this.originalStream.CanSeek), false, this.counting); this.counting.Write(stream2.Properties, 0, stream2.Properties.Length); return(stream2); } case ZipCompressionMethod.PPMd: this.counting.Write(this.writer.ppmdProperties.Properties, 0, 2); return(new PpmdStream(this.writer.ppmdProperties, this.counting, true)); case ZipCompressionMethod.None: return(counting); case ZipCompressionMethod.Deflate: return(new DeflateStream(this.counting, CompressionMode.Compress, this.writer.zipCompressionInfo.DeflateCompressionLevel, true)); } throw new NotSupportedException("CompressionMethod: " + this.writer.zipCompressionInfo.Compression); }
private Stream GetWriteStream(Stream originalStream) { counting = new CountingWritableStream(originalStream); switch (compressionInfo.Compression) { case ZipCompressionMethod.None: return(counting); case ZipCompressionMethod.Deflate: return(new DeflateStream(counting, CompressionMode.Compress, compressionInfo.DeflateCompressionLevel, true)); case ZipCompressionMethod.BZip2: return(new BZip2Stream(counting, CompressionMode.Compress, true)); case ZipCompressionMethod.LZMA: counting.WriteByte(9); counting.WriteByte(20); counting.WriteByte(5); counting.WriteByte(0); LzmaStream lzmaStream = new LzmaStream(new LzmaEncoderProperties(!originalStream.CanSeek), false, counting); counting.Write(lzmaStream.Properties, 0, lzmaStream.Properties.Length); return(lzmaStream); case ZipCompressionMethod.PPMd: counting.Write(writer.ppmdProperties.Properties, 0, 2); return(new PpmdStream(writer.ppmdProperties, counting, true)); default: throw new NotSupportedException("CompressionMethod: " + compressionInfo.Compression); } }
// Decompressed the data in this block public byte[] Decompress() { var decompressed = new byte[UncompressedSize]; using var lzmaStream = new LzmaStream(Properites, new MemoryStream(CompressedData)); lzmaStream.Read(decompressed, 0, decompressed.Length); return(decompressed); }
public void TestLzma2Decompress1Byte() { var properties = new byte[] { 0x01 }; var compressedData = new byte[] { 0x01, 0x00, 0x00, 0x58, 0x00 }; var lzma2Stream = new MemoryStream(compressedData); var decompressor = new LzmaStream(properties, lzma2Stream, 5, 1); Assert.Equal('X', decompressor.ReadByte()); }
public void TestLzma2Decompress1Byte() { byte[] properties = new byte[] { 0x01 }; byte[] compressedData = new byte[] { 0x01, 0x00, 0x00, 0x58, 0x00 }; MemoryStream lzma2Stream = new MemoryStream(compressedData); LzmaStream decompressor = new LzmaStream(properties, lzma2Stream, 5, 1); Assert.AreEqual('X', decompressor.ReadByte()); }
public static ZipReturn ReadHeaderOrPackedHeader(Stream stream, long baseOffset, out Header header) { header = null; using (BinaryReader br = new BinaryReader(stream, Encoding.UTF8, true)) { HeaderProperty hp = (HeaderProperty)br.ReadByte(); switch (hp) { case HeaderProperty.kEncodedHeader: { StreamsInfo streamsInfo = new StreamsInfo(); streamsInfo.Read(br); if (streamsInfo.Folders.Length > 1) { return(ZipReturn.ZipUnsupportedCompression); } Folder firstFolder = streamsInfo.Folders[0]; if (firstFolder.Coders.Length > 1) { return(ZipReturn.ZipUnsupportedCompression); } byte[] method = firstFolder.Coders[0].Method; if (!((method.Length == 3) && (method[0] == 3) && (method[1] == 1) && (method[2] == 1))) // LZMA { return(ZipReturn.ZipUnsupportedCompression); } stream.Seek(baseOffset + (long)streamsInfo.PackPosition, SeekOrigin.Begin); using (LzmaStream decoder = new LzmaStream(firstFolder.Coders[0].Properties, stream)) { ZipReturn zr = ReadHeaderOrPackedHeader(decoder, baseOffset, out header); if (zr != ZipReturn.ZipGood) { return(zr); } } return(ZipReturn.ZipGood); } case HeaderProperty.kHeader: { header = new Header(); header.Read(br); return(ZipReturn.ZipGood); } } return(ZipReturn.ZipCentralDirError); } }
public void SimpleReadWithSmallBuffer() { using var packStream = GetType().OpenSiblingResourceStream("r.lzma"); using var unpackTarget = new MemoryStream(); using (var lzmaStream = new LzmaStream(packStream, CompressionMode.Decompress, bufferSize: 10)) lzmaStream.CopyTo(unpackTarget); unpackTarget.Seek(0, SeekOrigin.Begin); using var unpackReader = new StreamReader(unpackTarget); var unpacked = unpackReader.ReadToEnd(); Assert.AreEqual(GetReference(), unpacked); }
private Stream GetWriteStream(Stream writeStream) { counting = new CountingWritableSubStream(writeStream); Stream output = counting; switch (writer.compression) { case ZipCompressionMethod.None: { return(output); } case ZipCompressionMethod.Deflate: { return(new System.IO.Compression.DeflateStream(counting, System.IO.Compression.CompressionMode.Compress, true)); } #if BZIP2 case ZipCompressionMethod.BZip2: { return(new BZip2Stream(counting, CompressionMode.Compress, true)); } #endif #if LZMA case ZipCompressionMethod.LZMA: { counting.WriteByte(9); counting.WriteByte(20); counting.WriteByte(5); counting.WriteByte(0); LzmaStream lzmaStream = new LzmaStream(new LzmaEncoderProperties(!originalStream.CanSeek), false, counting); counting.Write(lzmaStream.Properties, 0, lzmaStream.Properties.Length); return(lzmaStream); } #endif #if PPMd case ZipCompressionMethod.PPMd: { counting.Write(writer.ppmdProperties.Properties, 0, 2); return(new PpmdStream(writer.ppmdProperties, counting, true)); } #endif default: { throw new NotSupportedException("CompressionMethod: " + writer.compression); } } }
private void Pack(LzmaCompressionParameters lzmaCompressionParameters) { using var rawStream = GetType().OpenSiblingResourceStream("flowers.bmp"); using var packTarget = new MemoryStream(); using (var lzmaStream = new LzmaStream(packTarget, lzmaCompressionParameters)) rawStream.CopyTo(lzmaStream); using var packSource = new MemoryStream(packTarget.ToArray()); using var rawTarget = new MemoryStream(); using (var unpackLzmaStream = new LzmaStream(packSource, CompressionMode.Decompress)) unpackLzmaStream.CopyTo(rawTarget); Assert.IsTrue(IsEqual(rawStream, rawTarget)); }
public ZipReturn ZipFileCreate(string newFilename, bool compressOutput, int dictionarySize = 1 << 24, int numFastBytes = 64) { if (ZipOpen != ZipOpenType.Closed) { return(ZipReturn.ZipFileAlreadyOpen); } DirUtil.CreateDirForFile(newFilename); _zipFileInfo = new FileInfo(newFilename); int errorCode = FileStream.OpenFileWrite(newFilename, out _zipFs); if (errorCode != 0) { ZipFileClose(); return(ZipReturn.ZipErrorOpeningFile); } ZipOpen = ZipOpenType.OpenWrite; _signatureHeader = new SignatureHeader(); _header = new Header(); using (BinaryWriter bw = new BinaryWriter(_zipFs, Encoding.UTF8, true)) { _signatureHeader.Write(bw); } _baseOffset = _zipFs.Position; _compressed = compressOutput; _unpackedStreamSize = 0; if (_compressed) { LzmaEncoderProperties ep = new LzmaEncoderProperties(true, dictionarySize, numFastBytes); LzmaStream lzs = new LzmaStream(ep, false, _zipFs); _codeMSbytes = lzs.Properties; _lzmaStream = lzs; /* * ZstandardStream zss = new ZstandardStream(_zipFs, 22, true); * _codeMSbytes = new byte[] { 1, 4, 18, 0, 0 }; * _lzmaStream = zss; */ _packStreamStart = (ulong)_zipFs.Position; } return(ZipReturn.ZipGood); }
private static async Task <long> CountOutputBytes(byte[] data) { using (var outStream = new MemoryStream()) { using (var lzmaStream = new LzmaStream(new LzmaEncoderProperties(), false, outStream)) { using (var inStream = new MemoryStream(data)) { await inStream.CopyToAsync(lzmaStream); } } return(outStream.Length); } }
/// <summary>解压缩数据流</summary> /// <param name="inStream">输入流</param> /// <param name="outStream">输出流。如果不指定,则内部实例化一个内存流</param> /// <remarks>返回输出流,注意此时指针位于末端</remarks> public static Stream DecompressLzma(this Stream inStream, Stream outStream = null) { if (outStream == null) { outStream = new MemoryStream(); } // 第三个参数为true,保持数据流打开,内部不应该干涉外部,不要关闭外部的数据流 using (var stream = new LzmaStream(inStream, CompressionMode.Decompress)) { stream.CopyTo(outStream); stream.Close(); } return(outStream); }
internal static MemoryStream SevenZipDecompress(Stream compressedStream, long outputSize) { //manually read the LZMA properties var props = new byte[5]; compressedStream.Read(props, 0, 5); var ms = new MemoryStream((int)outputSize); using (var lzmaStream = new LzmaStream(props, compressedStream, -1, outputSize)) { lzmaStream.CopyTo(ms); } ms.Position = 0; return(ms); }
/// <summary>压缩数据流</summary> /// <param name="inStream">输入流</param> /// <param name="outStream">输出流。如果不指定,则内部实例化一个内存流</param> /// <param name="level">压缩等级</param> /// <remarks>返回输出流,注意此时指针位于末端</remarks> public static Stream CompressLzma(this Stream inStream, Stream outStream = null, Int32 level = 4) { if (outStream == null) { outStream = new MemoryStream(); } // 第三个参数为true,保持数据流打开,内部不应该干涉外部,不要关闭外部的数据流 using (var stream = new LzmaStream(outStream, CompressionMode.Compress, level)) { inStream.CopyTo(stream); stream.Flush(); stream.Close(); } return(outStream); }
private static void DecompressLzma(Stream compressed, Stream target) { var lzmaProperties = new byte[5]; compressed.Seek(4, SeekOrigin.Current); compressed.Read(lzmaProperties, 0, 5); var lzmaStream = new LzmaStream(lzmaProperties, compressed); int readBytes; var buffer = new byte[512]; do { readBytes = lzmaStream.Read(buffer, 0, buffer.Length); target.Write(buffer, 0, readBytes); } while (readBytes > 0); target.Flush(); target.Seek(0, SeekOrigin.Begin); }
internal static MemoryStream SevenZipCompress(Stream decompressedStream, SevenZipCompressionLevel sevenZipCompressionLevel) { //256 starting capacity so we can write the LZMA properties var ms = new MemoryStream(256); var dictionarySize = (int)sevenZipCompressionLevel; //explicit using-scope so we can readjust the position of the output stream before returning. //the dispose function of the LzmaStream changes the output stream and we don't want to call it after changing //the stream position and returning using (var lzmaStream = new LzmaStream(new LzmaEncoderProperties(false, dictionarySize), false, ms)) { //the LzmaStream does not write the properties to the output stream and only writes the compressed data //meaning we have to manually write the properties to the output streams so it can be decompressed later ms.Write(lzmaStream.Properties, 0, lzmaStream.Properties.Length); decompressedStream.CopyTo(lzmaStream); } ms.Position = 0; return(ms); }
private Stream ReadBlockData(BlockTableEntry fileBlock, bool raw = false) { var ret = (Stream)SourceFile.CreateViewStream(fileBlock.DirectoryOffset, fileBlock.BlockSize, MemoryMappedFileAccess.Read); if (!raw) { if (Deflate) { ret = new DeflateStream(ret, CompressionMode.Decompress, false); } if (Rar) { byte[] properties = new byte[5]; ret = new BufferedStream(ret, 16777216); ret.Read(properties, 0, properties.Length); ret = new LzmaStream(properties, ret, ret.Length - properties.Length, UncompressedSize, null, false); } } return(ret); }
private static uint UncompressScene(ref MemoryStream image, uint offset, out MemoryStream sceneBuffer, uint size) { // throw new NotImplementedException(); sceneBuffer = new MemoryStream(); //BinaryReader bin = new BinaryReader() MemoryStream rawSceneBuffer = new MemoryStream(); image.Position = offset; int _size = (int)Convert.ToInt64(size); CopyStream(image, rawSceneBuffer, _size); rawSceneBuffer.Position = 0; BinaryReader reader = new BinaryReader(rawSceneBuffer); UInt32 Magic = reader.ReadUInt32(); rawSceneBuffer.Position = 0; if (Magic == Common.FourCC("LZMA", false)) { rawSceneBuffer.Position = 4; UInt32 RealSize = reader.ReadUInt32(); UInt32 CompressedSize = reader.ReadUInt32(); byte[] properties = new byte[5]; properties = reader.ReadBytes(5); //rawSceneBuffer.Position = 0; LzmaStream lzmaStream = new LzmaStream(properties, rawSceneBuffer, size, CompressedSize); //_ = lzmaStream.Position; throw new NotImplementedException(); } else if (Magic == Common.FourCC("bvcd", false)) { rawSceneBuffer.CopyTo(sceneBuffer); return(size); } return(0); }
public ZipReturn ZipFileCreate(string newFilename, bool compressOutput) { if (ZipOpen != ZipOpenType.Closed) { return(ZipReturn.ZipFileAlreadyOpen); } DirUtil.CreateDirForFile(newFilename); _zipFileInfo = new FileInfo(newFilename); int errorCode = FileStream.OpenFileWrite(newFilename, out _zipFs); if (errorCode != 0) { ZipFileClose(); return(ZipReturn.ZipErrorOpeningFile); } ZipOpen = ZipOpenType.OpenWrite; _signatureHeader = new SignatureHeader(); _header = new Header(); BinaryWriter bw = new BinaryWriter(_zipFs); _signatureHeader.Write(bw); _compressed = compressOutput; _unpackedStreamSize = 0; if (_compressed) { LzmaEncoderProperties ep = new LzmaEncoderProperties(true, 1 << 24, 64); _lzmaStream = new LzmaStream(ep, false, _zipFs); _codeMSbytes = _lzmaStream.Properties; _packStreamStart = (ulong)_zipFs.Position; } return(ZipReturn.ZipGood); }
private byte[] createReplayData() { byte[] content = new ASCIIEncoding().GetBytes(replayStringContent); using (var outStream = new MemoryStream()) { using (var lzma = new LzmaStream(new LzmaEncoderProperties(false, 1 << 21, 255), false, outStream)) { outStream.Write(lzma.Properties); long fileSize = content.Length; for (int i = 0; i < 8; i++) { outStream.WriteByte((byte)(fileSize >> (8 * i))); } lzma.Write(content); } return(outStream.ToArray()); } }
public Score ReadReplayFile(string replayFilename) { Score score; using (Stream s = storage.GetStream(Path.Combine(replay_folder, replayFilename))) using (SerializationReader sr = new SerializationReader(s)) { var ruleset = Ruleset.GetRuleset((PlayMode)sr.ReadByte()); var processor = ruleset.CreateScoreProcessor(); score = processor.GetScore(); /* score.Pass = true;*/ var version = sr.ReadInt32(); /* score.FileChecksum = */ var beatmapHash = sr.ReadString(); score.Beatmap = beatmaps.Query <BeatmapInfo>().Where(b => b.Hash == beatmapHash).FirstOrDefault(); /* score.PlayerName = */ sr.ReadString(); /* var localScoreChecksum = */ sr.ReadString(); /* score.Count300 = */ sr.ReadUInt16(); /* score.Count100 = */ sr.ReadUInt16(); /* score.Count50 = */ sr.ReadUInt16(); /* score.CountGeki = */ sr.ReadUInt16(); /* score.CountKatu = */ sr.ReadUInt16(); /* score.CountMiss = */ sr.ReadUInt16(); score.TotalScore = sr.ReadInt32(); score.MaxCombo = sr.ReadUInt16(); /* score.Perfect = */ sr.ReadBoolean(); /* score.EnabledMods = (Mods)*/ sr.ReadInt32(); /* score.HpGraphString = */ sr.ReadString(); /* score.Date = */ sr.ReadDateTime(); var compressedReplay = sr.ReadByteArray(); if (version >= 20140721) { /*OnlineId =*/ sr.ReadInt64(); } else if (version >= 20121008) { /*OnlineId =*/ sr.ReadInt32(); } using (var replayInStream = new MemoryStream(compressedReplay)) { byte[] properties = new byte[5]; if (replayInStream.Read(properties, 0, 5) != 5) { throw (new Exception("input .lzma is too short")); } long outSize = 0; for (int i = 0; i < 8; i++) { int v = replayInStream.ReadByte(); if (v < 0) { throw (new Exception("Can't Read 1")); } outSize |= ((long)(byte)v) << (8 * i); } long compressedSize = replayInStream.Length - replayInStream.Position; using (var lzma = new LzmaStream(properties, replayInStream, compressedSize, outSize)) using (var reader = new StreamReader(lzma)) score.Replay = new LegacyReplay(reader); } } return(score); }
public static void VirusTotalFileFromRepo(DbFile file) { try { if (!Context.VirusTotalEnabled) { Failed?.Invoke("VirusTotal is not usable"); return; } if (vTotal == null) { Failed?.Invoke("VirusTotal is not initalized"); } FileReport fResult = null; UpdateProgress?.Invoke("Requesting existing report to VirusTotal", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif Task.Run(async() => { fResult = await vTotal.GetFileReportAsync(file.Sha256); }).Wait(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.VirusTotalFileFromRepo({0}): VirusTotal took {1} seconds to answer for SHA256 request", file, stopwatch.Elapsed.TotalSeconds); #endif if (fResult.ResponseCode == FileReportResponseCode.NotPresent) { Failed?.Invoke(fResult.VerboseMsg); return; } if (fResult.ResponseCode != FileReportResponseCode.Queued) { if (fResult.ResponseCode == FileReportResponseCode.Present) { if (fResult.Positives > 0) { file.HasVirus = true; if (fResult.Scans != null) { foreach (KeyValuePair <string, ScanEngine> engine in fResult.Scans) { if (!engine.Value.Detected) { continue; } file.Virus = engine.Value.Result; file.VirusTotalTime = engine.Value.Update; dbCore.DbOps.UpdateFile(file); ScanFinished?.Invoke(file); return; } } } else { // If no scan has been done, mark as false. // If a positive has already existed don't overwrite it. file.HasVirus = false; file.Virus = null; file.VirusTotalTime = DateTime.UtcNow; dbCore.DbOps.UpdateFile(file); ScanFinished?.Invoke(file); return; } } string repoPath; AlgoEnum algorithm; if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".gz"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".gz"); algorithm = AlgoEnum.GZip; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".bz2"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".bz2"); algorithm = AlgoEnum.BZip2; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lzma"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lzma"); algorithm = AlgoEnum.LZMA; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lz"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lz"); algorithm = AlgoEnum.LZip; } else { Failed?.Invoke($"Cannot find file with hash {file.Sha256} in the repository"); return; } UpdateProgress?.Invoke("Uncompressing file...", null, 0, 0); var inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); Stream zStream = null; switch (algorithm) { case AlgoEnum.GZip: zStream = new GZipStream(inFs, CompressionMode.Decompress); break; case AlgoEnum.BZip2: zStream = new BZip2Stream(inFs, CompressionMode.Decompress); break; case AlgoEnum.LZMA: byte[] properties = new byte[5]; inFs.Read(properties, 0, 5); inFs.Seek(8, SeekOrigin.Current); zStream = new LzmaStream(properties, inFs, inFs.Length - 13, file.Length); break; case AlgoEnum.LZip: zStream = new LZipStream(inFs, CompressionMode.Decompress); break; } ScanResult sResult = null; #if DEBUG stopwatch.Restart(); #endif // Cannot use zStream directly, VirusTotal.NET requests the size *sigh* string tmpFile = Path.Combine(Settings.Current.TemporaryFolder, Path.GetTempFileName()); var outFs = new FileStream(tmpFile, FileMode.Create, FileAccess.ReadWrite); zStream?.CopyTo(outFs); zStream?.Close(); outFs.Seek(0, SeekOrigin.Begin); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.VirusTotalFileFromRepo({0}): Uncompressing took {1} seconds", file, stopwatch.Elapsed.TotalSeconds); #endif UpdateProgress?.Invoke("Uploading file to VirusTotal...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif Task.Run(async() => { sResult = await vTotal.ScanFileAsync(outFs, file.Sha256); // Keep filename private, sorry! }).Wait(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.VirusTotalFileFromRepo({0}): Upload to VirusTotal took {1} seconds", file, stopwatch.Elapsed.TotalSeconds); #endif outFs.Close(); File.Delete(tmpFile); if (sResult == null || sResult.ResponseCode == ScanFileResponseCode.Error) { if (sResult == null) { Failed?.Invoke("Cannot send file to VirusTotal"); } else { Failed(sResult.VerboseMsg); } return; } // Seems that we are faster than them, getting a lot of "not queued" responses... Thread.Sleep(2500); Task.Run(async() => { fResult = await vTotal.GetFileReportAsync(file.Sha256); }).Wait(); } UpdateProgress?.Invoke("Waiting for VirusTotal analysis...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif int counter = 0; while (fResult.ResponseCode == FileReportResponseCode.Queued) { // Timeout... if (counter == 10) { break; } // Wait 15 seconds so we fall in the 4 requests/minute Thread.Sleep(15000); Task.Run(async() => { fResult = await vTotal.GetFileReportAsync(file.Sha256); }).Wait(); counter++; } #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.VirusTotalFileFromRepo({0}): VirusTotal took {1} seconds to do the analysis", file, stopwatch.Elapsed.TotalSeconds); #endif if (fResult.ResponseCode != FileReportResponseCode.Present) { Failed?.Invoke(fResult.VerboseMsg); return; } if (fResult.Positives > 0) { file.HasVirus = true; if (fResult.Scans == null) { return; } foreach (KeyValuePair <string, ScanEngine> engine in fResult.Scans) { if (!engine.Value.Detected) { continue; } file.Virus = engine.Value.Result; file.VirusTotalTime = engine.Value.Update; dbCore.DbOps.UpdateFile(file); ScanFinished?.Invoke(file); return; } } else { // If no scan has been done, mark as false. // If a positive has already existed don't overwrite it. file.HasVirus = false; file.Virus = null; file.VirusTotalTime = DateTime.UtcNow; dbCore.DbOps.UpdateFile(file); ScanFinished?.Invoke(file); } } catch (Exception ex) { Failed?.Invoke($"Exception {ex.InnerException.Message} when calling VirusTotal"); #if DEBUG Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException); #endif } }
public Score Parse(Stream stream) { var score = new Score { ScoreInfo = new ScoreInfo(), Replay = new Replay() }; using (SerializationReader sr = new SerializationReader(stream)) { currentRuleset = GetRuleset(sr.ReadByte()); score.ScoreInfo = new ScoreInfo { Ruleset = currentRuleset.RulesetInfo }; var version = sr.ReadInt32(); var workingBeatmap = GetBeatmap(sr.ReadString()); if (workingBeatmap is DummyWorkingBeatmap) { throw new BeatmapNotFoundException(); } currentBeatmap = workingBeatmap.Beatmap; score.ScoreInfo.Beatmap = currentBeatmap.BeatmapInfo; score.ScoreInfo.User = new User { Username = sr.ReadString() }; // MD5Hash sr.ReadString(); var count300 = (int)sr.ReadUInt16(); var count100 = (int)sr.ReadUInt16(); var count50 = (int)sr.ReadUInt16(); var countGeki = (int)sr.ReadUInt16(); var countKatu = (int)sr.ReadUInt16(); var countMiss = (int)sr.ReadUInt16(); switch (currentRuleset.LegacyID) { case 0: score.ScoreInfo.Statistics[HitResult.Great] = count300; score.ScoreInfo.Statistics[HitResult.Good] = count100; score.ScoreInfo.Statistics[HitResult.Meh] = count50; score.ScoreInfo.Statistics[HitResult.Miss] = countMiss; break; case 1: score.ScoreInfo.Statistics[HitResult.Great] = count300; score.ScoreInfo.Statistics[HitResult.Good] = count100; score.ScoreInfo.Statistics[HitResult.Miss] = countMiss; break; case 2: score.ScoreInfo.Statistics[HitResult.Perfect] = count300; score.ScoreInfo.Statistics[HitResult.Miss] = countMiss; break; case 3: score.ScoreInfo.Statistics[HitResult.Perfect] = countGeki; score.ScoreInfo.Statistics[HitResult.Great] = count300; score.ScoreInfo.Statistics[HitResult.Good] = countKatu; score.ScoreInfo.Statistics[HitResult.Ok] = count100; score.ScoreInfo.Statistics[HitResult.Meh] = count50; score.ScoreInfo.Statistics[HitResult.Miss] = countMiss; break; } score.ScoreInfo.TotalScore = sr.ReadInt32(); score.ScoreInfo.MaxCombo = sr.ReadUInt16(); /* score.Perfect = */ sr.ReadBoolean(); score.ScoreInfo.Mods = currentRuleset.ConvertLegacyMods((LegacyMods)sr.ReadInt32()).ToArray(); /* score.HpGraphString = */ sr.ReadString(); score.ScoreInfo.Date = sr.ReadDateTime(); var compressedReplay = sr.ReadByteArray(); if (version >= 20140721) { score.ScoreInfo.OnlineScoreID = sr.ReadInt64(); } else if (version >= 20121008) { score.ScoreInfo.OnlineScoreID = sr.ReadInt32(); } if (compressedReplay?.Length > 0) { using (var replayInStream = new MemoryStream(compressedReplay)) { byte[] properties = new byte[5]; if (replayInStream.Read(properties, 0, 5) != 5) { throw new IOException("input .lzma is too short"); } long outSize = 0; for (int i = 0; i < 8; i++) { int v = replayInStream.ReadByte(); if (v < 0) { throw new IOException("Can't Read 1"); } outSize |= (long)(byte)v << (8 * i); } long compressedSize = replayInStream.Length - replayInStream.Position; using (var lzma = new LzmaStream(properties, replayInStream, compressedSize, outSize)) using (var reader = new StreamReader(lzma)) readLegacyReplay(score.Replay, reader); } } } CalculateAccuracy(score.ScoreInfo); return(score); }
public Score Parse(Stream stream) { var score = new Score { Replay = new Replay() }; WorkingBeatmap workingBeatmap; using (SerializationReader sr = new SerializationReader(stream)) { currentRuleset = GetRuleset(sr.ReadByte()); var scoreInfo = new ScoreInfo { Ruleset = currentRuleset.RulesetInfo }; score.ScoreInfo = scoreInfo; int version = sr.ReadInt32(); workingBeatmap = GetBeatmap(sr.ReadString()); if (workingBeatmap is DummyWorkingBeatmap) { throw new BeatmapNotFoundException(); } scoreInfo.User = new APIUser { Username = sr.ReadString() }; // MD5Hash sr.ReadString(); scoreInfo.SetCount300(sr.ReadUInt16()); scoreInfo.SetCount100(sr.ReadUInt16()); scoreInfo.SetCount50(sr.ReadUInt16()); scoreInfo.SetCountGeki(sr.ReadUInt16()); scoreInfo.SetCountKatu(sr.ReadUInt16()); scoreInfo.SetCountMiss(sr.ReadUInt16()); scoreInfo.TotalScore = sr.ReadInt32(); scoreInfo.MaxCombo = sr.ReadUInt16(); /* score.Perfect = */ sr.ReadBoolean(); scoreInfo.Mods = currentRuleset.ConvertFromLegacyMods((LegacyMods)sr.ReadInt32()).ToArray(); // lazer replays get a really high version number. if (version < LegacyScoreEncoder.FIRST_LAZER_VERSION) { scoreInfo.Mods = scoreInfo.Mods.Append(currentRuleset.CreateMod <ModClassic>()).ToArray(); } currentBeatmap = workingBeatmap.GetPlayableBeatmap(currentRuleset.RulesetInfo, scoreInfo.Mods); scoreInfo.BeatmapInfo = currentBeatmap.BeatmapInfo; /* score.HpGraphString = */ sr.ReadString(); scoreInfo.Date = sr.ReadDateTime(); byte[] compressedReplay = sr.ReadByteArray(); if (version >= 20140721) { scoreInfo.OnlineID = sr.ReadInt64(); } else if (version >= 20121008) { scoreInfo.OnlineID = sr.ReadInt32(); } if (compressedReplay?.Length > 0) { using (var replayInStream = new MemoryStream(compressedReplay)) { byte[] properties = new byte[5]; if (replayInStream.Read(properties, 0, 5) != 5) { throw new IOException("input .lzma is too short"); } long outSize = 0; for (int i = 0; i < 8; i++) { int v = replayInStream.ReadByte(); if (v < 0) { throw new IOException("Can't Read 1"); } outSize |= (long)(byte)v << (8 * i); } long compressedSize = replayInStream.Length - replayInStream.Position; using (var lzma = new LzmaStream(properties, replayInStream, compressedSize, outSize)) using (var reader = new StreamReader(lzma)) readLegacyReplay(score.Replay, reader); } } } PopulateAccuracy(score.ScoreInfo); // before returning for database import, we must restore the database-sourced BeatmapInfo. // if not, the clone operation in GetPlayableBeatmap will cause a dereference and subsequent database exception. score.ScoreInfo.BeatmapInfo = workingBeatmap.BeatmapInfo; return(score); }
private static void ParseFrames(BinaryReader reader, Replay replay) { byte[] rawReplayData = reader.ReadByteArray(); MemoryStream frameStream = new MemoryStream(rawReplayData); byte[] properties = new byte[5]; frameStream.Read(properties, 0, 5); long outSize = new BinaryReader(frameStream).ReadInt64(); long compressedSize = frameStream.Length - frameStream.Position; LzmaStream lzmaStream = new LzmaStream(properties, frameStream, compressedSize, outSize); StreamReader sr = new StreamReader(lzmaStream); string[] framesData = sr.ReadToEnd().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); float frameTime = 0; Keys lastKeys = Keys.None; foreach (string rawFrame in framesData) { string[] frameData = rawFrame.Split('|'); float timeDiff = Single.Parse(frameData[0]); if (timeDiff < 0) { continue; } if (replay.Frames.Count != 0) // Sometimes the first frame will have a really high number instead of 0, so lets just skip it { frameTime += timeDiff; } ReplayFrame frame = new ReplayFrame { time = frameTime, x = Single.Parse(frameData[1]), y = Single.Parse(frameData[2]), keys = (Keys)Int16.Parse(frameData[3]) }; // if any key is pressed that wasnt pressed last frame if ((lastKeys & Keys.K1) < (frame.keys & Keys.K1) || (lastKeys & Keys.K2) < (frame.keys & Keys.K2) || (lastKeys & Keys.M1) < (frame.keys & Keys.M1) || (lastKeys & Keys.M2) < (frame.keys & Keys.M2)) { frame.action = FrameAction.Click; } // if any key is pressed that was pressed last frame if (((lastKeys & Keys.K1) == (frame.keys & Keys.K1) && (frame.keys & Keys.K1) > 0) || ((lastKeys & Keys.K2) == (frame.keys & Keys.K2) && (frame.keys & Keys.K2) > 0) || ((lastKeys & Keys.M1) == (frame.keys & Keys.M1) && (frame.keys & Keys.M1) > 0) || ((lastKeys & Keys.M2) == (frame.keys & Keys.M2) && (frame.keys & Keys.M2) > 0)) { frame.action = FrameAction.Hold; } // if any key isnt pressed that was pressed last frame if ((lastKeys & Keys.K1) > (frame.keys & Keys.K1) || (lastKeys & Keys.K2) > (frame.keys & Keys.K2) || (lastKeys & Keys.M1) > (frame.keys & Keys.M1) || (lastKeys & Keys.M2) > (frame.keys & Keys.M2)) { frame.action = FrameAction.Release; } if (lastKeys != frame.keys) { replay.Actions.Add(frame); } lastKeys = frame.keys; replay.Frames.Add(frame); } }
public Score Parse(Stream stream) { Score score; using (SerializationReader sr = new SerializationReader(stream)) { score = new Score { Ruleset = rulesets.GetRuleset(sr.ReadByte()) }; currentRuleset = score.Ruleset.CreateInstance(); /* score.Pass = true;*/ var version = sr.ReadInt32(); /* score.FileChecksum = */ var beatmapHash = sr.ReadString(); score.Beatmap = beatmaps.QueryBeatmap(b => b.MD5Hash == beatmapHash); currentBeatmap = beatmaps.GetWorkingBeatmap(score.Beatmap).Beatmap; /* score.PlayerName = */ score.User = new User { Username = sr.ReadString() }; /* var localScoreChecksum = */ sr.ReadString(); /* score.Count300 = */ sr.ReadUInt16(); /* score.Count100 = */ sr.ReadUInt16(); /* score.Count50 = */ sr.ReadUInt16(); /* score.CountGeki = */ sr.ReadUInt16(); /* score.CountKatu = */ sr.ReadUInt16(); /* score.CountMiss = */ sr.ReadUInt16(); score.TotalScore = sr.ReadInt32(); score.MaxCombo = sr.ReadUInt16(); /* score.Perfect = */ sr.ReadBoolean(); /* score.EnabledMods = (Mods)*/ score.Mods = currentRuleset.ConvertLegacyMods((LegacyMods)sr.ReadInt32()).ToArray(); /* score.HpGraphString = */ sr.ReadString(); /* score.Date = */ sr.ReadDateTime(); var compressedReplay = sr.ReadByteArray(); if (version >= 20140721) { /*OnlineId =*/ sr.ReadInt64(); } else if (version >= 20121008) { /*OnlineId =*/ sr.ReadInt32(); } using (var replayInStream = new MemoryStream(compressedReplay)) { byte[] properties = new byte[5]; if (replayInStream.Read(properties, 0, 5) != 5) { throw new IOException("input .lzma is too short"); } long outSize = 0; for (int i = 0; i < 8; i++) { int v = replayInStream.ReadByte(); if (v < 0) { throw new IOException("Can't Read 1"); } outSize |= (long)(byte)v << (8 * i); } long compressedSize = replayInStream.Length - replayInStream.Position; using (var lzma = new LzmaStream(properties, replayInStream, compressedSize, outSize)) using (var reader = new StreamReader(lzma)) { score.Replay = new Replay { User = score.User }; readLegacyReplay(score.Replay, reader); } } } return(score); }
public override void SetBaseStream(Stream stream) { BaseStream = new LzmaStream(new[] { _dictionarySize }, stream); }
public static void ClamScanFileFromRepo(DbFile file) { try { if (Context.ClamdVersion == null) { Failed?.Invoke("clamd is not usable"); return; } if (clam == null) { Failed?.Invoke("clamd is not initalized"); } string repoPath; AlgoEnum algorithm; if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".gz"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".gz"); algorithm = AlgoEnum.GZip; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".bz2"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".bz2"); algorithm = AlgoEnum.BZip2; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lzma"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lzma"); algorithm = AlgoEnum.LZMA; } else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lz"))) { repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(), file.Sha256[1].ToString(), file.Sha256[2].ToString(), file.Sha256[3].ToString(), file.Sha256[4].ToString(), file.Sha256 + ".lz"); algorithm = AlgoEnum.LZip; } else { Failed?.Invoke($"Cannot find file with hash {file.Sha256} in the repository"); return; } ClamScanResult result = null; Stream zStream = null; if (Settings.Current.ClamdIsLocal) { if (algorithm == AlgoEnum.LZMA || algorithm == AlgoEnum.LZip) { string tmpFile = Path.Combine(Settings.Current.TemporaryFolder, Path.GetTempFileName()); FileStream outFs = new FileStream(tmpFile, FileMode.Create, FileAccess.Write); FileStream inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); if (algorithm == AlgoEnum.LZMA) { byte[] properties = new byte[5]; inFs.Read(properties, 0, 5); inFs.Seek(8, SeekOrigin.Current); zStream = new LzmaStream(properties, inFs, inFs.Length - 13, file.Length); } else { zStream = new LZipStream(inFs, CompressionMode.Decompress); } UpdateProgress?.Invoke("Uncompressing file...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif zStream.CopyTo(outFs); zStream.Close(); outFs.Close(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanFileFromRepo({0}): Uncompressing took {1} seconds", file, stopwatch.Elapsed.TotalSeconds); #endif UpdateProgress?.Invoke("Requesting local scan to clamd server...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif Task.Run(async() => { result = await clam.ScanFileOnServerMultithreadedAsync(tmpFile); }) .Wait(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanFileFromRepo({0}): Clamd took {1} seconds to scan", file, stopwatch.Elapsed.TotalSeconds); #endif File.Delete(tmpFile); } else { UpdateProgress?.Invoke("Requesting local scan to clamd server...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif Task.Run(async() => { result = await clam.ScanFileOnServerMultithreadedAsync(repoPath); }) .Wait(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanFileFromRepo({0}): Clamd took {1} seconds to scan", file, stopwatch.Elapsed.TotalSeconds); #endif } } else { FileStream inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); switch (algorithm) { case AlgoEnum.GZip: zStream = new GZipStream(inFs, CompressionMode.Decompress); break; case AlgoEnum.BZip2: zStream = new BZip2Stream(inFs, CompressionMode.Decompress); break; case AlgoEnum.LZMA: byte[] properties = new byte[5]; inFs.Read(properties, 0, 5); inFs.Seek(8, SeekOrigin.Current); zStream = new LzmaStream(properties, inFs, inFs.Length - 13, file.Length); break; case AlgoEnum.LZip: zStream = new LZipStream(inFs, CompressionMode.Decompress); break; } UpdateProgress?.Invoke("Uploading file to clamd server...", null, 0, 0); #if DEBUG stopwatch.Restart(); #endif Task.Run(async() => { result = await clam.SendAndScanFileAsync(zStream); }).Wait(); #if DEBUG stopwatch.Stop(); Console.WriteLine("Core.ClamScanFileFromRepo({0}): Clamd took {1} seconds to scan", file, stopwatch.Elapsed.TotalSeconds); #endif zStream.Close(); } if (result.InfectedFiles != null && result.InfectedFiles.Count > 0) { file.HasVirus = true; file.Virus = result.InfectedFiles[0].VirusName; } else if (file.HasVirus == null) { // If no scan has been done, mark as false. // If a positive has already existed don't overwrite it. file.HasVirus = false; file.Virus = null; } file.ClamTime = DateTime.UtcNow; dbCore.DbOps.UpdateFile(file); ScanFinished?.Invoke(file); } catch (ThreadAbortException) { } catch (Exception ex) { Failed?.Invoke($"Exception {ex.Message} when calling clamd"); #if DEBUG Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException); #endif } }
internal ZipReturn LocalFileOpenReadStream(Stream zipFs, bool raw, out Stream readStream, out ulong streamSize, out ushort compressionMethod) { streamSize = 0; compressionMethod = _compressionMethod; readStream = null; zipFs.Seek((long)_dataLocation, SeekOrigin.Begin); switch (_compressionMethod) { case 0: readStream = zipFs; streamSize = _compressedSize; // same as UncompressedSize break; case 8: if (raw) { readStream = zipFs; streamSize = _compressedSize; } else { //readStream = new ZlibBaseStream(zipFs, CompressionMode.Decompress, CompressionLevel.Default, ZlibStreamFlavor.DEFLATE, true); readStream = new System.IO.Compression.DeflateStream(zipFs, System.IO.Compression.CompressionMode.Decompress, true); streamSize = UncompressedSize; } break; case 9: readStream = new Deflate64Stream(zipFs, System.IO.Compression.CompressionMode.Decompress); streamSize = UncompressedSize; break; //case 10: // readStream = new BlastStream(zipFs); // streamSize = UncompressedSize; // break; case 12: readStream = new CBZip2InputStream(zipFs, false); streamSize = UncompressedSize; break; case 14: { zipFs.ReadByte(); // Major version zipFs.ReadByte(); // Minor version int headerSize = zipFs.ReadByte() + (zipFs.ReadByte() << 8); byte[] header = new byte[headerSize]; zipFs.Read(header, 0, headerSize); readStream = new LzmaStream(header, zipFs); streamSize = UncompressedSize; break; } case 20: case 93: readStream = new ZstdSharp.DecompressionStream(zipFs); streamSize = UncompressedSize; break; case 98: { int headerSize = 2; byte[] header = new byte[headerSize]; zipFs.Read(header, 0, headerSize); readStream = new PpmdStream(new PpmdProperties(header), zipFs, false); streamSize = UncompressedSize; break; } } return(readStream == null ? ZipReturn.ZipErrorGettingDataStream : ZipReturn.ZipGood); }