static void Main(string[] args) { string base64; using (var reader = File.OpenText(args[0])) { // Skip the first line, which has some header information // TODO: Use it instead, to validate the rest of the data. reader.ReadLine(); base64 = reader.ReadToEnd(); } byte[] bytes = Convert.FromBase64String(base64); int startOfBody = FindStartOfBody(bytes); using (var input = new MemoryStream(bytes, startOfBody, bytes.Length - startOfBody)) { using (var bzip2 = new BZip2Stream(input, SharpCompress.Compressors.CompressionMode.Decompress, true)) { using (var output = File.OpenWrite(args[1])) { bzip2.CopyTo(output); } } } }
/// <summary> /// Decompresssed a block of data using the BZip2 algorithm. /// </summary> /// <param name="inData">The compressed data block.</param> /// <returns>The decompressed data.</returns> public static byte[] DecompressBZip2(byte[] inData) { using var ms = new MemoryStream(inData); using var input = new BZip2Stream(ms, CompressionMode.Decompress, false); using var decompressedData = new MemoryStream(); input.CopyTo(decompressedData); return(decompressedData.ToArray()); }
private void prepareLocalCache() { string cacheFilePath = storage.GetFullPath(cache_database_name); string compressedCacheFilePath = $"{cacheFilePath}.bz2"; cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); cacheDownloadRequest.Failed += ex => { File.Delete(compressedCacheFilePath); File.Delete(cacheFilePath); Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database); }; cacheDownloadRequest.Finished += () => { try { using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) using (var outStream = File.OpenWrite(cacheFilePath)) using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) bz2.CopyTo(outStream); // set to null on completion to allow lookups to begin using the new source cacheDownloadRequest = null; } catch (Exception ex) { Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database); File.Delete(cacheFilePath); } finally { File.Delete(compressedCacheFilePath); } }; Task.Run(async() => { try { await cacheDownloadRequest.PerformAsync(); } catch { // Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway. } }); }
/// <summary> /// Decompresssed a block of data using the BZip2 algorithm. /// </summary> /// <param name="inData">The compressed data block.</param> /// <returns>The decompressed data.</returns> public static byte[] DecompressBZip2(byte[] inData) { using (MemoryStream ms = new MemoryStream(inData)) { using (BZip2Stream input = new BZip2Stream(ms, CompressionMode.Decompress)) { using (MemoryStream decompressedData = new MemoryStream()) { input.CopyTo(decompressedData); return(decompressedData.ToArray()); } } } }
/// <inheritdoc/> public ConcurrentDictionary <string, ConcurrentQueue <string> > Scan(Scanner scanner, Stream stream, string file) { // If the BZip2 file itself fails try { string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); Directory.CreateDirectory(tempPath); using (BZip2Stream bz2File = new BZip2Stream(stream, CompressionMode.Decompress, true)) { // If an individual entry fails try { string tempFile = Path.Combine(tempPath, Guid.NewGuid().ToString()); using (FileStream fs = File.OpenWrite(tempFile)) { bz2File.CopyTo(fs); } } catch { } } // Collect and format all found protections var protections = scanner.GetProtections(tempPath); // If temp directory cleanup fails try { Directory.Delete(tempPath, true); } catch { } // Remove temporary path references Utilities.StripFromKeys(protections, tempPath); return(protections); } catch { } return(null); }
private void prepareLocalCache() { string cacheFilePath = storage.GetFullPath(cache_database_name); string compressedCacheFilePath = $"{cacheFilePath}.bz2"; cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2"); cacheDownloadRequest.Failed += ex => { File.Delete(compressedCacheFilePath); File.Delete(cacheFilePath); Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database); }; cacheDownloadRequest.Finished += () => { try { using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) using (var outStream = File.OpenWrite(cacheFilePath)) using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) bz2.CopyTo(outStream); // set to null on completion to allow lookups to begin using the new source cacheDownloadRequest = null; } catch (Exception ex) { Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database); File.Delete(cacheFilePath); } finally { File.Delete(compressedCacheFilePath); } }; cacheDownloadRequest.PerformAsync(); }
/// <summary> /// Gets the specified URL in raw format /// </summary> public byte[] Get(String url) { NameValueCollection parameters = new NameValueCollection(); try { var requestEventArgs = new RestRequestEventArgs("GET", url, null, null, null); this.Requesting?.Invoke(this, requestEventArgs); if (requestEventArgs.Cancel) { s_tracer.TraceVerbose("HTTP request cancelled"); return(null); } // Invoke var httpWebReq = this.CreateHttpRequest(url, requestEventArgs.Query); httpWebReq.Method = "GET"; // Get the responst byte[] retVal = null; Exception requestException = null; var httpTask = httpWebReq.GetResponseAsync().ContinueWith(o => { if (o.IsFaulted) { requestException = o.Exception.InnerExceptions.First(); } else { try { this.Responding?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, o.Result.ContentLength)); byte[] buffer = new byte[2048]; int br = 1; using (var ms = new MemoryStream()) using (var httpStream = o.Result.GetResponseStream()) { while (br > 0) { br = httpStream.Read(buffer, 0, 2048); ms.Write(buffer, 0, br); // Raise event this.FireProgressChanged(o.Result.ContentType, ms.Length / (float)o.Result.ContentLength); } ms.Seek(0, SeekOrigin.Begin); switch (o.Result.Headers["Content-Encoding"]) { case "deflate": using (var dfs = new DeflateStream(ms, CompressionMode.Decompress, leaveOpen: true)) using (var oms = new MemoryStream()) { dfs.CopyTo(oms); retVal = oms.ToArray(); } break; case "gzip": using (var gzs = new GZipStream(ms, CompressionMode.Decompress, leaveOpen: true)) using (var oms = new MemoryStream()) { gzs.CopyTo(oms); retVal = oms.ToArray(); } break; case "bzip2": using (var lzmas = new BZip2Stream(ms, CompressionMode.Decompress, leaveOpen: true)) using (var oms = new MemoryStream()) { lzmas.CopyTo(oms); retVal = oms.ToArray(); } break; case "lzma": using (var lzmas = new LZipStream(ms, CompressionMode.Decompress, leaveOpen: true)) using (var oms = new MemoryStream()) { lzmas.CopyTo(oms); retVal = oms.ToArray(); } break; default: retVal = ms.ToArray(); break; } } } catch (Exception e) { s_tracer.TraceError("Error downloading {0}: {1}", url, e.Message); } } }, TaskContinuationOptions.LongRunning); httpTask.Wait(); if (requestException != null) { throw requestException; } this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, 0)); return(retVal); } catch (WebException e) { switch (this.ValidateResponse(e.Response)) { case ServiceClientErrorType.Valid: return(this.Get(url)); default: throw new RestClientException <byte[]>( null, e, e.Status, e.Response); } } catch (Exception e) { s_tracer.TraceError("Error invoking HTTP: {0}", e.Message); this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 500, 0)); throw; } }