static void Main(string[] args)
{
string base64;
using (var reader = File.OpenText(args[0]))
{
// Skip the first line, which has some header information
// TODO: Use it instead, to validate the rest of the data.
reader.ReadLine();
base64 = reader.ReadToEnd();
}
byte[] bytes = Convert.FromBase64String(base64);
int startOfBody = FindStartOfBody(bytes);
using (var input = new MemoryStream(bytes, startOfBody, bytes.Length - startOfBody))
{
using (var bzip2 = new BZip2Stream(input, SharpCompress.Compressors.CompressionMode.Decompress, true))
{
using (var output = File.OpenWrite(args[1]))
{
bzip2.CopyTo(output);
}
}
}
}
/// <summary>
/// Decompresssed a block of data using the BZip2 algorithm.
/// </summary>
/// <param name="inData">The compressed data block.</param>
/// <returns>The decompressed data.</returns>
public static byte[] DecompressBZip2(byte[] inData)
{
using var ms = new MemoryStream(inData);
using var input = new BZip2Stream(ms, CompressionMode.Decompress, false);
using var decompressedData = new MemoryStream();
input.CopyTo(decompressedData);
return(decompressedData.ToArray());
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
Task.Run(async() =>
{
try
{
await cacheDownloadRequest.PerformAsync();
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
});
}
/// <summary>
/// Decompresssed a block of data using the BZip2 algorithm.
/// </summary>
/// <param name="inData">The compressed data block.</param>
/// <returns>The decompressed data.</returns>
public static byte[] DecompressBZip2(byte[] inData)
{
using (MemoryStream ms = new MemoryStream(inData))
{
using (BZip2Stream input = new BZip2Stream(ms, CompressionMode.Decompress))
{
using (MemoryStream decompressedData = new MemoryStream())
{
input.CopyTo(decompressedData);
return(decompressedData.ToArray());
}
}
}
}
/// <inheritdoc/>
public ConcurrentDictionary <string, ConcurrentQueue <string> > Scan(Scanner scanner, Stream stream, string file)
{
// If the BZip2 file itself fails
try
{
string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
Directory.CreateDirectory(tempPath);
using (BZip2Stream bz2File = new BZip2Stream(stream, CompressionMode.Decompress, true))
{
// If an individual entry fails
try
{
string tempFile = Path.Combine(tempPath, Guid.NewGuid().ToString());
using (FileStream fs = File.OpenWrite(tempFile))
{
bz2File.CopyTo(fs);
}
}
catch { }
}
// Collect and format all found protections
var protections = scanner.GetProtections(tempPath);
// If temp directory cleanup fails
try
{
Directory.Delete(tempPath, true);
}
catch { }
// Remove temporary path references
Utilities.StripFromKeys(protections, tempPath);
return(protections);
}
catch { }
return(null);
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
cacheDownloadRequest.PerformAsync();
}
/// <summary>
/// Gets the specified URL in raw format
/// </summary>
public byte[] Get(String url)
{
NameValueCollection parameters = new NameValueCollection();
try
{
var requestEventArgs = new RestRequestEventArgs("GET", url, null, null, null);
this.Requesting?.Invoke(this, requestEventArgs);
if (requestEventArgs.Cancel)
{
s_tracer.TraceVerbose("HTTP request cancelled");
return(null);
}
// Invoke
var httpWebReq = this.CreateHttpRequest(url, requestEventArgs.Query);
httpWebReq.Method = "GET";
// Get the responst
byte[] retVal = null;
Exception requestException = null;
var httpTask = httpWebReq.GetResponseAsync().ContinueWith(o =>
{
if (o.IsFaulted)
{
requestException = o.Exception.InnerExceptions.First();
}
else
{
try
{
this.Responding?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, o.Result.ContentLength));
byte[] buffer = new byte[2048];
int br = 1;
using (var ms = new MemoryStream())
using (var httpStream = o.Result.GetResponseStream())
{
while (br > 0)
{
br = httpStream.Read(buffer, 0, 2048);
ms.Write(buffer, 0, br);
// Raise event
this.FireProgressChanged(o.Result.ContentType, ms.Length / (float)o.Result.ContentLength);
}
ms.Seek(0, SeekOrigin.Begin);
switch (o.Result.Headers["Content-Encoding"])
{
case "deflate":
using (var dfs = new DeflateStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
dfs.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "gzip":
using (var gzs = new GZipStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
gzs.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "bzip2":
using (var lzmas = new BZip2Stream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
lzmas.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "lzma":
using (var lzmas = new LZipStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
lzmas.CopyTo(oms);
retVal = oms.ToArray();
}
break;
default:
retVal = ms.ToArray();
break;
}
}
}
catch (Exception e)
{
s_tracer.TraceError("Error downloading {0}: {1}", url, e.Message);
}
}
}, TaskContinuationOptions.LongRunning);
httpTask.Wait();
if (requestException != null)
{
throw requestException;
}
this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, 0));
return(retVal);
}
catch (WebException e)
{
switch (this.ValidateResponse(e.Response))
{
case ServiceClientErrorType.Valid:
return(this.Get(url));
default:
throw new RestClientException <byte[]>(
null,
e,
e.Status,
e.Response);
}
}
catch (Exception e)
{
s_tracer.TraceError("Error invoking HTTP: {0}", e.Message);
this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 500, 0));
throw;
}
}
