/// <summary>
/// Decompresssed a block of data using the LZMA algorithm.
/// </summary>
/// <param name="inData">The compressed data block.</param>
/// <returns>The decompressed data.</returns>
public static byte[] DecompressLZMA(byte[] inData)
{
using var ms = new MemoryStream(inData);
using var input = new LZipStream(ms, CompressionMode.Decompress);
using var decompressedData = new MemoryStream();
input.CopyTo(decompressedData);
return(decompressedData.ToArray());
}
public void DecompressFile(string source, string destination)
{
var inFs = new FileStream(source, FileMode.Open, FileAccess.Read);
var outFs = new FileStream(destination, FileMode.Create, FileAccess.Write);
Stream zStream = new LZipStream(inFs, CompressionMode.Decompress);
zStream.CopyTo(outFs);
outFs.Close();
zStream.Close();
inFs.Close();
}
public int OnExecute(IConsole console)
{
try
{
using (var istm = Util.OpenInputStream(InputFile))
using (var ostm = Util.OpenOutputStream(OutputFile, true))
{
using (var izstm = new LZipStream(istm, CompressionMode.Decompress))
{
izstm.CopyTo(ostm);
}
}
}
catch (Exception e)
{
console.Error.WriteLine($"failed lzip decompression:{e}");
return(1);
}
return(0);
}
/// <summary>
/// Gets the specified URL in raw format
/// </summary>
public byte[] Get(String url)
{
NameValueCollection parameters = new NameValueCollection();
try
{
var requestEventArgs = new RestRequestEventArgs("GET", url, null, null, null);
this.Requesting?.Invoke(this, requestEventArgs);
if (requestEventArgs.Cancel)
{
s_tracer.TraceVerbose("HTTP request cancelled");
return(null);
}
// Invoke
var httpWebReq = this.CreateHttpRequest(url, requestEventArgs.Query);
httpWebReq.Method = "GET";
// Get the responst
byte[] retVal = null;
Exception requestException = null;
var httpTask = httpWebReq.GetResponseAsync().ContinueWith(o =>
{
if (o.IsFaulted)
{
requestException = o.Exception.InnerExceptions.First();
}
else
{
try
{
this.Responding?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, o.Result.ContentLength));
byte[] buffer = new byte[2048];
int br = 1;
using (var ms = new MemoryStream())
using (var httpStream = o.Result.GetResponseStream())
{
while (br > 0)
{
br = httpStream.Read(buffer, 0, 2048);
ms.Write(buffer, 0, br);
// Raise event
this.FireProgressChanged(o.Result.ContentType, ms.Length / (float)o.Result.ContentLength);
}
ms.Seek(0, SeekOrigin.Begin);
switch (o.Result.Headers["Content-Encoding"])
{
case "deflate":
using (var dfs = new DeflateStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
dfs.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "gzip":
using (var gzs = new GZipStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
gzs.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "bzip2":
using (var lzmas = new BZip2Stream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
lzmas.CopyTo(oms);
retVal = oms.ToArray();
}
break;
case "lzma":
using (var lzmas = new LZipStream(ms, CompressionMode.Decompress, leaveOpen: true))
using (var oms = new MemoryStream())
{
lzmas.CopyTo(oms);
retVal = oms.ToArray();
}
break;
default:
retVal = ms.ToArray();
break;
}
}
}
catch (Exception e)
{
s_tracer.TraceError("Error downloading {0}: {1}", url, e.Message);
}
}
}, TaskContinuationOptions.LongRunning);
httpTask.Wait();
if (requestException != null)
{
throw requestException;
}
this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 200, 0));
return(retVal);
}
catch (WebException e)
{
switch (this.ValidateResponse(e.Response))
{
case ServiceClientErrorType.Valid:
return(this.Get(url));
default:
throw new RestClientException <byte[]>(
null,
e,
e.Status,
e.Response);
}
}
catch (Exception e)
{
s_tracer.TraceError("Error invoking HTTP: {0}", e.Message);
this.Responded?.Invoke(this, new RestResponseEventArgs("GET", url, null, null, null, 500, 0));
throw;
}
}
/// <summary>
/// Performs an installation
/// </summary>
public virtual bool Install(AppletPackage package, bool isUpgrade = false)
{
this.m_tracer.TraceWarning("Installing {0}", package.Meta);
// TODO: Verify package hash / signature
if (!this.VerifyPackage(package))
{
throw new SecurityException("Applet failed validation");
}
else if (!this.m_appletCollection.VerifyDependencies(package.Meta))
{
this.m_tracer.TraceWarning($"Applet {package.Meta} depends on : [{String.Join(", ", package.Meta.Dependencies.Select(o => o.ToString()))}] which are missing or incompatible");
}
var appletSection = ApplicationContext.Current.Configuration.GetSection <AppletConfigurationSection>();
String appletPath = Path.Combine(appletSection.AppletDirectory, package.Meta.Id);
try
{
// Desearialize an prep for install
this.m_tracer.TraceInfo("Installing applet {0} (IsUpgrade={1})", package.Meta, isUpgrade);
ApplicationContext.Current.SetProgress(package.Meta.GetName("en"), 0.0f);
// TODO: Verify the package
// Copy
if (!Directory.Exists(appletSection.AppletDirectory))
{
Directory.CreateDirectory(appletSection.AppletDirectory);
}
if (File.Exists(appletPath))
{
if (!isUpgrade)
{
throw new InvalidOperationException(Strings.err_duplicate_package_name);
}
// Unload the loaded applet version
var existingApplet = this.m_appletCollection.FirstOrDefault(o => o.Info.Id == package.Meta.Id);
if (existingApplet != null)
{
this.UnInstallInternal(existingApplet);
}
}
var mfst = package.Unpack();
// Migrate data.
if (mfst.DataSetup != null)
{
foreach (var itm in mfst.DataSetup.Action)
{
Type idpType = typeof(IDataPersistenceService <>);
idpType = idpType.MakeGenericType(new Type[] { itm.Element.GetType() });
var svc = ApplicationContext.Current.GetService(idpType);
idpType.GetMethod(itm.ActionName).Invoke(svc, new object[] { itm.Element, TransactionMode.Commit, AuthenticationContext.SystemPrincipal });
}
}
// Now export all the binary files out
var assetDirectory = Path.Combine(appletSection.AppletDirectory, "assets", mfst.Info.Id);
if (!Directory.Exists(assetDirectory))
{
Directory.CreateDirectory(assetDirectory);
}
else
{
Directory.Delete(assetDirectory, true);
}
for (int i = 0; i < mfst.Assets.Count; i++)
{
var itm = mfst.Assets[i];
var itmPath = Path.Combine(assetDirectory, itm.Name);
ApplicationContext.Current.SetProgress($"Installing {package.Meta.GetName("en")}", 0.1f + (float)(0.8 * (float)i / mfst.Assets.Count));
// Get dir name and create
if (!Directory.Exists(Path.GetDirectoryName(itmPath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(itmPath));
}
// Extract content
if (itm.Content is byte[])
{
if (Encoding.UTF8.GetString(itm.Content as byte[], 0, 4) == "LZIP")
{
using (var fs = File.Create(itmPath))
using (var ms = new MemoryStream(itm.Content as byte[]))
using (var lzs = new LZipStream(new NonDisposingStream(ms), SharpCompress.Compressors.CompressionMode.Decompress))
lzs.CopyTo(fs);
}
else
{
File.WriteAllBytes(itmPath, itm.Content as byte[]);
}
itm.Content = null;
}
else if (itm.Content is String)
{
File.WriteAllText(itmPath, itm.Content as String);
itm.Content = null;
}
}
// Serialize the data to disk
using (FileStream fs = File.Create(appletPath))
mfst.Save(fs);
// For now sign with SHA256
SHA256 sha = SHA256.Create();
package.Meta.Hash = sha.ComputeHash(File.ReadAllBytes(appletPath));
// HACK: Re-re-remove
appletSection.Applets.RemoveAll(o => o.Id == package.Meta.Id);
appletSection.Applets.Add(package.Meta.AsReference());
ApplicationContext.Current.SetProgress(package.Meta.GetName("en"), 0.98f);
if (ApplicationContext.Current.ConfigurationPersister.IsConfigured)
{
ApplicationContext.Current.ConfigurationPersister.Save(ApplicationContext.Current.Configuration);
}
this.LoadApplet(mfst);
}
catch (Exception e)
{
this.m_tracer.TraceError("Error installing applet {0} : {1}", package.Meta.ToString(), e);
// Remove
if (File.Exists(appletPath))
{
File.Delete(appletPath);
}
throw;
}
return(true);
}
public static void VirusTotalFileFromRepo(DbFile file)
{
try
{
if (!Context.VirusTotalEnabled)
{
Failed?.Invoke("VirusTotal is not usable");
return;
}
if (vTotal == null)
{
Failed?.Invoke("VirusTotal is not initalized");
}
FileReport fResult = null;
UpdateProgress?.Invoke("Requesting existing report to VirusTotal", null, 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
Task.Run(async() =>
{
fResult = await vTotal.GetFileReportAsync(file.Sha256);
}).Wait();
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.VirusTotalFileFromRepo({0}): VirusTotal took {1} seconds to answer for SHA256 request",
file, stopwatch.Elapsed.TotalSeconds);
#endif
if (fResult.ResponseCode == FileReportResponseCode.NotPresent)
{
Failed?.Invoke(fResult.VerboseMsg);
return;
}
if (fResult.ResponseCode != FileReportResponseCode.Queued)
{
if (fResult.ResponseCode == FileReportResponseCode.Present)
{
if (fResult.Positives > 0)
{
file.HasVirus = true;
if (fResult.Scans != null)
{
foreach (KeyValuePair <string, ScanEngine> engine in fResult.Scans)
{
if (!engine.Value.Detected)
{
continue;
}
file.Virus = engine.Value.Result;
file.VirusTotalTime = engine.Value.Update;
dbCore.DbOps.UpdateFile(file);
ScanFinished?.Invoke(file);
return;
}
}
}
else
{
// If no scan has been done, mark as false.
// If a positive has already existed don't overwrite it.
file.HasVirus = false;
file.Virus = null;
file.VirusTotalTime = DateTime.UtcNow;
dbCore.DbOps.UpdateFile(file);
ScanFinished?.Invoke(file);
return;
}
}
string repoPath;
AlgoEnum algorithm;
if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".gz")))
{
repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".gz");
algorithm = AlgoEnum.GZip;
}
else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".bz2")))
{
repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".bz2");
algorithm = AlgoEnum.BZip2;
}
else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".lzma")))
{
repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".lzma");
algorithm = AlgoEnum.LZMA;
}
else if (File.Exists(Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".lz")))
{
repoPath = Path.Combine(Settings.Current.RepositoryPath, file.Sha256[0].ToString(),
file.Sha256[1].ToString(), file.Sha256[2].ToString(),
file.Sha256[3].ToString(), file.Sha256[4].ToString(),
file.Sha256 + ".lz");
algorithm = AlgoEnum.LZip;
}
else
{
Failed?.Invoke($"Cannot find file with hash {file.Sha256} in the repository");
return;
}
UpdateProgress?.Invoke("Uncompressing file...", null, 0, 0);
var inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read);
Stream zStream = null;
switch (algorithm)
{
case AlgoEnum.GZip:
zStream = new GZipStream(inFs, CompressionMode.Decompress);
break;
case AlgoEnum.BZip2:
zStream = new BZip2Stream(inFs, CompressionMode.Decompress);
break;
case AlgoEnum.LZMA:
byte[] properties = new byte[5];
inFs.Read(properties, 0, 5);
inFs.Seek(8, SeekOrigin.Current);
zStream = new LzmaStream(properties, inFs, inFs.Length - 13, file.Length);
break;
case AlgoEnum.LZip:
zStream = new LZipStream(inFs, CompressionMode.Decompress);
break;
}
ScanResult sResult = null;
#if DEBUG
stopwatch.Restart();
#endif
// Cannot use zStream directly, VirusTotal.NET requests the size *sigh*
string tmpFile = Path.Combine(Settings.Current.TemporaryFolder, Path.GetTempFileName());
var outFs = new FileStream(tmpFile, FileMode.Create, FileAccess.ReadWrite);
zStream?.CopyTo(outFs);
zStream?.Close();
outFs.Seek(0, SeekOrigin.Begin);
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.VirusTotalFileFromRepo({0}): Uncompressing took {1} seconds", file,
stopwatch.Elapsed.TotalSeconds);
#endif
UpdateProgress?.Invoke("Uploading file to VirusTotal...", null, 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
Task.Run(async() =>
{
sResult = await vTotal.ScanFileAsync(outFs, file.Sha256); // Keep filename private, sorry!
}).Wait();
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.VirusTotalFileFromRepo({0}): Upload to VirusTotal took {1} seconds", file,
stopwatch.Elapsed.TotalSeconds);
#endif
outFs.Close();
File.Delete(tmpFile);
if (sResult == null ||
sResult.ResponseCode == ScanFileResponseCode.Error)
{
if (sResult == null)
{
Failed?.Invoke("Cannot send file to VirusTotal");
}
else
{
Failed(sResult.VerboseMsg);
}
return;
}
// Seems that we are faster than them, getting a lot of "not queued" responses...
Thread.Sleep(2500);
Task.Run(async() =>
{
fResult = await vTotal.GetFileReportAsync(file.Sha256);
}).Wait();
}
UpdateProgress?.Invoke("Waiting for VirusTotal analysis...", null, 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
int counter = 0;
while (fResult.ResponseCode == FileReportResponseCode.Queued)
{
// Timeout...
if (counter == 10)
{
break;
}
// Wait 15 seconds so we fall in the 4 requests/minute
Thread.Sleep(15000);
Task.Run(async() =>
{
fResult = await vTotal.GetFileReportAsync(file.Sha256);
}).Wait();
counter++;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.VirusTotalFileFromRepo({0}): VirusTotal took {1} seconds to do the analysis",
file, stopwatch.Elapsed.TotalSeconds);
#endif
if (fResult.ResponseCode != FileReportResponseCode.Present)
{
Failed?.Invoke(fResult.VerboseMsg);
return;
}
if (fResult.Positives > 0)
{
file.HasVirus = true;
if (fResult.Scans == null)
{
return;
}
foreach (KeyValuePair <string, ScanEngine> engine in fResult.Scans)
{
if (!engine.Value.Detected)
{
continue;
}
file.Virus = engine.Value.Result;
file.VirusTotalTime = engine.Value.Update;
dbCore.DbOps.UpdateFile(file);
ScanFinished?.Invoke(file);
return;
}
}
else
{
// If no scan has been done, mark as false.
// If a positive has already existed don't overwrite it.
file.HasVirus = false;
file.Virus = null;
file.VirusTotalTime = DateTime.UtcNow;
dbCore.DbOps.UpdateFile(file);
ScanFinished?.Invoke(file);
}
}
catch (Exception ex)
{
Failed?.Invoke($"Exception {ex.InnerException.Message} when calling VirusTotal");
#if DEBUG
Console.WriteLine("Exception {0}\n{1}", ex.Message, ex.InnerException);
#endif
}
}
