public static async Task GetMetaData()
{
List <Task> tlist = new List <Task>();
vehicles.ForEach(a =>
{
try
{
var newurl = String.Format(url, a);
var file = Path.Combine(Settings.GetDataDirectory(), "LogMessages" + a + ".xml.xz");
if (File.Exists(file))
{
if (new FileInfo(file).LastWriteTime.AddDays(7) > DateTime.Now)
{
return;
}
}
var dltask = Download.getFilefromNetAsync(newurl, file);
tlist.Add(dltask);
}
catch (Exception ex) { log.Error(ex); }
});
await Task.WhenAll(tlist);
vehicles.ForEach(a =>
{
try
{
var fileout = Path.Combine(Settings.GetDataDirectory(), "LogMessages" + a + ".xml");
var file = Path.Combine(Settings.GetDataDirectory(), "LogMessages" + a + ".xml.xz");
if (File.Exists(file))
{
using (var read = File.OpenRead(file))
{
if (XZStream.IsXZStream(read))
{
read.Position = 0;
var stream = new XZStream(read);
using (var outst = File.OpenWrite(fileout))
{
try
{
stream.CopyTo(outst);
}
catch (XZIndexMarkerReachedException)
{
// ignore
}
}
}
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
});
}
/// <summary>
/// Attempt to extract a stream from an archive
/// </summary>
/// <param name="entryName">Name of the entry to be extracted</param>
/// <param name="realEntry">Output representing the entry name that was found</param>
/// <returns>MemoryStream representing the entry, null on error</returns>
public override (MemoryStream, string) CopyToStream(string entryName)
{
MemoryStream ms = new MemoryStream();
string realEntry;
try
{
// Decompress the _filename stream
realEntry = Path.GetFileNameWithoutExtension(this.Filename);
var xz = new XZStream(File.OpenRead(this.Filename));
// Write the file out
byte[] xbuffer = new byte[_bufferSize];
int xlen;
while ((xlen = xz.Read(xbuffer, 0, _bufferSize)) > 0)
{
ms.Write(xbuffer, 0, xlen);
ms.Flush();
}
// Dispose of the streams
xz.Dispose();
}
catch (Exception ex)
{
logger.Error(ex);
ms = null;
realEntry = null;
}
return(ms, realEntry);
}
/// Main function that decompresses a xz file and serialized it
// into a json for mongodb storage.
public static void Main(string[] args)
{
if (args.Length != 1)
{
Console.Write("syntax is ./FundUniquePackages.exe <file>.xz");
return;
}
var fs = File.OpenRead(args[0]);
if (fs.Name.EndsWith(".xz"))
{
using (Stream zipStream = new XZStream(fs))
{
using (StreamReader unzip = new StreamReader(zipStream))
{
while (!unzip.EndOfStream)
{
streamReadHelper(unzip);
}
}
}
} else {
using (StreamReader fsStream = new StreamReader(fs))
{
while (!fsStream.EndOfStream)
{
streamReadHelper(fsStream);
}
}
}
File.WriteAllText("FileSystemBinaries.json", JsonConvert.SerializeObject(serializationTransform(filesystemToBinaryInfo)));
File.WriteAllText("BinaryVariations.json", JsonConvert.SerializeObject(serializationTransform(binaryNameToBinaryInfo)));
//File.WriteAllText("SimilarityArray.json", generateSimilarityMatrix());
}
private static void Template(string xzFileName, string originFileName, bool useSpan)
{
byte[] decompDigest;
byte[] originDigest;
string xzFile = Path.Combine(TestSetup.SampleDir, xzFileName);
string originFile = Path.Combine(TestSetup.SampleDir, originFileName);
using (MemoryStream decompMs = new MemoryStream())
{
XZDecompressOptions decompOpts = new XZDecompressOptions();
using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read))
using (XZStream xz = new XZStream(compFs, decompOpts))
{
#if !NETFRAMEWORK
if (useSpan)
{
byte[] buffer = new byte[64 * 1024];
int bytesRead;
do
{
bytesRead = xz.Read(buffer.AsSpan());
decompMs.Write(buffer.AsSpan(0, bytesRead));
} while (0 < bytesRead);
}
else
#endif
{
xz.CopyTo(decompMs);
}
decompMs.Flush();
xz.GetProgress(out ulong finalIn, out ulong finalOut);
Assert.AreEqual(compFs.Length, xz.TotalIn);
Assert.AreEqual(decompMs.Length, xz.TotalOut);
Assert.AreEqual((ulong)compFs.Length, finalIn);
Assert.AreEqual((ulong)decompMs.Length, finalOut);
}
decompMs.Position = 0;
using (HashAlgorithm hash = SHA256.Create())
{
decompDigest = hash.ComputeHash(decompMs);
}
}
using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (HashAlgorithm hash = SHA256.Create())
{
originDigest = hash.ComputeHash(originFs);
}
}
Assert.IsTrue(decompDigest.SequenceEqual(originDigest));
}
public void CanReadStream()
{
XZStream xz = new XZStream(CompressedStream);
using (var sr = new StreamReader(xz))
{
string uncompressed = sr.ReadToEnd();
Assert.That(uncompressed, Is.EqualTo(Original));
}
}
/// <summary>
/// Generate a list of DatItem objects from the header values in an archive
/// </summary>
/// <returns>List of DatItem objects representing the found data</returns>
public override List <BaseFile> GetChildren()
{
if (_children == null || _children.Count == 0)
{
_children = new List <BaseFile>();
string gamename = Path.GetFileNameWithoutExtension(this.Filename);
BaseFile possibleTxz = GetTorrentXZFileInfo();
// If it was, then add it to the outputs and continue
if (possibleTxz != null && possibleTxz.Filename != null)
{
_children.Add(possibleTxz);
}
else
{
try
{
// Create a blank item for the entry
BaseFile xzEntryRom = new BaseFile();
// Perform a quickscan, if flagged to
if (this.AvailableHashes == Hash.CRC)
{
xzEntryRom.Filename = gamename;
using (BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename)))
{
br.BaseStream.Seek(-8, SeekOrigin.End);
xzEntryRom.CRC = br.ReadBytesBigEndian(4);
xzEntryRom.Size = br.ReadInt32BigEndian();
}
}
// Otherwise, use the stream directly
else
{
var xzStream = new XZStream(File.OpenRead(this.Filename));
xzEntryRom = xzStream.GetInfo(hashes: this.AvailableHashes);
xzEntryRom.Filename = gamename;
xzStream.Dispose();
}
// Fill in comon details and add to the list
xzEntryRom.Parent = gamename;
_children.Add(xzEntryRom);
}
catch (Exception ex)
{
logger.Error(ex);
return(null);
}
}
}
return(_children);
}
public void CanReadStream()
{
XZStream xz = new XZStream(CompressedStream);
using (var sr = new StreamReader(xz))
{
string uncompressed = sr.ReadToEnd();
Assert.Equal(uncompressed, Original);
}
}
public void NativeXZ()
{
byte[] buffer = _srcFileBytesDict[DecompMethod.NativeXZ];
using (MemoryStream decompMs = new MemoryStream((int)_magicFileLen))
using (MemoryStream compMs = new MemoryStream(buffer))
using (XZStream xzs = new XZStream(compMs, new XZDecompressOptions()))
{
xzs.CopyTo(decompMs);
}
}
public override byte[] Decompress(byte[] compressedData)
{
var inStream = new MemoryStream(compressedData);
var outStream = new MemoryStream();
using (var decompress = new XZStream(inStream))
{
CopyTo(decompress, outStream);
}
return(outStream.ToArray());
}
/// <inheritdoc/>
public override byte[] Decompress(byte[] compressedData, int blockLength)
{
XZDecompressOptions decompOpts = new XZDecompressOptions();
using (MemoryStream inputStream = new MemoryStream(compressedData, 0, blockLength))
using (MemoryStream outputStream = new MemoryStream())
using (XZStream xzStream = new XZStream(inputStream, decompOpts))
{
xzStream.CopyTo(outputStream);
xzStream.Flush();
return(outputStream.ToArray());
}
}
/// <summary>
/// Extracts an zip file contained in fileEntry.
/// </summary>
/// <param name="fileEntry"> FileEntry to extract </param>
/// <returns> Extracted files </returns>
public IEnumerable <FileEntry> Extract(FileEntry fileEntry, ExtractorOptions options, ResourceGovernor governor)
{
XZStream?xzStream = null;
try
{
xzStream = new XZStream(fileEntry.Content);
}
catch (Exception e)
{
Logger.Debug(Extractor.DEBUG_STRING, ArchiveFileType.XZ, fileEntry.FullPath, string.Empty, e.GetType());
}
if (xzStream != null)
{
var newFilename = Path.GetFileNameWithoutExtension(fileEntry.Name);
var newFileEntry = new FileEntry(newFilename, xzStream, fileEntry);
// SharpCompress does not expose metadata without a full read, so we need to decompress first,
// and then abort if the bytes exceeded the governor's capacity.
var streamLength = xzStream.Index?.Records?.Select(r => r.UncompressedSize)
.Aggregate((ulong?)0, (a, b) => a + b);
// BUG: Technically, we're casting a ulong to a long, but we don't expect 9 exabyte steams, so
// low risk.
if (streamLength.HasValue)
{
governor.CheckResourceGovernor((long)streamLength.Value);
}
if (Extractor.IsQuine(newFileEntry))
{
Logger.Info(Extractor.IS_QUINE_STRING, fileEntry.Name, fileEntry.FullPath);
throw new OverflowException();
}
foreach (var extractedFile in Context.Extract(newFileEntry, options, governor))
{
yield return(extractedFile);
}
xzStream.Dispose();
}
else
{
if (options.ExtractSelfOnFail)
{
yield return(fileEntry);
}
}
}
public static string DecompressXZFile(string sourceFilePath, string destinationDirectory, TaskLoggingHelper logger)
{
var destFile = Path.Combine(destinationDirectory, Path.GetFileName(sourceFilePath)) + ".uncompressed";
if (File.Exists(destFile))
{
LogMessage($"Uncompressed tar file already exists: {destFile}", logger);
return(destFile);
}
using (Stream xz = new XZStream(File.OpenRead(sourceFilePath)))
using (Stream outputStream = new FileStream(destFile, FileMode.CreateNew))
{
xz.CopyTo(outputStream);
}
return(destFile);
}
/// <summary>
/// Write an input file to a torrent XZ archive
/// </summary>
/// <param name="inputStream">Input stream to be moved</param>
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">DatItem representing the new information</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public override bool Write(Stream inputStream, string outDir, Rom rom)
{
bool success = false;
// If the stream is not readable, return
if (!inputStream.CanRead)
{
return(success);
}
// Make sure the output directory exists
if (!Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
}
outDir = Path.GetFullPath(outDir);
// Now get the Rom info for the file so we have hashes and size
rom = new Rom(inputStream.GetInfo(keepReadOpen: true));
// Get the output file name
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));
outfile = outfile.Replace(".gz", ".xz");
// Check to see if the folder needs to be created
if (!Directory.Exists(Path.GetDirectoryName(outfile)))
{
Directory.CreateDirectory(Path.GetDirectoryName(outfile));
}
// If the output file exists, don't try to write again
if (!File.Exists(outfile))
{
// Compress the input stream
XZStream outputStream = new XZStream(FileExtensions.TryCreate(outfile));
inputStream.CopyTo(outputStream);
// Dispose of everything
outputStream.Dispose();
}
return(true);
}
private static (bool, string?) ExtractLinux(FileInfo archiveFile, string destination)
{
using var stream = archiveFile.OpenRead();
bool isXz = XZStream.IsXZStream(stream);
stream.Position = 0;
if (isXz)
{
using var xzStream = new XZStream(stream);
using var reader = TarReader.Open(xzStream);
return(Extract(reader, destination));
}
else
{
using var reader = TarReader.Open(stream);
return(Extract(reader, destination));
}
}
private static async Task ReadTarXzAsync(
FileStream fileStream,
string outputDir,
Func <string, string> getEntryPath,
bool shouldExtract)
{
using (var xzStream = new XZStream(fileStream))
using (var tarReader = TarReader.Open(xzStream))
{
var createdDirs = new HashSet <string>();
while (tarReader.MoveToNextEntry())
{
if (!shouldExtract)
{
return;
}
if (tarReader.Entry.IsDirectory)
{
continue;
}
var entryPath = getEntryPath(tarReader.Entry.Key);
if (entryPath == null)
{
continue;
}
var fullEntryPath = Path.GetFullPath(Path.Combine(outputDir, entryPath));
var entryDir = Path.GetDirectoryName(fullEntryPath);
if (createdDirs.Add(entryDir))
{
Directory.CreateDirectory(entryDir);
}
using (var entryStream = tarReader.OpenEntryStream())
using (var outputStream = new FileStream(fullEntryPath, FileMode.Create))
{
await entryStream.CopyToAsync(outputStream).ConfigureAwait(false);
}
}
}
}
public int OnExecute(IConsole console)
{
try
{
using (var istm = Util.OpenInputStream(InputFile))
using (var ostm = Util.OpenOutputStream(OutputFile, true))
{
using (var izstm = new XZStream(istm))
{
izstm.CopyTo(ostm);
}
}
}
catch (Exception e)
{
console.Error.WriteLine($"failed xz decompression:{e}");
return(1);
}
return(0);
}
/// <inheritdoc/>
public ConcurrentDictionary <string, ConcurrentQueue <string> > Scan(Scanner scanner, Stream stream, string file)
{
// If the xz file itself fails
try
{
string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
Directory.CreateDirectory(tempPath);
using (XZStream xzFile = new XZStream(stream))
{
// If an individual entry fails
try
{
string tempFile = Path.Combine(tempPath, Guid.NewGuid().ToString());
using (FileStream fs = File.OpenWrite(tempFile))
{
xzFile.CopyTo(fs);
}
}
catch { }
}
// Collect and format all found protections
var protections = scanner.GetProtections(tempPath);
// If temp directory cleanup fails
try
{
Directory.Delete(tempPath, true);
}
catch { }
// Remove temporary path references
Utilities.StripFromKeys(protections, tempPath);
return(protections);
}
catch { }
return(null);
}
/// <summary>
/// Attempt to extract a file as an archive
/// </summary>
/// <param name="outDir">Output directory for archive extraction</param>
/// <returns>True if the extraction was a success, false otherwise</returns>
public override bool CopyAll(string outDir)
{
bool encounteredErrors = true;
try
{
// Create the temp directory
Directory.CreateDirectory(outDir);
// Decompress the _filename stream
FileStream outstream = FileExtensions.TryCreate(Path.Combine(outDir, Path.GetFileNameWithoutExtension(this.Filename)));
var xz = new XZStream(File.OpenRead(this.Filename));
xz.CopyTo(outstream);
// Dispose of the streams
outstream.Dispose();
xz.Dispose();
encounteredErrors = false;
}
catch (EndOfStreamException ex)
{
// Catch this but don't count it as an error because SharpCompress is unsafe
logger.Verbose(ex);
}
catch (InvalidOperationException ex)
{
logger.Warning(ex);
encounteredErrors = true;
}
catch (Exception ex)
{
logger.Error(ex);
encounteredErrors = true;
}
return(encounteredErrors);
}
/// <inheritdoc/>
public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
{
XZCompressOptions compOpts = new XZCompressOptions
{
Level = (LzmaCompLevel)(int)Level,
ExtremeFlag = Extreme,
LeaveOpen = true
};
XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions
{
Threads = Threads,
};
inputStream.Position = 0;
outputStream.SetLength(0);
using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts))
{
inputStream.CopyTo(xzStream);
xzStream.Flush();
}
}
private static void CompressTemplate(string sampleFileName, bool useSpan, bool success, int threads, LzmaCompLevel level, bool extreme)
{
string destDir = Path.GetTempFileName();
File.Delete(destDir);
Directory.CreateDirectory(destDir);
try
{
string tempDecompFile = Path.Combine(destDir, Path.GetFileName(sampleFileName));
string tempXzFile = tempDecompFile + ".xz";
XZCompressOptions compOpts = new XZCompressOptions
{
Level = level,
ExtremeFlag = extreme,
LeaveOpen = true,
};
XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions
{
Threads = threads,
};
string sampleFile = Path.Combine(TestSetup.SampleDir, sampleFileName);
using (FileStream xzCompFs = new FileStream(tempXzFile, FileMode.Create, FileAccess.Write, FileShare.None))
using (FileStream sampleFs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
using (XZStream xzs = new XZStream(xzCompFs, compOpts, threadOpts))
{
#if !NETFRAMEWORK
if (useSpan)
{
byte[] buffer = new byte[64 * 1024];
int bytesRead;
do
{
bytesRead = sampleFs.Read(buffer.AsSpan());
xzs.Write(buffer.AsSpan(0, bytesRead));
} while (0 < bytesRead);
}
else
#endif
{
sampleFs.CopyTo(xzs);
}
xzs.Flush();
xzs.GetProgress(out ulong finalIn, out ulong finalOut);
Assert.AreEqual(sampleFs.Length, xzs.TotalIn);
Assert.AreEqual(xzCompFs.Length, xzs.TotalOut);
Assert.AreEqual((ulong)sampleFs.Length, finalIn);
Assert.AreEqual((ulong)xzCompFs.Length, finalOut);
}
Assert.IsTrue(TestHelper.RunXZ(tempXzFile) == 0);
byte[] decompDigest;
byte[] originDigest;
using (FileStream fs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (HashAlgorithm hash = SHA256.Create())
{
originDigest = hash.ComputeHash(fs);
}
}
using (FileStream fs = new FileStream(tempDecompFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (HashAlgorithm hash = SHA256.Create())
{
decompDigest = hash.ComputeHash(fs);
}
}
Assert.IsTrue(originDigest.SequenceEqual(decompDigest));
Assert.IsTrue(success);
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception)
{
Assert.IsFalse(success);
}
#pragma warning restore CA1031 // Do not catch general exception types
finally
{
if (Directory.Exists(destDir))
{
Directory.Delete(destDir, true);
}
}
}
/// <summary>
/// Opens a Reader for Non-seeking usage
/// </summary>
/// <param name="stream"></param>
/// <param name="options"></param>
/// <returns></returns>
public static IReader Open(Stream stream, ReaderOptions options = null)
{
stream.CheckNotNull("stream");
options = options ?? new ReaderOptions()
{
LeaveStreamOpen = false
};
RewindableStream rewindableStream = new RewindableStream(stream);
rewindableStream.StartRecording();
if (ZipArchive.IsZipFile(rewindableStream, options.Password))
{
rewindableStream.Rewind(true);
return(ZipReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (GZipArchive.IsGZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
GZipStream testStream = new GZipStream(rewindableStream, CompressionMode.Decompress);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.GZip));
}
rewindableStream.Rewind(true);
return(GZipReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (BZip2Stream.IsBZip2(rewindableStream))
{
rewindableStream.Rewind(false);
BZip2Stream testStream = new BZip2Stream(rewindableStream, CompressionMode.Decompress, true);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.BZip2));
}
}
rewindableStream.Rewind(false);
if (LZipStream.IsLZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
LZipStream testStream = new LZipStream(rewindableStream, CompressionMode.Decompress, true);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.LZip));
}
}
rewindableStream.Rewind(false);
if (RarArchive.IsRarFile(rewindableStream, options))
{
rewindableStream.Rewind(true);
return(RarReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (TarArchive.IsTarFile(rewindableStream))
{
rewindableStream.Rewind(true);
return(TarReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (XZStream.IsXZStream(rewindableStream))
{
rewindableStream.Rewind(true);
XZStream testStream = new XZStream(rewindableStream);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.Xz));
}
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Reader Formats: Zip, GZip, BZip2, Tar, Rar, LZip, XZ");
}
public static bool TryOpen(Stream stream, ReaderOptions options, ArchiveTypeMask archiveTypes, out IReader reader)
{
stream.CheckNotNull("stream");
options = options ?? new ReaderOptions()
{
LeaveStreamOpen = false
};
RewindableStream rewindableStream = new RewindableStream(stream);
rewindableStream.StartRecording();
if (archiveTypes.HasFlag(ArchiveTypeMask.Zip) && ZipArchive.IsZipFile(rewindableStream, options.Password))
{
rewindableStream.Rewind(true);
reader = ZipReader.Open(rewindableStream, options);
return(true);
}
rewindableStream.Rewind(false);
if (archiveTypes.HasFlag(ArchiveTypeMask.GZip) && GZipArchive.IsGZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
GZipStream testStream = new GZipStream(rewindableStream, CompressionMode.Decompress);
if (archiveTypes.HasFlag(ArchiveTypeMask.Tar) && TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
reader = new TarReader(rewindableStream, options, CompressionType.GZip);
return(true);
}
rewindableStream.Rewind(true);
reader = GZipReader.Open(rewindableStream, options);
return(true);
}
rewindableStream.Rewind(false);
if (archiveTypes.HasFlag(ArchiveTypeMask.BZip2) && BZip2Stream.IsBZip2(rewindableStream))
{
rewindableStream.Rewind(false);
BZip2Stream testStream = new BZip2Stream(new NonDisposingStream(rewindableStream), CompressionMode.Decompress, false);
if (archiveTypes.HasFlag(ArchiveTypeMask.Tar) && TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
reader = new TarReader(rewindableStream, options, CompressionType.BZip2);
return(true);
}
}
rewindableStream.Rewind(false);
if (archiveTypes.HasFlag(ArchiveTypeMask.LZip) && LZipStream.IsLZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
LZipStream testStream = new LZipStream(new NonDisposingStream(rewindableStream), CompressionMode.Decompress);
if (archiveTypes.HasFlag(ArchiveTypeMask.Tar) && TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
reader = new TarReader(rewindableStream, options, CompressionType.LZip);
return(true);
}
}
rewindableStream.Rewind(false);
if (archiveTypes.HasFlag(ArchiveTypeMask.Rar) && RarArchive.IsRarFile(rewindableStream, options))
{
rewindableStream.Rewind(true);
reader = RarReader.Open(rewindableStream, options);
return(true);
}
rewindableStream.Rewind(false);
if (archiveTypes.HasFlag(ArchiveTypeMask.Tar) && TarArchive.IsTarFile(rewindableStream))
{
rewindableStream.Rewind(true);
reader = TarReader.Open(rewindableStream, options);
return(true);
}
rewindableStream.Rewind(false);
if (XZStream.IsXZStream(rewindableStream))
{
rewindableStream.Rewind(true);
XZStream testStream = new XZStream(rewindableStream);
if (archiveTypes.HasFlag(ArchiveTypeMask.Tar) && TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
reader = new TarReader(rewindableStream, options, CompressionType.Xz);
return(true);
}
}
reader = null;
return(false);
}
public async Task Unpack(VirtualFile file, VirtualDirectory destination,
IProgressNotifier parentProgress = null, ChangeObservable observable = null)
{
if (Path.GetExtension(file.Name)?.Equals(".zip", StringComparison.OrdinalIgnoreCase) == true)
{
await UnpackZipFile().ConfigureAwait(false);
}
else if (file.Name.EndsWith(".tar.xz", StringComparison.OrdinalIgnoreCase))
{
UnpackTarXzFile(file);
}
else if (Path.GetExtension(file.Name)?.Equals(".xz", StringComparison.OrdinalIgnoreCase) == true)
{
UnpackXzFile(file);
}
else if (Path.GetExtension(file.Name)?.Equals(".tar", StringComparison.OrdinalIgnoreCase) == true)
{
UnpackTarFile(file);
}
else if (Path.GetExtension(file.Name)?.Equals(".sh", StringComparison.OrdinalIgnoreCase) == true &&
environmentService.Platform != OSPlatform.Windows)
{
await UnpackSelfExtractingShellScript().ConfigureAwait(false);
}
else
{
throw new UnsupportedArchiveFormatException(file.FullName);
}
async Task UnpackSelfExtractingShellScript()
{
ValidateShellScript();
StringBuilderUserInterface userInterface = new StringBuilderUserInterface(log, writeInformation: true, writeError: true);
try
{
using (parentProgress?.SpawnInfiniteProgress("Executing the shell script."))
using (IProcess process = processManager.StartProcess(file.FullName, $"-y -d \"{destination.FullName}\"", userInterface))
{
await process.WaitForExitAsync().ConfigureAwait(false);
if (process.ExitCode != 0)
{
throw new UnsupportedArchiveFormatException(file.FullName,
new FormattableException(
$"An error occured while executing the script.{Environment.NewLine}" +
$"{userInterface.Error}"));
}
}
}
catch (Exception e)
{
if (e is FormattableException)
{
throw;
}
throw new UnsupportedArchiveFormatException(file.FullName,
new FormattableException(
$"An exception occured while executing the script.{Environment.NewLine}" +
$"{e.Message}", e));
}
void ValidateShellScript()
{
StringBuilderUserInterface validationUserInterface = new StringBuilderUserInterface(log, writeInformation: true, writeError: true);
try
{
using (IProcess process = processManager.StartProcess(file.FullName, "--help", validationUserInterface))
{
process.WaitForExit();
}
}
catch (UnauthorizedAccessException e)
{
throw new UnsupportedArchiveFormatException(file.FullName,
new FormattableException(
$"An exception occured while inspecting the script.{Environment.NewLine}" +
$"This excpetion can occur when the file is not marked as executable.{Environment.NewLine}" +
$"{e.Message}", e));
}
catch (Exception e)
{
if (e is FormattableException)
{
throw;
}
throw new UnsupportedArchiveFormatException(file.FullName,
new FormattableException(
$"An exception occured while inspecting the script.{Environment.NewLine}" +
$"{e.Message}", e));
}
if (!Regex.IsMatch(validationUserInterface.Information,
@"(?=.*(?:usage|Usage)).*(?=.*-y)(?=.*-d)"))
{
throw new UnsupportedArchiveFormatException(file.FullName,
new FormattableException($"Did not find the expected usage information.{Environment.NewLine}" +
$"The expected information need to include the options '-y' and '-d'.{Environment.NewLine}" +
$"The following usage information was given:{Environment.NewLine}" +
$"{validationUserInterface.Information}"));
}
}
}
async Task UnpackZipFile()
{
using (Stream fileStream = file.OpenRead())
using (ZipFile zipFile = new ZipFile(fileStream))
using (IProgressNotifier mainProgress = parentProgress?.Spawn(2, $"Extracting {file.FullName} to {destination.FullName}."))
{
archiveResultBuilder.Clear();
using (mainProgress?.SpawnInfiniteProgress("Test archive integrity."))
{
ZipFile copy = zipFile;
await Task.Run(() =>
{
if (!copy.TestArchive(true, TestStrategy.FindAllErrors, ResultHandler))
{
throw new UnsupportedArchiveFormatException(
file.FullName, new FormattableException(archiveResultBuilder.ToString()));
}
}).ConfigureAwait(false);
}
double increment = (double)Constants.ProgressMaxResolution / zipFile.Count + 1;
using (IProgressNotifier extractProgress = mainProgress?.Spawn(Constants.ProgressMaxResolution, "Extract files"))
{
foreach (ZipEntry zipEntry in zipFile)
{
extractProgress?.TickIncrement(increment);
if (!zipEntry.IsFile)
{
continue; // Ignore directories
}
byte[] buffer = new byte[Constants.StreamCopyBufferSize]; // 4K is optimum
using (Stream zipStream = zipFile.GetInputStream(zipEntry))
{
string[] path = fileSystem.GetPath(zipEntry.Name);
VirtualDirectory fileDestination = destination.Directory(path.Take(path.Length - 1).ToArray());
VirtualFile entryFile = fileDestination.File(path.Last());
// Unzip file in buffered chunks. This is just as fast as unpacking to a buffer the full size
// of the file, but does not waste memory.
using (Stream streamWriter = entryFile.OpenWrite())
{
StreamUtils.Copy(zipStream, streamWriter, buffer);
}
}
}
}
}
}
VirtualFile UnpackXzFile(VirtualFile packedFile)
{
using (Stream fileStream = packedFile.OpenRead())
using (XZStream xzStream = new XZStream(fileStream))
using (parentProgress?.SpawnInfiniteProgress($"Extracting {packedFile.Name}..."))
{
string[] path = fileSystem.GetPath(packedFile.FullName);
string relativeFilePath = path.Last().Substring(0, path.Last().LastIndexOf(".xz", StringComparison.OrdinalIgnoreCase));
if (destination.FileExists(relativeFilePath))
{
destination.File(relativeFilePath).Delete();
}
VirtualFile destinationFile = destination.File(relativeFilePath);
observable?.OnNext(new Change(() => destinationFile.Restore()));
byte[] buffer = new byte[Constants.StreamCopyBufferSize]; // 4K is optimum
using (Stream streamWriter = destinationFile.OpenWrite())
{
StreamUtils.Copy(xzStream, streamWriter, buffer);
return(destinationFile);
}
}
}
void UnpackTarFile(VirtualFile packedFile)
{
//sharpcompress
using (Stream fileStream = packedFile.OpenRead())
using (TarArchive tarArchive = TarArchive.Open(fileStream))
{
double increment = (double)Constants.ProgressMaxResolution / tarArchive.Entries.Count;
using (IProgressNotifier extractProgress = parentProgress?.Spawn(Constants.ProgressMaxResolution, "Extracting .tar archive"))
{
foreach (TarArchiveEntry tarEntry in tarArchive.Entries)
{
extractProgress?.TickIncrement(increment);
if (tarEntry.IsDirectory)
{
continue; // Ignore directories
}
byte[] buffer = new byte[Constants.StreamCopyBufferSize]; // 4K is optimum
using (Stream tarStream = tarEntry.OpenEntryStream())
{
string[] path = fileSystem.GetPath(tarEntry.Key);
VirtualDirectory fileDestination = destination.Directory(path.Take(path.Length - 1).ToArray());
if (fileDestination.FileExists(path.Last()))
{
fileDestination.File(path.Last()).Delete();
}
VirtualFile entryFile = fileDestination.File(path.Last());
observable?.OnNext(new Change(() => entryFile.Restore()
//, $"Unpack {tarEntry.Key}."
));
//Unzip file in buffered chunks. This is just as fast as unpacking to a buffer the full size
// of the file, but does not waste memory.
using (Stream streamWriter = entryFile.OpenWrite())
{
StreamUtils.Copy(tarStream, streamWriter, buffer);
}
}
}
}
}
}
void UnpackTarXzFile(VirtualFile packedFile)
{
using (IProgressNotifier subProgress = parentProgress?.Spawn(2))
{
parentProgress = subProgress;
VirtualFile tarFile = UnpackXzFile(packedFile);
UnpackTarFile(tarFile);
tarFile.Delete();
}
}
}
