internal GZipWritableArchiveEntry(GZipArchive archive, Stream stream, string path, long size, DateTime? lastModified, bool closeStream)
: base(archive, null)
{
this.stream = stream;
this.path = path;
this.size = size;
this.lastModified = lastModified;
this.closeStream = closeStream;
}
internal GZipWritableArchiveEntry(int index, GZipArchive archive, Stream stream,
string path, long size, DateTime? lastModified)
: base(index, archive, null)
{
this.Stream = stream;
this.path = path;
this.size = size;
this.lastModified = lastModified;
}
internal GZipArchiveEntry(GZipArchive archive, GZipFilePart part)
: base(part)
{
this.archive = archive;
}
/// <summary>
/// Process the DAT and find all matches in input files and folders assuming they're a depot
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildDepot(
DatFile datFile,
List <string> inputs,
string outDir,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return(false);
}
// Check that the output directory exists
outDir = outDir.Ensure(create: true);
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
{
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
}
#endregion
bool success = true;
#region Rebuild from depots in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through and get only directories from the input paths
List <string> directories = new List <string>();
Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
logger.Verbose($"Adding depot: {input}");
lock (directories)
{
directories.Add(input);
}
}
});
// If we don't have any directories, we want to exit
if (directories.Count == 0)
{
return(success);
}
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = datFile.Items.SortedKeys.ToList();
foreach (string hash in keys)
{
// Pre-empt any issues that could arise from string length
if (hash.Length != Constants.SHA1Length)
{
continue;
}
logger.User($"Checking hash '{hash}'");
// Get the extension path for the hash
string subpath = Utilities.GetDepotPath(hash, datFile.Header.InputDepot.Depth);
// Find the first depot that includes the hash
string foundpath = null;
foreach (string directory in directories)
{
if (File.Exists(Path.Combine(directory, subpath)))
{
foundpath = Path.Combine(directory, subpath);
break;
}
}
// If we didn't find a path, then we continue
if (foundpath == null)
{
continue;
}
// If we have a path, we want to try to get the rom information
GZipArchive archive = new GZipArchive(foundpath);
BaseFile fileinfo = archive.GetTorrentGZFileInfo();
// If the file information is null, then we continue
if (fileinfo == null)
{
continue;
}
// Ensure we are sorted correctly (some other calls can change this)
datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// If there are no items in the hash, we continue
if (datFile.Items[hash] == null || datFile.Items[hash].Count == 0)
{
continue;
}
// Otherwise, we rebuild that file to all locations that we need to
bool usedInternally;
if (datFile.Items[hash][0].ItemType == ItemType.Disk)
{
usedInternally = RebuildIndividualFile(datFile, new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
else if (datFile.Items[hash][0].ItemType == ItemType.Media)
{
usedInternally = RebuildIndividualFile(datFile, new Media(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
else
{
usedInternally = RebuildIndividualFile(datFile, new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
// If we are supposed to delete the depot file, do so
if (delete && usedInternally)
{
File.Delete(foundpath);
}
}
watch.Stop();
#endregion
return(success);
}
/// <summary>
/// Opens a Reader for Non-seeking usage
/// </summary>
/// <param name="stream"></param>
/// <param name="options"></param>
/// <returns></returns>
public static IReader Open(Stream stream, ReaderOptions options = null)
{
stream.CheckNotNull(nameof(stream));
options = options ?? new ReaderOptions()
{
LeaveStreamOpen = false
};
RewindableStream rewindableStream = new RewindableStream(stream);
rewindableStream.StartRecording();
if (ZipArchive.IsZipFile(rewindableStream, options.Password))
{
rewindableStream.Rewind(true);
return(ZipReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (GZipArchive.IsGZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
GZipStream testStream = new GZipStream(rewindableStream, CompressionMode.Decompress);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.GZip));
}
rewindableStream.Rewind(true);
return(GZipReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (BZip2Stream.IsBZip2(rewindableStream))
{
rewindableStream.Rewind(false);
BZip2Stream testStream = new BZip2Stream(new NonDisposingStream(rewindableStream), CompressionMode.Decompress, false);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.BZip2));
}
}
rewindableStream.Rewind(false);
if (LZipStream.IsLZipFile(rewindableStream))
{
rewindableStream.Rewind(false);
LZipStream testStream = new LZipStream(new NonDisposingStream(rewindableStream), CompressionMode.Decompress);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.LZip));
}
}
rewindableStream.Rewind(false);
if (RarArchive.IsRarFile(rewindableStream, options))
{
rewindableStream.Rewind(true);
return(RarReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (TarArchive.IsTarFile(rewindableStream))
{
rewindableStream.Rewind(true);
return(TarReader.Open(rewindableStream, options));
}
rewindableStream.Rewind(false);
if (XZStream.IsXZStream(rewindableStream))
{
rewindableStream.Rewind(true);
XZStream testStream = new XZStream(rewindableStream);
if (TarArchive.IsTarFile(testStream))
{
rewindableStream.Rewind(true);
return(new TarReader(rewindableStream, options, CompressionType.Xz));
}
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Reader Formats: Zip, GZip, BZip2, Tar, Rar, LZip, XZ");
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
{
return(false);
}
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List <BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
internalDatItem = new Media(internalFileInfo);
}
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
internalDatItem = new Disk(internalFileInfo);
}
else
{
internalDatItem = new Rom(internalFileInfo);
}
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return(usedExternally || usedInternally);
}
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive) {
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null) {
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead/(float) archive.TotalSize;
if (prog > 1)
prog = 1;
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?) (args.CompressedBytesRead/(totalMilliseconds/1000.0)) : null,
prog*100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
public static bool isArchive(FileInfo file)
{
//FIXME This is broken and detects .gba ROMs as stuff
return(GZipArchive.IsGZipFile(file) || RarArchive.IsRarFile(file) || SevenZipArchive.IsSevenZipFile(file) ||
TarArchive.IsTarFile(file) || ZipArchive.IsZipFile(file));
}
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive)
{
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null)
{
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead / (float)archive.TotalSize;
if (prog > 1)
{
prog = 1;
}
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?)(args.CompressedBytesRead / (totalMilliseconds / 1000.0)) : null,
prog * 100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
private IArchive GetArchiveWithGzWorkaround(IAbsoluteFilePath sourceFile, string ext)
=> ext.Equals(".gz", StringComparison.OrdinalIgnoreCase)
? GZipArchive.Open(sourceFile.ToString())
: ArchiveFactory.Open(sourceFile.ToString());
