/// <inheritdoc />
public ICommandDispatcher <TCommand, TSuccess, TFailure> WithCommand(TCommand command, string commandNamespace)
{
_command = command;
_commandStream = _streamFactory.GetStream <TCommand>(_streamProvider, _streamId, commandNamespace);
return(this);
}
/// <summary>
/// Rent a Stream reader instance
/// </summary>
public Stream Rent()
{
if (!_pool.TryTake(out var stream))
{
stream = _factory.GetStream(false, false);
}
return(stream);
}
public override async Task OnActivateAsync()
{
//this will subscribe to kyc request events on the underlying stream provider
var streamProvider = GetStreamProvider(Constants.StreamProviderName);
_incomingStream = _streamFactory.GetStream <MockMessage>(streamProvider, this.GetPrimaryKey(), Constants.ActivityStreamNamespace);
await _incomingStream.SubscribeAsync(this);
await base.OnActivateAsync();
}
public static async Task <BA2Reader> Load(IStreamFactory streamFactory)
{
var rdr = new BA2Reader(await streamFactory.GetStream())
{
_streamFactory = streamFactory
};
await rdr.LoadHeaders();
return(rdr);
}
public static async ValueTask <BSAReader> LoadAsync(IStreamFactory factory)
{
await using var stream = await factory.GetStream().ConfigureAwait(false);
using var br = new BinaryReader(stream);
var bsa = new BSAReader {
_streamFactory = factory
};
bsa.LoadHeaders(br);
return(bsa);
}
public static async Task <Dictionary <RelativePath, T> > GatheringExtract <T>(IStreamFactory sFn,
Predicate <RelativePath> shouldExtract, Func <RelativePath, IStreamFactory, ValueTask <T> > mapfn)
{
if (sFn is NativeFileStreamFactory)
{
Utils.Log($"Extracting {sFn.Name}");
}
await using var archive = await sFn.GetStream();
var sig = await ArchiveSigs.MatchesAsync(archive);
archive.Position = 0;
switch (sig)
{
case Definitions.FileType.RAR_OLD:
case Definitions.FileType.RAR_NEW:
case Definitions.FileType._7Z:
case Definitions.FileType.ZIP:
{
if (sFn.Name.FileName.Extension == OMODExtension)
{
return(await GatheringExtractWithOMOD(archive, shouldExtract, mapfn));
}
else
{
return(await GatheringExtractWith7Zip <T>(archive, (Definitions.FileType) sig, shouldExtract,
mapfn));
}
}
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
return(await GatheringExtractWithBSA(sFn, (Definitions.FileType) sig, shouldExtract, mapfn));
case Definitions.FileType.TES3:
if (sFn.Name.FileName.Extension == BSAExtension)
{
return(await GatheringExtractWithBSA(sFn, (Definitions.FileType) sig, shouldExtract, mapfn));
}
else
{
throw new Exception($"Invalid file format {sFn.Name}");
}
default:
throw new Exception($"Invalid file format {sFn.Name}");
}
}
public static async ValueTask <TES3Reader> Load(IStreamFactory factory)
{
await using var fs = await factory.GetStream();
using var br = new BinaryReader(fs);
var rdr = new TES3Reader
{
_streamFactory = factory,
_versionNumber = br.ReadUInt32(),
_hashTableOffset = br.ReadUInt32(),
_fileCount = br.ReadUInt32()
};
rdr._files = new TES3FileEntry[rdr._fileCount];
for (int i = 0; i < rdr._fileCount; i++)
{
var file = new TES3FileEntry {
Index = i,
Archive = rdr,
Size = br.ReadUInt32(),
Offset = br.ReadUInt32()
};
rdr._files[i] = file;
}
for (int i = 0; i < rdr._fileCount; i++)
{
rdr._files[i].NameOffset = br.ReadUInt32();
}
var origPos = br.BaseStream.Position;
for (int i = 0; i < rdr._fileCount; i++)
{
br.BaseStream.Position = origPos + rdr._files[i].NameOffset;
rdr._files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3));
}
br.BaseStream.Position = rdr._hashTableOffset + 12;
for (int i = 0; i < rdr._fileCount; i++)
{
rdr._files[i].Hash1 = br.ReadUInt32();
rdr._files[i].Hash2 = br.ReadUInt32();
}
rdr._dataOffset = br.BaseStream.Position;
return(rdr);
}
private static string Decrypt(
string algorithmName,
IEnumerable <byte> key,
IEnumerable <byte> initialVector,
IEnumerable <byte> rawStringBytes,
IStreamFactory <MemoryStream> streamFactory = default)
{
streamFactory = streamFactory ?? new MemoryStreamFactory();
using (var symmetricAlgorithm = SymmetricAlgorithm.Create(algorithmName))
using (var encryptor = symmetricAlgorithm.CreateDecryptor(key.ToArray(), initialVector.ToArray()))
using (var stream = streamFactory.GetStream(Guid.NewGuid(), rawStringBytes))
using (var cryptoStreamReader = new CryptoStream(stream, encryptor, CryptoStreamMode.Read))
using (var streamReader = new StreamReader(cryptoStreamReader))
return(streamReader.ReadToEnd());
}
public async Task WriteLogAsync(Guid id, DateTime timeStamp, string message)
{
if (string.IsNullOrEmpty(message))
{
throw new ArgumentException("Message argument is required.");
}
await _semaphore.WaitAsync();
try
{
using var stream = _streamFactory.GetStream();
stream.Position = stream.Length;
using StreamWriter outputFile = new StreamWriter(stream);
await outputFile.WriteLineAsync($"{timeStamp:s} {id} {message}");
}
finally
{
_semaphore.Release();
}
}
private static IEnumerable <byte> Encrypt(
string algorithmName,
IEnumerable <byte> key,
IEnumerable <byte> initialVector,
string value,
IStreamFactory <MemoryStream> streamFactory = default)
{
streamFactory = streamFactory ?? new MemoryStreamFactory();
using (var symmetricAlgorithm = SymmetricAlgorithm.Create(algorithmName))
using (var encryptor = symmetricAlgorithm.CreateEncryptor(
key.ToArray(),
initialVector.ToArray()))
using (var stream = streamFactory.GetStream(Guid.NewGuid()))
{
using (var cryptoStreamWriter = new CryptoStream(stream, encryptor, CryptoStreamMode.Write))
using (var streamWriter = new StreamWriter(cryptoStreamWriter))
streamWriter.Write(value);
return(stream.ToArray());
}
}
private static async Task <Dictionary <RelativePath, T> > GatheringExtractWith7Zip <T>(WorkQueue queue, IStreamFactory sf, Definitions.FileType sig, Predicate <RelativePath> shouldExtract, Func <RelativePath, IExtractedFile, ValueTask <T> > mapfn,
AbsolutePath tempPath, HashSet <RelativePath> onlyFiles)
{
TempFile tmpFile = null;
var dest = tempPath.Combine(Guid.NewGuid().ToString());
dest.CreateDirectory();
TempFile spoolFile = null;
AbsolutePath source;
try
{
if (sf.Name is AbsolutePath abs)
{
source = abs;
}
else
{
spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString())
.WithExtension(source.Extension));
await using var s = await sf.GetStream();
await spoolFile.Path.WriteAllAsync(s);
source = spoolFile.Path;
}
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}",
$"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper {
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
};
if (onlyFiles != null)
{
//It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
IEnumerable <string> AllVariants(string input)
{
yield return($"\"{input}\"");
yield return($"\"\\{input}\"");
}
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] { "x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off" };
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var(_, line) = p;
if (line == null)
{
return;
}
if (line.Length <= 4 || line[3] != '%')
{
return;
}
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
var results = await dest.EnumerateFiles()
.PMap(queue, async f =>
{
var path = f.RelativeTo(dest);
if (!shouldExtract(path))
{
return(((RelativePath, T)) default);
public StreamPool(IStreamFactory factory, bool appendOnly)
{
_factory = factory;
_writer = new Lazy <Stream>(() => _factory.GetStream(true, appendOnly), LazyThreadSafetyMode.PublicationOnly);
}
public StreamPool(IStreamFactory factory, bool readOnly)
{
_factory = factory;
_writer = readOnly ? null : _factory.GetStream(false);
}
public StreamPool(IStreamFactory factory, bool appendOnly)
{
_factory = factory;
_writer = new Lazy <Stream>(() => _factory.GetStream(true, appendOnly), true);
}
public ValueTask <Stream> GetStream()
{
return(_factory.GetStream());
}
internal BinaryReader GetStream()
{
return(new BinaryReader(_streamFactory.GetStream().Result));
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
IPath relPath, int depth = 0)
{
Hash hash;
if (extractedFile is NativeFileStreamFactory)
{
hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty;
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.xxHashAsync();
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
{
return(vself);
}
await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = stream.Length,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash,
};
if (Consts.TextureExtensions.Contains(relPath.FileName.Extension))
{
self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false);
}
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
}
// Can't extract, so return
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
await WriteToCache(self);
return(self);
}
try
{
var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));
self.Children = list.Values.ToImmutableList();
}
catch (EndOfStreamException)
{
return(self);
}
catch (Exception)
{
Utils.Log($"Error while examining the contents of {relPath.FileName}");
throw;
}
await WriteToCache(self);
return(self);
}
public static async Task <Dictionary <RelativePath, T> > GatheringExtract <T>(WorkQueue queue, IStreamFactory sFn,
Predicate <RelativePath> shouldExtract, Func <RelativePath, IExtractedFile, ValueTask <T> > mapfn,
AbsolutePath?tempFolder = null,
HashSet <RelativePath> onlyFiles = null)
{
if (tempFolder == null)
{
tempFolder = TempFolder.BaseFolder;
}
if (sFn is NativeFileStreamFactory)
{
Utils.Log($"Extracting {sFn.Name}");
}
await using var archive = await sFn.GetStream();
var sig = await ArchiveSigs.MatchesAsync(archive);
archive.Position = 0;
Dictionary <RelativePath, T> results = new Dictionary <RelativePath, T>();
switch (sig)
{
case Definitions.FileType.RAR_OLD:
case Definitions.FileType.RAR_NEW:
case Definitions.FileType._7Z:
case Definitions.FileType.ZIP:
{
if (sFn.Name.FileName.Extension == OMODExtension)
{
results = await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
}
else
{
results = await GatheringExtractWith7Zip <T>(queue, sFn, (Definitions.FileType) sig, shouldExtract,
mapfn, tempFolder.Value, onlyFiles);
}
break;
}
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
results = await GatheringExtractWithBSA(sFn, (Definitions.FileType) sig, shouldExtract, mapfn);
break;
case Definitions.FileType.TES3:
if (sFn.Name.FileName.Extension == BSAExtension)
{
results = await GatheringExtractWithBSA(sFn, (Definitions.FileType) sig, shouldExtract, mapfn);
}
else
{
throw new Exception($"Invalid file format {sFn.Name}");
}
break;
default:
throw new Exception($"Invalid file format {sFn.Name}");
}
if (onlyFiles != null && onlyFiles.Count != results.Count)
{
throw new Exception(
$"Sanity check error extracting {sFn.Name} - {results.Count} results, expected {onlyFiles.Count}");
}
return(results);
}
