public static long DumpStream(Stream streamSource,
Stream streamTarget,
ProgressOutput progressOutputMethod)
{
int nChunkSize = 8192;
byte[] bytes = new byte[nChunkSize];
long lLength = 0;
while (true)
{
if (progressOutputMethod != null)
{
if (progressOutputMethod(lLength) == false)
break;
}
int n = streamSource.Read(bytes, 0, nChunkSize);
if (n != 0) // 2005/6/8
streamTarget.Write(bytes, 0, n);
if (n <= 0)
break;
lLength += n;
//if (n<1000)
// break;
}
if (progressOutputMethod != null)
{
progressOutputMethod(lLength);
}
return lLength;
}
bool ReadInternal()
{
bool result = true;
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Trying to read source listing {0}...", mSourceFile);
}
if (!File.Exists(mSourceFile))
{
result = false;
}
else
{
mPkgFile = new CaPackageFile();
using (var xml = new IO.XmlElementStream(mSourceFile, FileAccess.Read, this))
{
xml.InitializeAtRootElement();
PkgDefinition.Serialize(xml);
}
}
if (result == false)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFailed!");
}
}
return(result);
}
bool PackChunks(IO.EndianStream ecfStream)
{
bool success = true;
foreach (var chunk in EcfDefinition.Chunks)
{
if (ProgressOutput != null)
{
ProgressOutput.Write("\r\t\t{0} ", chunk.Id.ToString("X16"));
}
success = success && BuildChunkToStream(ecfStream, chunk);
if (!success)
{
break;
}
}
if (success && ProgressOutput != null)
{
ProgressOutput.Write("\r\t\t{0} \r", new string(' ', 16));
}
return(success);
}
bool ReadInternal()
{
bool result = true;
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Trying to read source listing {0}...", mSourceFile);
}
if (!File.Exists(mSourceFile))
{
result = false;
}
else
{
mEraFile = new EraFile();
mEraFile.BuildModeDefaultTimestamp = EraFile.GetMostRecentTimeStamp(mSourceFile);
using (var xml = new IO.XmlElementStream(mSourceFile, FA.Read, this))
{
xml.InitializeAtRootElement();
result &= mEraFile.ReadDefinition(xml);
}
}
if (result == false)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFailed!");
}
}
return(result);
}
bool ReadEraFromFile()
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Opening and reading ERA file {0}...",
mSourceFile);
}
if (ExpanderOptions.Test(EraFileExpanderOptions.DontLoadEntireEraIntoMemory))
{
mEraBaseStream = System.IO.File.OpenRead(mSourceFile);
}
else
{
byte[] era_bytes = System.IO.File.ReadAllBytes(mSourceFile);
if (ExpanderOptions.Test(EraFileExpanderOptions.Decrypt))
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Decrypting...");
}
DecryptFileBytes(era_bytes);
}
mEraBaseStream = new System.IO.MemoryStream(era_bytes, writable: false);
}
mEraStream = new IO.EndianStream(mEraBaseStream, Shell.EndianFormat.Big, this, permissions: FA.Read);
mEraStream.StreamMode = FA.Read;
return(ReadEraFromStream());
}
public bool ExpandTo(string workPath, string listingName)
{
if (mEraFile == null)
{
return(false);
}
if (!System.IO.Directory.Exists(workPath))
{
System.IO.Directory.CreateDirectory(workPath);
}
bool result = true;
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Outputting listing...");
}
try { SaveListing(workPath, listingName); }
catch (Exception ex)
{
if (VerboseOutput != null)
{
VerboseOutput.WriteLine("\tEncountered an error while outputting listing: {0}", ex);
}
result = false;
}
if (result && !ExpanderOptions.Test(EraFileExpanderOptions.OnlyDumpListing))
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Expanding archive to {0}...", workPath);
}
try { mEraFile.ExpandTo(mEraStream, workPath); }
catch (Exception ex)
{
if (VerboseOutput != null)
{
VerboseOutput.WriteLine("\tEncountered an error while expanding archive: {0}", ex);
}
result = false;
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Done");
}
}
mEraStream.Close();
return(result);
}
public static long DumpStream(Stream streamSource,
Stream streamTarget,
ProgressOutput progressOutputMethod)
{
int nChunkSize = 8192;
byte[] bytes = new byte[nChunkSize];
long lLength = 0;
while (true)
{
if (progressOutputMethod != null)
{
if (progressOutputMethod(lLength) == false)
{
break;
}
}
int n = streamSource.Read(bytes, 0, nChunkSize);
if (n != 0) // 2005/6/8
{
streamTarget.Write(bytes, 0, n);
}
if (n <= 0)
{
break;
}
lLength += n;
//if (n<1000)
// break;
}
if (progressOutputMethod != null)
{
progressOutputMethod(lLength);
}
return(lLength);
}
static IOutput DetermineOutput(Options options, IConsole console)
{
IOutput output = new VerboseOutput(console);
if (options.Silent)
{
output = new SilentOutput();
}
if (options.Progress)
{
output = new ProgressOutput(console);
}
if (options.NoColor)
{
return(output);
}
return(new ColorOutput(output));
}
bool ReadInternal()
{
bool result = true;
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Trying to read source listing {0}...", mSourceFile);
}
if (!File.Exists(mSourceFile))
{
result = false;
}
else
{
using (var xml = new IO.XmlElementStream(mSourceFile, FA.Read, this))
{
xml.InitializeAtRootElement();
EcfDefinition.Serialize(xml);
}
EcfDefinition.CullChunksPossiblyWithoutFileData((chunkIndex, chunk) =>
{
if (VerboseOutput != null)
{
VerboseOutput.WriteLine("\t\tCulling chunk #{0} since it has no associated file data",
chunkIndex);
}
});
}
if (result == false)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFailed!");
}
}
return(result);
}
bool ReadEcfFromFile()
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Opening and reading ECF file {0}...",
mSourceFile);
}
if (ExpanderOptions.Test(EcfFileExpanderOptions.DontLoadEntireEcfIntoMemory))
{
mEcfBaseStream = File.OpenRead(mSourceFile);
}
else
{
byte[] ecf_bytes = File.ReadAllBytes(mSourceFile);
mEcfBaseStream = new MemoryStream(ecf_bytes, writable: false);
}
mEcfStream = new IO.EndianStream(mEcfBaseStream, Shell.EndianFormat.Big, this, permissions: FileAccess.Read);
mEcfStream.StreamMode = FileAccess.Read;
return(ReadEcfFromStream());
}
public bool ExpandTo(string workPath, string listingName)
{
if (mEcfFile == null)
{
return(false);
}
if (!Directory.Exists(workPath))
{
Directory.CreateDirectory(workPath);
}
bool result = true;
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Outputting listing...");
}
try
{
PopulateEcfDefinitionFromEcfFile(workPath);
SaveListing(workPath, listingName);
}
catch (Exception ex)
{
if (VerboseOutput != null)
{
VerboseOutput.WriteLine("\tEncountered an error while outputting listing: {0}", ex);
}
result = false;
}
if (result && !ExpanderOptions.Test(EcfFileExpanderOptions.OnlyDumpListing) && WriteChunksToFile)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Expanding ECF to {0}...", workPath);
}
try
{
ExpandChunksToFiles();
}
catch (Exception ex)
{
if (VerboseOutput != null)
{
VerboseOutput.WriteLine("\tEncountered an error while expanding ECF: {0}", ex);
}
result = false;
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Done");
}
}
mEcfStream.Close();
return(result);
}
bool BuildInternal(string workPath, string pkgName, string outputPath)
{
string pkg_filename = Path.Combine(outputPath, pkgName);
if (File.Exists(pkg_filename))
{
var attrs = File.GetAttributes(pkg_filename);
if (attrs.HasFlag(FileAttributes.ReadOnly))
{
throw new IOException("PKG file is readonly, can't build: " + pkg_filename);
}
}
{
string sanitizedWorkPath = CaPackageFileDefinition.SanitizeWorkingEnvironmentPath(workPath);
if (sanitizedWorkPath != workPath && VerboseOutput != null)
{
VerboseOutput.WriteLine("Sanitized work path:");
VerboseOutput.WriteLine("\t{0} =>", workPath);
VerboseOutput.WriteLine("\t{0}", sanitizedWorkPath);
workPath = sanitizedWorkPath;
}
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Apply changes to PKG listing for work environment...");
}
if (PkgDefinition.RedefineForWorkingEnvironment(workPath
, BuilderOptions.Test(CaPackageFileBuilderOptions.AlwaysUseXmlOverXmb)
, VerboseOutput))
{
}
mPkgFile = new CaPackageFile();
mPkgFile.SetupHeaderAndEntries(PkgDefinition);
const int k_initial_buffer_size = 16 * IntegerMath.kMega; // 16MB
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Building {0} to {1}...", pkgName, outputPath);
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tAllocating memory...");
}
bool result = true;
using (var ms = new MemoryStream(k_initial_buffer_size))
using (var pkg_memory = new IO.EndianStream(ms, Shell.EndianFormat.Little, this, permissions: FileAccess.Write))
{
pkg_memory.StreamMode = FileAccess.Write;
// #TODO:
// create null bytes for the header and embedded file chunk descriptors
// previously just used Seek to do this, but it doesn't update Length.
long preamble_size = mPkgFile.CalculateHeaderAndFileChunksSize((CaPackageVersion)CaPackageFile.kCurrentVersion);
ms.SetLength(preamble_size);
ms.Seek(preamble_size, SeekOrigin.Begin);
// now we can start embedding the files
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tPacking files...");
}
result = false; //result && mPkgFile.Build(pkg_memory, workPath);
if (result)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFinializing...");
}
// seek back to the start of the PKG and write out the finalized header and file entries
ms.Seek(0, SeekOrigin.Begin);
mPkgFile.Serialize(pkg_memory);
Contract.Assert(pkg_memory.BaseStream.Position == preamble_size,
"Written PKG header size is greater than what we calculated");
using (var fs = new FileStream(pkg_filename, FileMode.Create, FileAccess.Write))
ms.WriteTo(fs);
}
}
return(result);
}
bool BuildInternal(string workPath, string outputPath)
{
EcfDefinition.WorkingDirectory = workPath;
string ecf_name = EcfDefinition.EcfName;
if (ecf_name.IsNotNullOrEmpty())
{
ecf_name = Path.GetFileNameWithoutExtension(mSourceFile);
}
string ecf_filename = Path.Combine(outputPath, ecf_name);
#if false // I'm no longer doing this since we don't strip the file ext off listing when expanding
// #TODO I bet a user could forget to include the preceding dot
if (EcfDefinition.EcfFileExtension.IsNotNullOrEmpty())
{
ecf_filename += EcfDefinition.EcfFileExtension;
}
#endif
if (File.Exists(ecf_filename))
{
var attrs = File.GetAttributes(ecf_filename);
if (attrs.HasFlag(FileAttributes.ReadOnly))
{
throw new IOException("ECF file is readonly, can't build: " + ecf_filename);
}
}
mEcfFile = new EcfFile();
mEcfFile.SetupHeaderAndChunks(EcfDefinition);
const FA k_mode = FA.Write;
const int k_initial_buffer_size = 8 * IntegerMath.kMega; // 8MB
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Building {0} to {1}...", ecf_name, outputPath);
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tAllocating memory...");
}
bool result = true;
using (var ms = new MemoryStream(k_initial_buffer_size))
using (var ecf_memory = new IO.EndianStream(ms, Shell.EndianFormat.Big, this, permissions: k_mode))
{
ecf_memory.StreamMode = k_mode;
ecf_memory.VirtualAddressTranslationInitialize(Shell.ProcessorSize.x32);
long preamble_size = mEcfFile.CalculateHeaderAndChunkEntriesSize();
ms.SetLength(preamble_size);
ms.Seek(preamble_size, SeekOrigin.Begin);
// now we can start embedding the files
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tPacking chunks...");
}
result = result && PackChunks(ecf_memory);
if (result)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFinializing...");
}
// seek back to the start of the ECF and write out the finalized header and chunk descriptors
ms.Seek(0, SeekOrigin.Begin);
mEcfFile.Serialize(ecf_memory);
Contract.Assert(ecf_memory.BaseStream.Position == preamble_size,
"Written ECF header size is NOT EQUAL what we calculated");
// Update sizes and checksums
ms.Seek(0, SeekOrigin.Begin);
mEcfFile.SerializeBegin(ecf_memory, isFinalizing: true);
mEcfFile.SerializeEnd(ecf_memory);
// finally, bake the ECF memory stream into a file
using (var fs = new FileStream(ecf_filename, FileMode.Create, FA.Write))
ms.WriteTo(fs);
}
}
return(result);
}
bool BuildInternal(string workPath, string eraName, string outputPath)
{
string era_filename = Path.Combine(outputPath, eraName);
if (!BuilderOptions.Test(EraFileBuilderOptions.Encrypt))
{
era_filename += EraFileExpander.kNameExtension;
}
else
{
era_filename += EraFileBuilder.kExtensionEncrypted;
}
mEraFile.FileName = era_filename;
if (File.Exists(era_filename))
{
var attrs = File.GetAttributes(era_filename);
if (attrs.HasFlag(FileAttributes.ReadOnly))
{
throw new IOException("ERA file is readonly, can't build: " + era_filename);
}
}
if (BuilderOptions.Test(EraFileBuilderOptions.AlwaysUseXmlOverXmb))
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Finding XML files to use over XMB references...");
}
mEraFile.TryToReferenceXmlOverXmbFies(workPath, VerboseOutput);
}
const FA k_mode = FA.Write;
const int k_initial_buffer_size = 24 * IntegerMath.kMega; // 24MB
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("Building {0} to {1}...", eraName, outputPath);
}
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tAllocating memory...");
}
bool result = true;
using (var ms = new MemoryStream(k_initial_buffer_size))
using (var era_memory = new IO.EndianStream(ms, Shell.EndianFormat.Big, this, permissions: k_mode))
{
era_memory.StreamMode = k_mode;
// we can use our custom VAT system to generate relative-offset (to a given chunk) information which ECFs use
era_memory.VirtualAddressTranslationInitialize(Shell.ProcessorSize.x32);
// create null bytes for the header and embedded file chunk descriptors
// previously just used Seek to do this, but it doesn't update Length.
long preamble_size = mEraFile.CalculateHeaderAndFileChunksSize();
ms.SetLength(preamble_size);
ms.Seek(preamble_size, SeekOrigin.Begin);
// now we can start embedding the files
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tPacking files...");
}
result &= mEraFile.Build(era_memory, workPath);
if (result)
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tFinializing...");
}
// seek back to the start of the ERA and write out the finalized header and file chunk descriptors
ms.Seek(0, SeekOrigin.Begin);
mEraFile.Serialize(era_memory);
// Right now we don't actually perform any file removing (eg, duplicates) until EraFile.Build so
// we also allow the written size to be LESS THAN the assumed preamble size
Contract.Assert(era_memory.BaseStream.Position <= preamble_size,
"Written ERA header size is greater than what we calculated");
// finally, bake the ERA memory stream into a file
if (BuilderOptions.Test(EraFileBuilderOptions.Encrypt))
{
if (ProgressOutput != null)
{
ProgressOutput.WriteLine("\tEncrypting...");
}
var era_bytes = ms.GetBuffer();
EncryptFileBytes(era_bytes, (int)ms.Length);
}
else // not encrypted
{
}
using (var fs = new FileStream(era_filename, FileMode.Create, FA.Write))
ms.WriteTo(fs);
}
}
return(result);
}
