public Database(Stream stream, bool readExisting)
{
if (stream.CanSeek == false || stream.CanRead == false)
{
throw new ArgumentException("stream must have seek / read access", "stream");
}
this.Stream = stream;
this.BaseOffset = this.Stream.Position;
this._Entries = new Dictionary<ResourceKey, Entry>();
this.Entries = new ReadOnlyDictionary<ResourceKey, Entry>(this._Entries);
this.OriginalEntries = new Dictionary<ResourceKey, StreamEntry>();
if (readExisting == true)
{
DatabasePackedFile dbpf = new DatabasePackedFile();
dbpf.Read(stream);
this.EndOfDataOffset = dbpf.IndexOffset;
foreach (DatabasePackedFile.Entry entry in dbpf.Entries)
{
this._Entries.Add(entry.Key, new StreamEntry()
{
Compressed = entry.Compressed,
Offset = entry.Offset,
CompressedSize = entry.CompressedSize,
DecompressedSize = entry.DecompressedSize,
CompressedFlags = entry.CompressionFlags,
Flags = entry.Flags,
});
}
}
else
{
this.EndOfDataOffset = 0;
}
}
private void button3_Click(object sender, EventArgs e)
{
progressBar1.Minimum = 0;
progressBar1.Value = 0;
string openPath = "";
if (comboBox1.Text.StartsWith("\\"))
{
openPath = MadScience.Helpers.findSims3Root() + comboBox1.Text;
}
else
{
openPath = comboBox1.Text;
}
if (openPath == "") return;
Stream input = File.OpenRead(openPath);
Database db = new Database(input, true);
input.Seek(0, SeekOrigin.Begin);
DatabasePackedFile dbpf = new DatabasePackedFile();
try
{
dbpf.Read(input);
}
catch (NotAPackageException)
{
MessageBox.Show("bad file: {0}", comboBox1.Text);
input.Close();
return;
}
this.currentFile = new FileInfo(openPath);
lookupTypes();
indexEntries.Clear();
listView1.Items.Clear();
txtPkgIndexType.Text = dbpf.IndexType.ToString();
txtPkgNumChunks.Text = dbpf.Entries.Count.ToString();
uint searchTypeID = 0;
if (comboBox2.Text != "") { searchTypeID = Gibbed.Helpers.StringHelpers.ParseHex32(comboBox2.Text); }
ulong instanceID = 0;
bool searchInstance = false;
if (textBox8.Text != "")
{
instanceID = Gibbed.Helpers.StringHelpers.ParseHex64(textBox8.Text);
searchInstance = true;
}
string tempChunk = "";
int count = 0;
progressBar1.Maximum = dbpf.Entries.Count;
for (int i = 0; i < dbpf.Entries.Count; i++)
{
ListViewItem item = new ListViewItem();
DatabasePackedFile.Entry entry = dbpf.Entries[i];
progressBar1.Value++;
bool searchChunk = false;
if (entry.Key.TypeId == searchTypeID)
{
if (searchInstance == true)
{
if (entry.Key.InstanceId == instanceID)
{
searchChunk = true;
}
}
else
{
searchChunk = true;
}
}
else
{
if (searchInstance == true)
{
if (entry.Key.InstanceId == instanceID)
{
searchChunk = true;
}
}
}
//if (searchTypeID == 0) { searchChunk = true; }
if (searchChunk)
{
tempChunk = Encoding.UTF8.GetString(db.GetResource(entry.Key));
bool addChunk = false;
// If everything is blank we just list... everything
if (textBox3.Text == "" && textBox6.Text == "" && textBox7.Text == "")
{
addChunk = true;
}
else
{
if (tempChunk.Contains(textBox3.Text))
{
if (textBox6.Text != "")
{
if (tempChunk.Contains(textBox6.Text))
{
if (textBox7.Text != "")
{
if (tempChunk.Contains(textBox7.Text))
{
addChunk = true;
}
}
else
{
addChunk = true;
}
}
}
else
{
addChunk = true;
}
}
}
if (addChunk == true)
{
item.Text = this.lookupList.lookup(entry.Key.TypeId).shortName;
item.SubItems.Add(entry.Key.TypeId.ToString("X8"));
item.SubItems.Add(entry.Key.GroupId.ToString("X8"));
item.SubItems.Add(entry.Key.InstanceId.ToString("X16"));
item.SubItems.Add(entry.DecompressedSize.ToString());
item.SubItems.Add(count.ToString());
listView1.Items.Add(item);
indexEntries.Add(entry);
count++;
}
}
}
input.Close();
}
private void loadFile(string filename)
{
toolStripStatusLabel1.Text = filename;
Stream input = File.OpenRead(filename);
DatabasePackedFile dbpf = new DatabasePackedFile();
try
{
dbpf.Read(input);
}
catch (NotAPackageException)
{
MessageBox.Show("bad file: {0}", filename);
input.Close();
return;
}
this.currentFile = new FileInfo(filename);
input.Seek(0, SeekOrigin.Begin);
lookupTypes();
indexEntries.Clear();
listView1.Items.Clear();
txtPkgIndexType.Text = dbpf.IndexType.ToString();
txtPkgNumChunks.Text = dbpf.Entries.Count.ToString();
for (int i = 0; i < dbpf.Entries.Count; i++)
{
ListViewItem item = new ListViewItem();
DatabasePackedFile.Entry entry = dbpf.Entries[i];
item.Text = this.lookupList.lookup(entry.Key.TypeId).shortName;
item.SubItems.Add(entry.Key.TypeId.ToString("X8"));
item.SubItems.Add(entry.Key.GroupId.ToString("X8"));
item.SubItems.Add(entry.Key.InstanceId.ToString("X16"));
item.SubItems.Add(entry.DecompressedSize.ToString());
item.SubItems.Add(i.ToString());
listView1.Items.Add(item);
indexEntries.Add(entry);
}
input.Close();
}
public PackageUnpack(Stream[] packageFiles, string destinationFolder, PleaseWait progress)
{
double e = 0.0;
FullTextIndex.ResetPath(destinationFolder);
if (Directory.Exists(destinationFolder))
{
if (progress != null)
{
progress.beginTask(e = 0.25, 1.0);
}
Directory.Delete(destinationFolder, true);
if (progress != null)
{
progress.endTask();
}
}
Directory.CreateDirectory(destinationFolder);
double totalSize = 0;
for (int i = 0; i < packageFiles.Length; i++)
{
totalSize += packageFiles[i].Length;
}
if (progress != null)
{
progress.beginTask(1.0 - e, totalSize);
}
foreach (var filestream in packageFiles)
{
DatabasePackedFile db = new DatabasePackedFile();
db.Read(filestream);
var DatabaseFiles = db.Indices.ToArray();
var group_sporemaster = "sporemaster".FNV();
var instance_names = "names".FNV();
foreach (var dbf in DatabaseFiles)
{
if (dbf.GroupId == group_sporemaster && dbf.InstanceId == instance_names)
{
byte[] data = unpack(filestream, dbf);
readNamesFile(data);
}
}
var locale_type = NameRegistry.Types.toHash("locale");
var prop_type = NameRegistry.Types.toHash("prop");
var rw4_type = NameRegistry.Types.toHash("rw4");
foreach (var dbf in DatabaseFiles)
{
// skip over automatically generated locale files
if (!(dbf.TypeId == locale_type && (
NameRegistry.Groups.toName(dbf.InstanceId).StartsWith("auto_") ||
NameRegistry.Groups.toName(dbf.InstanceId).EndsWith("_auto"))))
{
var fn = Path.Combine(destinationFolder, NameRegistry.getFileName(dbf.GroupId, dbf.InstanceId, dbf.TypeId));
Directory.CreateDirectory(Path.GetDirectoryName(fn));
byte[] data = unpack(filestream, dbf);
if (dbf.TypeId == prop_type)
{
writePropFile(data, fn);
}
else if (dbf.TypeId == rw4_type)
{
writeRW4File(data, fn);
}
else
{
writeBinaryFile(data, fn);
}
}
if (progress != null)
{
progress.addProgress(dbf.CompressedSize);
}
}
filestream.Close();
}
if (progress != null)
{
progress.endTask();
}
}
public static void Main(string[] args)
{
bool showHelp = false;
string currentProject = null;
var options = new OptionSet()
{
{ "h|help", "show this message and exit", v => showHelp = v != null },
{ "p|project=", "override current project", v => currentProject = v },
};
List <string> extras;
try
{
extras = options.Parse(args);
}
catch (OptionException e)
{
Console.Write("{0}: ", GetExecutableName());
Console.WriteLine(e.Message);
Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName());
return;
}
if (extras.Count != 0 || showHelp == true)
{
Console.WriteLine("Usage: {0} [OPTIONS]+", GetExecutableName());
Console.WriteLine();
Console.WriteLine("Options:");
options.WriteOptionDescriptions(Console.Out);
return;
}
Console.WriteLine("Loading project...");
var manager = Manager.Load(currentProject);
if (manager.ActiveProject == null)
{
Console.WriteLine("Nothing to do: no active project loaded.");
return;
}
var project = manager.ActiveProject;
var instanceHashes = project.LoadListsInstance32Names();
var installPath = project.InstallPath;
var listsPath = project.ListsPath;
if (installPath == null)
{
Console.WriteLine("Could not detect install path.");
return;
}
if (listsPath == null)
{
Console.WriteLine("Could not detect lists path.");
return;
}
installPath = installPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
listsPath = listsPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
Console.WriteLine("Searching for archives...");
var findPaths = new[]
{
"SimCity",
"SimCityData",
Path.Combine("SimCityUserData", "EcoGame"),
};
var inputPaths = new List <string>();
foreach (var findPath in findPaths)
{
var combinedPath = Path.Combine(installPath, findPath);
var discoveredPaths = Directory.GetFiles(combinedPath, "*.package", SearchOption.AllDirectories);
inputPaths.AddRange(discoveredPaths);
}
var outputPaths = new List <string>();
var breakdown = new Breakdown();
var tracking = new Tracking();
Console.WriteLine("Processing...");
for (int i = 0; i < inputPaths.Count; i++)
{
var inputPath = inputPaths[i];
var outputPath = GetListPath(installPath, inputPath);
if (outputPath == null)
{
throw new InvalidOperationException();
}
Console.WriteLine(outputPath);
outputPath = Path.Combine(listsPath, outputPath);
if (outputPaths.Contains(outputPath) == true)
{
throw new InvalidOperationException();
}
outputPaths.Add(outputPath);
if (File.Exists(inputPath + ".bak") == true)
{
inputPath += ".bak";
}
var dbpf = new DatabasePackedFile();
using (var input = File.OpenRead(inputPath))
{
dbpf.Read(input);
}
HandleEntries(dbpf.Entries.Select(e => e.Key.InstanceId).Distinct(),
instanceHashes,
tracking,
breakdown,
outputPath);
}
using (var output = new StreamWriter(Path.Combine(Path.Combine(listsPath, "files"), "status32.txt")))
{
output.WriteLine("{0}",
new Breakdown()
{
Known = tracking.Names.Distinct().Count(),
Total = tracking.Hashes.Distinct().Count(),
});
}
}
public static void Main(string[] args)
{
bool showHelp = false;
bool extractUnknowns = true;
string filterPattern = null;
bool overwriteFiles = false;
bool verbose = false;
var options = new OptionSet()
{
{ "o|overwrite", "overwrite existing files", v => overwriteFiles = v != null },
{ "nu|no-unknowns", "don't extract unknown files", v => extractUnknowns = v == null },
{ "f|filter=", "only extract files using pattern", v => filterPattern = v },
{ "v|verbose", "be verbose", v => verbose = v != null },
{ "h|help", "show this message and exit", v => showHelp = v != null },
};
List <string> extras;
try
{
extras = options.Parse(args);
}
catch (OptionException e)
{
Console.Write("{0}: ", GetExecutableName());
Console.WriteLine(e.Message);
Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName());
return;
}
if (extras.Count < 1 || extras.Count > 2 || showHelp == true)
{
Console.WriteLine("Usage: {0} [OPTIONS]+ input_package [output_dir]", GetExecutableName());
Console.WriteLine();
Console.WriteLine("Options:");
options.WriteOptionDescriptions(Console.Out);
return;
}
string inputPath = extras[0];
string outputPath = extras.Count > 1 ? extras[1] : Path.ChangeExtension(inputPath, null) + "_unpack";
inputPath = Path.GetFullPath(inputPath);
outputPath = Path.GetFullPath(outputPath);
Regex filter = null;
if (string.IsNullOrEmpty(filterPattern) == false)
{
filter = new Regex(filterPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
if (verbose == true)
{
Console.WriteLine("Loading project...");
}
var manager = ProjectData.Manager.Load();
if (manager.ActiveProject == null)
{
Console.WriteLine("Warning: no active project loaded.");
}
var typeLookup = new PackageTypeLookup();
if (manager.ActiveProject != null &&
string.IsNullOrEmpty(manager.ActiveProject.ListsPath) == false)
{
var typePath = Path.Combine(manager.ActiveProject.ListsPath, "package types.xml");
if (File.Exists(typePath) == true)
{
using (var input = File.OpenRead(typePath))
{
var serializer = new XmlSerializer(typeof(PackageTypeLookup));
typeLookup = (PackageTypeLookup)serializer.Deserialize(input);
}
}
}
var groupLookup = new PackageGroupLookup();
if (manager.ActiveProject != null &&
string.IsNullOrEmpty(manager.ActiveProject.ListsPath) == false)
{
var groupPath = Path.Combine(manager.ActiveProject.ListsPath, "package groups.xml");
if (File.Exists(groupPath) == true)
{
using (var input = File.OpenRead(groupPath))
{
var serializer = new XmlSerializer(typeof(PackageGroupLookup));
groupLookup = (PackageGroupLookup)serializer.Deserialize(input);
}
}
}
if (verbose == true)
{
Console.WriteLine("Reading package...");
}
var instance32Hashes = manager.LoadListsInstance32Names();
var instance64Hashes = manager.LoadListsInstance64Names();
using (var input = File.OpenRead(inputPath))
{
var dbpf = new DatabasePackedFile();
dbpf.Read(input);
var settings = new XmlWriterSettings()
{
Indent = true,
};
Directory.CreateDirectory(outputPath);
var xmlPath = Path.Combine(outputPath, "files.xml");
using (var xml = XmlWriter.Create(xmlPath, settings))
{
xml.WriteStartDocument();
xml.WriteStartElement("files");
if (dbpf.Entries.Count > 0)
{
if (verbose == true)
{
Console.WriteLine("Unpacking files...");
}
long current = 0;
long total = dbpf.Entries.Count;
var padding = total.ToString(CultureInfo.InvariantCulture).Length;
foreach (var entry in dbpf.Entries)
{
current++;
var typeInfo = typeLookup.GetTypeInfo(entry.Key.TypeId);
bool isUnknown = false;
string entryName;
if (instance32Hashes.Contains(entry.Key.InstanceId) == true)
{
entryName = instance32Hashes[entry.Key.InstanceId];
}
else if (instance64Hashes.Contains(entry.Key.InstanceId) == true)
{
entryName = instance64Hashes[entry.Key.InstanceId];
}
else
{
if (extractUnknowns == false)
{
continue;
}
isUnknown = true;
if ((entry.Key.InstanceId & 0xFFFFFFFF00000000ul) == 0)
{
entryName = "#" + entry.Key.InstanceId.ToString("X8", CultureInfo.InvariantCulture);
}
else
{
entryName = "#" + entry.Key.InstanceId.ToString("X16", CultureInfo.InvariantCulture);
}
}
if (typeInfo != null &&
string.IsNullOrEmpty(typeInfo.Extension) == false)
{
entryName += "." + typeInfo.Extension;
}
else
{
entryName += ".#" + entry.Key.TypeId.ToString("X8", CultureInfo.InvariantCulture);
}
var groupInfo = groupLookup.GetGroupInfo(entry.Key.GroupId);
if (groupInfo == null ||
string.IsNullOrEmpty(groupInfo.Name) == true)
{
if (entry.Key.GroupId == 0)
{
entryName = Path.Combine("default", entryName);
}
else
{
entryName = Path.Combine("__UNKNOWNGROUP",
"#" +
entry.Key.GroupId.ToString("X8",
CultureInfo.InvariantCulture),
entryName);
}
}
else
{
entryName = Path.Combine(groupInfo.Name, entryName);
}
if (filter != null &&
filter.IsMatch(entryName) == false)
{
continue;
}
if (isUnknown == true)
{
entryName = Path.Combine("__UNKNOWNNAME", entryName);
}
var entryPath = Path.Combine(outputPath, entryName);
xml.WriteStartElement("file");
xml.WriteAttributeString("group",
entry.Key.GroupId.ToString("X8", CultureInfo.InvariantCulture));
xml.WriteAttributeString("instance",
entry.Key.InstanceId.ToString("X16", CultureInfo.InvariantCulture));
xml.WriteAttributeString("type",
entry.Key.TypeId.ToString("X8", CultureInfo.InvariantCulture));
xml.WriteValue(GetRelativePathForFiles(xmlPath, entryPath));
xml.WriteEndElement();
if (overwriteFiles == false &&
File.Exists(entryPath) == true)
{
continue;
}
if (verbose == true)
{
Console.WriteLine("[{0}/{1}] {2}",
current.ToString(CultureInfo.InvariantCulture).PadLeft(padding),
total,
entryName);
}
var entryParent = Path.GetDirectoryName(entryPath);
if (string.IsNullOrEmpty(entryParent) == false)
{
Directory.CreateDirectory(entryParent);
}
using (var output = File.Create(entryPath))
{
input.Seek(entry.Offset, SeekOrigin.Begin);
if (entry.IsCompressed == true)
{
var compressedBytes = input.ReadBytes(entry.CompressedSize);
var uncompressedBytes = RefPack.Decompression.Decompress(compressedBytes);
if (uncompressedBytes.Length != entry.UncompressedSize)
{
throw new InvalidOperationException();
}
output.WriteBytes(uncompressedBytes);
}
else
{
output.WriteFromStream(input, entry.UncompressedSize);
}
}
}
}
xml.WriteEndElement();
xml.WriteEndDocument();
}
}
}
private bool searchForKey(string keyString)
{
// Validate keystring
if (validateKey(keyString) == false)
{
return(false);
}
string sims3root = MadScience.Helpers.findSims3Root();
if (String.IsNullOrEmpty(sims3root))
{
return(false);
}
toolStripProgressBar1.Minimum = 0;
toolStripProgressBar1.Value = 0;
toolStripStatusLabel1.Text = "Searching for image... please wait";
statusStrip1.Refresh();
Stream input = File.OpenRead(Path.Combine(sims3root, MadScience.Helpers.getGameSubPath("\\GameData\\Shared\\Packages\\FullBuild2.package")));
Database db = new Database(input, true);
input.Seek(0, SeekOrigin.Begin);
DatabasePackedFile dbpf = new DatabasePackedFile();
try
{
dbpf.Read(input);
}
catch (MadScience.Exceptions.NotAPackageException)
{
MessageBox.Show("bad file: {0}", Path.Combine(sims3root, "\\GameData\\Shared\\Packages\\FullBuild2.package"));
input.Close();
return(false);
}
// Split the input key
keyString = keyString.Replace("key:", "");
string[] temp = keyString.Split(":".ToCharArray());
uint typeID = MadScience.StringHelpers.ParseHex32("0x" + temp[0]);
uint groupID = MadScience.StringHelpers.ParseHex32("0x" + temp[1]);
ulong instanceID = MadScience.StringHelpers.ParseHex64("0x" + temp[2]);
toolStripProgressBar1.Maximum = dbpf.Entries.Count;
bool foundMatch = false;
for (int i = 0; i < dbpf.Entries.Count; i++)
{
DatabasePackedFile.Entry entry = dbpf.Entries[i];
toolStripProgressBar1.Value++;
if (entry.Key.typeId == typeID && entry.Key.groupId == groupID && entry.Key.instanceId == instanceID)
{
foundMatch = true;
MadScience.DDSPreview ddsP = new MadScience.DDSPreview();
ddsP.loadDDS(db.GetResourceStream(entry.Key));
ddsP.ShowDialog();
break;
}
}
toolStripProgressBar1.Value = 0;
toolStripStatusLabel1.Text = "";
statusStrip1.Refresh();
input.Close();
return(foundMatch);
}
private void OnOpen(object sender, EventArgs e)
{
if (this.openDialog.ShowDialog() != DialogResult.OK)
{
return;
}
if (this.openDialog.InitialDirectory != null)
{
this.openDialog.InitialDirectory = null;
}
Stream input = this.openDialog.OpenFile();
DatabasePackedFile dbpf = new DatabasePackedFile();
dbpf.Read(input);
this.AddMapNames(dbpf, input);
this.DatabaseFiles = dbpf.Entries.ToArray();
Dictionary <string, TreeNode> categoryNodes = new Dictionary <string, TreeNode>();
Dictionary <uint, TreeNode> unknownTypeNodes = new Dictionary <uint, TreeNode>();
this.typeList.Nodes.Clear();
this.typeList.BeginUpdate();
TreeNode knownNode = new TreeNode("Known");
TreeNode unknownNode = new TreeNode("Unknown");
for (int i = 0; i < this.DatabaseFiles.Length; i++)
{
DatabasePackedFile.Entry index = this.DatabaseFiles[i];
TreeNode typeNode = null;
if (Lookup.Types.ContainsKey(index.Key.TypeId) == true)
{
TypeLookup type = Lookup.Types[index.Key.TypeId];
TreeNode categoryNode;
if (categoryNodes.ContainsKey(type.Category) == false)
{
categoryNode = new TreeNode();
categoryNode.Text = type.Category;
categoryNode.Tag = new Dictionary <uint, TreeNode>();
knownNode.Nodes.Add(categoryNode);
categoryNodes[type.Category] = categoryNode;
}
else
{
categoryNode = categoryNodes[type.Category];
}
Dictionary <uint, TreeNode> typeNodes = categoryNode.Tag as Dictionary <uint, TreeNode>;
if (typeNodes.ContainsKey(index.Key.TypeId) == false)
{
typeNode = new TreeNode();
typeNode.Text = type.Description == null ? type.Directory : type.Description;
typeNode.Tag = new List <DatabasePackedFile.Entry>();
typeNode.NodeFont = this.MonospaceFont;
categoryNode.Nodes.Add(typeNode);
typeNodes[index.Key.TypeId] = typeNode;
}
else
{
typeNode = typeNodes[index.Key.TypeId];
}
}
else
{
if (unknownTypeNodes.ContainsKey(index.Key.TypeId) == false)
{
typeNode = new TreeNode();
typeNode.Text = "#" + index.Key.TypeId.ToString("X8");
typeNode.Tag = new List <DatabasePackedFile.Entry>();
typeNode.NodeFont = this.MonospaceFont;
unknownNode.Nodes.Add(typeNode);
unknownTypeNodes[index.Key.TypeId] = typeNode;
}
else
{
typeNode = unknownTypeNodes[index.Key.TypeId];
}
}
List <DatabasePackedFile.Entry> files = typeNode.Tag as List <DatabasePackedFile.Entry>;
files.Add(index);
}
if (knownNode.Nodes.Count > 0)
{
this.typeList.Nodes.Add(knownNode);
}
if (unknownNode.Nodes.Count > 0)
{
this.typeList.Nodes.Add(unknownNode);
}
this.typeList.Sort();
this.typeList.EndUpdate();
if (knownNode.Nodes.Count > 0)
{
knownNode.Expand();
}
else if (unknownNode.Nodes.Count > 0)
{
unknownNode.Expand();
}
}