/*public void AddGuessAndSubMap(MetaValueGuess guess, MetaMap map)
* {
* if (map == null)
* {
* AddGuess(guess);
* return;
* }
*
* // Don't overwrite good guesses with null pointers
* if (_guesses.ContainsKey(guess.Offset))
* {
* if (guess.Pointer == 0 || guess.Pointer == 0xFFFFFFFF || guess.Pointer == 0xCDCDCDCD)
* return;
* }
*
* MetaMap previousMap;
* if (_submaps.TryGetValue(guess.Offset, out previousMap))
* {
* if (map.EstimatedSize > previousMap.EstimatedSize)
* _guesses[guess.Offset] = guess;
* }
* else
* {
* _guesses[guess.Offset] = guess;
* }
* AssociateSubMap(guess.Offset, map);
* }*/
public void AssociateSubMap(int offset, MetaMap map)
{
if (map == null)
{
return;
}
if (!_guesses.ContainsKey(offset))
{
return;
}
// Bail out if this will cause a cyclic dependency to happen
if (map == this || _ancestors.Contains(map) || map._ancestors.Contains(this))
{
return;
}
// If a map has already been associated with the offset, then merge it,
// otherwise just insert it
MetaMap oldMap;
if (_submaps.TryGetValue(offset, out oldMap))
{
oldMap.MergeWith(map);
}
else
{
_submaps[offset] = map;
}
// Combine the map's ancestor set with ours
map._ancestors.UnionWith(_ancestors);
map._ancestors.Add(this);
}
public void MergeWith(MetaMap other)
{
if (other == this || _ancestors.Contains(other) || other._ancestors.Contains(this))
{
return;
}
foreach (MetaValueGuess guess in other._guesses.Values)
{
MetaMap otherMap = other.GetSubMap(guess.Offset);
MetaMap myMap = GetSubMap(guess.Offset);
if (otherMap != null || myMap == null)
{
if (AddGuess(guess))
{
AssociateSubMap(guess.Offset, otherMap);
}
}
}
foreach (var estimate in other._sizeEstimates)
{
EstimateSize(estimate.Key, estimate.Value);
}
}
public void Fold(int firstBlockSize)
{
if (firstBlockSize <= 0)
{
return;
}
// Repeatedly fold the last guess in the list until the key is less than firstBlockSize
while (_guesses.Count > 0 && _guesses.Keys[_guesses.Count - 1] >= firstBlockSize)
{
// Grab the guess at the end and remove it along with its submap
MetaValueGuess guess = _guesses.Values[_guesses.Count - 1];
_guesses.RemoveAt(_guesses.Count - 1);
MetaMap subMap = GetSubMap(guess.Offset);
if (subMap != null)
{
_submaps.Remove(guess.Offset);
}
// Wrap its offset
guess.Offset %= firstBlockSize;
// Add them back in
if (AddGuess(guess))
{
AssociateSubMap(guess.Offset, subMap);
}
}
}
/*public void AddGuessAndSubMap(MetaValueGuess guess, MetaMap map)
{
if (map == null)
{
AddGuess(guess);
return;
}
// Don't overwrite good guesses with null pointers
if (_guesses.ContainsKey(guess.Offset))
{
if (guess.Pointer == 0 || guess.Pointer == 0xFFFFFFFF || guess.Pointer == 0xCDCDCDCD)
return;
}
MetaMap previousMap;
if (_submaps.TryGetValue(guess.Offset, out previousMap))
{
if (map.EstimatedSize > previousMap.EstimatedSize)
_guesses[guess.Offset] = guess;
}
else
{
_guesses[guess.Offset] = guess;
}
AssociateSubMap(guess.Offset, map);
}*/
public void AssociateSubMap(int offset, MetaMap map)
{
if (map == null)
return;
if (!_guesses.ContainsKey(offset))
return;
// Bail out if this will cause a cyclic dependency to happen
if (map == this || _ancestors.Contains(map) || map._ancestors.Contains(this))
return;
// If a map has already been associated with the offset, then merge it,
// otherwise just insert it
MetaMap oldMap;
if (_submaps.TryGetValue(offset, out oldMap))
oldMap.MergeWith(map);
else
_submaps[offset] = map;
// Combine the map's ancestor set with ours
map._ancestors.UnionWith(_ancestors);
map._ancestors.Add(this);
}
private void WritePlugin(MetaMap map, int size, IPluginVisitor writer)
{
for (int offset = 0; offset < size; offset += 4)
{
MetaValueGuess guess = map.GetGuess(offset);
if (guess != null)
{
switch (guess.Type)
{
case MetaValueType.DataReference:
if (offset <= size - 0x14)
{
writer.VisitDataReference("Unknown", (uint) offset, "bytes", false, 4, 0);
offset += 0x10;
continue;
}
break;
case MetaValueType.TagReference:
if (offset <= size - 0x10)
{
writer.VisitTagReference("Unknown", (uint) offset, false, true, true, 0);
offset += 0xC;
continue;
}
break;
case MetaValueType.Reflexive:
if (offset <= size - 0xC)
{
MetaMap subMap = map.GetSubMap(offset);
if (subMap != null)
{
int subMapSize = subMap.GetBestSizeEstimate();
writer.EnterReflexive("Unknown", (uint) offset, false, (uint)subMapSize, 4, 0);
WritePlugin(subMap, subMapSize, writer);
writer.LeaveReflexive();
offset += 0x8;
continue;
}
}
break;
}
}
// Just write an unknown value depending upon how much space we have left
if (offset <= size - 4)
writer.VisitUndefined("Unknown", (uint) offset, false, 0);
else if (offset <= size - 2)
writer.VisitInt16("Unknown", (uint) offset, false, 0);
else
writer.VisitInt8("Unknown", (uint) offset, false, 0);
}
}
private void worker_DoWork(object sender, DoWorkEventArgs e, IList<MapEntry> generatorMaps, string outputPath,
BackgroundWorker worker)
{
var globalMaps = new Dictionary<string, MetaMap>();
DateTime startTime = DateTime.Now;
string gameIdentifier = "";
for (int i = 0; i < generatorMaps.Count; i++)
{
var tagMaps = new Dictionary<ITag, MetaMap>();
IReader reader;
KeyValuePair<ICacheFile, EngineDescription> cacheData = LoadMap(generatorMaps[i].LocalMapPath, out reader);
ICacheFile cacheFile = cacheData.Key;
var analyzer = new MetaAnalyzer(cacheFile);
if (gameIdentifier == "")
gameIdentifier = cacheData.Value.Settings.GetSetting<string>("shortName");
var mapsToProcess = new Queue<MetaMap>();
foreach (ITag tag in cacheFile.Tags)
{
if (tag.MetaLocation == null)
continue;
var map = new MetaMap();
tagMaps[tag] = map;
mapsToProcess.Enqueue(map);
reader.SeekTo(tag.MetaLocation.AsOffset());
analyzer.AnalyzeArea(reader, tag.MetaLocation.AsPointer(), map);
}
GenerateSubMaps(mapsToProcess, analyzer, reader, cacheFile);
var classMaps = new Dictionary<string, MetaMap>();
foreach (ITag tag in cacheFile.Tags)
{
if (tag.MetaLocation == null)
continue;
MetaMap map = tagMaps[tag];
EstimateMapSize(map, tag.MetaLocation.AsPointer(), analyzer.GeneratedMemoryMap, 1);
string magicStr = CharConstant.ToString(tag.Class.Magic);
MetaMap oldClassMap;
if (classMaps.TryGetValue(magicStr, out oldClassMap))
oldClassMap.MergeWith(map);
else
classMaps[magicStr] = map;
}
foreach (var map in classMaps)
{
MetaMap globalMap;
if (globalMaps.TryGetValue(map.Key, out globalMap))
globalMap.MergeWith(map.Value);
else
globalMaps[map.Key] = map.Value;
}
reader.Close();
worker.ReportProgress(100*i/(generatorMaps.Count - 1));
}
string badChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
foreach (var map in globalMaps)
{
string filename = badChars.Aggregate(map.Key, (current, badChar) => current.Replace(badChar, '_'));
filename += ".xml";
string path = Path.Combine(outputPath, filename);
var settings = new XmlWriterSettings
{
Indent = true,
IndentChars = "\t"
};
using (XmlWriter writer = XmlWriter.Create(path, settings))
{
var pluginWriter = new AssemblyPluginWriter(writer, gameIdentifier);
int size = map.Value.GetBestSizeEstimate();
FoldSubMaps(map.Value);
pluginWriter.EnterPlugin(size);
pluginWriter.EnterRevisions();
pluginWriter.VisitRevision(new PluginRevision("Assembly", 1, "Generated plugin from scratch."));
pluginWriter.LeaveRevisions();
WritePlugin(map.Value, size, pluginWriter);
pluginWriter.LeavePlugin();
}
}
DateTime endTime = DateTime.Now;
e.Result = endTime.Subtract(startTime);
}
private void GenerateSubMaps(Queue<MetaMap> maps, MetaAnalyzer analyzer, IReader reader, ICacheFile cacheFile)
{
var generatedMaps = new Dictionary<uint, MetaMap>();
while (maps.Count > 0)
{
MetaMap map = maps.Dequeue();
foreach (MetaValueGuess guess in map.Guesses.Where(guess => guess.Type == MetaValueType.Reflexive))
{
MetaMap subMap;
if (!generatedMaps.TryGetValue(guess.Pointer, out subMap))
{
subMap = new MetaMap();
reader.SeekTo(cacheFile.MetaArea.PointerToOffset(guess.Pointer));
analyzer.AnalyzeArea(reader, guess.Pointer, subMap);
maps.Enqueue(subMap);
generatedMaps[guess.Pointer] = subMap;
}
map.AssociateSubMap(guess.Offset, subMap);
}
}
}
private void FoldSubMaps(MetaMap map)
{
foreach (MetaValueGuess guess in map.Guesses)
{
if (guess.Type != MetaValueType.Reflexive) continue;
MetaMap subMap = map.GetSubMap(guess.Offset);
if (subMap == null) continue;
//var entryCount = (int)guess.Data1;
int firstBlockSize = subMap.GetBestSizeEstimate();
//if (firstBlockSize > 0 && !subMap.IsFolded(firstBlockSize))
//{
subMap.Fold(firstBlockSize);
FoldSubMaps(subMap);
//}
}
}
private void EstimateMapSize(MetaMap map, uint mapAddress, MemoryMap memMap, int entryCount)
{
bool alreadyVisited = map.HasSizeEstimates;
int newSize = memMap.EstimateBlockSize(mapAddress);
map.EstimateSize(newSize/entryCount);
map.Truncate(newSize);
if (alreadyVisited) return;
foreach (MetaValueGuess guess in map.Guesses)
{
if (guess.Type != MetaValueType.Reflexive) continue;
MetaMap subMap = map.GetSubMap(guess.Offset);
if (subMap != null)
EstimateMapSize(subMap, guess.Pointer, memMap, (int) guess.Data1);
}
}
public void AnalyzeArea(IReader reader, long startAddress, MetaMap resultMap)
{
// Right now, this method only searches for the signatures of a few complex meta values.
// Reflexives: int32 entry count + uint32 address + 4 bytes of padding
// Data references: int32 size + 8 bytes of padding + uint32 address
// Tag references: int32 class id + 8 bytes of padding + uint32 datum index
// ASCII strings: characters with the values 0 or 0x20 - 0x7F
// End at the next-highest address
long endAddress = _memMap.GetNextHighestAddress(startAddress);
if (endAddress == 0xFFFFFFFF)
{
throw new InvalidOperationException("Invalid start address for area analysis");
}
uint size = (uint)(endAddress - startAddress); // The size of the block of data
int paddingLength = 0; // The number of 4-byte padding values that have been read
uint prePadding = 0; // The last non-padding uint32 that was read
MetaValueGuess pendingGuess = null; // If not null and padding is encountered, this guess is confirmed
for (int offset = 0; offset < size; offset += 4)
{
uint value = reader.ReadUInt32();
long expValue = _expander.Expand(value);
if (IsPadding(value))
{
if (paddingLength == 0 && pendingGuess != null)
{
resultMap.AddGuess(pendingGuess);
// Add the address to the memory map
long address = pendingGuess.Pointer;
_memMap.AddAddress(address, (int)reader.Position);
}
// More padding! :D
paddingLength++;
pendingGuess = null;
}
else
{
pendingGuess = null;
if (offset <= size - 8 &&
prePadding > 0 &&
prePadding < 0x80000000
// && (value & 3) == 0
&& IsValidAddress(expValue) &&
expValue + prePadding > value &&
IsValidAddress(expValue + prePadding - 1) &&
!_memMap.BlockCrossesBoundary(expValue, (int)prePadding))
{
// Either a reflexive or a data reference
// Check the padding to determine which (see the comments at the beginning of this method)
if (paddingLength == 2 && offset >= 12 && (prePadding & 3) == 0)
{
// Found a data reference
uint dataSize = prePadding;
pendingGuess = new MetaValueGuess(offset - 12, MetaValueType.DataReference, expValue, dataSize);
// Guess with Pointer = address, Data1 = data size
}
else if (paddingLength == 0 && offset >= 4)
{
// Found a reflexive!
uint entryCount = prePadding;
pendingGuess = new MetaValueGuess(offset - 4, MetaValueType.Reflexive, expValue, entryCount);
// Guess with Pointer = address, Data1 = entry count
}
}
if (paddingLength == 2 && offset >= 12 &&
(_classIds.Contains((int)prePadding) || (prePadding == 0xFFFFFFFF && value == 0xFFFFFFFF)))
{
// Found a tag reference
uint classId = prePadding;
uint datumIndex = value;
var guess = new MetaValueGuess(offset - 12, MetaValueType.TagReference, datumIndex, classId);
// Guess with Pointer = datum index, Data1 = class id
resultMap.AddGuess(guess);
}
// This obviously isn't a padding value because IsPadding returned false,
// so update padding run information accordingly
prePadding = value;
paddingLength = 0;
}
}
}
public void AnalyzeArea(IReader reader, uint startAddress, MetaMap resultMap)
{
// Right now, this method only searches for the signatures of a few complex meta values.
// Reflexives: int32 entry count + uint32 address + 4 bytes of padding
// Data references: int32 size + 8 bytes of padding + uint32 address
// Tag references: int32 class id + 8 bytes of padding + uint32 datum index
// ASCII strings: characters with the values 0 or 0x20 - 0x7F
// End at the next-highest address
uint endAddress = _memMap.GetNextHighestAddress(startAddress);
if (endAddress == 0xFFFFFFFF)
throw new InvalidOperationException("Invalid start address for area analysis");
uint size = endAddress - startAddress; // The size of the block of data
int paddingLength = 0; // The number of 4-byte padding values that have been read
uint prePadding = 0; // The last non-padding uint32 that was read
MetaValueGuess pendingGuess = null; // If not null and padding is encountered, this guess is confirmed
for (int offset = 0; offset < size; offset += 4)
{
uint value = reader.ReadUInt32();
if (IsPadding(value))
{
if (paddingLength == 0 && pendingGuess != null)
{
resultMap.AddGuess(pendingGuess);
// Add the address to the memory map
uint address = pendingGuess.Pointer;
_memMap.AddAddress(address, (int) reader.Position);
}
// More padding! :D
paddingLength++;
pendingGuess = null;
}
else
{
pendingGuess = null;
if (offset <= size - 8
&& prePadding > 0
&& prePadding < 0x80000000
&& (value & 3) == 0
&& IsValidAddress(value)
&& value + prePadding > value
&& IsValidAddress(value + prePadding - 1)
&& !_memMap.BlockCrossesBoundary(value, (int) prePadding))
{
// Either a reflexive or a data reference
// Check the padding to determine which (see the comments at the beginning of this method)
if (paddingLength == 2 && offset >= 12 && (prePadding & 3) == 0)
{
// Found a data reference
uint dataSize = prePadding;
uint address = value;
pendingGuess = new MetaValueGuess(offset - 12, MetaValueType.DataReference, address, dataSize);
// Guess with Pointer = address, Data1 = data size
}
else if (paddingLength == 0 && offset >= 4)
{
// Found a reflexive!
uint entryCount = prePadding;
uint address = value;
pendingGuess = new MetaValueGuess(offset - 4, MetaValueType.Reflexive, address, entryCount);
// Guess with Pointer = address, Data1 = entry count
}
}
if (paddingLength == 2 && offset >= 12 &&
(_classIds.Contains((int) prePadding) || (prePadding == 0xFFFFFFFF && value == 0xFFFFFFFF)))
{
// Found a tag reference
uint classId = prePadding;
uint datumIndex = value;
var guess = new MetaValueGuess(offset - 12, MetaValueType.TagReference, datumIndex, classId);
// Guess with Pointer = datum index, Data1 = class id
resultMap.AddGuess(guess);
}
// This obviously isn't a padding value because IsPadding returned false,
// so update padding run information accordingly
prePadding = value;
paddingLength = 0;
}
}
}
public void MergeWith(MetaMap other)
{
if (other == this || _ancestors.Contains(other) || other._ancestors.Contains(this))
return;
foreach (MetaValueGuess guess in other._guesses.Values)
{
MetaMap otherMap = other.GetSubMap(guess.Offset);
MetaMap myMap = GetSubMap(guess.Offset);
if (otherMap != null || myMap == null)
{
if (AddGuess(guess))
AssociateSubMap(guess.Offset, otherMap);
}
}
foreach (var estimate in other._sizeEstimates)
EstimateSize(estimate.Key, estimate.Value);
}
