public static bool TryUnHash(ulong id, out Idstring pck)
{
id = SwapEndianness(id);
pck = HashIndex.Get(id);
return(true);
}
private void ProcessFile(string path)
{
Idstring p_ids = HashIndex.Get(Path.GetFileNameWithoutExtension(path));
var t_ids = new Tuple <Idstring, Idstring, Idstring>(p_ids, new Idstring(0), HashIndex.Get(Path.GetExtension(path)));
if (!this._browser.RawFiles.ContainsKey(t_ids))
{
this.error_output.Write(string.Format("File with path {0} does not exist!\n", path));
this.error_output.Flush();
return;
}
FileEntry file = this._browser.RawFiles[t_ids];
if (file.BundleEntries.Count == 0 || this.ExtractedPaths.Contains(p_ids))
{
return;
}
try
{
if (Path.GetExtension(path) == ".object")
{
string model_file = Path.Combine(Path.GetDirectoryName(path), Path.GetFileNameWithoutExtension(path)).Replace("\\", "/");
Idstring m_ids = HashIndex.Get(model_file);
//error_output.WriteLine(string.Format("Attempt to ouput model file {0}", model_file));
var t_m_ids = new Tuple <Idstring, Idstring, Idstring>(m_ids, new Idstring(0), HashIndex.Get("model"));
if (this._browser.RawFiles.ContainsKey(t_m_ids))
{
this.WriteFile(this._browser.RawFiles[t_m_ids]);
}
string cooked_physics = Path.Combine(Path.GetDirectoryName(path), Path.GetFileNameWithoutExtension(path)).Replace("\\", "/");
Idstring c_ids = HashIndex.Get(cooked_physics);
var t_c_ids = new Tuple <Idstring, Idstring, Idstring>(c_ids, new Idstring(0), HashIndex.Get("cooked_physics"));
//error_output.WriteLine(string.Format("Attempt to ouput cooked_physics file {0}", cooked_physics));
if (this._browser.RawFiles.ContainsKey(t_c_ids))
{
this.WriteFile(this._browser.RawFiles[t_c_ids]);
}
}
if (this.FileProcessors.ContainsKey(file._extension.ToString()))
{
this.FileProcessors[file._extension.ToString()].Invoke(file);
}
else
{
this.WriteFile(file);
}
}
catch (Exception exc)
{
this.error_output.Write("Exception occured on file: {0}\n", file.Path);
this.error_output.Write(exc.Message + "\n");
this.error_output.Write(exc.StackTrace + "\n");
this.error_output.Flush();
}
}
public StringEntry(BinaryReader br)
{
br.ReadUInt64();
ulong HashedID = br.ReadUInt64();
br.ReadUInt32();
this.StringPosition = br.ReadUInt32();
this.ID = HashIndex.Get(HashedID);
}
public MassUnitHeader(BinaryReader br)
{
ulong UnitPathHash = br.ReadUInt64();
this.Unit = HashIndex.Get(UnitPathHash);
br.ReadSingle(); // Unknown.
this.InstanceCount = br.ReadUInt32();
br.ReadUInt32(); // Unknown
this.Offset = br.ReadUInt32();
br.BaseStream.Seek(8, SeekOrigin.Current);
}
public MassUnitHeader(BinaryReader br)
{
Positions = new List <Vector3>();
Rotations = new List <Quaternion>();
ulong UnitPathHash = br.ReadUInt64();
this.Unit = HashIndex.Get(UnitPathHash);
br.ReadSingle(); // Visiblity count?
this.InstanceCount = br.ReadUInt32();
br.ReadUInt32(); // Unknown
this.Offset = br.ReadUInt32();
br.BaseStream.Seek(8, SeekOrigin.Current);
}
public override double Score <F>(IClassifier <L, F> classifier, GeneralDataset <L, F> data)
{
labelIndex = new HashIndex <L>();
labelIndex.AddAll(classifier.Labels());
labelIndex.AddAll(data.labelIndex.ObjectsList());
ClearCounts();
int[] labelsArr = data.GetLabelsArray();
for (int i = 0; i < data.Size(); i++)
{
IDatum <L, F> d = data.GetRVFDatum(i);
L guess = classifier.ClassOf(d);
AddGuess(guess, labelIndex.Get(labelsArr[i]));
}
FinalizeCounts();
return(GetFMeasure());
}
public void execute(PackageBrowser browser)
{
this.ExtractedPaths = new HashSet <Idstring>();
this.error_output = new StreamWriter("./heist_extractor.log");
this._browser = browser;
System.Diagnostics.Stopwatch clock = new System.Diagnostics.Stopwatch();
clock.Start();
this.error_output.Write("Heist Extractor executed" + "\n");
this.error_output.Flush();
Idstring ids = HashIndex.Get(this.heist_world);
Idstring ids_ext = HashIndex.Get("world");
var tids = new Tuple <Idstring, Idstring, Idstring>(ids, new Idstring(0), ids_ext);
if (browser.RawFiles.ContainsKey(tids))
{
this.ProcessWorld(browser.RawFiles[tids]);
}
else
{
Console.WriteLine("World File does not exist");
}
//this.ProcessFolder(browser.Root);
//Path.Combine(Definitions.HashDir, hashlist_tag)
using (StreamWriter str = new StreamWriter(new FileStream(Path.Combine(this.OutputPath, "add.xml"), FileMode.Create, FileAccess.Write)))
{
str.Write("<table>\n");
foreach (Idstring path in this.ExtractedPaths)
{
string[] split = path.ToString().Split('.');
str.Write(String.Format("\t<{0} path=\"{1}\" force=\"true\"/>\n", split[1], split[0]));
}
str.Write("</table>\n");
}
clock.Stop();
this.error_output.Write("Scrape operation took {0} seconds" + "\n", clock.Elapsed.TotalSeconds.ToString());
this.error_output.Close();
}
/*private void ProcessFolder(IParent folder)
* {
* foreach (IChild child in folder.Children.Values)
* {
* if (child is FileEntry)
* this.ProcessFile(child as FileEntry);
* else if (child is IParent)
* this.ProcessFolder(child as IParent);
* }
* }*/
private void WriteFile(FileEntry entry, byte[] byt = null)
{
Idstring ids = HashIndex.Get(entry.Path);
if (entry.BundleEntries.Count == 0 || this.ExtractedPaths.Contains(ids))
{
return;
}
string path = Path.Combine(this.OutputPath, entry.Path);
string folder = Path.GetDirectoryName(path);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
byte[] bytes = byt ?? entry.FileBytes() ?? new byte[0];
File.WriteAllBytes(path, bytes);
this.ExtractedPaths.Add(ids);
}
private void ProcessWorld(FileEntry file)
{
foreach (KeyValuePair <string, IChild> child in file.Parent.Children)
{
if (child.Value is FileEntry)
{
this.WriteFile(child.Value as FileEntry);
}
}
this.WriteFile(file);
this.ProcessScriptData(file, new List <XMLTagLookup> {
new XMLTagLookup {
node_name = "environment_values", value = new[] { "environment" }, Converter = (hash) => { return(hash + ".environment"); }
}
});
string continents_file = Path.Combine(Path.GetDirectoryName(file.Path), "continents").Replace("\\", "/");
Idstring ids = HashIndex.Get(continents_file);
var t_ids = new Tuple <Idstring, Idstring, Idstring>(ids, new Idstring(0), HashIndex.Get("continents"));
if (this._browser.RawFiles.ContainsKey(t_ids))
{
FileEntry c_file = this._browser.RawFiles[t_ids];
this.WriteFile(c_file);
string xml = ScriptActions.GetConverter("scriptdata", "script_cxml").export(c_file.FileStream(), true);
XmlDocument doc = new XmlDocument();
try
{
doc.LoadXml(xml);
foreach (XmlNode child in doc.ChildNodes[0])
{
this.ProcessFile(Path.Combine(Path.GetDirectoryName(file.Path), string.Format("{0}/{0}.continent", child.Attributes.GetNamedItem("name").Value)).Replace("\\", "/"));
}
}
catch (Exception exc)
{
this.error_output.Write("Exception occured on file: {0}\n", c_file.Path);
if (xml != null)
{
this.error_output.Write(xml + "\n");
}
this.error_output.Write(exc.Message + "\n");
this.error_output.Write(exc.StackTrace + "\n");
this.error_output.Flush();
return;
}
}
else
{
this.error_output.Write("Continents file {0} does not exist!\n", continents_file);
}
string mission_file = Path.Combine(Path.GetDirectoryName(file.Path), "mission").Replace("\\", "/");
Idstring m_ids = HashIndex.Get(mission_file);
var t_m_ids = new Tuple <Idstring, Idstring, Idstring>(m_ids, new Idstring(0), HashIndex.Get("mission"));
if (this._browser.RawFiles.ContainsKey(t_m_ids))
{
FileEntry m_file = this._browser.RawFiles[t_m_ids];
this.WriteFile(m_file);
string xml = ScriptActions.GetConverter("scriptdata", "script_cxml").export(m_file.FileStream(), true);
XmlDocument doc = new XmlDocument();
try
{
doc.LoadXml(xml);
foreach (XmlNode child in doc.ChildNodes[0])
{
this.ProcessFile(Path.Combine(Path.GetDirectoryName(file.Path), string.Format("{0}.mission", child.Attributes.GetNamedItem("file").Value)).Replace("\\", "/"));
}
}
catch (Exception exc)
{
this.error_output.Write("Exception occured on file: {0}\n", m_file.Path);
if (xml != null)
{
this.error_output.Write(xml + "\n");
}
this.error_output.Write(exc.Message + "\n");
this.error_output.Write(exc.StackTrace + "\n");
this.error_output.Flush();
return;
}
}
else
{
this.error_output.Write("Mission file {0} does not exist!\n", continents_file);
}
this.error_output.Flush();
}
/// <summary>
/// Provides some testing and opportunities for exploration of the
/// probabilities of a BaseLexicon.
/// </summary>
/// <remarks>
/// Provides some testing and opportunities for exploration of the
/// probabilities of a BaseLexicon. What's here currently probably
/// only works for the English Penn Treeebank, as it uses default
/// constructors. Of the words given to test on,
/// the first is treated as sentence initial, and the rest as not
/// sentence initial.
/// </remarks>
/// <param name="args">
/// The command line arguments:
/// java BaseLexicon treebankPath fileRange unknownWordModel words
/// </param>
public static void Main(string[] args)
{
if (args.Length < 3)
{
log.Info("java BaseLexicon treebankPath fileRange unknownWordModel words*");
return;
}
System.Console.Out.Write("Training BaseLexicon from " + args[0] + ' ' + args[1] + " ... ");
Treebank tb = new DiskTreebank();
tb.LoadPath(args[0], new NumberRangesFileFilter(args[1], true));
// TODO: change this interface so the lexicon creates its own indices?
IIndex <string> wordIndex = new HashIndex <string>();
IIndex <string> tagIndex = new HashIndex <string>();
Options op = new Options();
op.lexOptions.useUnknownWordSignatures = System.Convert.ToInt32(args[2]);
Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon lex = new Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon(op, wordIndex, tagIndex);
lex.InitializeTraining(tb.Count);
lex.Train(tb);
lex.FinishTraining();
System.Console.Out.WriteLine("done.");
System.Console.Out.WriteLine();
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(4);
IList <string> impos = new List <string>();
for (int i = 3; i < args.Length; i++)
{
if (lex.IsKnown(args[i]))
{
System.Console.Out.WriteLine(args[i] + " is a known word. Log probabilities [log P(w|t)] for its taggings are:");
for (IEnumerator <IntTaggedWord> it = lex.RuleIteratorByWord(wordIndex.AddToIndex(args[i]), i - 3, null); it.MoveNext();)
{
IntTaggedWord iTW = it.Current;
System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(lex.Score(iTW, i - 3, wordIndex.Get(iTW.word), null)));
}
}
else
{
string sig = lex.GetUnknownWordModel().GetSignature(args[i], i - 3);
System.Console.Out.WriteLine(args[i] + " is an unknown word. Signature with uwm " + lex.GetUnknownWordModel().GetUnknownLevel() + ((i == 3) ? " init" : "non-init") + " is: " + sig);
impos.Clear();
IList <string> lis = new List <string>(tagIndex.ObjectsList());
lis.Sort();
foreach (string tStr in lis)
{
IntTaggedWord iTW = new IntTaggedWord(args[i], tStr, wordIndex, tagIndex);
double score = lex.Score(iTW, 1, args[i], null);
if (score == float.NegativeInfinity)
{
impos.Add(tStr);
}
else
{
System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(score));
}
}
if (impos.Count > 0)
{
System.Console.Out.WriteLine(args[i] + " impossible tags: " + impos);
}
}
System.Console.Out.WriteLine();
}
}
protected void ReadFile(BinaryReader _br)
{
br = _br;
uint bnk_count = br.ReadUInt32();
//Skip second count
br.BaseStream.Position += 4;
uint bnk_offset = br.ReadUInt32();
uint section_pointer = br.ReadUInt32();
uint unknown1 = br.ReadUInt32();
uint sound_count = br.ReadUInt32();
//Skip second count
br.BaseStream.Position += 4;
uint sound_offset = br.ReadUInt32();
//Skips section pointer, unknown1, unknown2
br.BaseStream.Position += 12;
uint u_count = br.ReadUInt32();
//Skip second count
br.BaseStream.Position += 4;
uint u_offset = br.ReadUInt32();
br.BaseStream.Position = bnk_offset;
for (int i = 0; i < bnk_count; i++)
{
br.BaseStream.Position += 4;
uint position = br.ReadUInt32();
this.SeekPush();
br.BaseStream.Position = position;
Soundbanks.Add(this.ReadString());
this.SeekPop();
}
br.BaseStream.Position = sound_offset;
Dictionary <ulong, uint> sound_lookups = new Dictionary <ulong, uint>();
for (int i = 0; i < sound_count; i++)
{
uint id = (uint)br.ReadUInt64();
ulong hash = br.ReadUInt64();
if (sound_lookups.ContainsKey(hash))
{
uint other_id = sound_lookups[hash];
continue;
}
sound_lookups.Add(hash, id);
}
br.BaseStream.Position = u_offset;
for (int i = 0; i < u_count; i++)
{
ulong hash = br.ReadUInt64();
br.BaseStream.Position += 4;
uint string_pos = br.ReadUInt32();
this.SeekPush();
br.BaseStream.Position = string_pos;
string str = this.ReadString();
this.SeekPop();
if (!sound_lookups.ContainsKey(hash))
{
continue;
}
uint id = sound_lookups[hash];
if (SoundLookups.ContainsKey(id))
{
continue;
}
Idstring ids = HashIndex.Get(hash);
SoundLookups.Add(id, new Tuple <string, Idstring>(id.ToString() != str ? str : null, ids));
}
}
public MassUnitHeader(ulong Unit, List <Vector3> Positions, List <Quaternion> Rotations) : this(HashIndex.Get(Unit), Positions, Rotations)
{
}
