public static bool TryUnHash(ulong id, out Idstring pck) { id = SwapEndianness(id); pck = HashIndex.Get(id); return(true); }
private void ProcessFile(string path) { Idstring p_ids = HashIndex.Get(Path.GetFileNameWithoutExtension(path)); var t_ids = new Tuple <Idstring, Idstring, Idstring>(p_ids, new Idstring(0), HashIndex.Get(Path.GetExtension(path))); if (!this._browser.RawFiles.ContainsKey(t_ids)) { this.error_output.Write(string.Format("File with path {0} does not exist!\n", path)); this.error_output.Flush(); return; } FileEntry file = this._browser.RawFiles[t_ids]; if (file.BundleEntries.Count == 0 || this.ExtractedPaths.Contains(p_ids)) { return; } try { if (Path.GetExtension(path) == ".object") { string model_file = Path.Combine(Path.GetDirectoryName(path), Path.GetFileNameWithoutExtension(path)).Replace("\\", "/"); Idstring m_ids = HashIndex.Get(model_file); //error_output.WriteLine(string.Format("Attempt to ouput model file {0}", model_file)); var t_m_ids = new Tuple <Idstring, Idstring, Idstring>(m_ids, new Idstring(0), HashIndex.Get("model")); if (this._browser.RawFiles.ContainsKey(t_m_ids)) { this.WriteFile(this._browser.RawFiles[t_m_ids]); } string cooked_physics = Path.Combine(Path.GetDirectoryName(path), Path.GetFileNameWithoutExtension(path)).Replace("\\", "/"); Idstring c_ids = HashIndex.Get(cooked_physics); var t_c_ids = new Tuple <Idstring, Idstring, Idstring>(c_ids, new Idstring(0), HashIndex.Get("cooked_physics")); //error_output.WriteLine(string.Format("Attempt to ouput cooked_physics file {0}", cooked_physics)); if (this._browser.RawFiles.ContainsKey(t_c_ids)) { this.WriteFile(this._browser.RawFiles[t_c_ids]); } } if (this.FileProcessors.ContainsKey(file._extension.ToString())) { this.FileProcessors[file._extension.ToString()].Invoke(file); } else { this.WriteFile(file); } } catch (Exception exc) { this.error_output.Write("Exception occured on file: {0}\n", file.Path); this.error_output.Write(exc.Message + "\n"); this.error_output.Write(exc.StackTrace + "\n"); this.error_output.Flush(); } }
public StringEntry(BinaryReader br) { br.ReadUInt64(); ulong HashedID = br.ReadUInt64(); br.ReadUInt32(); this.StringPosition = br.ReadUInt32(); this.ID = HashIndex.Get(HashedID); }
public MassUnitHeader(BinaryReader br) { ulong UnitPathHash = br.ReadUInt64(); this.Unit = HashIndex.Get(UnitPathHash); br.ReadSingle(); // Unknown. this.InstanceCount = br.ReadUInt32(); br.ReadUInt32(); // Unknown this.Offset = br.ReadUInt32(); br.BaseStream.Seek(8, SeekOrigin.Current); }
public MassUnitHeader(BinaryReader br) { Positions = new List <Vector3>(); Rotations = new List <Quaternion>(); ulong UnitPathHash = br.ReadUInt64(); this.Unit = HashIndex.Get(UnitPathHash); br.ReadSingle(); // Visiblity count? this.InstanceCount = br.ReadUInt32(); br.ReadUInt32(); // Unknown this.Offset = br.ReadUInt32(); br.BaseStream.Seek(8, SeekOrigin.Current); }
public override double Score <F>(IClassifier <L, F> classifier, GeneralDataset <L, F> data) { labelIndex = new HashIndex <L>(); labelIndex.AddAll(classifier.Labels()); labelIndex.AddAll(data.labelIndex.ObjectsList()); ClearCounts(); int[] labelsArr = data.GetLabelsArray(); for (int i = 0; i < data.Size(); i++) { IDatum <L, F> d = data.GetRVFDatum(i); L guess = classifier.ClassOf(d); AddGuess(guess, labelIndex.Get(labelsArr[i])); } FinalizeCounts(); return(GetFMeasure()); }
public void execute(PackageBrowser browser) { this.ExtractedPaths = new HashSet <Idstring>(); this.error_output = new StreamWriter("./heist_extractor.log"); this._browser = browser; System.Diagnostics.Stopwatch clock = new System.Diagnostics.Stopwatch(); clock.Start(); this.error_output.Write("Heist Extractor executed" + "\n"); this.error_output.Flush(); Idstring ids = HashIndex.Get(this.heist_world); Idstring ids_ext = HashIndex.Get("world"); var tids = new Tuple <Idstring, Idstring, Idstring>(ids, new Idstring(0), ids_ext); if (browser.RawFiles.ContainsKey(tids)) { this.ProcessWorld(browser.RawFiles[tids]); } else { Console.WriteLine("World File does not exist"); } //this.ProcessFolder(browser.Root); //Path.Combine(Definitions.HashDir, hashlist_tag) using (StreamWriter str = new StreamWriter(new FileStream(Path.Combine(this.OutputPath, "add.xml"), FileMode.Create, FileAccess.Write))) { str.Write("<table>\n"); foreach (Idstring path in this.ExtractedPaths) { string[] split = path.ToString().Split('.'); str.Write(String.Format("\t<{0} path=\"{1}\" force=\"true\"/>\n", split[1], split[0])); } str.Write("</table>\n"); } clock.Stop(); this.error_output.Write("Scrape operation took {0} seconds" + "\n", clock.Elapsed.TotalSeconds.ToString()); this.error_output.Close(); }
/*private void ProcessFolder(IParent folder) * { * foreach (IChild child in folder.Children.Values) * { * if (child is FileEntry) * this.ProcessFile(child as FileEntry); * else if (child is IParent) * this.ProcessFolder(child as IParent); * } * }*/ private void WriteFile(FileEntry entry, byte[] byt = null) { Idstring ids = HashIndex.Get(entry.Path); if (entry.BundleEntries.Count == 0 || this.ExtractedPaths.Contains(ids)) { return; } string path = Path.Combine(this.OutputPath, entry.Path); string folder = Path.GetDirectoryName(path); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } byte[] bytes = byt ?? entry.FileBytes() ?? new byte[0]; File.WriteAllBytes(path, bytes); this.ExtractedPaths.Add(ids); }
private void ProcessWorld(FileEntry file) { foreach (KeyValuePair <string, IChild> child in file.Parent.Children) { if (child.Value is FileEntry) { this.WriteFile(child.Value as FileEntry); } } this.WriteFile(file); this.ProcessScriptData(file, new List <XMLTagLookup> { new XMLTagLookup { node_name = "environment_values", value = new[] { "environment" }, Converter = (hash) => { return(hash + ".environment"); } } }); string continents_file = Path.Combine(Path.GetDirectoryName(file.Path), "continents").Replace("\\", "/"); Idstring ids = HashIndex.Get(continents_file); var t_ids = new Tuple <Idstring, Idstring, Idstring>(ids, new Idstring(0), HashIndex.Get("continents")); if (this._browser.RawFiles.ContainsKey(t_ids)) { FileEntry c_file = this._browser.RawFiles[t_ids]; this.WriteFile(c_file); string xml = ScriptActions.GetConverter("scriptdata", "script_cxml").export(c_file.FileStream(), true); XmlDocument doc = new XmlDocument(); try { doc.LoadXml(xml); foreach (XmlNode child in doc.ChildNodes[0]) { this.ProcessFile(Path.Combine(Path.GetDirectoryName(file.Path), string.Format("{0}/{0}.continent", child.Attributes.GetNamedItem("name").Value)).Replace("\\", "/")); } } catch (Exception exc) { this.error_output.Write("Exception occured on file: {0}\n", c_file.Path); if (xml != null) { this.error_output.Write(xml + "\n"); } this.error_output.Write(exc.Message + "\n"); this.error_output.Write(exc.StackTrace + "\n"); this.error_output.Flush(); return; } } else { this.error_output.Write("Continents file {0} does not exist!\n", continents_file); } string mission_file = Path.Combine(Path.GetDirectoryName(file.Path), "mission").Replace("\\", "/"); Idstring m_ids = HashIndex.Get(mission_file); var t_m_ids = new Tuple <Idstring, Idstring, Idstring>(m_ids, new Idstring(0), HashIndex.Get("mission")); if (this._browser.RawFiles.ContainsKey(t_m_ids)) { FileEntry m_file = this._browser.RawFiles[t_m_ids]; this.WriteFile(m_file); string xml = ScriptActions.GetConverter("scriptdata", "script_cxml").export(m_file.FileStream(), true); XmlDocument doc = new XmlDocument(); try { doc.LoadXml(xml); foreach (XmlNode child in doc.ChildNodes[0]) { this.ProcessFile(Path.Combine(Path.GetDirectoryName(file.Path), string.Format("{0}.mission", child.Attributes.GetNamedItem("file").Value)).Replace("\\", "/")); } } catch (Exception exc) { this.error_output.Write("Exception occured on file: {0}\n", m_file.Path); if (xml != null) { this.error_output.Write(xml + "\n"); } this.error_output.Write(exc.Message + "\n"); this.error_output.Write(exc.StackTrace + "\n"); this.error_output.Flush(); return; } } else { this.error_output.Write("Mission file {0} does not exist!\n", continents_file); } this.error_output.Flush(); }
/// <summary> /// Provides some testing and opportunities for exploration of the /// probabilities of a BaseLexicon. /// </summary> /// <remarks> /// Provides some testing and opportunities for exploration of the /// probabilities of a BaseLexicon. What's here currently probably /// only works for the English Penn Treeebank, as it uses default /// constructors. Of the words given to test on, /// the first is treated as sentence initial, and the rest as not /// sentence initial. /// </remarks> /// <param name="args"> /// The command line arguments: /// java BaseLexicon treebankPath fileRange unknownWordModel words /// </param> public static void Main(string[] args) { if (args.Length < 3) { log.Info("java BaseLexicon treebankPath fileRange unknownWordModel words*"); return; } System.Console.Out.Write("Training BaseLexicon from " + args[0] + ' ' + args[1] + " ... "); Treebank tb = new DiskTreebank(); tb.LoadPath(args[0], new NumberRangesFileFilter(args[1], true)); // TODO: change this interface so the lexicon creates its own indices? IIndex <string> wordIndex = new HashIndex <string>(); IIndex <string> tagIndex = new HashIndex <string>(); Options op = new Options(); op.lexOptions.useUnknownWordSignatures = System.Convert.ToInt32(args[2]); Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon lex = new Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon(op, wordIndex, tagIndex); lex.InitializeTraining(tb.Count); lex.Train(tb); lex.FinishTraining(); System.Console.Out.WriteLine("done."); System.Console.Out.WriteLine(); NumberFormat nf = NumberFormat.GetNumberInstance(); nf.SetMaximumFractionDigits(4); IList <string> impos = new List <string>(); for (int i = 3; i < args.Length; i++) { if (lex.IsKnown(args[i])) { System.Console.Out.WriteLine(args[i] + " is a known word. Log probabilities [log P(w|t)] for its taggings are:"); for (IEnumerator <IntTaggedWord> it = lex.RuleIteratorByWord(wordIndex.AddToIndex(args[i]), i - 3, null); it.MoveNext();) { IntTaggedWord iTW = it.Current; System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(lex.Score(iTW, i - 3, wordIndex.Get(iTW.word), null))); } } else { string sig = lex.GetUnknownWordModel().GetSignature(args[i], i - 3); System.Console.Out.WriteLine(args[i] + " is an unknown word. Signature with uwm " + lex.GetUnknownWordModel().GetUnknownLevel() + ((i == 3) ? " init" : "non-init") + " is: " + sig); impos.Clear(); IList <string> lis = new List <string>(tagIndex.ObjectsList()); lis.Sort(); foreach (string tStr in lis) { IntTaggedWord iTW = new IntTaggedWord(args[i], tStr, wordIndex, tagIndex); double score = lex.Score(iTW, 1, args[i], null); if (score == float.NegativeInfinity) { impos.Add(tStr); } else { System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(score)); } } if (impos.Count > 0) { System.Console.Out.WriteLine(args[i] + " impossible tags: " + impos); } } System.Console.Out.WriteLine(); } }
protected void ReadFile(BinaryReader _br) { br = _br; uint bnk_count = br.ReadUInt32(); //Skip second count br.BaseStream.Position += 4; uint bnk_offset = br.ReadUInt32(); uint section_pointer = br.ReadUInt32(); uint unknown1 = br.ReadUInt32(); uint sound_count = br.ReadUInt32(); //Skip second count br.BaseStream.Position += 4; uint sound_offset = br.ReadUInt32(); //Skips section pointer, unknown1, unknown2 br.BaseStream.Position += 12; uint u_count = br.ReadUInt32(); //Skip second count br.BaseStream.Position += 4; uint u_offset = br.ReadUInt32(); br.BaseStream.Position = bnk_offset; for (int i = 0; i < bnk_count; i++) { br.BaseStream.Position += 4; uint position = br.ReadUInt32(); this.SeekPush(); br.BaseStream.Position = position; Soundbanks.Add(this.ReadString()); this.SeekPop(); } br.BaseStream.Position = sound_offset; Dictionary <ulong, uint> sound_lookups = new Dictionary <ulong, uint>(); for (int i = 0; i < sound_count; i++) { uint id = (uint)br.ReadUInt64(); ulong hash = br.ReadUInt64(); if (sound_lookups.ContainsKey(hash)) { uint other_id = sound_lookups[hash]; continue; } sound_lookups.Add(hash, id); } br.BaseStream.Position = u_offset; for (int i = 0; i < u_count; i++) { ulong hash = br.ReadUInt64(); br.BaseStream.Position += 4; uint string_pos = br.ReadUInt32(); this.SeekPush(); br.BaseStream.Position = string_pos; string str = this.ReadString(); this.SeekPop(); if (!sound_lookups.ContainsKey(hash)) { continue; } uint id = sound_lookups[hash]; if (SoundLookups.ContainsKey(id)) { continue; } Idstring ids = HashIndex.Get(hash); SoundLookups.Add(id, new Tuple <string, Idstring>(id.ToString() != str ? str : null, ids)); } }
public MassUnitHeader(ulong Unit, List <Vector3> Positions, List <Quaternion> Rotations) : this(HashIndex.Get(Unit), Positions, Rotations) { }