// NOTE: When printUniqueRules is set to true, the number of unique grammar/lexicon rules will be written in the output file
// this should only be used for debug reasons as BitPar will crash when parsing these lines.
internal void SaveFile(string path, List <string> content, ExtractMode extractmode, bool printUniqueRules)
{
FileStream file = new FileStream(path, FileMode.Create, FileAccess.Write);
StreamWriter writer = new StreamWriter(file);
foreach (string entry in content)
{
writer.WriteLine(entry);
}
if (printUniqueRules)
{
writer.WriteLine("============================================================================");
if (extractmode.Equals(ExtractMode.GRAMMAR))
{
writer.WriteLine("Number of unique grammar rules: " + grammarRulesCounter);
}
else if (extractmode.Equals(ExtractMode.LEXICON))
{
writer.WriteLine("Number of unique lexicon rules: " + lexiconRulesCounter);
}
else
{
throw new ArgumentException("Illegal ExtractMode enumerator was passed");
}
}
writer.Close();
file.Close();
}
/// <summary>
/// Constructor used for integrity checks (test mode)
/// </summary>
public ArchiveExtractCallback(SevenZipArchive archive, Dictionary <uint, ArchiveEntry> entries, string password)
{
_archive = archive;
_entries = entries;
_password = string.IsNullOrEmpty(password) ? archive.Password : password;
_options = ExtractOptions.NoAbortOnFailure;
_extractMode = ExtractMode.IntegrityCheck;
}
/* PSEUDOCODE:
* func TreeNode ExtractNode(string str)
* {
* create a node
* remove parenthesis surrounding the input string
* assign the left-most tag/word as the value of the node
* retrieve a string list of the children (based on input string)
* add each child to the node by calling this function (recursively)
* }
*/
// TODO_LOW: to improve clarity of this implementation two control flow branches could be constructed (one for non-terminals and another for terminals)
private TreeNode ExtractNode(string str, ExtractMode extractmode)
{
TreeNode res = new TreeNode();
bool isTerminal = false;
// step 0 - check if the incoming string is a terminal
if (!str.Contains(' '))
{
isTerminal = true;
}
// step 1 - remove surrounding parenthesis (for non-terminals)
if (str[0] == '(' && !isTerminal)
{
str = str.Remove(0, 1);
}
if (str[str.Length - 1] == ')' && !isTerminal)
{
str = str.Remove(str.Length - 1, 1);
}
// step 2 - set the value of the node
if (!isTerminal)
{
res.Value = str.Substring(0, str.IndexOf(' '));
}
else
{
res.Value = str;
}
// step 2.5 - remove the value of the node from the input string
if (!isTerminal)
{
str = str.Substring(str.IndexOf(' ') + 1);
}
else
{
str = "";
}
// step 3 - make a List<string> of node children
List <string> childNodes = GetChildrenList(str);
for (int i = 0; i < childNodes.Count; i++)
{
if (!childNodes[i].Contains(' ') && extractmode.Equals(ExtractMode.GRAMMAR)) // if the childnode is a terminal and the extraction mode is set to "grammar"
{
break;
}
else
{
res.addChild(ExtractNode(childNodes[i], extractmode));
}
}
return(res);
}
/// <summary>
/// Constructor used to extract a file to a stream
/// </summary>
public ArchiveExtractCallback(SevenZipArchive archive, Dictionary <uint, ArchiveEntry> entries, string password, Stream targetStream, ExtractOptions options)
{
_archive = archive;
_entries = entries;
_password = string.IsNullOrEmpty(password) ? archive.Password : password;
_targetStream = targetStream;
_options = options;
_extractMode = ExtractMode.ExtractToStream;
}
/// <summary>
/// Constructor used to extract files to the file system
/// </summary>
public ArchiveExtractCallback(SevenZipArchive archive, Dictionary <uint, ArchiveEntry> entries, string password, string targetDirectory, ExtractOptions options)
{
_archive = archive;
_entries = entries;
_directoryEntries = new SortedDictionary <string, ArchiveEntry>(new DirectoryNameComparer());
_password = string.IsNullOrEmpty(password) ? archive.Password : password;
_targetDirectory = targetDirectory;
_options = options;
_extractMode = ExtractMode.ExtractToFile;
}
public string ExtractInfoFromPrimaryKey(string primaryKey, ExtractMode mode)
{
if (String.IsNullOrWhiteSpace(primaryKey))
{
return(null);
}
if (!primaryKey.Contains('-'))
{
return(null);
}
return(primaryKey.Split('-')[(int)mode]);
}
public void ExtractObjct(Asset.AssetObjectInfo obj, TypeTree typeTree, string outputPath, ExtractMode mode = ExtractMode.Auto)
{
outputPath = outputPath + "\\" + AssetToolUtility.ClassIDToClassName(obj.classID) + "\\";
switch (mode) {
case ExtractMode.Auto:
ExtractAuto(obj, typeTree, outputPath);
break;
case ExtractMode.OnlyRawBits:
extractOnlyRawBits(obj, typeTree, outputPath);
break;
case ExtractMode.OnlyRawText:
extractOnlyRawText(obj, typeTree, outputPath);
break;
case ExtractMode.RawTextOrRawBits:
extractRawTextOrRawBits(obj, typeTree, outputPath);
break;
}
}
public void GetStream(uint index, out ISequentialOutStream outStream, ExtractMode extractMode)
{
if (!files.ContainsKey(index))
{
outStream = null;
mode = ExtractMode.Skip;
return;
}
if (extractMode == ExtractMode.Extract)
{
if (stream != null)
{
stream.Dispose();
}
current = index;
var args = new ExtractFileEventArgs(owner.Archive, files[current], ExtractionStage.Extracting);
owner.ExtractFile(owner, args);
if (!args.ContinueOperation)
{
throw new IOException("User aborted!");
}
var ostream = new SevenZipOutFileStream(files[index].Destination, (long)files[index].Size, owner.priority);
ostream.ProgressHandler += owner.ExtractProgress;
stream = ostream;
}
else
{
if (stream != null)
{
stream.Dispose();
}
stream = new SevenZipNullStream();
}
outStream = stream;
mode = extractMode;
}
public void PrepareOperation(ExtractMode askExtractMode)
{
}
private void ProcessPattern(ExtractMode mode, string pattern, string text, string inputFile)
{
Regex r = new Regex(pattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline);
MatchCollection matches = r.Matches(text);
foreach (Match match in matches)
{
GroupCollection groups = match.Groups;
if (groups.Count < 2)
throw new Exception(String.Format(
"Invalid pattern '{0}'.\nTwo groups are required at least.\nSource: {1}",
pattern, match.Value));
// Initialisation
string context = String.Empty;
string msgid = String.Empty;
string msgidPlural = String.Empty;
switch (mode)
{
case ExtractMode.Msgid:
msgid = Unescape(groups[1].Value);
break;
case ExtractMode.MsgidConcat:
MatchCollection matches2 = Regex.Matches(groups[0].Value, CsharpStringPattern);
StringBuilder sb = new StringBuilder();
foreach (Match match2 in matches2)
{
sb.Append(Unescape(match2.Value));
}
msgid = sb.ToString();
break;
case ExtractMode.MsgidFromResx:
string controlId = Unescape(groups[1].Value);
msgid = ExtractResourceString(controlId);
if (String.IsNullOrEmpty(msgid))
{
if (Options.Verbose)
Trace.WriteLine(String.Format(
"Warning: cannot extract string for control '{0}' ({1})",
controlId, inputFile));
continue;
}
if (controlId == msgid)
continue; // Text property was initialized by controlId and was not changed so this text is not usable in application
break;
case ExtractMode.MsgidPlural:
if (groups.Count < 3)
throw new Exception(String.Format("Invalid 'GetPluralString' call.\nSource: {0}", match.Value));
msgid = Unescape(groups[1].Value);
msgidPlural = Unescape(groups[2].Value);
break;
case ExtractMode.ContextMsgid:
if (groups.Count < 3)
throw new Exception(String.Format("Invalid get context message call.\nSource: {0}", match.Value));
context = Unescape(groups[1].Value);
msgid = Unescape(groups[2].Value);
if (groups.Count == 4)
msgidPlural = Unescape(groups[3].Value);
break;
}
if (String.IsNullOrEmpty(msgid))
{
if (Options.Verbose)
Trace.Write(String.Format("WARN: msgid is empty in {0}\r\n", inputFile));
}
else
{
MergeWithEntry(context, msgid, msgidPlural, inputFile, CalcLineNumber(text, match.Index));
}
}
}
public void ExtractDesign(int[] itemSerials)
{
_mode = ExtractMode.ItemSerials;
_itemSerials = itemSerials;
ThreadPool.QueueUserWorkItem(new WaitCallback(StartExtraction));
}
public void ExtractDesign()
{
_mode = ExtractMode.Area;
ThreadPool.QueueUserWorkItem(new WaitCallback(StartExtraction));
}
public void Extract(Asset asset, TypeTreeDataBase typeTreeDB, string outputPath, ExtractMode mode = ExtractMode.Auto)
{
foreach (var objinfo in asset.ObjectInfos) {
string className = AssetToolUtility.ClassIDToClassName(objinfo.classID);
var path = outputPath + "/Class " +objinfo.classID+" "+ className+"/";
var typeTree = typeTreeDB.GetType(asset.AssetVersion, objinfo.classID);
ExtractObjct(objinfo, typeTree, outputPath, mode);
}
}
public void ExtractObjct(Asset.AssetObjectInfo obj, TypeTree typeTree, string outputPath, ExtractMode mode = ExtractMode.Auto)
{
outputPath = outputPath + "\\" + AssetToolUtility.ClassIDToClassName(obj.classID) + "\\";
switch (mode)
{
case ExtractMode.Auto:
ExtractAuto(obj, typeTree, outputPath);
break;
case ExtractMode.OnlyRawBits:
extractOnlyRawBits(obj, typeTree, outputPath);
break;
case ExtractMode.OnlyRawText:
extractOnlyRawText(obj, typeTree, outputPath);
break;
case ExtractMode.RawTextOrRawBits:
extractRawTextOrRawBits(obj, typeTree, outputPath);
break;
}
}
public void PrepareOperation(ExtractMode askExtractMode)
{
}
public void Extract(UInt32[] indices, UInt32 numItems, ExtractMode testMode, IArchiveExtractCallback extractCallback)
{
inArchive.Extract(indices, numItems, testMode, extractCallback);
}
public void Extract(Asset asset, TypeTreeDataBase typeTreeDB, string outputPath, ExtractMode mode = ExtractMode.Auto)
{
foreach (var objinfo in asset.ObjectInfos)
{
string className = AssetToolUtility.ClassIDToClassName(objinfo.classID);
var path = outputPath + "/Class " + objinfo.classID + " " + className + "/";
var typeTree = typeTreeDB.GetType(asset.AssetVersion, objinfo.classID);
ExtractObjct(objinfo, typeTree, outputPath, mode);
}
}
public void GetStream(uint index, out ISequentialOutStream outStream, ExtractMode extractMode)
{
if (!files.ContainsKey(index)) {
outStream = null;
mode = ExtractMode.Skip;
return;
}
if (extractMode == ExtractMode.Extract) {
if (stream != null) {
stream.Dispose();
}
current = index;
var args = new ExtractFileEventArgs(owner.Archive, files[current], ExtractionStage.Extracting);
owner.ExtractFile(owner, args);
if (!args.ContinueOperation) {
throw new IOException("User aborted!");
}
var ostream = new SevenZipOutFileStream(files[index].Destination, (long)files[index].Size, owner.priority);
ostream.ProgressHandler += owner.ExtractProgress;
stream = ostream;
}
else {
if (stream != null) {
stream.Dispose();
}
stream = new SevenZipNullStream();
}
outStream = stream;
mode = extractMode;
}
/* PSEUDOCODE:
* func void CountRules (TreeNode node, ...)
* {
* if node has children
* create rule node --> children
* store the rule
* call CountRules for each child
* }
*/
// TODO_LOW: this method could return a "RuleGatherer" which stores the rules of one tree (a node and all of sub-nodes)
// this structure could then be inserted into a "RuleManager" or counter that extracts the rules
// implementing this requires additional passing of data but makes CountRules more loosely coupled from the storage of rules
// TODO_LOW: see if there's a way to increase readability of this implementation
private void CountRules(TreeNode node, Dictionary <string, int> ruleset, ExtractMode extractmode)
{
if (node.hasChildren())
{
string rule, lhs, rhs = "";
List <string> children = new List <string>();
switch (extractmode)
{
case ExtractMode.GRAMMAR:
// step 1 - create rule for node --> children
// step 1.1 - getting the left hand side of the rule
lhs = node.Value;
// step 1.2 - getting the right hand side of the rule
for (int i = 0; i < node.countChildren(); i++)
{
children.Add(node.getChild(i).Value);
}
children.Sort(); // sort the children strings so that permutations of the same rule is not stored separately
for (int i = 0; i < node.countChildren(); i++)
{
rhs += children[i] + " ";
}
rhs = rhs.Trim();
// step 1.3 - concatenate lhs and rhs to make the key for the rule
rule = lhs + " " + rhs;
// step 2 - store the rule
if (ruleset.ContainsKey(rule))
{
ruleset[rule] = ruleset[rule] + 1;
}
else
{
ruleset.Add(rule, 1);
}
// step 3 - call CountRules for each child
for (int i = 0; i < node.countChildren(); i++)
{
CountRules(node.getChild(i), ruleset, extractmode);
}
break;
case ExtractMode.LEXICON:
// step 1 - create rule for node --> children
// step 1.1 - getting the left hand side of the rule
lhs = node.Value;
for (int i = 0; i < node.countChildren(); i++)
{
// step 1.2 - getting the right hand side of the rule
rhs = node.getChild(i).Value;
// step 1.3 - concatenate lhs and rhs to make the key for the rule
rule = lhs + " " + rhs;
// step 2 - store the rule
if (ruleset.ContainsKey(rule))
{
ruleset[rule] = ruleset[rule] + 1;
}
else
{
ruleset.Add(rule, 1);
}
// step 3 - call CountRules for each child
CountRules(node.getChild(i), ruleset, extractmode);
}
break;
default:
throw new ArgumentException("Illegal ExtractMode enumerator was passed");
}
}
}
// formats the rules of the ruleset into a specific string format (which will later be written to file)
internal List <string> FormatRules(Dictionary <string, int> ruleset, out int ruleCounter, ExtractMode extractmode, Separator separator)
{
ruleCounter = 0; // counts the number of times a unique rule occurs
List <string> res = new List <string>();
List <string> rhslist = new List <string>(); // list that holds unique "right hand side" keys, used when formatting the lexicon
string sep = "";
if (separator.Equals(Separator.WHITESPACE))
{
sep = " ";
}
else if (separator.Equals(Separator.TAB))
{
sep = "\t";
}
else
{
throw new ArgumentException("Illegal Separator enumerator was passed");
}
switch (extractmode)
{
case ExtractMode.GRAMMAR:
foreach (KeyValuePair <string, int> entry in ruleset)
{
string rule = entry.Key;
rule = rule.Replace(' ', '\t');
string count = entry.Value.ToString();
res.Add(count + sep + rule);
ruleCounter++;
}
return(res);
case ExtractMode.LEXICON:
foreach (KeyValuePair <string, int> entry in ruleset)
{
string[] rule = entry.Key.Split(' ');
string lhs = rule[0];
string rhs = rule[1];
string count = entry.Value.ToString();
// if the right hand side rule has not been added to res
if (!rhslist.Contains(rhs))
{
res.Add(rhs + sep + lhs + sep + count);
rhslist.Add(rhs);
ruleCounter++;
}
// if the right hand side rule has previously been added to res
else
{
string searchStr = rhs + sep;
string resStr = res.FirstOrDefault(elem => elem.StartsWith(searchStr));
int index = res.IndexOf(resStr);
res[index] = res[index] + sep + lhs + sep + count;
ruleCounter++;
}
}
return(res);
default:
throw new ArgumentException("Illegal ExtractMode enumerator was passed");
}
}
internal void ExtractRules(string[] sentences, Dictionary <string, int> ruleset, ExtractMode extractmode)
{
for (int i = 0; i < sentences.Length; i++)
{
TreeNode node = ExtractNode(sentences[i], extractmode);
CountRules(node, ruleset, extractmode);
}
}
public void Extract(UInt32[] indices, UInt32 numItems, ExtractMode testMode, IArchiveExtractCallback extractCallback)
{
inArchive.Extract(indices, numItems, testMode, extractCallback);
}
private void ProcessPattern(ExtractMode mode, string pattern, string text, string inputFile)
{
Regex r = new Regex(pattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline);
MatchCollection matches = r.Matches(text);
foreach (Match match in matches)
{
GroupCollection groups = match.Groups;
if (groups.Count < 2)
{
throw new Exception(String.Format(
"Invalid pattern '{0}'.\nTwo groups are required at least.\nSource: {1}",
pattern, match.Value));
}
// Initialisation
string context = String.Empty;
string msgid = String.Empty;
string msgidPlural = String.Empty;
switch (mode)
{
case ExtractMode.Msgid:
msgid = Unescape(groups[1].Value);
break;
case ExtractMode.MsgidConcat:
MatchCollection matches2 = Regex.Matches(groups[0].Value, CsharpStringPattern);
StringBuilder sb = new StringBuilder();
foreach (Match match2 in matches2)
{
sb.Append(Unescape(match2.Value));
}
msgid = sb.ToString();
break;
case ExtractMode.MsgidFromResx:
string controlId = Unescape(groups[1].Value);
msgid = ExtractResourceString(controlId);
if (String.IsNullOrEmpty(msgid))
{
if (Options.Verbose)
{
Trace.WriteLine(String.Format(
"Warning: cannot extract string for control '{0}' ({1})",
controlId, inputFile));
}
continue;
}
if (controlId == msgid)
{
continue; // Text property was initialized by controlId and was not changed so this text is not usable in application
}
break;
case ExtractMode.MsgidPlural:
if (groups.Count < 3)
{
throw new Exception(String.Format("Invalid 'GetPluralString' call.\nSource: {0}", match.Value));
}
msgid = Unescape(groups[1].Value);
msgidPlural = Unescape(groups[2].Value);
break;
case ExtractMode.ContextMsgid:
if (groups.Count < 3)
{
throw new Exception(String.Format("Invalid get context message call.\nSource: {0}", match.Value));
}
context = Unescape(groups[1].Value);
msgid = Unescape(groups[2].Value);
if (groups.Count == 4)
{
msgidPlural = Unescape(groups[3].Value);
}
break;
}
if (String.IsNullOrEmpty(msgid))
{
if (Options.Verbose)
{
Trace.Write(String.Format("WARN: msgid is empty in {0}\r\n", inputFile));
}
}
else
{
MergeWithEntry(context, msgid, msgidPlural, inputFile, CalcLineNumber(text, match.Index));
}
}
}
/// <summary>
/// Constructor used for getting the unpacked size of an archive
/// </summary>
public ArchiveExtractCallback(string password)
{
_password = password;
_extractMode = ExtractMode.UnpackedSize;
}
