public ParsedContent GetContents()
{
var res = contents;
contents = new ParsedContent();
return(res);
}
/// <summary>
/// Convert the WikiPage to a DiscordEmbed.
/// </summary>
/// <returns>A DiscordEmbed containing the info gathered about the page.</returns>
public DiscordEmbed ToDiscordEmbed()
{
string categoryS = "";
if (Categories != null && Categories.Length > 0)
{
categoryS = Categories[0];
for (int i = 1; i < Categories.Length; i++)
{
categoryS += $" | {Categories[i]}";
}
categoryS += " -- ";
}
var embed = new DiscordEmbedBuilder
{
Title = $"__**{Title}**__",
Url = Util.GetWikiURL(Title),
Description = ParsedContent.Limit(1024),
Color = new DiscordColor(255, 211, 50)
};
embed.WithFooter($"{categoryS}Last edit by User:{LastChangeUser}", Util.GetImageURL("Artifact_Cutout.png"));
embed.WithTimestamp(LastChangeDate);
return(embed);
}
private static ParsedContent ParseContent(SiteContentVersion content, List <SiteContent> contents, int profileTypeId)
{
if (content == null || string.IsNullOrWhiteSpace(content.ContentText))
{
return(null);
}
var parsedContent = new ParsedContent()
{
Html = string.Empty,
Scripts = string.Empty,
Styles = string.Empty
};
var doc = new HtmlDocument();
doc.LoadHtml(content.ContentText);
foreach (var script in doc.DocumentNode.Descendants("script").ToArray())
{
parsedContent.Scripts += script.InnerHtml;
script.Remove();
}
foreach (var style in doc.DocumentNode.Descendants("style").ToArray())
{
parsedContent.Styles += style.InnerHtml;
style.Remove();
}
foreach (var link in doc.DocumentNode.Descendants("a").ToArray())
{
var href = link.GetAttributeValue("href", string.Empty);
if (!string.IsNullOrEmpty(href))
{
var linkContent = contents.FindByPermalinkOrUrl(href);
if (linkContent != null)
{
if (linkContent.ProfileTypes.Any() && linkContent.ProfileTypes.All(p => p.ProfileTypeID != profileTypeId))
{
link.Attributes["href"].Remove();
link.SetAttributeValue("class", "disabled");
}
else if (linkContent.ContentType == ContentType.File && linkContent.Site != null && linkContent.FileInfo != null)
{
link.SetAttributeValue("data-track", "true");
link.SetAttributeValue("data-label", linkContent.Site.SiteName);
link.SetAttributeValue("data-value", linkContent.FileInfo.Name);
}
}
}
}
parsedContent.Html = doc.DocumentNode.InnerHtml;
return(parsedContent);
}
public Parantheses(string content)
{
Operators = new List <Operator>();
Paranthesis = new List <Parantheses>();
Content = content;
ContentArray = Content.ToCharArray();
Parse();
ContentArray = ParsedContent.ToCharArray();
}
static void Main(string[] args)
{
FunctionTable.LoadData();
Analyse analyse = new Analyse();
//analyse.Files.Add(@"F:\Games\Call Of Cthulhu DCoTE\Testing\0.bin");
//analyse.Files.Add(@"F:\Games\Call Of Cthulhu DCoTE\Testing\1.bin");
//analyse.Files.Add(@"F:\Games\Call Of Cthulhu DCoTE\Testing\2.bin");
//analyse.Files.Add(@"F:\Games\Call Of Cthulhu DCoTE\Testing\3.bin");
//analyse.Files.Add(@"F:\Games\Call Of Cthulhu DCoTE\Testing\4.bin");
//analyse.Files.Add(args[0]);
if (args.Length > 1)
{
analyse.SaveAsNew = (args[1] != "asold");
analyse.BuildNew = (args[1] == "buildnew");
}
analyse.NewBase = 605;
//1AE = 430
//25D = 605
//2F8 = 760
if (args.Length > 2)
{
short parseShort = 0;
short.TryParse(args[2], out parseShort);
analyse.NewBase = parseShort;
}
ParsedContent parsedContent = analyse.AnalyseFile(args[0]);
analyse.PostAnalysis(parsedContent);
Console.WriteLine("Finished.");
//Console.ReadLine();
}
public ParsedContent Read(Stream stream)
{
var res = new ParsedContent();
var info = new AtomCollection(Channel.ChannelInfo.Extra);
var processed = false;
var eos = false;
while (!eos)
{
var start_pos = stream.Position;
try {
switch (state)
{
case ReaderState.Header:
{
var bin = ReadBytes(stream, 13);
var header = new FileHeader(bin);
if (header.IsValid)
{
Logger.Info("FLV Header found");
fileHeader = header;
bin = header.Binary;
res.ContentHeader = new Content(position, bin);
res.Contents = null;
info.SetChanInfoType("FLV");
info.SetChanInfoStreamType("video/x-flv");
info.SetChanInfoStreamExt(".flv");
res.ChannelInfo = new ChannelInfo(info);
position = bin.Length;
tags.Clear();
state = ReaderState.Body;
}
else
{
throw new BadDataException();
}
}
break;
case ReaderState.Body:
{
var bin = ReadBytes(stream, 11);
var read_valid = false;
var body = new FLVTag(bin);
if (body.IsValidHeader)
{
body.ReadBody(stream);
body.ReadFooter(stream);
if (body.IsValidFooter)
{
read_valid = true;
bin = body.Binary;
if (res.Contents == null)
{
res.Contents = new List <Content>();
}
res.Contents.Add(new Content(position, bin));
tags.AddLast(new TagDesc {
Timestamp = body.Timestamp / 1000.0, DataSize = body.DataSize
});
var timespan = tags.Last.Value.Timestamp - tags.First.Value.Timestamp;
if (timespan >= 30.0)
{
var sz = tags.Take(tags.Count - 1).Sum(t => t.DataSize);
info.SetChanInfoBitrate((int)(sz * 8 / timespan + 900) / 1000);
res.ChannelInfo = new ChannelInfo(info);
while (tags.Count > 1)
{
tags.RemoveFirst();
}
}
position += bin.Length;
}
}
if (!read_valid)
{
stream.Position = start_pos;
var header = new FileHeader(ReadBytes(stream, 13));
if (header.IsValid)
{
Logger.Info("New FLV Header found");
read_valid = true;
fileHeader = header;
bin = header.Binary;
res.ContentHeader = new Content(0, bin);
res.Contents = null;
info.SetChanInfoType("FLV");
info.SetChanInfoStreamType("video/x-flv");
info.SetChanInfoStreamExt(".flv");
res.ChannelInfo = new ChannelInfo(info);
tags.Clear();
position = bin.Length;
}
}
if (!read_valid)
{
throw new BadDataException();
}
}
break;
}
processed = true;
}
catch (EndOfStreamException) {
if (!processed)
{
throw;
}
stream.Position = start_pos;
eos = true;
}
catch (BadDataException) {
stream.Position = start_pos + 1;
}
}
return(res);
}
public ParsedContent Read(Stream stream)
{
var chunks = 0;
var res = new ParsedContent();
var pos = Channel.ContentPosition;
try {
while (chunks < 8)
{
var chunk = ASFChunk.Read(stream);
chunks++;
switch (chunk.KnownType)
{
case ASFChunk.ChunkType.Header: {
var header = ASFHeader.Read(chunk);
var info = new AtomCollection(Channel.ChannelInfo.Extra);
info.SetChanInfoBitrate(header.Bitrate);
if (header.Streams.Any(type => type == ASFHeader.StreamType.Video))
{
info.SetChanInfoType("WMV");
info.SetChanInfoStreamType("video/x-ms-wmv");
info.SetChanInfoStreamExt(".wmv");
}
else if (header.Streams.Any(type => type == ASFHeader.StreamType.Audio))
{
info.SetChanInfoType("WMA");
info.SetChanInfoStreamType("audio/x-ms-wma");
info.SetChanInfoStreamExt(".wma");
}
else
{
info.SetChanInfoType("ASF");
info.SetChanInfoStreamType("video/x-ms-asf");
info.SetChanInfoStreamExt(".asf");
}
res.ChannelInfo = new ChannelInfo(info);
res.ContentHeader = new Content(pos, chunk.ToByteArray());
pos += chunk.TotalLength;
}
break;
case ASFChunk.ChunkType.Data:
if (res.Contents == null)
{
res.Contents = new System.Collections.Generic.List <Content>();
}
res.Contents.Add(new Content(pos, chunk.ToByteArray()));
pos += chunk.TotalLength;
break;
case ASFChunk.ChunkType.Unknown:
break;
}
}
}
catch (EndOfStreamException) {
if (chunks == 0)
{
throw;
}
}
return(res);
}
public ParsedContent Read(Stream stream)
{
var res = new ParsedContent();
var bodies = new List <Element>();
var info = new AtomCollection(Channel.ChannelInfo.Extra);
var processed = false;
var eos = false;
while (!eos)
{
var start_pos = stream.Position;
try {
var elt = Element.ReadHeader(stream);
if (ebml.MaxIDLength < elt.ID.Length ||
ebml.MaxSizeLength < elt.Size.Length)
{
throw new BadDataException();
}
switch (state)
{
case ReaderState.EBML:
if (elt.ID.BinaryEquals(Elements.EBML))
{
elt.ReadBody(stream);
ebml = new EBML(elt);
state = ReaderState.Segment;
}
else
{
throw new BadDataException();
}
break;
case ReaderState.Segment:
if (elt.ID.BinaryEquals(Elements.Segment))
{
segment = new Segment(elt);
state = ReaderState.EndOfHeader;
}
else if (elt.ID.BinaryEquals(Elements.EBML))
{
stream.Position = elt.Position;
state = ReaderState.EBML;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Void) ||
elt.ID.BinaryEquals(Elements.CRC32))
{
elt.ReadBody(stream);
}
else
{
throw new BadDataException();
}
break;
case ReaderState.EndOfHeader:
if (elt.ID.BinaryEquals(Elements.Segment))
{
stream.Position = elt.Position;
state = ReaderState.Segment;
continue;
}
else if (elt.ID.BinaryEquals(Elements.EBML))
{
stream.Position = elt.Position;
state = ReaderState.EBML;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Cluster))
{
stream.Position = elt.Position;
state = ReaderState.Cluster;
clusters.Clear();
MemoryStream header;
using (header = new MemoryStream()) {
ebml.Element.Write(header);
segment.Element.Write(header);
foreach (var c in segment.HeaderElements)
{
c.Write(header);
}
}
res.ContentHeader = new Content(0, header.ToArray());
if (ebml.DocType == "webm")
{
info.SetChanInfoType("WEBM");
info.SetChanInfoStreamType("video/webm");
info.SetChanInfoStreamExt(".webm");
}
else
{
info.SetChanInfoType("MKV");
info.SetChanInfoStreamType("video/x-matroska");
info.SetChanInfoStreamExt(".mkv");
}
res.ChannelInfo = new ChannelInfo(info);
}
else
{
elt.ReadBody(stream);
segment.AddHeader(elt);
}
break;
case ReaderState.Cluster:
if (elt.ID.BinaryEquals(Elements.Segment))
{
stream.Position = elt.Position;
state = ReaderState.Segment;
continue;
}
else if (elt.ID.BinaryEquals(Elements.EBML))
{
stream.Position = elt.Position;
state = ReaderState.EBML;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Cluster))
{
if (clusters.Count > 0)
{
var timespan = clusters.Sum(c => c.Timespan);
if (timespan >= 30.0)
{
var sz = clusters.Sum(c => c.Timespan > 0 ? c.BlockSize : 0);
var kbps = (int)((sz * 8 / timespan + 900) / 1000.0);
info.SetChanInfoBitrate(kbps);
res.ChannelInfo = new ChannelInfo(info);
while (clusters.Count > 1)
{
clusters.RemoveFirst();
}
}
}
var cluster = new Cluster(elt);
clusters.AddLast(cluster);
bodies.Add(elt);
state = ReaderState.Timecode;
}
else if (elt.ID.BinaryEquals(Elements.Void) ||
elt.ID.BinaryEquals(Elements.CRC32))
{
elt.ReadBody(stream);
bodies.Add(elt);
}
else
{
throw new BadDataException();
}
break;
case ReaderState.Timecode:
if (elt.ID.BinaryEquals(Elements.Segment))
{
stream.Position = elt.Position;
state = ReaderState.Segment;
continue;
}
else if (elt.ID.BinaryEquals(Elements.EBML))
{
stream.Position = elt.Position;
state = ReaderState.EBML;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Cluster))
{
stream.Position = elt.Position;
state = ReaderState.Cluster;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Timecode))
{
elt.ReadBody(stream);
if (clusters.Last != null)
{
clusters.Last.Value.Timecode =
Element.ReadUInt(new MemoryStream(elt.Data), elt.Data.Length) * (segment.TimecodeScale / 1000000000.0);
if (clusters.Count > 1)
{
clusters.Last.Previous.Value.Timespan = clusters.Last.Value.Timecode - clusters.Last.Previous.Value.Timecode;
}
}
bodies.Add(elt);
state = ReaderState.Block;
}
else if (elt.ID.BinaryEquals(Elements.SimpleBlock) ||
elt.ID.BinaryEquals(Elements.BlockGroup))
{
stream.Position = elt.Position;
state = ReaderState.Block;
continue;
}
else
{
elt.ReadBody(stream);
bodies.Add(elt);
}
break;
case ReaderState.Block:
if (elt.ID.BinaryEquals(Elements.Segment))
{
stream.Position = elt.Position;
state = ReaderState.Segment;
continue;
}
else if (elt.ID.BinaryEquals(Elements.EBML))
{
stream.Position = elt.Position;
state = ReaderState.EBML;
continue;
}
else if (elt.ID.BinaryEquals(Elements.Cluster))
{
stream.Position = elt.Position;
state = ReaderState.Cluster;
continue;
}
else if ((elt.ID.BinaryEquals(Elements.SimpleBlock) ||
elt.ID.BinaryEquals(Elements.BlockGroup)) &&
(clusters.Last.Value.BlockID == null ||
elt.ID.BinaryEquals(clusters.Last.Value.BlockID)))
{
elt.ReadBody(stream);
clusters.Last.Value.BlockSize += elt.Size.Value;
clusters.Last.Value.BlockID = elt.ID.Binary;
bodies.Add(elt);
}
else if (clusters.Last.Value.BlockID == null)
{
elt.ReadBody(stream);
bodies.Add(elt);
}
else
{
stream.Position = elt.Position;
state = ReaderState.Cluster;
continue;
}
break;
}
processed = true;
}
catch (EndOfStreamException) {
if (!processed)
{
throw;
}
stream.Position = start_pos;
eos = true;
}
catch (BadDataException) {
stream.Position = start_pos + 1;
}
}
if (res.ContentHeader != null)
{
position =
res.ContentHeader.Position +
res.ContentHeader.Data.Length;
}
if (bodies.Count > 0)
{
res.Contents = new List <Content>();
foreach (var body in bodies)
{
MemoryStream s;
using (s = new MemoryStream()) {
body.Write(s);
}
var data = s.ToArray();
res.Contents.Add(new Content(position, data));
position += data.Length;
}
}
return(res);
}
private ParsedContent Parse(Uri uri, Stream contentStream)
{
var result = new ParsedContent();
var document = new HtmlDocument();
document.Load(contentStream);
var robotsMetaNode = document.DocumentNode.SelectSingleNode("html/head/meta[@name=\"ROBOTS\"]");
if (robotsMetaNode != null)
{
var robotsMetaContent = robotsMetaNode.GetAttributeValue("content", null);
if (robotsMetaContent.IndexOf("noindex", StringComparison.InvariantCultureIgnoreCase) != -1)
{
result.NoIndex = true;
}
if (robotsMetaContent.IndexOf("nofollow", StringComparison.InvariantCultureIgnoreCase) != -1)
{
result.NoFollow = true;
return(result);
}
}
var canonicalNode = document.DocumentNode.SelectSingleNode("html/head/link[@rel=\"canonical\"]");
if (canonicalNode != null)
{
var canonicalHref = canonicalNode.GetAttributeValue("href", null);
if (canonicalHref != null && Uri.TryCreate(canonicalHref, UriKind.Absolute, out var canonicalUri))
{
result.CanonicalUri = canonicalUri;
}
}
string baseHref = null;
var baseNode = document.DocumentNode.SelectSingleNode("html/head/base");
if (baseNode != null)
{
baseHref = baseNode.GetAttributeValue("href", null);
}
var anchorNodes = document.DocumentNode.SelectNodes("//a");
if (anchorNodes != null)
{
var crawledLinks = new List <CrawlLink>();
foreach (var anchor in anchorNodes)
{
var href = anchor.GetAttributeValue("href", null);
if (href == null)
{
continue;
}
var anchorLocation = uri.BuildUriFromHref(href, baseHref);
if (anchorLocation == null)
{
//Invalid links are ignored
continue;
}
crawledLinks.Add(new CrawlLink
{
Location = anchorLocation,
Title = anchor.GetAttributeValue("title", null),
Text = anchor.InnerText,
Relationship = anchor.GetAttributeValue("rel", null),
});
}
result.Links = crawledLinks;
}
return(result);
}
private void processSingleMessage(string folderName, Message inMessage)
{
ParsedPstMessage parsedMessage = new ParsedPstMessage()
{
FolderName = folderName,
};
try
{
parsedMessage.CreationTime = convertPtypTimeToDateTime(inMessage.GetProperty(MAPIProperties.PidTagCreationTime).Value.Value.ToInt64());
parsedMessage.ModificationTime = convertPtypTimeToDateTime(inMessage.GetProperty(MAPIProperties.PidTagLastModificationTime).Value.Value.ToInt64());
}
catch { }
try
{
parsedMessage.Subject = inMessage.GetProperty(MAPIProperties.PidTagSubject).Value.Value.ToUnicode().Trim();
}
catch { }
try
{
parsedMessage.Sender = inMessage.GetProperty(MAPIProperties.PidTagSenderEmailAddress).Value.Value.ToUnicode().Trim();
}
catch { }
try
{
parsedMessage.Body = inMessage.GetProperty(MAPIProperties.PidTagBody).Value.Value.ToUnicode().Trim();
}
catch { }
try
{
foreach (Recipient recipient in inMessage.GetRecipients())
{
try
{
string recipientAddress = recipient.GetProperty(MAPIProperties.PidTagEmailAddress).Value.Value.ToUnicode();
parsedMessage.Recipients.Add(recipientAddress);
}
catch (Exception ex)
{
Output.AddLogEntry("recipient.GetProperty(): " + ex.Message, true);
}
}
}
catch (Exception ex)
{
Output.AddLogEntry("message.GetRecipients(): " + ex.Message, true);
}
// TODO: reenable fetching of attachements when rest is working properly -->
//try
//{
// foreach (Attachment attachment in inMessage.GetAttachments())
// {
// try
// {
// string attFilename = attachment.GetProperty(MAPIProperties.PidTagAttachLongFilename).Value.Value.ToUnicode();
// parsedMessage.AttachmentNames.Add(attFilename);
// }
// catch (Exception ex)
// {
// log.Error(ex.Message);
// Errors.Add(ex.Message);
// }
// }
//}
//catch (Exception ex)
//{
// updateLogAndJournal(null, "message.GetAttachments(): " + ex.Message, true);
//}
ParsedContent.Add(parsedMessage);
//string filePart = $"{parsedMessage.Sender} - {parsedMessage.Subject}";
//foreach (char invalidChar in System.IO.Path.GetInvalidFileNameChars())
//{
// filePart = filePart.Replace(invalidChar, '_');
//}
//string outFile = $"{outDir}\\{filePart}";
////outFile = $"{outDir}\\{msgCounter++.ToString()}";
//string outFileTxt = outFile + ".txt";
//string outFilePdf = outFile + ".pdf";
//string result = parsedMessage.MsgAsString;
//File.WriteAllText(outFileTxt, result);
//// TODO: reenable after test -->
//converter.Convert(outFileTxt, outFilePdf);
//if (!KeepIntermediateFiles)
//{
// File.Delete(outFileTxt);
//}
////extractAndConvertAttachements(inFile, outFile);
}
public void BuildVariables(ParsedContent parsedContent)
{
Variables = new Dictionary <short, Variable>();
ValueList = new List <ValueViewModel>();
this.ValueList = new List <ValueViewModel>();
foreach (Value value in parsedContent.ValuesList)
{
ValueViewModel valueViewModel = new ValueViewModel()
{
Address = value.Address,
AddressHex = (short)(value.Address - parsedContent.BaseAddress + parsedContent.StartOfValues),
AddressHexBase = (short)(value.Address + parsedContent.StartOfValues),
DataType = value.DataType,
Reference = value.Reference,
IsMe = value.IsMe,
IsPlayer = value.IsPlayer
};
if (value.SubValues.Count > 0)
{
if (valueViewModel.DataType == DataTypeType.Float)
{
valueViewModel.SubValue1 = BitConverter.ToSingle(BitConverter.GetBytes((Int32)(value.SubValues[0])), 0);
}
else
{
valueViewModel.SubValue1 = value.SubValues[0];
}
}
if (value.SubValues.Count > 1)
{
valueViewModel.SubValue2 = value.SubValues[1];
}
if (value.SubValues.Count > 2)
{
valueViewModel.SubValue3 = value.SubValues[2];
}
if (value.SubValues.Count > 3)
{
valueViewModel.SubValue4 = value.SubValues[3];
}
if (value.SubValues.Count > 4)
{
valueViewModel.SubValue5 = value.SubValues[4];
}
this.ValueList.Add(valueViewModel);
//build variables list
if (!Variables.ContainsKey(value.Address))
{
Variable v = new Variable()
{
Address = value.Address,
DataType = value.DataType.ToString(),
//Name = "var" + value.DataType.ToString() + value.Address.ToString()
};
if (value.IsMe ||
value.IsPlayer)
{
v.Used = true;
v.Static = true;
}
switch (v.DataType.ToLower())
{
case ("int"):
v.Name = "nVar" + v.Address.ToString();
break;
case ("string"):
v.Name = "szVar" + v.Address.ToString();
break;
case ("point"):
v.Name = "ptVar" + v.Address.ToString();
v.Static = false; //points are never static
break;
case ("character"):
v.Name = "cVar" + v.Address.ToString();
break;
case ("float"):
v.Name = "fVar" + v.Address.ToString();
break;
case ("quaternion"):
v.Name = "qVar" + v.Address.ToString();
break;
}
if (FixTable.isStaticFix(parsedContent.ScriptName, v.Name))
{
v.Static = false;
}
Variables.Add(value.Address, v);
}
}
//work out which variables are static or unused
for (int i = 0; i < parsedContent.OpCodeList.Count; i++)
{
Operation operation = parsedContent.OpCodeList[i];
if (operation.OpCode == OpCodeType.OP_GETTOP)
{
if (Variables.ContainsKey(operation.DataIndex.Value))
{
Variables[operation.DataIndex.Value].Static = false;
Variables[operation.DataIndex.Value].Used = true;
}
}
else if (operation.OpCode == OpCodeType.OP_PUSH)
{
if (Variables.ContainsKey(operation.DataIndex.Value))
{
Variables[operation.DataIndex.Value].Used = true;
}
}
}
}
