public LogNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
try { logLevel = LogLevel.FromString(xmlNode.Attributes["LogLevel"].Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
try { logLevel = LogLevel.FromString(xmlNode.Attributes["log_level"].Value); }
catch (Exception e2) {
if (e2.GetType() == typeof(ThreadAbortException))
throw e2;
logLevel = LogLevel.Debug;
}
}
try { message = xmlNode.Attributes["Message"].Value; }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
try { message = xmlNode.Attributes["message"].Value; }
catch (Exception e2) {
if (e2.GetType() == typeof(ThreadAbortException))
throw e2;
logger.Error("Missing MESSAGE attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
loadSuccess = true;
}
public DistanceNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "string1":
string1 = attr.Value;
break;
case "string2":
string2 = attr.Value;
break;
}
}
// Validate STRING1 attribute
if (string1 == null) {
logger.Error("Missing STRING1 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate STRING2 attribute
if (string2 == null) {
logger.Error("Missing STRING2 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public SleepNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode)
{
logger.Debug("executing set: " + xmlNode.OuterXml);
}
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "length":
try { _length = int.Parse(attr.Value); }
catch (Exception) {
_length = 100;
}
break;
}
}
// get the innervalue
string innerValue = xmlNode.InnerText.Trim();
// Validate length attribute
if (_length <= 0)
{
logger.Error("The LENGTH attribute must be greater than 0: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public LoopNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
Logger.Debug("ScriptableScraperProvider: Executing loop: {0}", xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "on":
LoopingVariable = attr.Value;
break;
case "limit":
try
{
Limit = int.Parse(attr.Value);
}
catch (Exception e)
{
Logger.Error("ScriptableScraperProvider: Invalid value for LIMIT attribute on: {0}", xmlNode.OuterXml, e);
}
break;
}
}
// Validate ON attribute
if (LoopingVariable == null)
{
Logger.Error("ScriptableScraperProvider: Missing ON attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
public DistanceNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "string1":
String1 = attr.Value;
break;
case "string2":
String2 = attr.Value;
break;
}
}
// Validate STRING1 attribute
if (String1 == null)
{
Logger.Error("ScriptableScraperProvider: Missing STRING1 attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
// Validate STRING2 attribute
if (String2 == null)
{
Logger.Error("ScriptableScraperProvider: Missing STRING2 attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
public SleepNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode) logger.Debug("executing set: " + xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "length":
try { _length = int.Parse(attr.Value); }
catch (Exception) {
_length = 100;
}
break;
}
}
// get the innervalue
string innerValue = xmlNode.InnerText.Trim();
// Validate length attribute
if (_length <= 0) {
logger.Error("The LENGTH attribute must be greater than 0: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public LoopNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode) logger.Debug("executing loop: " + xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "on":
loopingVariable = attr.Value;
break;
case "limit":
try {
limit = int.Parse(attr.Value);
}
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
logger.Error("Invalid value for LIMIT attribute on: " + xmlNode.OuterXml);
}
break;
}
}
// Validate ON attribute
if (loopingVariable == null) {
logger.Error("Missing ON attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public SortNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "direction":
string dirStr = attr.Value.ToLower().Trim();
if (dirStr == "desc" || dirStr == "descending")
direction = DirectionType.DESCENDING;
else if (dirStr == "asc" || dirStr == "ascending")
direction = DirectionType.ASCENDING;
else {
logger.Error("Invalid sort direction on: " + xmlNode.OuterXml);
}
break;
case "by":
sortBy = attr.Value;
break;
}
}
// Validate BY attribute
if (sortBy == null) {
logger.Error("Missing BY attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public ParseNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "input":
input = attr.Value;
break;
case "regex":
pattern = attr.Value;
break;
case "xpath":
xpath = attr.Value;
break;
}
}
// Validate INPUT attribute
if (input == null) {
logger.Error("Missing INPUT attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate REGEX/XPATH attribute
if (pattern == null && xpath == null) {
logger.Error("Missing REGEX or XPATH attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public string Parse(ScriptableScraper context, string value, string options)
{
DateTime parsedDateTime;
// use sensible default (language of scraper)
if (options.IsNullOrWhiteSpace())
{
CultureInfo ci = CultureInfo.CreateSpecificCulture(context.Language);
if (DateTime.TryParse(value, ci, DateTimeStyles.None, out parsedDateTime))
{
// store the value as the invariant datetime format
value = parsedDateTime.ToString(CultureInfo.InvariantCulture.DateTimeFormat);
}
else
{
logger.Error("Could not parse \"date\" modifier using script language \"" + context.Language + "\"");
}
}
// use exact format as specified in options
else
{
if (DateTime.TryParseExact(value, new string[] { options }, CultureInfo.InvariantCulture.DateTimeFormat, DateTimeStyles.None, out parsedDateTime))
{
// store the value as the invariant datetime format
value = parsedDateTime.ToString(CultureInfo.InvariantCulture.DateTimeFormat);
}
else
{
logger.Error("Could not parse \"date\" modifier using options \"" + options + "\"");
}
}
return value;
}
public string Parse(ScriptableScraper context, string value, string options)
{
DateTime parsedDateTime;
// use sensible default (language of scraper)
if (options.IsNullOrWhiteSpace())
{
CultureInfo ci = CultureInfo.CreateSpecificCulture(context.Language);
if (DateTime.TryParse(value, ci, DateTimeStyles.None, out parsedDateTime))
{
// store the value as the invariant datetime format
value = parsedDateTime.ToString(CultureInfo.InvariantCulture.DateTimeFormat);
}
else
{
logger.Error("Could not parse \"date\" modifier using script language \"" + context.Language + "\"");
}
}
// use exact format as specified in options
else
{
if (DateTime.TryParseExact(value, new string[] { options }, CultureInfo.InvariantCulture.DateTimeFormat, DateTimeStyles.None, out parsedDateTime))
{
// store the value as the invariant datetime format
value = parsedDateTime.ToString(CultureInfo.InvariantCulture.DateTimeFormat);
}
else
{
logger.Error("Could not parse \"date\" modifier using options \"" + options + "\"");
}
}
return(value);
}
public SetNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode) logger.Debug("executing set: " + xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "value":
value = attr.Value;
break;
}
}
// get the innervalue
string innerValue = xmlNode.InnerText.Trim();
// Validate TEST attribute
if (value == null) {
value = innerValue;
if (innerValue.Equals(String.Empty)) {
logger.Error("Missing VALUE attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
} else if (!innerValue.Equals(String.Empty)) {
logger.Error("Ambiguous assignment on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public SleepNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
Logger.Debug("ScriptableScraperProvider: Executing set: {0}", xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "length":
if (int.TryParse(attr.Value, out var length))
{
Length = length;
}
else
{
Length = 100;
}
break;
}
}
// get the inner value
string innerValue = xmlNode.InnerText.Trim();
// Validate length attribute
if (Length <= 0)
{
Logger.Error("ScriptableScraperProvider: The LENGTH attribute must be greater than 0: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
public static ScraperGame getGame(ScriptableScraper scraper, Dictionary <string, string> paramList)
{
if (scraper == null)
{
return(null);
}
Dictionary <string, string> results = scraper.Execute("get_details", paramList);
if (results == null)
{
return(null);
}
string grade;
string title;
string yearmade;
string company;
string description;
string genre;
results.TryGetValue("game.grade", out grade);
results.TryGetValue("game.title", out title);
results.TryGetValue("game.yearmade", out yearmade);
results.TryGetValue("game.company", out company);
results.TryGetValue("game.description", out description);
results.TryGetValue("game.genre", out genre);
return(new ScraperGame(title, company, yearmade, grade, description, genre));
}
public string Parse(ScriptableScraper context, string value, string options)
{
value = Regex.Replace(value, @"\n", string.Empty); // Remove all linebreaks
value = Regex.Replace(value, @"<br\s*/?>", "\n", RegexOptions.IgnoreCase); // Replace HTML breaks with \n
value = Regex.Replace(value, @"</p>", "\n\n", RegexOptions.IgnoreCase); // Replace paragraph tags with \n\n
value = Regex.Replace(value, @"<.+?>", string.Empty); // Remove all other tags
value = Regex.Replace(value, @"\n{3,}", "\n\n"); // Trim newlines
value = Regex.Replace(value, @"\t{2,}", " ").Trim(); // Trim whitespace
return HttpUtility.HtmlDecode(value);
}
public string Parse(ScriptableScraper context, string value, string options)
{
value = Regex.Replace(value, @"\n", string.Empty); // Remove all linebreaks
value = Regex.Replace(value, @"<br\s*/?>", "\n", RegexOptions.IgnoreCase); // Replace HTML breaks with \n
value = Regex.Replace(value, @"</p>", "\n\n", RegexOptions.IgnoreCase); // Replace paragraph tags with \n\n
value = Regex.Replace(value, @"<.+?>", string.Empty); // Remove all other tags
value = Regex.Replace(value, @"\n{3,}", "\n\n"); // Trim newlines
value = Regex.Replace(value, @"\t{2,}", " ").Trim(); // Trim whitespace
return(HttpUtility.HtmlDecode(value));
}
public MathNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
string resultTypeStr = null;
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "value1":
value1 = attr.Value;
break;
case "value2":
value2 = attr.Value;
break;
case "result_type":
resultTypeStr = attr.Value;
break;
}
}
// Validate VALUE1 attribute
if (value1 == null)
{
logger.Error("Missing VALUE1 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate VALUE2 attribute
if (value2 == null)
{
logger.Error("Missing VALUE2 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate RESULT_TYPE attribute
if (resultTypeStr != null && resultTypeStr.ToUpper().Equals("FLOAT"))
{
resultType = ResultTypeEnum.FLOAT;
}
else
{
resultType = ResultTypeEnum.INT;
}
}
public ScraperNode(XmlNode xmlNode, ScriptableScraper context)
{
this.xmlNode = xmlNode;
children = new List <ScraperNode>();
this.Context = context;
LoadSuccess = LoadChildren();
// try to load our node attribute
foreach (Attribute currAttr in this.GetType().GetCustomAttributes(true))
{
if (currAttr is ScraperNodeAttribute)
{
nodeSettings = (ScraperNodeAttribute)currAttr;
continue;
}
}
if (nodeSettings.LoadNameAttribute)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "name":
Name = attr.Value;
break;
}
}
// Validate NAME attribute
if (Name == null)
{
Logger.Error("ScriptableScraperProvider: Missing NAME attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
// if it's a bad variable name we fail as well
if (Name.Contains(" "))
{
Logger.Error("ScriptableScraperProvider: Invalid NAME attribute (no spaces allowed) \"{0}\" for {}", Name, xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
}
public ReplaceNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "input":
input = attr.Value;
break;
case "pattern":
pattern = attr.Value;
break;
case "with":
replacement = attr.Value;
break;
}
}
// Validate INPUT attribute
if (input == null)
{
logger.Error("Missing INPUT attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate PATTERN attribute
if (pattern == null)
{
logger.Error("Missing PATTERN attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate WITH attribute
if (replacement == null)
{
logger.Error("Missing WITH attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public ReplaceNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "input":
Input = attr.Value;
break;
case "pattern":
Pattern = attr.Value;
break;
case "with":
With = attr.Value;
break;
}
}
// Validate INPUT attribute
if (Input == null)
{
Logger.Error("ScriptableScraperProvider: Missing INPUT attribute on: {0}", xmlNode.OuterXml);
return;
}
// Validate PATTERN attribute
if (Pattern == null)
{
Logger.Error("ScriptableScraperProvider: Missing PATTERN attribute on: {0}", xmlNode.OuterXml);
return;
}
// Validate WITH attribute
if (With == null)
{
Logger.Error("ScriptableScraperProvider: Missing WITH attribute on: {0}", xmlNode.OuterXml);
return;
}
LoadSuccess = true;
}
public bool Load(string script)
{
bool debugMode = MovingPicturesCore.Settings.DataSourceDebugActive;
scraper = new ScriptableScraper(script, debugMode);
if (!scraper.LoadSuccessful)
{
scraper = null;
return(false);
}
providesMovieDetails = scraper.ScriptType.Contains("MovieDetailsFetcher");
providesCoverArt = scraper.ScriptType.Contains("MovieCoverFetcher");
providesBackdrops = scraper.ScriptType.Contains("MovieBackdropFetcher");
return(true);
}
public IfNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "test":
test = attr.Value;
break;
}
}
// Validate TEST attribute
if (test == null) {
logger.Error("Missing TEST attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public LogNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
try { logLevel = LogLevel.FromString(xmlNode.Attributes["LogLevel"].Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
{
throw e;
}
try { logLevel = LogLevel.FromString(xmlNode.Attributes["log_level"].Value); }
catch (Exception e2) {
if (e2.GetType() == typeof(ThreadAbortException))
{
throw e2;
}
logLevel = LogLevel.Debug;
}
}
try { message = xmlNode.Attributes["Message"].Value; }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
{
throw e;
}
try { message = xmlNode.Attributes["message"].Value; }
catch (Exception e2) {
if (e2.GetType() == typeof(ThreadAbortException))
{
throw e2;
}
logger.Error("Missing MESSAGE attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
loadSuccess = true;
}
public Scraper(string script)
{
if (string.IsNullOrEmpty(script))
{
return;
}
//try and load a valid scraper
Stream scriptStream;
try
{
Assembly asm = Assembly.GetExecutingAssembly();
scriptStream = asm.GetManifestResourceStream(script);
}
catch (Exception ex)
{
scriptStream = null;
Logger.LogError("Error loading script {0} - {1}", script, ex.Message);
}
if (scriptStream == null)
{
return;
}
string scriptTxt = "";
using (StreamReader sr = new StreamReader(scriptStream))
{
scriptTxt = sr.ReadToEnd();
}
scraper = new ScriptableScraper(scriptTxt, false);
if (!scraper.LoadSuccessful)
{
Logger.LogError("Import_Scraper - Error loading script {0}", script);
scraper = null;
return; //problem with scraper
}
name = scraper.Name;
idString = scraper.ID.ToString();
isReady = true;
}
public LoopNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode)
{
logger.Debug("executing loop: " + xmlNode.OuterXml);
}
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "on":
loopingVariable = attr.Value;
break;
case "limit":
try {
limit = int.Parse(attr.Value);
}
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
{
throw e;
}
logger.Error("Invalid value for LIMIT attribute on: " + xmlNode.OuterXml);
}
break;
}
}
// Validate ON attribute
if (loopingVariable == null)
{
logger.Error("Missing ON attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public string Parse(ScriptableScraper context, string value, string options)
{
// if we have an encoding string try to build an encoding object
Encoding encoding = null;
if (options != string.Empty)
{
try { encoding = Encoding.GetEncoding(options.ToLower()); }
catch (ArgumentException)
{
encoding = null;
logger.Error("Scraper script tried to use an invalid encoding for \"safe\" modifier");
}
}
if (encoding != null) {
return HttpUtility.UrlEncode(value, encoding).Replace("+", "%20");
}
else {
return HttpUtility.UrlEncode(value).Replace("+", "%20");
}
}
public SetNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
if (context.DebugMode)
{
logger.Debug("executing set: " + xmlNode.OuterXml);
}
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "value":
value = attr.Value;
break;
}
}
// get the innervalue
string innerValue = xmlNode.InnerText.Trim();
// Validate TEST attribute
if (value == null)
{
value = innerValue;
if (innerValue.Equals(String.Empty))
{
logger.Error("Missing VALUE attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
else if (!innerValue.Equals(String.Empty))
{
logger.Error("Ambiguous assignment on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public static List <string> getCoverUrls(ScriptableScraper scraper, Dictionary <string, string> paramList)
{
List <string> imageResults = new List <string>();
imageResults.Add("");
imageResults.Add("");
if (scraper == null)
{
return(imageResults);
}
Dictionary <string, string> results;
results = scraper.Execute("get_cover_art", paramList);
if (results == null)
{
return(imageResults);
}
string covers = "";
if (results.TryGetValue("game.covers", out covers))
{
string[] coverurls = covers.Split('|');
for (int i = 1; i < coverurls.Length; i++)
{
if (i == 1)
{
imageResults[0] = coverurls[i];
}
else if (i == 2)
{
imageResults[1] = coverurls[i];
}
}
}
return(imageResults);
}
public IfNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "test":
test = attr.Value;
break;
}
}
// Validate TEST attribute
if (test == null)
{
logger.Error("Missing TEST attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public ReplaceNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "input":
input = attr.Value;
break;
case "pattern":
pattern = attr.Value;
break;
case "with":
replacement = attr.Value;
break;
}
}
// Validate INPUT attribute
if (input == null) {
logger.Error("Missing INPUT attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate PATTERN attribute
if (pattern == null) {
logger.Error("Missing PATTERN attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate WITH attribute
if (replacement == null) {
logger.Error("Missing WITH attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public LogNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
try
{
if (Enum.TryParse <LoggingLevel>(xmlNode.Attributes["LogLevel"].Value, out var lvl))
{
LogLevel = lvl;
}
else if (Enum.TryParse <LoggingLevel>(xmlNode.Attributes["log_level"].Value, out var lvl2))
{
LogLevel = lvl2;
}
}
catch
{
LogLevel = LoggingLevel.Debug;
}
try
{
Message = xmlNode.Attributes["Message"].Value;
}
catch
{
try
{
Message = xmlNode.Attributes["message"].Value;
}
catch (Exception e)
{
Logger.Error("ScriptableScraperProvider: Missing MESSAGE attribute on: {0}", xmlNode.OuterXml, e);
LoadSuccess = false;
return;
}
}
LoadSuccess = true;
}
public SortNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "direction":
string dirStr = attr.Value.ToLower().Trim();
if (dirStr == "desc" || dirStr == "descending")
{
direction = DirectionType.DESCENDING;
}
else if (dirStr == "asc" || dirStr == "ascending")
{
direction = DirectionType.ASCENDING;
}
else
{
logger.Error("Invalid sort direction on: " + xmlNode.OuterXml);
}
break;
case "by":
sortBy = attr.Value;
break;
}
}
// Validate BY attribute
if (sortBy == null)
{
logger.Error("Missing BY attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
public ParseNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "input":
Input = attr.Value;
break;
case "regex":
Pattern = attr.Value;
break;
case "xpath":
Xpath = attr.Value;
break;
}
}
// Validate INPUT attribute
if (Input == null)
{
Logger.Error("ScriptableScraperProvider: Missing INPUT attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
// Validate REGEX/XPATH attribute
if (Pattern == null && Xpath == null)
{
Logger.Error("ScriptableScraperProvider: Missing REGEX or XPATH attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
public SetNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
Logger.Debug("ScriptableScraperProvider: Executing set: {0}", xmlNode.OuterXml);
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes)
{
switch (attr.Name)
{
case "value":
Value = attr.Value;
break;
}
}
// get the inner value
string innerValue = xmlNode.InnerText.Trim();
// Validate VALUE attribute
if (Value == null)
{
Value = innerValue;
if (innerValue.Equals(String.Empty))
{
Logger.Error("ScriptableScraperProvider: Missing VALUE attribute on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
else if (!innerValue.Equals(String.Empty))
{
Logger.Error("ScriptableScraperProvider: Ambiguous assignment on: {0}", xmlNode.OuterXml);
LoadSuccess = false;
return;
}
}
public string Parse(ScriptableScraper context, string value, string options)
{
// if we have an encoding string try to build an encoding object
Encoding encoding = null;
if (options != string.Empty)
{
try { encoding = Encoding.GetEncoding(options.ToLower()); }
catch (ArgumentException)
{
encoding = null;
logger.Error("Scraper script tried to use an invalid encoding for \"safe\" modifier");
}
}
if (encoding != null)
{
return(HttpUtility.UrlEncode(value, encoding).Replace("+", "%20"));
}
else
{
return(HttpUtility.UrlEncode(value).Replace("+", "%20"));
}
}
public ActionNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public DivideNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public SubtractNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public MultiplyNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public AddNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public MathNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Load attributes
string resultTypeStr = null;
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "value1":
value1 = attr.Value;
break;
case "value2":
value2 = attr.Value;
break;
case "result_type":
resultTypeStr = attr.Value;
break;
}
}
// Validate VALUE1 attribute
if (value1 == null) {
logger.Error("Missing VALUE1 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate VALUE2 attribute
if (value2 == null) {
logger.Error("Missing VALUE2 attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
// Validate RESULT_TYPE attribute
if (resultTypeStr != null && resultTypeStr.ToUpper().Equals("FLOAT"))
resultType = ResultTypeEnum.FLOAT;
else
resultType = ResultTypeEnum.INT;
}
public MultiplyNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public SubtractNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
}
public string Parse(ScriptableScraper context, string value, string options)
{
return HttpUtility.HtmlDecode(value);
}
public static List <string> getScreenUrls(ScriptableScraper scraper, Dictionary <string, string> paramList)
{
List <string> imageResults = new List <string>();
imageResults.Add("");
imageResults.Add("");
if (scraper == null)
{
return(imageResults);
}
Dictionary <string, string> results;
results = scraper.Execute("get_screenshots", paramList);
if (results == null)
{
return(imageResults);
}
string screens = "";
string screenshotsbaseurl = "";
results.TryGetValue("game.screenshotsbaseurl", out screenshotsbaseurl);
if (results.TryGetValue("game.screenshots", out screens))
{
string[] screenurls = screens.Split('|');
bool foundTitleScreen = false;
bool foundInGame = false;
for (int i = 1; i < screenurls.Length; i++)
{
if (foundTitleScreen && foundInGame)
{
break;
}
if (!screenurls[i].ToLower().StartsWith("http://"))
{
screenurls[i] = screenshotsbaseurl + screenurls[i];
}
if (screenurls[i].Contains("screenshot-start-screen"))
{
imageResults[0] = screenurls[i];
foundTitleScreen = true;
}
else if (screenurls[i].Contains("screenshot-title-screen"))
{
if (!foundTitleScreen)
{
imageResults[0] = screenurls[i];
}
}
else if (imageResults[0] == "")
{
imageResults[0] = screenurls[i];
}
else if (imageResults[1] == "")
{
imageResults[1] = screenurls[i];
foundInGame = true;
}
}
}
return(imageResults);
}
public static List <Bitmap> getScreens(ScriptableScraper scraper, Dictionary <string, string> paramList, out Bitmap title, out Bitmap inGame, Conf_OnlineLookup.onProgress progress, int progressStart, int progressEnd, bool unattended)
{
List <Bitmap> imageResults = new List <Bitmap>();
title = null;
inGame = null;
if (scraper == null)
{
return(imageResults);
}
Dictionary <string, string> results;
results = scraper.Execute("get_screenshots", paramList);
if (results == null)
{
return(imageResults);
}
string screens = "";
string screenshotsbaseurl = "";
results.TryGetValue("game.screenshotsbaseurl", out screenshotsbaseurl);
if (results.TryGetValue("game.screenshots", out screens))
{
string[] screenurls = screens.Split('|');
bool foundTitleScreen = false;
bool foundInGame = false;
int increment = progressEnd - progressStart;
if (screenurls.Length > 0)
{
increment = (int)Math.Floor(increment / (double)screenurls.Length);
}
for (int i = 1; i < screenurls.Length; i++)
{
progressStart += increment;
progress.Invoke(string.Format("Getting screenshot {0}/{1}...", i.ToString(), screenurls.Length.ToString()), progressStart);
if (!screenurls[i].ToLower().StartsWith("http://"))
{
screenurls[i] = screenshotsbaseurl + screenurls[i];
}
bool isTitle = false;
bool isInGame = false;
if (!foundTitleScreen || !foundInGame)
{
if (screenurls[i].Contains("screenshot-start-screen"))
{
isTitle = true;
foundTitleScreen = true;
}
else if (screenurls[i].Contains("screenshot-title-screen"))
{
if (!foundTitleScreen)
{
isTitle = true;
}
}
else if (title == null)
{
isTitle = true;
}
else if (inGame == null)
{
isInGame = true;
foundInGame = true;
}
}
if (unattended) //only download image if we need to
{
if (isTitle)
{
title = ImageHandler.Instance.BitmapFromWeb(screenurls[i]);
}
else if (isInGame)
{
inGame = ImageHandler.Instance.BitmapFromWeb(screenurls[i]);
}
if (foundInGame && foundTitleScreen) //best matches found, no point in continuing search
{
break;
}
}
else //if not unattended then download all images to allow user selection
{
Bitmap thePic = ImageHandler.Instance.BitmapFromWeb(screenurls[i]);
imageResults.Add(thePic);
if (isTitle)
{
title = thePic;
}
else if (isInGame)
{
inGame = thePic;
}
}
}
}
return(imageResults);
}
//Background scraper for use in Configuration
public static List <ScraperResult> getSearchResults(ScriptableScraper scraper, Dictionary <string, string> paramList)
{
List <ScraperResult> results = new List <ScraperResult>();
if (scraper == null)
{
return(results);
}
Dictionary <string, string> scraperResults = scraper.Execute("search", paramList);
if (scraperResults == null)
{
return(results);
}
string respage = "";
scraperResults.TryGetValue("search_page", out respage);
int count = 0;
// The movie result is only valid if the script supplies a unique site
while (scraperResults.ContainsKey("game[" + count + "].site_id"))
{
string siteId;
string title;
string yearmade;
string system;
string xsiteId;
string xgamePattern;
string prefix = "game[" + count + "].";
count++;
if (!scraperResults.TryGetValue(prefix + "site_id", out siteId))
{
continue;
}
if (scraperResults.TryGetValue(prefix + "system", out system))
{
system = system.Trim();
}
scraperResults.TryGetValue(prefix + "title", out title);
scraperResults.TryGetValue(prefix + "yearmade", out yearmade);
scraperResults.TryGetValue(prefix + "extraGameUrl", out xsiteId);
scraperResults.TryGetValue(prefix + "extraGamePattern", out xgamePattern);
if (!string.IsNullOrEmpty(xsiteId))
{
siteId = xsiteId;
}
results.Add(new ScraperResult(siteId, title, system, yearmade));
//additional field added as mobygames combines results from different platforms so we need
//to split them to get the correct details/thumbs for our platform
if (!string.IsNullOrEmpty(xgamePattern))
{
//split combined results from mobygames
foreach (Match match in new Regex(@"<a href=""/game/([^""]*)"">([^<]*)</a> \(<em>(\d{4})").Matches(xgamePattern))
{
results.Add(new ScraperResult(match.Groups[1].Value, title, match.Groups[2].Value, match.Groups[3].Value));
}
}
}
return(results);
}
public static List <Bitmap> getCovers(ScriptableScraper scraper, Dictionary <string, string> paramList, out Bitmap front, out Bitmap back, Conf_OnlineLookup.onProgress progress, int progressStart, int progressEnd, bool unattended)
{
List <Bitmap> imageResults = new List <Bitmap>();
front = null;
back = null;
if (scraper == null)
{
return(imageResults);
}
Dictionary <string, string> results = scraper.Execute("get_cover_art", paramList);
if (results == null)
{
return(imageResults);
}
string covers = "";
if (results.TryGetValue("game.covers", out covers))
{
string[] coverurls = covers.Split('|');
//calculate percentage increment for progress bar based on number of results
int increment = progressEnd - progressStart;
if (coverurls.Length > 0)
{
increment = (int)Math.Floor(increment / (double)coverurls.Length);
}
for (int i = 1; i < coverurls.Length; i++)
{
progressStart += increment;
progress.Invoke(string.Format("Getting cover art {0}/{1}...", i.ToString(), coverurls.Length.ToString()), progressStart);
bool isFront = false;
bool isBack = false;
//Bitmap pic = ImageHandler.Instance.BitmapFromWeb(coverurls[i]);
//imageResults.Add(pic);
if (i == 1)
{
isFront = true;
}
else if (i == 2)
{
isBack = true;
}
if (unattended)
{
if (isFront)
{
front = ImageHandler.Instance.BitmapFromWeb(coverurls[i]);
}
else if (isBack)
{
back = ImageHandler.Instance.BitmapFromWeb(coverurls[i]);
}
//no point in getting additional thumbs as import is unattended
if (front != null && back != null)
{
break;
}
}
else
{
Bitmap thePic = ImageHandler.Instance.BitmapFromWeb(coverurls[i]);
imageResults.Add(thePic);
if (isFront)
{
front = thePic;
}
else if (isBack)
{
back = thePic;
}
}
}
}
return(imageResults);
}
public RetrieveNode(XmlNode xmlNode, ScriptableScraper context)
: base(xmlNode, context)
{
// Set default attribute valuess
_useCaching = true;
allowUnsafeHeader = false;
maxRetries = 5;
timeout = 5000;
timeoutIncrement = 2000;
_method = "GET";
acceptLanguage = context.Language;
// Load attributes
foreach (XmlAttribute attr in xmlNode.Attributes) {
switch (attr.Name) {
case "url":
url = attr.Value;
break;
case "file":
file = attr.Value;
break;
case "useragent":
userAgent = attr.Value;
break;
case "accept_language":
acceptLanguage = attr.Value;
break;
case "allow_unsafe_header":
try { allowUnsafeHeader = bool.Parse(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "use_caching":
try { _useCaching = bool.Parse(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "encoding":
// grab encoding, if not specified it will try to set
// the encoding using information from the response header.
try { encoding = Encoding.GetEncoding(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "retries":
try { maxRetries = int.Parse(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "timeout":
try { timeout = int.Parse(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "timeout_increment":
try { timeoutIncrement = int.Parse(attr.Value); }
catch (Exception e) {
if (e.GetType() == typeof(ThreadAbortException))
throw e;
}
break;
case "cookies":
cookies = attr.Value;
break;
case "method":
_method = attr.Value.Trim().ToUpper();
break;
}
}
// Validate URL / FILE attribute
if (url == null && file == null) {
logger.Error("Missing URL or FILE attribute on: " + xmlNode.OuterXml);
loadSuccess = false;
return;
}
}
