示例#1
0
        public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
        {
            var releases = new List <ReleaseInfo>();

            searchBlock Search = Definition.Search;

            // init template context
            var variables = getTemplateVariablesFromConfigData();

            variables[".Query.Type"]       = query.QueryType;
            variables[".Query.Q"]          = query.SearchTerm;
            variables[".Query.Series"]     = null;
            variables[".Query.Ep"]         = query.Episode;
            variables[".Query.Season"]     = query.Season;
            variables[".Query.Movie"]      = null;
            variables[".Query.Year"]       = null;
            variables[".Query.Limit"]      = query.Limit;
            variables[".Query.Offset"]     = query.Offset;
            variables[".Query.Extended"]   = query.Extended;
            variables[".Query.Categories"] = query.Categories;
            variables[".Query.APIKey"]     = query.ApiKey;
            variables[".Query.TVDBID"]     = null;
            variables[".Query.TVRageID"]   = query.RageID;
            variables[".Query.IMDBID"]     = query.ImdbID;
            variables[".Query.TVMazeID"]   = null;
            variables[".Query.TraktID"]    = null;

            variables[".Query.Episode"] = query.GetEpisodeSearchString();
            variables[".Categories"]    = MapTorznabCapsToTrackers(query);

            var KeywordTokens    = new List <string>();
            var KeywordTokenKeys = new List <string> {
                "Q", "Series", "Movie", "Year"
            };

            foreach (var key in KeywordTokenKeys)
            {
                var Value = (string)variables[".Query." + key];
                if (!string.IsNullOrWhiteSpace(Value))
                {
                    KeywordTokens.Add(Value);
                }
            }

            if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
            {
                KeywordTokens.Add((string)variables[".Query.Episode"]);
            }
            variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
            variables[".Keywords"]       = variables[".Query.Keywords"];

            // build search URL
            var searchUrl       = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
            var queryCollection = new NameValueCollection();

            if (Search.Inputs != null)
            {
                foreach (var Input in Search.Inputs)
                {
                    var value = applyGoTemplateText(Input.Value, variables);
                    if (Input.Key == "$raw")
                    {
                        searchUrl += value;
                    }
                    else
                    {
                        queryCollection.Add(Input.Key, value);
                    }
                }
            }
            searchUrl += "&" + queryCollection.GetQueryString();

            // send HTTP request
            var response = await RequestBytesWithCookies(searchUrl);

            var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);

            try
            {
                var SearchResultParser   = new HtmlParser();
                var SearchResultDocument = SearchResultParser.Parse(results);

                var             RowsDom = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
                List <IElement> Rows    = new List <IElement>();
                foreach (var RowDom in RowsDom)
                {
                    Rows.Add(RowDom);
                }

                // merge following rows for After selector
                var After = Definition.Search.Rows.After;
                if (After > 0)
                {
                    for (int i = 0; i < Rows.Count; i += 1)
                    {
                        var CurrentRow = Rows[i];
                        for (int j = 0; j < After; j += 1)
                        {
                            var          MergeRowIndex = i + j + 1;
                            var          MergeRow      = Rows[MergeRowIndex];
                            List <INode> MergeNodes    = new List <INode>();
                            foreach (var node in MergeRow.QuerySelectorAll("td"))
                            {
                                MergeNodes.Add(node);
                            }
                            CurrentRow.Append(MergeNodes.ToArray());
                        }
                        Rows.RemoveRange(i + 1, After);
                    }
                }

                foreach (var Row in Rows)
                {
                    try
                    {
                        var release = new ReleaseInfo();
                        release.MinimumRatio    = 1;
                        release.MinimumSeedTime = 48 * 60 * 60;

                        // Parse fields
                        foreach (var Field in Search.Fields)
                        {
                            string value = handleSelector(Field.Value, Row);
                            value = ParseUtil.NormalizeSpace(value);
                            try
                            {
                                switch (Field.Key)
                                {
                                case "download":
                                    if (value.StartsWith("magnet:"))
                                    {
                                        release.MagnetUri = new Uri(value);
                                        release.Link      = release.MagnetUri;
                                    }
                                    else
                                    {
                                        release.Link = resolvePath(value);
                                    }
                                    break;

                                case "details":
                                    var url = resolvePath(value);
                                    release.Guid = url;
                                    if (release.Comments == null)
                                    {
                                        release.Comments = url;
                                    }
                                    break;

                                case "comments":
                                    var CommentsUrl = resolvePath(value);
                                    release.Comments = CommentsUrl;
                                    if (release.Guid == null)
                                    {
                                        release.Guid = CommentsUrl;
                                    }
                                    break;

                                case "title":
                                    release.Title = value;
                                    break;

                                case "description":
                                    release.Description = value;
                                    break;

                                case "category":
                                    release.Category = MapTrackerCatToNewznab(value);
                                    break;

                                case "size":
                                    release.Size = ReleaseInfo.GetBytes(value);
                                    break;

                                case "leechers":
                                    if (release.Peers == null)
                                    {
                                        release.Peers = ParseUtil.CoerceInt(value);
                                    }
                                    else
                                    {
                                        release.Peers += ParseUtil.CoerceInt(value);
                                    }
                                    break;

                                case "seeders":
                                    release.Seeders = ParseUtil.CoerceInt(value);
                                    if (release.Peers == null)
                                    {
                                        release.Peers = release.Seeders;
                                    }
                                    else
                                    {
                                        release.Peers += release.Seeders;
                                    }
                                    break;

                                case "date":
                                    release.PublishDate = DateTimeUtil.FromUnknown(value);
                                    break;

                                case "files":
                                    release.Files = ParseUtil.CoerceLong(value);
                                    break;

                                case "grabs":
                                    release.Grabs = ParseUtil.CoerceLong(value);
                                    break;

                                case "downloadvolumefactor":
                                    release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
                                    break;

                                case "uploadvolumefactor":
                                    release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
                                    break;

                                default:
                                    break;
                                }
                            }
                            catch (Exception ex)
                            {
                                throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
                            }
                        }

                        // if DateHeaders is set go through the previous rows and look for the header selector
                        var DateHeaders = Definition.Search.Rows.Dateheaders;
                        if (DateHeaders != null)
                        {
                            var    PrevRow = Row.PreviousElementSibling;
                            string value   = null;
                            while (PrevRow != null)
                            {
                                try
                                {
                                    value = handleSelector(DateHeaders, PrevRow);
                                    break;
                                }
                                catch (Exception ex)
                                {
                                    // do nothing
                                }
                                PrevRow = PrevRow.PreviousElementSibling;
                            }

                            if (value == null)
                            {
                                throw new Exception(string.Format("No date header row found for {0}", release.ToString()));
                            }

                            release.PublishDate = DateTimeUtil.FromUnknown(value);
                        }

                        releases.Add(release);
                    }
                    catch (Exception ex)
                    {
                        logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex));
                    }
                }
            }
            catch (Exception ex)
            {
                OnParseError(results, ex);
            }

            return(releases);
        }
示例#2
0
        public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
        {
            var releases = new List <ReleaseInfo>();

            searchBlock Search = Definition.Search;

            // init template context
            var variables = getTemplateVariablesFromConfigData();

            variables[".Query.Type"]       = query.QueryType;
            variables[".Query.Q"]          = query.SearchTerm;
            variables[".Query.Series"]     = null;
            variables[".Query.Ep"]         = query.Episode;
            variables[".Query.Season"]     = query.Season;
            variables[".Query.Movie"]      = null;
            variables[".Query.Year"]       = null;
            variables[".Query.Limit"]      = query.Limit;
            variables[".Query.Offset"]     = query.Offset;
            variables[".Query.Extended"]   = query.Extended;
            variables[".Query.Categories"] = query.Categories;
            variables[".Query.APIKey"]     = query.ApiKey;
            variables[".Query.TVDBID"]     = null;
            variables[".Query.TVRageID"]   = query.RageID;
            variables[".Query.IMDBID"]     = query.ImdbID;
            variables[".Query.TVMazeID"]   = null;
            variables[".Query.TraktID"]    = null;

            variables[".Query.Episode"] = query.GetEpisodeSearchString();
            variables[".Categories"]    = MapTorznabCapsToTrackers(query);

            var KeywordTokens    = new List <string>();
            var KeywordTokenKeys = new List <string> {
                "Q", "Series", "Movie", "Year"
            };

            foreach (var key in KeywordTokenKeys)
            {
                var Value = (string)variables[".Query." + key];
                if (!string.IsNullOrWhiteSpace(Value))
                {
                    KeywordTokens.Add(Value);
                }
            }

            if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
            {
                KeywordTokens.Add((string)variables[".Query.Episode"]);
            }
            variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
            variables[".Keywords"]       = variables[".Query.Keywords"];

            // build search URL
            var searchUrl       = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
            var queryCollection = new NameValueCollection();

            if (Search.Inputs != null)
            {
                foreach (var Input in Search.Inputs)
                {
                    var value = applyGoTemplateText(Input.Value, variables);
                    if (Input.Key == "$raw")
                    {
                        searchUrl += value;
                    }
                    else
                    {
                        queryCollection.Add(Input.Key, value);
                    }
                }
            }
            searchUrl += "&" + queryCollection.GetQueryString();

            // send HTTP request
            var response = await RequestBytesWithCookies(searchUrl);

            var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);

            try
            {
                var SearchResultParser   = new HtmlParser();
                var SearchResultDocument = SearchResultParser.Parse(results);

                var Rows = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
                foreach (var Row in Rows)
                {
                    try
                    {
                        var release = new ReleaseInfo();
                        release.MinimumRatio    = 1;
                        release.MinimumSeedTime = 48 * 60 * 60;

                        // Parse fields
                        foreach (var Field in Search.Fields)
                        {
                            string value = handleSelector(Field.Value, Row);
                            value = ParseUtil.NormalizeSpace(value);
                            try
                            {
                                switch (Field.Key)
                                {
                                case "download":
                                    release.Link = resolvePath(value);
                                    break;

                                case "details":
                                    var url = resolvePath(value);
                                    release.Guid = url;
                                    if (release.Comments == null)
                                    {
                                        release.Comments = url;
                                    }
                                    break;

                                case "comments":
                                    release.Comments = resolvePath(value);
                                    break;

                                case "title":
                                    release.Title = value;
                                    break;

                                case "description":
                                    release.Description = value;
                                    break;

                                case "category":
                                    release.Category = MapTrackerCatToNewznab(value);
                                    break;

                                case "size":
                                    release.Size = ReleaseInfo.GetBytes(value);
                                    break;

                                case "leechers":
                                    if (release.Peers == null)
                                    {
                                        release.Peers = ParseUtil.CoerceInt(value);
                                    }
                                    else
                                    {
                                        release.Peers += ParseUtil.CoerceInt(value);
                                    }
                                    break;

                                case "seeders":
                                    release.Seeders = ParseUtil.CoerceInt(value);
                                    if (release.Peers == null)
                                    {
                                        release.Peers = release.Seeders;
                                    }
                                    else
                                    {
                                        release.Peers += release.Seeders;
                                    }

                                    break;

                                case "date":
                                    release.PublishDate = DateTimeUtil.FromUnknown(value);
                                    break;

                                case "files":
                                    release.Files = ParseUtil.CoerceLong(value);
                                    break;

                                case "grabs":
                                    release.Grabs = ParseUtil.CoerceLong(value);
                                    break;

                                case "downloadvolumefactor":
                                    release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
                                    break;

                                case "uploadvolumefactor":
                                    release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
                                    break;

                                default:
                                    break;
                                }
                            }
                            catch (Exception ex)
                            {
                                throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
                            }
                        }

                        releases.Add(release);
                    }
                    catch (Exception ex)
                    {
                        logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex));
                    }
                }
            }
            catch (Exception ex)
            {
                OnParseError(results, ex);
            }

            return(releases);
        }