Example #1
0
        //Function responsible of clearing database of queries before the crawler can start.
        public void emptyDatabase()
        {
            dbConn dbConnection = new dbConn();

            dbConnection.dbTruncate();
            dbConnection.dbClose();
        }
Example #2
0
        //Get keywords from content
        static public void regexKeywords(string response)
        {
            //Fetch keywords from body
            string keywords = "";
            string pattern  = "(<meta itemprop=\"|<link itemprop=\"thumbnailUrl\")(.*?)\">";

            string videoId     = "(<meta itemprop=\"videoId\" content=\")(.*?)\">";
            string title       = "(<meta itemprop=\"name\" content=\")(.*?)\">";
            string description = "(<meta itemprop=\"description\" content=\")(.*?)\">";
            string genre       = "(<meta itemprop=\"genre\" content=\")(.*?)\">";
            string thumbnail   = "(<link itemprop=\"thumbnailUrl\" href=\")(.*?)\">";

            MatchCollection collection;
            MatchCollection collectionUrl;
            MatchCollection collectionTitle;
            MatchCollection collectionDescription;
            MatchCollection collectionGenre;
            MatchCollection collectionThumbnail;

            dbConn dbConnection = new dbConn();

            try
            {
                collection = Regex.Matches(response, pattern);
                string foundUrl         = "https://www.youtube.com/watch?v=";
                string foundTitle       = "";
                string foundDescription = "";
                string foundGenre       = "";
                string foundThumbnail   = "";
                foreach (Match m in collection)
                {
                    //Check matched resulsts
                    keywords = m.Value;

                    collectionUrl = Regex.Matches(m.Value, videoId);
                    foreach (Match m2 in collectionUrl)
                    {
                        foundUrl += m2.Groups[2].Value;
                    }
                    collectionTitle = Regex.Matches(m.Value, title);
                    foreach (Match m2 in collectionTitle)
                    {
                        foundTitle = m2.Groups[2].Value;
                    }
                    collectionDescription = Regex.Matches(m.Value, description);
                    foreach (Match m2 in collectionDescription)
                    {
                        foundDescription = m2.Groups[2].Value;
                    }
                    collectionGenre = Regex.Matches(m.Value, genre);
                    foreach (Match m2 in collectionGenre)
                    {
                        foundGenre = m2.Groups[2].Value;
                    }
                    collectionThumbnail = Regex.Matches(m.Value, thumbnail);
                    foreach (Match m2 in collectionThumbnail)
                    {
                        foundThumbnail = m2.Groups[2].Value;
                    }
                }

                //keywords in database
                dbConnection.insertQuery("INSERT INTO video(Url, Title, Description, Genre, Thumbnail) VALUES('" + foundUrl + "', '" + foundTitle + "', '" + foundDescription + "', '" + foundGenre + "', '" + foundThumbnail + "')");
            }
            catch (NullReferenceException e)
            {
                Debug.WriteLine("getKeywords() geeft NullReferenceException: " + e.Message);
            }
            dbConnection.dbClose();
        }