예제 #1
0
        public static void Consume(Object obj)
        {
            try
            {
                //File.AppendAllText("Consumer.txt", "Consuming : " + QueueLength + Environment.NewLine);
                //VideoWrapper video = pair.Value;

                VideoWrapper video = obj as VideoWrapper;

                ChannelVideo.parseVideo(video, multithreadingChannelName);
                ///Done Crawling video description
                QueueLength--;
                Console.WriteLine("consuming" + QueueLength);
                if (WaitForComplete)
                {
                    if (QueueLength == 0)
                    {
                        Event.Set();
                    }
                }
                //break;
            }
            catch (Exception ex)
            {
                //File.AppendAllText("Exception In Consume.txt", ex.Message + Environment.NewLine);
            }
        }
예제 #2
0
        public static void ExtractFromUserFavourite(string pChannelName, string pUserId, int pStartIndex)
        {
            try
            {
                string channelFileName    = ConfigurationManager.AppSettings["channelsFileName"].ToString();
                string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
                string channelCleanedName = Common.CleanFileName(pChannelName);
                //File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
                //For Debugging
                if (ConfigurationManager.AppSettings["ExtractAllVideosFlag"].ToString().Equals("False", StringComparison.InvariantCultureIgnoreCase))
                {
                    int totalVideo = Int32.Parse(ConfigurationManager.AppSettings["totalVideos"].ToString());
                    if (totalVideo <= recordCount)
                    {
                        //Constant.tempFiles.Add(videFileNameXML);
                        return;
                    }
                }
                //string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + pChannelName + "&start-index=1&max-results=10&v=2";
                WebRequest      nameRequest;
                HttpWebResponse nameResponse;
                Stream          nameStream;
                StreamReader    nameReader;

                //File.WriteAllText(pChannelName + "/" + channelFileNameXML, xmlData);

                //Other type of extraction here
                //Extract Playlists
                string favouriteUrl = "https://gdata.youtube.com/feeds/api/users/" + pUserId + "/favorites?start-index=" + pStartIndex + "&v=2";    //This will return all Playlists of this user
                nameRequest  = WebRequest.Create(favouriteUrl);
                nameResponse = (HttpWebResponse)nameRequest.GetResponse();

                nameStream = nameResponse.GetResponseStream();
                nameReader = new StreamReader(nameStream);

                string xmlData = nameReader.ReadToEnd();

                XmlDocument doc = new XmlDocument();
                //doc.Load(pChannelName + "/" + channelFileNameXML);


                File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
                Constant.tempFiles.Add(channelFileNameXML);

                doc = new XmlDocument();
                doc.Load(channelCleanedName + "/" + channelFileNameXML);
                XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
                namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");

                XmlNamespaceManager openSearchNameSpace = new XmlNamespaceManager(doc.NameTable);
                openSearchNameSpace.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
                XmlNode totalRecordNode = doc.SelectSingleNode("//openSearch:totalResults", openSearchNameSpace);
                if (totalRecordNode.InnerText.Equals("0"))
                {
                    return;
                }
                if (totalRecordNode != null && !totalRecordNode.InnerText.Equals("0"))
                {
                    XmlNode titleNode = doc.SelectSingleNode("//Atom:title", namespaceManager);
                    File.AppendAllText(channelCleanedName + "/" + channelFileName, titleNode.InnerText + "\r\n");
                    Dictionary <string, PlaylistWrapper> dictionaryPlayList = new Dictionary <string, PlaylistWrapper>();

                    XmlNodeList listNodes = doc.SelectNodes("//Atom:entry", namespaceManager);
                    if (listNodes.Count == 0)
                    {
                        return;
                    }
                    StringBuilder strBuilder = new StringBuilder();
                    strBuilder.Append("\tFavourite Videos:\r\n");
                    string title  = String.Empty;
                    string key    = String.Empty;
                    string url    = String.Empty;
                    string apiURL = String.Empty;
                    Dictionary <string, VideoWrapper> dictionaryVideoWrapper = new Dictionary <string, VideoWrapper>();
                    foreach (XmlNode n in listNodes)
                    {
                        foreach (XmlNode node in n.ChildNodes)
                        {
                            if (node.Name.Equals("title"))
                            {
                                title = node.InnerText;
                            }
                            else if (node.Name.Equals("link"))
                            {
                                if (node.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
                                {
                                    url = node.Attributes["href"].Value.Split(new Char[] { '&' }, StringSplitOptions.RemoveEmptyEntries)[0];
                                    key = url.Split(new Char[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[1];
                                }
                            }
                        }


                        strBuilder.Append("\t\tVideo Name: " + title + "\r\n");
                        if (!dictionaryVideoWrapper.ContainsKey(key))
                        {
                            VideoWrapper vWrapper = new VideoWrapper();
                            vWrapper.setVideoKey(key);
                            vWrapper.setVideoName(title);
                            vWrapper.setVideoUrl(url);
                            dictionaryVideoWrapper.Add(key, vWrapper);
                            updatedFlag = true;
                            recordCount++;
                        }
                        if (updatedFlag)
                        {
                            File.AppendAllText(channelCleanedName + "/" + channelFileName, strBuilder.ToString());
                        }
                        updatedFlag = false;
                        strBuilder.Remove(0, strBuilder.Length);
                    }
                    ChannelVideo.parseVideo(dictionaryVideoWrapper, pChannelName);
                    ChannelComment.CrawlComments(dictionaryVideoWrapper, pChannelName);
                    pStartIndex += 25;
                    ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
                }
                Common.RemoveTempFiles(Constant.tempFiles, channelCleanedName);
            }
            catch (Exception ex)
            {
                //ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
            }
        }