public static void Consume(Object obj)
{
try
{
//File.AppendAllText("Consumer.txt", "Consuming : " + QueueLength + Environment.NewLine);
//VideoWrapper video = pair.Value;
VideoWrapper video = obj as VideoWrapper;
ChannelVideo.parseVideo(video, multithreadingChannelName);
///Done Crawling video description
QueueLength--;
Console.WriteLine("consuming" + QueueLength);
if (WaitForComplete)
{
if (QueueLength == 0)
{
Event.Set();
}
}
//break;
}
catch (Exception ex)
{
//File.AppendAllText("Exception In Consume.txt", ex.Message + Environment.NewLine);
}
}
public static void ExtractFromUserFavourite(string pChannelName, string pUserId, int pStartIndex)
{
try
{
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
string channelCleanedName = Common.CleanFileName(pChannelName);
//File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//For Debugging
if (ConfigurationManager.AppSettings["ExtractAllVideosFlag"].ToString().Equals("False", StringComparison.InvariantCultureIgnoreCase))
{
int totalVideo = Int32.Parse(ConfigurationManager.AppSettings["totalVideos"].ToString());
if (totalVideo <= recordCount)
{
//Constant.tempFiles.Add(videFileNameXML);
return;
}
}
//string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + pChannelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest;
HttpWebResponse nameResponse;
Stream nameStream;
StreamReader nameReader;
//File.WriteAllText(pChannelName + "/" + channelFileNameXML, xmlData);
//Other type of extraction here
//Extract Playlists
string favouriteUrl = "https://gdata.youtube.com/feeds/api/users/" + pUserId + "/favorites?start-index=" + pStartIndex + "&v=2"; //This will return all Playlists of this user
nameRequest = WebRequest.Create(favouriteUrl);
nameResponse = (HttpWebResponse)nameRequest.GetResponse();
nameStream = nameResponse.GetResponseStream();
nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
XmlDocument doc = new XmlDocument();
//doc.Load(pChannelName + "/" + channelFileNameXML);
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
Constant.tempFiles.Add(channelFileNameXML);
doc = new XmlDocument();
doc.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNamespaceManager openSearchNameSpace = new XmlNamespaceManager(doc.NameTable);
openSearchNameSpace.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
XmlNode totalRecordNode = doc.SelectSingleNode("//openSearch:totalResults", openSearchNameSpace);
if (totalRecordNode.InnerText.Equals("0"))
{
return;
}
if (totalRecordNode != null && !totalRecordNode.InnerText.Equals("0"))
{
XmlNode titleNode = doc.SelectSingleNode("//Atom:title", namespaceManager);
File.AppendAllText(channelCleanedName + "/" + channelFileName, titleNode.InnerText + "\r\n");
Dictionary <string, PlaylistWrapper> dictionaryPlayList = new Dictionary <string, PlaylistWrapper>();
XmlNodeList listNodes = doc.SelectNodes("//Atom:entry", namespaceManager);
if (listNodes.Count == 0)
{
return;
}
StringBuilder strBuilder = new StringBuilder();
strBuilder.Append("\tFavourite Videos:\r\n");
string title = String.Empty;
string key = String.Empty;
string url = String.Empty;
string apiURL = String.Empty;
Dictionary <string, VideoWrapper> dictionaryVideoWrapper = new Dictionary <string, VideoWrapper>();
foreach (XmlNode n in listNodes)
{
foreach (XmlNode node in n.ChildNodes)
{
if (node.Name.Equals("title"))
{
title = node.InnerText;
}
else if (node.Name.Equals("link"))
{
if (node.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
url = node.Attributes["href"].Value.Split(new Char[] { '&' }, StringSplitOptions.RemoveEmptyEntries)[0];
key = url.Split(new Char[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[1];
}
}
}
strBuilder.Append("\t\tVideo Name: " + title + "\r\n");
if (!dictionaryVideoWrapper.ContainsKey(key))
{
VideoWrapper vWrapper = new VideoWrapper();
vWrapper.setVideoKey(key);
vWrapper.setVideoName(title);
vWrapper.setVideoUrl(url);
dictionaryVideoWrapper.Add(key, vWrapper);
updatedFlag = true;
recordCount++;
}
if (updatedFlag)
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, strBuilder.ToString());
}
updatedFlag = false;
strBuilder.Remove(0, strBuilder.Length);
}
ChannelVideo.parseVideo(dictionaryVideoWrapper, pChannelName);
ChannelComment.CrawlComments(dictionaryVideoWrapper, pChannelName);
pStartIndex += 25;
ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
}
Common.RemoveTempFiles(Constant.tempFiles, channelCleanedName);
}
catch (Exception ex)
{
//ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
}
}
