public static void ExtractFromUserFavourite(string pChannelName, string pUserId, int pStartIndex)
{
try
{
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
string channelCleanedName = Common.CleanFileName(pChannelName);
//File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//For Debugging
if (ConfigurationManager.AppSettings["ExtractAllVideosFlag"].ToString().Equals("False", StringComparison.InvariantCultureIgnoreCase))
{
int totalVideo = Int32.Parse(ConfigurationManager.AppSettings["totalVideos"].ToString());
if (totalVideo <= recordCount)
{
//Constant.tempFiles.Add(videFileNameXML);
return;
}
}
//string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + pChannelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest;
HttpWebResponse nameResponse;
Stream nameStream;
StreamReader nameReader;
//File.WriteAllText(pChannelName + "/" + channelFileNameXML, xmlData);
//Other type of extraction here
//Extract Playlists
string favouriteUrl = "https://gdata.youtube.com/feeds/api/users/" + pUserId + "/favorites?start-index=" + pStartIndex + "&v=2"; //This will return all Playlists of this user
nameRequest = WebRequest.Create(favouriteUrl);
nameResponse = (HttpWebResponse)nameRequest.GetResponse();
nameStream = nameResponse.GetResponseStream();
nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
XmlDocument doc = new XmlDocument();
//doc.Load(pChannelName + "/" + channelFileNameXML);
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
Constant.tempFiles.Add(channelFileNameXML);
doc = new XmlDocument();
doc.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNamespaceManager openSearchNameSpace = new XmlNamespaceManager(doc.NameTable);
openSearchNameSpace.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
XmlNode totalRecordNode = doc.SelectSingleNode("//openSearch:totalResults", openSearchNameSpace);
if (totalRecordNode.InnerText.Equals("0"))
{
return;
}
if (totalRecordNode != null && !totalRecordNode.InnerText.Equals("0"))
{
XmlNode titleNode = doc.SelectSingleNode("//Atom:title", namespaceManager);
File.AppendAllText(channelCleanedName + "/" + channelFileName, titleNode.InnerText + "\r\n");
Dictionary <string, PlaylistWrapper> dictionaryPlayList = new Dictionary <string, PlaylistWrapper>();
XmlNodeList listNodes = doc.SelectNodes("//Atom:entry", namespaceManager);
if (listNodes.Count == 0)
{
return;
}
StringBuilder strBuilder = new StringBuilder();
strBuilder.Append("\tFavourite Videos:\r\n");
string title = String.Empty;
string key = String.Empty;
string url = String.Empty;
string apiURL = String.Empty;
Dictionary <string, VideoWrapper> dictionaryVideoWrapper = new Dictionary <string, VideoWrapper>();
foreach (XmlNode n in listNodes)
{
foreach (XmlNode node in n.ChildNodes)
{
if (node.Name.Equals("title"))
{
title = node.InnerText;
}
else if (node.Name.Equals("link"))
{
if (node.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
url = node.Attributes["href"].Value.Split(new Char[] { '&' }, StringSplitOptions.RemoveEmptyEntries)[0];
key = url.Split(new Char[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[1];
}
}
}
strBuilder.Append("\t\tVideo Name: " + title + "\r\n");
if (!dictionaryVideoWrapper.ContainsKey(key))
{
VideoWrapper vWrapper = new VideoWrapper();
vWrapper.setVideoKey(key);
vWrapper.setVideoName(title);
vWrapper.setVideoUrl(url);
dictionaryVideoWrapper.Add(key, vWrapper);
updatedFlag = true;
recordCount++;
}
if (updatedFlag)
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, strBuilder.ToString());
}
updatedFlag = false;
strBuilder.Remove(0, strBuilder.Length);
}
ChannelVideo.parseVideo(dictionaryVideoWrapper, pChannelName);
ChannelComment.CrawlComments(dictionaryVideoWrapper, pChannelName);
pStartIndex += 25;
ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
}
Common.RemoveTempFiles(Constant.tempFiles, channelCleanedName);
}
catch (Exception ex)
{
//ExtractFromUserFavourite(pChannelName, pUserId, pStartIndex);
}
}
public static bool ParseChannel(string pChannelName, string pAppName, string pDevKey, int pLevel)
{
///ReInitialize All Data
///
channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
///
multithreadingChannelName = pChannelName;
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
//File.AppendAllText(pChannelName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//This Request will give us 10 channels from index 1, which is searched by adding its name.
//e.g.https://gdata.youtube.com/feeds/api/channels?q=" + pChannelName + "&start-index=1&max-results=10&v=2
//q=<Channel Name>
//start-index = <start Index of Search result> (by default 1st Index is '1')
//max-result = <page size (containing number of channels)>
//v = Not known yet. :S
//e.g.https://gdata.youtube.com/feeds/api/channels?q=" + pChannelName + "&start-index=1&max-results=10&v=2
string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + pChannelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest = WebRequest.Create(channelUrl);
HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
Stream nameStream = nameResponse.GetResponseStream();
StreamReader nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
File.WriteAllText(pChannelName + "/" + channelFileNameXML, xmlData);
XmlDocument doc = new XmlDocument();
doc.Load(pChannelName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNodeList listResult = doc.SelectNodes(channelTitleXPath, namespaceManager);
int count = 0;
foreach (XmlNode node in listResult)
{
count++;
if (node.FirstChild.InnerText.Equals(pChannelName, StringComparison.InvariantCultureIgnoreCase))
{
break;
}
}
XmlNodeList entryNode = doc.SelectSingleNode(channelAtomEntry + "[" + count + "]", namespaceManager).ChildNodes;
foreach (XmlNode n in entryNode)
{
if (n.Name.Equals("title"))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Name: " + n.InnerText + "\r\n");
}
else if (n.Name.Equals("yt:channelStatistics"))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Subscribers Count: " + n.Attributes["subscriberCount"].Value + "\r\n");
File.AppendAllText(pChannelName + "/" + channelFileName, "Views Count: " + n.Attributes["viewCount"].Value + "\r\n");
}
else if (n.Name.Equals("link"))
{
if (n.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel URL: " + n.Attributes["href"].Value + "\r\n");
channelUrlMain = n.Attributes["href"].Value;
}
}
else if (n.Name.Equals("id"))
{
string id = n.InnerText;
string[] arrId = n.InnerText.Split(new Char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
bool indexFound = false;
for (int i = 0; i < arrId.Length; i++)
{
if (arrId[i].Equals("Channel", StringComparison.CurrentCultureIgnoreCase))
{
indexFound = true;
continue;
}
if (indexFound)
{
channelId = arrId[i];
break;
}
}
}
}
//"AI39si7SUChDwy6-ms_bz7rY-mzkqWc9vouhT_XZfh_xery5HjOujHc4USzQJ-M6XeWPCmGtaMzBgs3QP5S4O3vFBHoxmCfIjA"
YouTubeRequestSettings settings = new YouTubeRequestSettings(pAppName, pDevKey);
YouTubeRequest request = new YouTubeRequest(settings);
Uri videoEntryUrl = new Uri("http://gdata.youtube.com/feeds/api/users/" + pChannelName);
Feed <Video> videoFeed = request.Get <Video>(videoEntryUrl);
foreach (Entry e in videoFeed.Entries)
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Description: " + e.Summary + "\r\n");
break;
}
Constant.tempFiles.Add(channelFileNameXML);
//Constant.tempFiles.Add(pChannelName + "/" + log);
Dictionary <string, VideoWrapper> videoDictionary = new Dictionary <string, VideoWrapper>();
File.AppendAllText(pChannelName + "/" + channelFileName, "Video Lists \r\n");
startIndex = 1;
//File.AppendAllText(pChannelName + "/" + log, "\tEntering WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
WriteVideoLists(pChannelName, channelId, startIndex, videoDictionary, Enumeration.VideoRequestType.All);
//File.AppendAllText(pChannelName + "/" + log, "\t\tTotal Dictionary Items : " + videoDictionary.Count + Environment.NewLine);
//File.AppendAllText(pChannelName + "/" + log, "\r\n\tLeft WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//File.AppendAllText(pChannelName + "/" + "Count.txt", "Count After complete Request Response (Expected 1000) : " + recordCount + "\r\n");
//File.AppendAllText(pChannelName + "/" + log, "Leaving Parse Channel at : " + DateTime.Now);
//ChannelVideo.parseVideo(videoDictionary, pChannelName);
///Done Crawling video description
///Crawl Comments
///
ChannelComment.CrawlComments(videoDictionary, pChannelName);
///Done Crawling Comments
///
///Remove All Temporary Files here
///
Common.RemoveTempFiles(Constant.tempFiles, pChannelName);
///Done
///
//Commenting Temporarily on Paolo Request.. <04/19/2013>
if (ConfigurationManager.AppSettings["BothLevelsFlag"].ToString().Equals("true", StringComparison.CurrentCultureIgnoreCase))
{
/////ReInitialize All Data
/////
channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
/////ReInitializing Done
dictionary = new Dictionary <string, VideoCommentWrapper>();
dictionary = GlobalConstants.commentDictionary;
int testCount = 0;
foreach (KeyValuePair <string, VideoCommentWrapper> pair in dictionary)
{
GlobalConstants.commentDictionary = new Dictionary <string, VideoCommentWrapper>();
VideoCommentWrapper videoComment = pair.Value;
ParseChannelLevel2(videoComment, pAppName, pDevKey);
testCount++;
//if (testCount > 3)
// break;
}
}
return(true);
}
public static void ParseChannelLevel2(VideoCommentWrapper commentWrapper, string pAppName, string pDevKey)
{
try
{
string channelName = commentWrapper.userName;//displayName is not working in 1 case.. Got Luv2bbeautiful ., contains space + .
string userId = commentWrapper.userName;
string channelCleanedName = Common.CleanFileName(channelName);
if (!Directory.Exists(channelCleanedName))
{
Directory.CreateDirectory(channelCleanedName);
}
else
{
Directory.Delete(channelCleanedName, true);
Directory.CreateDirectory(channelCleanedName);
}
//Search for Channel...
// If Channel doesn't exit Search for following things
//1. Favourties, 2. Uploaded , 3. Playlist
multithreadingChannelName = channelCleanedName;
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
//File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
string videoUrl = "https://gdata.youtube.com/feeds/api/users/" + channelName + "/uploads?&start-index=" + startIndex;
HttpWebRequest nameRequest1 = (HttpWebRequest)WebRequest.Create(videoUrl);
nameRequest1.KeepAlive = false;
nameRequest1.ProtocolVersion = HttpVersion.Version10;
HttpWebResponse nameResponse1 = (HttpWebResponse)nameRequest1.GetResponse();
Stream nameStream1 = nameResponse1.GetResponseStream();
StreamReader nameReader1 = new StreamReader(nameStream1);
string xmlData1 = nameReader1.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData1);
//Constant.tempFiles.Add(channelFileNameXML);
XmlDocument doc1 = new XmlDocument();
doc1.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager2 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager2.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
//XmlNodeList listResult1 = doc1.SelectNodes(channelAtomEntry, namespaceManager2);
////Getting total Record
XmlNamespaceManager namespaceManager3 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager3.AddNamespace("openSearch", "http://a9.com/-/spec/opensearchrss/1.0/");
XmlNode nodeTotal = doc1.SelectSingleNode("//openSearch:totalResults", namespaceManager3);
int total = Int32.Parse(nodeTotal.InnerText);
if (total != 0)
{
string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + channelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest = WebRequest.Create(channelUrl);
HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
Stream nameStream = nameResponse.GetResponseStream();
StreamReader nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
XmlDocument doc = new XmlDocument();
doc.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager1 = new XmlNamespaceManager(doc.NameTable);
namespaceManager1.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
XmlNode node1 = doc.SelectSingleNode("//openSearch:totalResults", namespaceManager1);
//This means Channel Exists
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNodeList listResult = doc.SelectNodes(channelTitleXPath, namespaceManager);
int count = 0;
foreach (XmlNode node in listResult)
{
count++;
if (node.FirstChild.InnerText.Equals(channelName, StringComparison.InvariantCultureIgnoreCase))
{
break;
}
}
XmlNodeList entryNode = doc.SelectSingleNode(channelAtomEntry + "[" + count + "]", namespaceManager).ChildNodes;
foreach (XmlNode n in entryNode)
{
if (n.Name.Equals("title"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Name: " + n.InnerText + "\r\n");
}
else if (n.Name.Equals("yt:channelStatistics"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Subscribers Count: " + n.Attributes["subscriberCount"].Value + "\r\n");
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Views Count: " + n.Attributes["viewCount"].Value + "\r\n");
}
//else if (n.Name.Equals("summary"))
//{
// File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Description: " + n.InnerText + "\r\n");
//}
else if (n.Name.Equals("link"))
{
if (n.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel URL: " + n.Attributes["href"].Value + "\r\n");
channelUrlMain = n.Attributes["href"].Value;
}
}
else if (n.Name.Equals("id"))
{
string id = n.InnerText;
string[] arrId = n.InnerText.Split(new Char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
bool indexFound = false;
for (int i = 0; i < arrId.Length; i++)
{
if (arrId[i].Equals("Channel", StringComparison.CurrentCultureIgnoreCase))
{
indexFound = true;
continue;
}
if (indexFound)
{
channelId = arrId[i];
break;
}
}
}
}
//"AI39si7SUChDwy6-ms_bz7rY-mzkqWc9vouhT_XZfh_xery5HjOujHc4USzQJ-M6XeWPCmGtaMzBgs3QP5S4O3vFBHoxmCfIjA"
YouTubeRequestSettings settings = new YouTubeRequestSettings(pAppName, pDevKey);
YouTubeRequest request = new YouTubeRequest(settings);
Uri videoEntryUrl = new Uri("http://gdata.youtube.com/feeds/api/users/" + channelName);
Feed <Video> videoFeed = request.Get <Video>(videoEntryUrl);
foreach (Entry e in videoFeed.Entries)
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Description: " + e.Summary + "\r\n");
break;
}
Constant.tempFiles.Add(channelFileNameXML);
Dictionary <string, VideoWrapper> videoDictionary = new Dictionary <string, VideoWrapper>();
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Video Lists \r\n");
startIndex = 1;
//File.AppendAllText(channelCleanedName + "/" + log, "\tEntering WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
WriteVideoLists(channelName, channelId, startIndex, videoDictionary, Enumeration.VideoRequestType.All);
//File.AppendAllText(channelCleanedName + "/" + log, "\t\tTotal Dictionary Items : " + videoDictionary.Count + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + log, "\r\n\tLeft WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + "Count.txt", "Count After complete Request Response (Expected 1000) : " + recordCount + "\r\n");
//File.AppendAllText(channelCleanedName + "/" + log, "Leaving Parse Channel at : " + DateTime.Now);
///Crawl Comments
///
ChannelComment.CrawlComments(videoDictionary, channelName);
///Done Crawling Comments
//ChannelVideo.parseVideo(videoDictionary, channelName);
///Done Crawling video description
///Remove All Temporary Files here
///
Common.RemoveTempFiles(Constant.tempFiles, channelName);
///Done
///
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
}
//Extract Playlists
ExtractFromPlaylist(channelName, userId, 1);
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
//Extract Favourites
ExtractFromUserFavourite(channelName, userId, 1);
}
catch (Exception ex)
{
Thread.Sleep(10000);
ParseChannelLevel2(commentWrapper, pAppName, pDevKey);
}
}