public static void DownloadHtmls(string pChannelName, VideoWrapper pVideo, Dictionary <int, string> pHtmlFiles, int pPageNo)
{
string url = string.Empty;
try
{
url = ConfigurationManager.AppSettings["VideoAllCommentsUrl"].ToString() + pVideo.getVideoKey() + "&page=" + pPageNo;
//string url = "http://www.youtube.com/all_comments?v=LMiNEC1M-zY" + "&page=" + pPageNo;
///Base Case
///
HtmlWeb hwObject = new HtmlWeb();
//hwObject.UseCookies = false; // Experimental
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " going to hit URL at page # " + pPageNo + ".. " + DateTime.Now + Environment.NewLine);
HtmlDocument doc = hwObject.Load(url);
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " got response of page # " + pPageNo + ".." + DateTime.Now + Environment.NewLine);
HtmlNodeCollection totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']");
if (totalCollection == null)
{
return;
}
int totalCollectionCount = totalCollection.Count;
if (totalCollectionCount <= 0)
{
return;
}
///Base Case Ended
///
//Code Added by Me Right Now ....
///
totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']//div[@class='content']");
string videoUrl = "https://www.youtube.com/watch?v=" + pVideo.getVideoKey();
bool videoUrlFlag = false;
bool breakLoop = false;
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " starting to extract data.." + Environment.NewLine);
foreach (HtmlNode node in totalCollection)
{
//string[] userArr = node.InnerText.Split(new Char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
string user = string.Empty;
string displayName = string.Empty;
string date = string.Empty;
string comment = string.Empty;
HtmlNode nodeData = node.ParentNode;
string dataId = nodeData.Attributes[2].Value.Trim();
string authorId = nodeData.Attributes[1].Value.Trim();
HtmlNodeCollection childNodes = node.ChildNodes;
int divCount = 0;
foreach (HtmlNode child in childNodes)
{
if (child.Name.Equals("p"))
{
bool userFlag = false;
//bool dateFlag = false;
HtmlNodeCollection col = child.ChildNodes;
foreach (HtmlNode n in col)
{
if (n.Name.Equals("span") && !userFlag)
{
foreach (HtmlNode nNode in n.ChildNodes)
{
if (nNode.Name.Equals("a"))
{
user = nNode.Attributes["href"].Value.Split(new Char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1];
break;
}
}
displayName = n.InnerText.Trim();
userFlag = true;
}
else if (n.Name.Equals("span"))
{
date = n.InnerText.Trim();
//dateFlag = true;
break;
}
}
}
else if (child.Name.Equals("div"))
{
if (divCount == 0)
{
//That means Its Comment Text
comment = child.InnerText.Trim();
divCount++;
}
}
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " starting to write data in file.." + Environment.NewLine);
if (!displayName.Equals("") && !comment.Equals("") && !dataId.Equals("") && !authorId.Equals("") && !user.Equals("") && !GlobalConstants.commentDictionary.ContainsKey(dataId))
{
VideoCommentWrapper commentWrapper = new VideoCommentWrapper();
commentWrapper.authorId = authorId;
commentWrapper.commentId = dataId;
commentWrapper.commentText = comment;
commentWrapper.time = date;
commentWrapper.displayName = displayName;
commentWrapper.userName = user;
GlobalConstants.commentDictionary.Add(dataId, commentWrapper);
string videoFileName = pVideo.getVideoName();
//videoFile = videoName;
string videoName = Common.CleanFileName(videoFileName + "-" + fileComment) + ".txt";
if (!Directory.Exists(pChannelName + "/" + "Comments"))
{
Directory.CreateDirectory(pChannelName + "/" + "Comments");
}
commentCount++;
if (!videoUrlFlag)
{
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Video Url : " + videoUrl + Environment.NewLine + "\r\n");
videoUrlFlag = true;
}
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "User name : " + displayName + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment Date : " + date + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment : " + comment + Environment.NewLine);
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " ended writing data in file.." + Environment.NewLine);
if (parseAllComments.Equals("false", StringComparison.CurrentCultureIgnoreCase))
{
if (totalCommentsParse <= commentCount)
{
breakLoop = true;
break;
}
}
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " extracted all data.." + Environment.NewLine);
////Ended Added
////Commented by Me
//File.AppendAllText(pChannelName + "/CommentsTimeLog.txt", "Start Download Time for file : " + pVideo.getVideoName() + "-" + pPageNo + ": " + DateTime.Now + Environment.NewLine);
//WebRequest nameRequest = WebRequest.Create(url);
//HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
//Stream nameStream = nameResponse.GetResponseStream();
//StreamReader nameReader = new StreamReader(nameStream);
//string htmlData = nameReader.ReadToEnd();
//if (htmlData != null && !htmlData.Equals(""))
//{
// string videoName = pChannelName + "/Comments/" + Common.CleanFileName(pVideo.getVideoName()) + "-" + pPageNo + ".html";
// string dictionaryValue = Common.CleanFileName(pVideo.getVideoName()) + "-" + pPageNo + ".html";
// if (!Directory.Exists(pChannelName + "/Comments/"))
// {
// Directory.CreateDirectory(pChannelName + "/Comments/");
// }
// File.WriteAllText(videoName, htmlData);
// File.AppendAllText(pChannelName + "/CommentsTimeLog.txt", "End Download Time for file : " + pVideo.getVideoName() + "-" + pPageNo + ": " + DateTime.Now + Environment.NewLine + Environment.NewLine);
// //tempFiles.Add("/Comments/" + dictionaryValue);
// pHtmlFiles.Add(pPageNo, dictionaryValue);
//}
////Comment Ended
pPageNo++;
if (parseAllComments.Equals("true", StringComparison.CurrentCultureIgnoreCase))
{
DownloadHtmls(pChannelName, pVideo, pHtmlFiles, pPageNo); //Recursive Call
}
}
catch (Exception ex)
{
//Delete Cookies
//pPageNo++;
//File.AppendAllText(pChannelName + "/Comments/" + "ExceptionLogs.txt", "Exception : at URL : " + url + " -> Exception Message : " + ex.Message);
DownloadHtmls(pChannelName, pVideo, pHtmlFiles, pPageNo);
}
}
public static void GetAllComments(VideoWrapper pVideoWrapper, string pChannelName, Dictionary <int, string> pHtmlFiles)
{
string videoUrl = "https://www.youtube.com/watch?v=" + pVideoWrapper.getVideoKey();
bool videoUrlFlag = false;
List <string> tempFiles = new List <string>();
foreach (KeyValuePair <int, string> pair in pHtmlFiles)
{
try
{
string videoName = pair.Value;
//string videoName = "Machinima PlayStation Viewer's Choice LiveStream!-1";
//Stream stream = File.OpenRead("New folder/Machinima PlayStation Viewer's Choice LiveStream!-1.html");
Stream stream = File.OpenRead(pChannelName + "/Comments/" + videoName);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
StreamReader reader = new StreamReader(stream);
doc.LoadHtml(reader.ReadToEnd().ToString());
bool breakLoop = false;
HtmlNodeCollection totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']//div[@class='content']");
foreach (HtmlNode node in totalCollection)
{
//string[] userArr = node.InnerText.Split(new Char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
string user = string.Empty;
string displayName = string.Empty;
string date = string.Empty;
string comment = string.Empty;
HtmlNode nodeData = node.ParentNode;
string dataId = nodeData.Attributes[2].Value.Trim();
string authorId = nodeData.Attributes[1].Value.Trim();
HtmlNodeCollection childNodes = node.ChildNodes;
int divCount = 0;
foreach (HtmlNode child in childNodes)
{
if (child.Name.Equals("p"))
{
bool userFlag = false;
//bool dateFlag = false;
HtmlNodeCollection col = child.ChildNodes;
foreach (HtmlNode n in col)
{
if (n.Name.Equals("span") && !userFlag)
{
foreach (HtmlNode nNode in n.ChildNodes)
{
if (nNode.Name.Equals("a"))
{
user = nNode.Attributes["href"].Value.Split(new Char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1];
break;
}
}
displayName = n.InnerText.Trim();
userFlag = true;
}
else if (n.Name.Equals("span"))
{
date = n.InnerText.Trim();
//dateFlag = true;
break;
}
}
}
else if (child.Name.Equals("div"))
{
if (divCount == 0)
{
//That means Its Comment Text
comment = child.InnerText.Trim();
divCount++;
}
}
}
if (!displayName.Equals("") && !comment.Equals("") && !dataId.Equals("") && !authorId.Equals("") && !user.Equals("") && !GlobalConstants.commentDictionary.ContainsKey(dataId))
{
VideoCommentWrapper commentWrapper = new VideoCommentWrapper();
commentWrapper.authorId = authorId;
commentWrapper.commentId = dataId;
commentWrapper.commentText = comment;
commentWrapper.time = date;
commentWrapper.displayName = displayName;
commentWrapper.userName = user;
GlobalConstants.commentDictionary.Add(dataId, commentWrapper);
string videoFileName = pVideoWrapper.getVideoName();
//videoFile = videoName;
videoName = Common.CleanFileName(videoFileName + "-" + fileComment) + ".txt";
if (!Directory.Exists(pChannelName + "/" + "Comments"))
{
Directory.CreateDirectory(pChannelName + "/" + "Comments");
}
commentCount++;
if (!videoUrlFlag)
{
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Video Url : " + videoUrl + Environment.NewLine + "\r\n");
videoUrlFlag = true;
}
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "User name : " + displayName + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment Date : " + date + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment : " + comment + Environment.NewLine);
}
if (parseAllComments.Equals("false", StringComparison.CurrentCultureIgnoreCase))
{
if (totalCommentsParse <= commentCount)
{
breakLoop = true;
break;
}
}
}
reader.Close();
if (breakLoop)
{
break;
}
}
catch (Exception ex)
{
//File.AppendAllText("Logs Exception Comments.txt", ex.Message + Environment.NewLine + Environment.NewLine);
continue;
}
}
foreach (KeyValuePair <int, string> file in pHtmlFiles)
{
tempFiles.Add("/Comments/" + file.Value);
}
Common.RemoveTempFiles(tempFiles, pChannelName);
}
public static void ParseChannelLevel2(VideoCommentWrapper commentWrapper, string pAppName, string pDevKey)
{
try
{
string channelName = commentWrapper.userName;//displayName is not working in 1 case.. Got Luv2bbeautiful ., contains space + .
string userId = commentWrapper.userName;
string channelCleanedName = Common.CleanFileName(channelName);
if (!Directory.Exists(channelCleanedName))
{
Directory.CreateDirectory(channelCleanedName);
}
else
{
Directory.Delete(channelCleanedName, true);
Directory.CreateDirectory(channelCleanedName);
}
//Search for Channel...
// If Channel doesn't exit Search for following things
//1. Favourties, 2. Uploaded , 3. Playlist
multithreadingChannelName = channelCleanedName;
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
//File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
string videoUrl = "https://gdata.youtube.com/feeds/api/users/" + channelName + "/uploads?&start-index=" + startIndex;
HttpWebRequest nameRequest1 = (HttpWebRequest)WebRequest.Create(videoUrl);
nameRequest1.KeepAlive = false;
nameRequest1.ProtocolVersion = HttpVersion.Version10;
HttpWebResponse nameResponse1 = (HttpWebResponse)nameRequest1.GetResponse();
Stream nameStream1 = nameResponse1.GetResponseStream();
StreamReader nameReader1 = new StreamReader(nameStream1);
string xmlData1 = nameReader1.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData1);
//Constant.tempFiles.Add(channelFileNameXML);
XmlDocument doc1 = new XmlDocument();
doc1.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager2 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager2.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
//XmlNodeList listResult1 = doc1.SelectNodes(channelAtomEntry, namespaceManager2);
////Getting total Record
XmlNamespaceManager namespaceManager3 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager3.AddNamespace("openSearch", "http://a9.com/-/spec/opensearchrss/1.0/");
XmlNode nodeTotal = doc1.SelectSingleNode("//openSearch:totalResults", namespaceManager3);
int total = Int32.Parse(nodeTotal.InnerText);
if (total != 0)
{
string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + channelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest = WebRequest.Create(channelUrl);
HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
Stream nameStream = nameResponse.GetResponseStream();
StreamReader nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
XmlDocument doc = new XmlDocument();
doc.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager1 = new XmlNamespaceManager(doc.NameTable);
namespaceManager1.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
XmlNode node1 = doc.SelectSingleNode("//openSearch:totalResults", namespaceManager1);
//This means Channel Exists
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNodeList listResult = doc.SelectNodes(channelTitleXPath, namespaceManager);
int count = 0;
foreach (XmlNode node in listResult)
{
count++;
if (node.FirstChild.InnerText.Equals(channelName, StringComparison.InvariantCultureIgnoreCase))
{
break;
}
}
XmlNodeList entryNode = doc.SelectSingleNode(channelAtomEntry + "[" + count + "]", namespaceManager).ChildNodes;
foreach (XmlNode n in entryNode)
{
if (n.Name.Equals("title"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Name: " + n.InnerText + "\r\n");
}
else if (n.Name.Equals("yt:channelStatistics"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Subscribers Count: " + n.Attributes["subscriberCount"].Value + "\r\n");
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Views Count: " + n.Attributes["viewCount"].Value + "\r\n");
}
//else if (n.Name.Equals("summary"))
//{
// File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Description: " + n.InnerText + "\r\n");
//}
else if (n.Name.Equals("link"))
{
if (n.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel URL: " + n.Attributes["href"].Value + "\r\n");
channelUrlMain = n.Attributes["href"].Value;
}
}
else if (n.Name.Equals("id"))
{
string id = n.InnerText;
string[] arrId = n.InnerText.Split(new Char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
bool indexFound = false;
for (int i = 0; i < arrId.Length; i++)
{
if (arrId[i].Equals("Channel", StringComparison.CurrentCultureIgnoreCase))
{
indexFound = true;
continue;
}
if (indexFound)
{
channelId = arrId[i];
break;
}
}
}
}
//"AI39si7SUChDwy6-ms_bz7rY-mzkqWc9vouhT_XZfh_xery5HjOujHc4USzQJ-M6XeWPCmGtaMzBgs3QP5S4O3vFBHoxmCfIjA"
YouTubeRequestSettings settings = new YouTubeRequestSettings(pAppName, pDevKey);
YouTubeRequest request = new YouTubeRequest(settings);
Uri videoEntryUrl = new Uri("http://gdata.youtube.com/feeds/api/users/" + channelName);
Feed<Video> videoFeed = request.Get<Video>(videoEntryUrl);
foreach (Entry e in videoFeed.Entries)
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Description: " + e.Summary + "\r\n");
break;
}
Constant.tempFiles.Add(channelFileNameXML);
Dictionary<string, VideoWrapper> videoDictionary = new Dictionary<string, VideoWrapper>();
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Video Lists \r\n");
startIndex = 1;
//File.AppendAllText(channelCleanedName + "/" + log, "\tEntering WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
WriteVideoLists(channelName, channelId, startIndex, videoDictionary, Enumeration.VideoRequestType.All);
//File.AppendAllText(channelCleanedName + "/" + log, "\t\tTotal Dictionary Items : " + videoDictionary.Count + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + log, "\r\n\tLeft WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + "Count.txt", "Count After complete Request Response (Expected 1000) : " + recordCount + "\r\n");
//File.AppendAllText(channelCleanedName + "/" + log, "Leaving Parse Channel at : " + DateTime.Now);
///Crawl Comments
///
ChannelComment.CrawlComments(videoDictionary, channelName);
///Done Crawling Comments
//ChannelVideo.parseVideo(videoDictionary, channelName);
///Done Crawling video description
///Remove All Temporary Files here
///
Common.RemoveTempFiles(Constant.tempFiles, channelName);
///Done
///
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
}
//Extract Playlists
ExtractFromPlaylist(channelName, userId, 1);
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
//Extract Favourites
ExtractFromUserFavourite(channelName, userId, 1);
}
catch (Exception ex)
{
Thread.Sleep(10000);
ParseChannelLevel2(commentWrapper, pAppName, pDevKey);
}
}
public static bool ParseChannel(string pChannelName, string pAppName, string pDevKey, int pLevel)
{
///ReInitialize All Data
///
channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
///
multithreadingChannelName = pChannelName;
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
//File.AppendAllText(pChannelName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//This Request will give us 10 channels from index 1, which is searched by adding its name.
//e.g.https://gdata.youtube.com/feeds/api/channels?q=" + pChannelName + "&start-index=1&max-results=10&v=2
//q=<Channel Name>
//start-index = <start Index of Search result> (by default 1st Index is '1')
//max-result = <page size (containing number of channels)>
//v = Not known yet. :S
//e.g.https://gdata.youtube.com/feeds/api/channels?q=" + pChannelName + "&start-index=1&max-results=10&v=2
string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + pChannelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest = WebRequest.Create(channelUrl);
HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
Stream nameStream = nameResponse.GetResponseStream();
StreamReader nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
File.WriteAllText(pChannelName + "/" + channelFileNameXML, xmlData);
XmlDocument doc = new XmlDocument();
doc.Load(pChannelName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNodeList listResult = doc.SelectNodes(channelTitleXPath, namespaceManager);
int count = 0;
foreach (XmlNode node in listResult)
{
count++;
if (node.FirstChild.InnerText.Equals(pChannelName, StringComparison.InvariantCultureIgnoreCase))
{
break;
}
}
XmlNodeList entryNode = doc.SelectSingleNode(channelAtomEntry + "[" + count + "]", namespaceManager).ChildNodes;
foreach (XmlNode n in entryNode)
{
if (n.Name.Equals("title"))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Name: " + n.InnerText + "\r\n");
}
else if (n.Name.Equals("yt:channelStatistics"))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Subscribers Count: " + n.Attributes["subscriberCount"].Value + "\r\n");
File.AppendAllText(pChannelName + "/" + channelFileName, "Views Count: " + n.Attributes["viewCount"].Value + "\r\n");
}
else if (n.Name.Equals("link"))
{
if (n.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel URL: " + n.Attributes["href"].Value + "\r\n");
channelUrlMain = n.Attributes["href"].Value;
}
}
else if (n.Name.Equals("id"))
{
string id = n.InnerText;
string[] arrId = n.InnerText.Split(new Char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
bool indexFound = false;
for (int i = 0; i < arrId.Length; i++)
{
if (arrId[i].Equals("Channel", StringComparison.CurrentCultureIgnoreCase))
{
indexFound = true;
continue;
}
if (indexFound)
{
channelId = arrId[i];
break;
}
}
}
}
//"AI39si7SUChDwy6-ms_bz7rY-mzkqWc9vouhT_XZfh_xery5HjOujHc4USzQJ-M6XeWPCmGtaMzBgs3QP5S4O3vFBHoxmCfIjA"
YouTubeRequestSettings settings = new YouTubeRequestSettings(pAppName, pDevKey);
YouTubeRequest request = new YouTubeRequest(settings);
Uri videoEntryUrl = new Uri("http://gdata.youtube.com/feeds/api/users/" + pChannelName);
Feed <Video> videoFeed = request.Get <Video>(videoEntryUrl);
foreach (Entry e in videoFeed.Entries)
{
File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Description: " + e.Summary + "\r\n");
break;
}
Constant.tempFiles.Add(channelFileNameXML);
//Constant.tempFiles.Add(pChannelName + "/" + log);
Dictionary <string, VideoWrapper> videoDictionary = new Dictionary <string, VideoWrapper>();
File.AppendAllText(pChannelName + "/" + channelFileName, "Video Lists \r\n");
startIndex = 1;
//File.AppendAllText(pChannelName + "/" + log, "\tEntering WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
WriteVideoLists(pChannelName, channelId, startIndex, videoDictionary, Enumeration.VideoRequestType.All);
//File.AppendAllText(pChannelName + "/" + log, "\t\tTotal Dictionary Items : " + videoDictionary.Count + Environment.NewLine);
//File.AppendAllText(pChannelName + "/" + log, "\r\n\tLeft WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//File.AppendAllText(pChannelName + "/" + "Count.txt", "Count After complete Request Response (Expected 1000) : " + recordCount + "\r\n");
//File.AppendAllText(pChannelName + "/" + log, "Leaving Parse Channel at : " + DateTime.Now);
//ChannelVideo.parseVideo(videoDictionary, pChannelName);
///Done Crawling video description
///Crawl Comments
///
ChannelComment.CrawlComments(videoDictionary, pChannelName);
///Done Crawling Comments
///
///Remove All Temporary Files here
///
Common.RemoveTempFiles(Constant.tempFiles, pChannelName);
///Done
///
//Commenting Temporarily on Paolo Request.. <04/19/2013>
if (ConfigurationManager.AppSettings["BothLevelsFlag"].ToString().Equals("true", StringComparison.CurrentCultureIgnoreCase))
{
/////ReInitialize All Data
/////
channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
/////ReInitializing Done
dictionary = new Dictionary <string, VideoCommentWrapper>();
dictionary = GlobalConstants.commentDictionary;
int testCount = 0;
foreach (KeyValuePair <string, VideoCommentWrapper> pair in dictionary)
{
GlobalConstants.commentDictionary = new Dictionary <string, VideoCommentWrapper>();
VideoCommentWrapper videoComment = pair.Value;
ParseChannelLevel2(videoComment, pAppName, pDevKey);
testCount++;
//if (testCount > 3)
// break;
}
}
return(true);
}
public static void ParseChannelLevel2(VideoCommentWrapper commentWrapper, string pAppName, string pDevKey)
{
try
{
string channelName = commentWrapper.userName;//displayName is not working in 1 case.. Got Luv2bbeautiful ., contains space + .
string userId = commentWrapper.userName;
string channelCleanedName = Common.CleanFileName(channelName);
if (!Directory.Exists(channelCleanedName))
{
Directory.CreateDirectory(channelCleanedName);
}
else
{
Directory.Delete(channelCleanedName, true);
Directory.CreateDirectory(channelCleanedName);
}
//Search for Channel...
// If Channel doesn't exit Search for following things
//1. Favourties, 2. Uploaded , 3. Playlist
multithreadingChannelName = channelCleanedName;
string channelFileName = ConfigurationManager.AppSettings["channelsFileName"].ToString();
string channelFileNameXML = ConfigurationManager.AppSettings["channelsFileNameXML"].ToString();
//File.AppendAllText(channelCleanedName + "/" + log, "Entered Inside Parse Channel at : " + DateTime.Now + Environment.NewLine + Environment.NewLine);
string videoUrl = "https://gdata.youtube.com/feeds/api/users/" + channelName + "/uploads?&start-index=" + startIndex;
HttpWebRequest nameRequest1 = (HttpWebRequest)WebRequest.Create(videoUrl);
nameRequest1.KeepAlive = false;
nameRequest1.ProtocolVersion = HttpVersion.Version10;
HttpWebResponse nameResponse1 = (HttpWebResponse)nameRequest1.GetResponse();
Stream nameStream1 = nameResponse1.GetResponseStream();
StreamReader nameReader1 = new StreamReader(nameStream1);
string xmlData1 = nameReader1.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData1);
//Constant.tempFiles.Add(channelFileNameXML);
XmlDocument doc1 = new XmlDocument();
doc1.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager2 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager2.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
//XmlNodeList listResult1 = doc1.SelectNodes(channelAtomEntry, namespaceManager2);
////Getting total Record
XmlNamespaceManager namespaceManager3 = new XmlNamespaceManager(doc1.NameTable);
namespaceManager3.AddNamespace("openSearch", "http://a9.com/-/spec/opensearchrss/1.0/");
XmlNode nodeTotal = doc1.SelectSingleNode("//openSearch:totalResults", namespaceManager3);
int total = Int32.Parse(nodeTotal.InnerText);
if (total != 0)
{
string channelUrl = ConfigurationManager.AppSettings["ChannelSearchUrl"].ToString() + channelName + "&start-index=1&max-results=10&v=2";
WebRequest nameRequest = WebRequest.Create(channelUrl);
HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
Stream nameStream = nameResponse.GetResponseStream();
StreamReader nameReader = new StreamReader(nameStream);
string xmlData = nameReader.ReadToEnd();
File.WriteAllText(channelCleanedName + "/" + channelFileNameXML, xmlData);
XmlDocument doc = new XmlDocument();
doc.Load(channelCleanedName + "/" + channelFileNameXML);
XmlNamespaceManager namespaceManager1 = new XmlNamespaceManager(doc.NameTable);
namespaceManager1.AddNamespace("openSearch", "http://a9.com/-/spec/opensearch/1.1/");
XmlNode node1 = doc.SelectSingleNode("//openSearch:totalResults", namespaceManager1);
//This means Channel Exists
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(doc.NameTable);
namespaceManager.AddNamespace("Atom", "http://www.w3.org/2005/Atom");
XmlNodeList listResult = doc.SelectNodes(channelTitleXPath, namespaceManager);
int count = 0;
foreach (XmlNode node in listResult)
{
count++;
if (node.FirstChild.InnerText.Equals(channelName, StringComparison.InvariantCultureIgnoreCase))
{
break;
}
}
XmlNodeList entryNode = doc.SelectSingleNode(channelAtomEntry + "[" + count + "]", namespaceManager).ChildNodes;
foreach (XmlNode n in entryNode)
{
if (n.Name.Equals("title"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Name: " + n.InnerText + "\r\n");
}
else if (n.Name.Equals("yt:channelStatistics"))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Subscribers Count: " + n.Attributes["subscriberCount"].Value + "\r\n");
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Views Count: " + n.Attributes["viewCount"].Value + "\r\n");
}
//else if (n.Name.Equals("summary"))
//{
// File.AppendAllText(pChannelName + "/" + channelFileName, "Channel Description: " + n.InnerText + "\r\n");
//}
else if (n.Name.Equals("link"))
{
if (n.Attributes["rel"].Value.Equals("alternate", StringComparison.CurrentCultureIgnoreCase))
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel URL: " + n.Attributes["href"].Value + "\r\n");
channelUrlMain = n.Attributes["href"].Value;
}
}
else if (n.Name.Equals("id"))
{
string id = n.InnerText;
string[] arrId = n.InnerText.Split(new Char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
bool indexFound = false;
for (int i = 0; i < arrId.Length; i++)
{
if (arrId[i].Equals("Channel", StringComparison.CurrentCultureIgnoreCase))
{
indexFound = true;
continue;
}
if (indexFound)
{
channelId = arrId[i];
break;
}
}
}
}
//"AI39si7SUChDwy6-ms_bz7rY-mzkqWc9vouhT_XZfh_xery5HjOujHc4USzQJ-M6XeWPCmGtaMzBgs3QP5S4O3vFBHoxmCfIjA"
YouTubeRequestSettings settings = new YouTubeRequestSettings(pAppName, pDevKey);
YouTubeRequest request = new YouTubeRequest(settings);
Uri videoEntryUrl = new Uri("http://gdata.youtube.com/feeds/api/users/" + channelName);
Feed <Video> videoFeed = request.Get <Video>(videoEntryUrl);
foreach (Entry e in videoFeed.Entries)
{
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Channel Description: " + e.Summary + "\r\n");
break;
}
Constant.tempFiles.Add(channelFileNameXML);
Dictionary <string, VideoWrapper> videoDictionary = new Dictionary <string, VideoWrapper>();
File.AppendAllText(channelCleanedName + "/" + channelFileName, "Video Lists \r\n");
startIndex = 1;
//File.AppendAllText(channelCleanedName + "/" + log, "\tEntering WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
WriteVideoLists(channelName, channelId, startIndex, videoDictionary, Enumeration.VideoRequestType.All);
//File.AppendAllText(channelCleanedName + "/" + log, "\t\tTotal Dictionary Items : " + videoDictionary.Count + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + log, "\r\n\tLeft WriteVideoList at: " + DateTime.Now + Environment.NewLine + Environment.NewLine);
//File.AppendAllText(channelCleanedName + "/" + "Count.txt", "Count After complete Request Response (Expected 1000) : " + recordCount + "\r\n");
//File.AppendAllText(channelCleanedName + "/" + log, "Leaving Parse Channel at : " + DateTime.Now);
///Crawl Comments
///
ChannelComment.CrawlComments(videoDictionary, channelName);
///Done Crawling Comments
//ChannelVideo.parseVideo(videoDictionary, channelName);
///Done Crawling video description
///Remove All Temporary Files here
///
Common.RemoveTempFiles(Constant.tempFiles, channelName);
///Done
///
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
}
//Extract Playlists
ExtractFromPlaylist(channelName, userId, 1);
///ReInitialize All Data
///
//channelName = "";
channelId = "";
startIndex = 1;
recordCount = 0;
updatedFlag = false;
///ReInitializing Done
//Extract Favourites
ExtractFromUserFavourite(channelName, userId, 1);
}
catch (Exception ex)
{
Thread.Sleep(10000);
ParseChannelLevel2(commentWrapper, pAppName, pDevKey);
}
}
public static void GetAllComments(VideoWrapper pVideoWrapper, string pChannelName, Dictionary<int, string> pHtmlFiles)
{
string videoUrl = "https://www.youtube.com/watch?v=" + pVideoWrapper.getVideoKey();
bool videoUrlFlag = false;
List<string> tempFiles = new List<string>();
foreach (KeyValuePair<int, string> pair in pHtmlFiles)
{
try
{
string videoName = pair.Value;
//string videoName = "Machinima PlayStation Viewer's Choice LiveStream!-1";
//Stream stream = File.OpenRead("New folder/Machinima PlayStation Viewer's Choice LiveStream!-1.html");
Stream stream = File.OpenRead(pChannelName + "/Comments/" + videoName);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
StreamReader reader = new StreamReader(stream);
doc.LoadHtml(reader.ReadToEnd().ToString());
bool breakLoop = false;
HtmlNodeCollection totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']//div[@class='content']");
foreach (HtmlNode node in totalCollection)
{
//string[] userArr = node.InnerText.Split(new Char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
string user = string.Empty;
string displayName = string.Empty;
string date = string.Empty;
string comment = string.Empty;
HtmlNode nodeData = node.ParentNode;
string dataId = nodeData.Attributes[2].Value.Trim();
string authorId = nodeData.Attributes[1].Value.Trim();
HtmlNodeCollection childNodes = node.ChildNodes;
int divCount = 0;
foreach (HtmlNode child in childNodes)
{
if (child.Name.Equals("p"))
{
bool userFlag = false;
//bool dateFlag = false;
HtmlNodeCollection col = child.ChildNodes;
foreach (HtmlNode n in col)
{
if (n.Name.Equals("span") && !userFlag)
{
foreach (HtmlNode nNode in n.ChildNodes)
{
if (nNode.Name.Equals("a"))
{
user = nNode.Attributes["href"].Value.Split(new Char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1];
break;
}
}
displayName = n.InnerText.Trim();
userFlag = true;
}
else if (n.Name.Equals("span"))
{
date = n.InnerText.Trim();
//dateFlag = true;
break;
}
}
}
else if (child.Name.Equals("div"))
{
if (divCount == 0)
{
//That means Its Comment Text
comment = child.InnerText.Trim();
divCount++;
}
}
}
if (!displayName.Equals("") && !comment.Equals("") && !dataId.Equals("") && !authorId.Equals("") && !user.Equals("") && !GlobalConstants.commentDictionary.ContainsKey(dataId))
{
VideoCommentWrapper commentWrapper = new VideoCommentWrapper();
commentWrapper.authorId = authorId;
commentWrapper.commentId = dataId;
commentWrapper.commentText = comment;
commentWrapper.time = date;
commentWrapper.displayName = displayName;
commentWrapper.userName = user;
GlobalConstants.commentDictionary.Add(dataId, commentWrapper);
string videoFileName = pVideoWrapper.getVideoName();
//videoFile = videoName;
videoName = Common.CleanFileName(videoFileName + "-" + fileComment) + ".txt";
if (!Directory.Exists(pChannelName + "/" + "Comments"))
{
Directory.CreateDirectory(pChannelName + "/" + "Comments");
}
commentCount++;
if (!videoUrlFlag)
{
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Video Url : " + videoUrl + Environment.NewLine + "\r\n");
videoUrlFlag = true;
}
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "User name : " + displayName + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment Date : " + date + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment : " + comment + Environment.NewLine);
}
if (parseAllComments.Equals("false", StringComparison.CurrentCultureIgnoreCase))
{
if (totalCommentsParse <= commentCount)
{
breakLoop = true;
break;
}
}
}
reader.Close();
if (breakLoop)
break;
}
catch (Exception ex)
{
//File.AppendAllText("Logs Exception Comments.txt", ex.Message + Environment.NewLine + Environment.NewLine);
continue;
}
}
foreach (KeyValuePair<int, string> file in pHtmlFiles)
{
tempFiles.Add("/Comments/" + file.Value);
}
Common.RemoveTempFiles(tempFiles, pChannelName);
}
public static void DownloadHtmls(string pChannelName, VideoWrapper pVideo, Dictionary<int, string> pHtmlFiles, int pPageNo)
{
string url = string.Empty;
try
{
url = ConfigurationManager.AppSettings["VideoAllCommentsUrl"].ToString() + pVideo.getVideoKey() + "&page=" + pPageNo;
//string url = "http://www.youtube.com/all_comments?v=LMiNEC1M-zY" + "&page=" + pPageNo;
///Base Case
///
HtmlWeb hwObject = new HtmlWeb();
//hwObject.UseCookies = false; // Experimental
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " going to hit URL at page # " + pPageNo + ".. " + DateTime.Now + Environment.NewLine);
HtmlDocument doc = hwObject.Load(url);
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " got response of page # " + pPageNo + ".." + DateTime.Now + Environment.NewLine);
HtmlNodeCollection totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']");
if (totalCollection == null)
return;
int totalCollectionCount = totalCollection.Count;
if (totalCollectionCount <= 0)
return;
///Base Case Ended
///
//Code Added by Me Right Now ....
///
totalCollection = doc.DocumentNode.SelectNodes("//ul[@id='all-comments']//li[@class='comment']//div[@class='content']");
string videoUrl = "https://www.youtube.com/watch?v=" + pVideo.getVideoKey();
bool videoUrlFlag = false;
bool breakLoop = false;
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " starting to extract data.." + Environment.NewLine);
foreach (HtmlNode node in totalCollection)
{
//string[] userArr = node.InnerText.Split(new Char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
string user = string.Empty;
string displayName = string.Empty;
string date = string.Empty;
string comment = string.Empty;
HtmlNode nodeData = node.ParentNode;
string dataId = nodeData.Attributes[2].Value.Trim();
string authorId = nodeData.Attributes[1].Value.Trim();
HtmlNodeCollection childNodes = node.ChildNodes;
int divCount = 0;
foreach (HtmlNode child in childNodes)
{
if (child.Name.Equals("p"))
{
bool userFlag = false;
//bool dateFlag = false;
HtmlNodeCollection col = child.ChildNodes;
foreach (HtmlNode n in col)
{
if (n.Name.Equals("span") && !userFlag)
{
foreach (HtmlNode nNode in n.ChildNodes)
{
if (nNode.Name.Equals("a"))
{
user = nNode.Attributes["href"].Value.Split(new Char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1];
break;
}
}
displayName = n.InnerText.Trim();
userFlag = true;
}
else if (n.Name.Equals("span"))
{
date = n.InnerText.Trim();
//dateFlag = true;
break;
}
}
}
else if (child.Name.Equals("div"))
{
if (divCount == 0)
{
//That means Its Comment Text
comment = child.InnerText.Trim();
divCount++;
}
}
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " starting to write data in file.." + Environment.NewLine);
if (!displayName.Equals("") && !comment.Equals("") && !dataId.Equals("") && !authorId.Equals("") && !user.Equals("") && !GlobalConstants.commentDictionary.ContainsKey(dataId))
{
VideoCommentWrapper commentWrapper = new VideoCommentWrapper();
commentWrapper.authorId = authorId;
commentWrapper.commentId = dataId;
commentWrapper.commentText = comment;
commentWrapper.time = date;
commentWrapper.displayName = displayName;
commentWrapper.userName = user;
GlobalConstants.commentDictionary.Add(dataId, commentWrapper);
string videoFileName = pVideo.getVideoName();
//videoFile = videoName;
string videoName = Common.CleanFileName(videoFileName + "-" + fileComment) + ".txt";
if (!Directory.Exists(pChannelName + "/" + "Comments"))
{
Directory.CreateDirectory(pChannelName + "/" + "Comments");
}
commentCount++;
if (!videoUrlFlag)
{
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Video Url : " + videoUrl + Environment.NewLine + "\r\n");
videoUrlFlag = true;
}
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "User name : " + displayName + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment Date : " + date + Environment.NewLine);
File.AppendAllText(pChannelName + "/" + "Comments" + "/" + videoName, "Comment : " + comment + Environment.NewLine);
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " ended writing data in file.." + Environment.NewLine);
if (parseAllComments.Equals("false", StringComparison.CurrentCultureIgnoreCase))
{
if (totalCommentsParse <= commentCount)
{
breakLoop = true;
break;
}
}
}
//File.AppendAllText("ThreadsLog.txt", "Thread " + Thread.CurrentThread.GetHashCode() + " extracted all data.." + Environment.NewLine);
////Ended Added
////Commented by Me
//File.AppendAllText(pChannelName + "/CommentsTimeLog.txt", "Start Download Time for file : " + pVideo.getVideoName() + "-" + pPageNo + ": " + DateTime.Now + Environment.NewLine);
//WebRequest nameRequest = WebRequest.Create(url);
//HttpWebResponse nameResponse = (HttpWebResponse)nameRequest.GetResponse();
//Stream nameStream = nameResponse.GetResponseStream();
//StreamReader nameReader = new StreamReader(nameStream);
//string htmlData = nameReader.ReadToEnd();
//if (htmlData != null && !htmlData.Equals(""))
//{
// string videoName = pChannelName + "/Comments/" + Common.CleanFileName(pVideo.getVideoName()) + "-" + pPageNo + ".html";
// string dictionaryValue = Common.CleanFileName(pVideo.getVideoName()) + "-" + pPageNo + ".html";
// if (!Directory.Exists(pChannelName + "/Comments/"))
// {
// Directory.CreateDirectory(pChannelName + "/Comments/");
// }
// File.WriteAllText(videoName, htmlData);
// File.AppendAllText(pChannelName + "/CommentsTimeLog.txt", "End Download Time for file : " + pVideo.getVideoName() + "-" + pPageNo + ": " + DateTime.Now + Environment.NewLine + Environment.NewLine);
// //tempFiles.Add("/Comments/" + dictionaryValue);
// pHtmlFiles.Add(pPageNo, dictionaryValue);
//}
////Comment Ended
pPageNo++;
if(parseAllComments.Equals("true", StringComparison.CurrentCultureIgnoreCase))
DownloadHtmls(pChannelName, pVideo, pHtmlFiles, pPageNo); //Recursive Call
}
catch (Exception ex)
{
//Delete Cookies
//pPageNo++;
//File.AppendAllText(pChannelName + "/Comments/" + "ExceptionLogs.txt", "Exception : at URL : " + url + " -> Exception Message : " + ex.Message);
DownloadHtmls(pChannelName, pVideo, pHtmlFiles, pPageNo);
}
}
