private void ExtendVideo(string videoId, bool useCache, int level)
{
Interlocked.Decrement(ref _newlyFoundVideosCount);
if (level >= 5)
{
return;
}
try
{
if (useCache)
{
if (_videoCache.Contains(videoId))
{
return;
}
}
C.WriteLine("working.videoId[{0}]:{1}", level, videoId);
var video = Service.GetVideo(videoId);
if (video == null)
{
_videoCache.Add(videoId);
return;
}
if (useCache)
{
if (_videoCache.Contains(videoId))
{
return;
}
}
SqlStorage.Save(video);
_videoCache.Add(videoId);
if (video.NextVideos != null)
{
Interlocked.Add(ref _newlyFoundVideosCount, video.NextVideos?.Count ?? 0);
C.WriteLine("--- working.Found :" + _newlyFoundVideosCount);
foreach (var nv in video.NextVideos)
{
if (_videoCache.Contains(nv.Id))
{
continue;
}
var nvId = nv.Id;
var levelCopy = level + 1;
ThreadPool.QueueUserWorkItem(x => ExtendVideo(nvId, true, levelCopy));
}
}
}
catch (Exception e)
{
lock (_syncException)
{
C.WriteLine("ERROR:" + videoId + " " + e.Message);
File.AppendAllLines("errors.txt", new[] { videoId });
}
}
}
public void Browse(string channelId)
{
//lock (sync)
{
if (_cache.Contains(channelId))
{
return;
}
C.WriteLine(" queue.size::" + _queue.Count);
C.WriteLine("browsing channel with Id: " + channelId);
var channel = Service.GetChannelInfo(channelId);
SqlStorage.Save(channel);
if (_cache.Contains(channelId))
{
return;
}
_cache.Add(channelId);
if (channel.FriendChannels != null)
{
foreach (var info in channel.FriendChannels)
{
if (!_cache.Contains(info.Id))
{
C.WriteLine("queueing related channel ... " + info.Id);
_queue.Add(new KeyValuePair <string, string>(info.Id, channelId));
}
}
}
}
}
public void AnalyzeChannelsOnDb()
{
//var channelIdList = Sql.Execute<string>("select Id from Channels");
var channelIdList =
Sql.Execute <string>(
"select ChannelId from Videos with (NOLOCK) where ChannelId not in (select DISTINCT(Id) from Channels) ORDER BY NEWID()")
.Distinct().ToArray();
C.WriteLine("found channels " + channelIdList.Length);
var videoIds = Sql.Execute <string>("select Id from Videos");
foreach (var channelId in channelIdList)
{
try
{
var channel = Service.GetChannelInfo(channelId);
if (channel == null)
{
continue;
}
SqlStorage.Save(channel);
C.WriteLine("started channel : " + channel.Name);
if (channel.Uploads == null)
{
continue;
}
foreach (var v in channel.Uploads)
{
if (string.IsNullOrEmpty(v.Id))
{
continue;
}
if (videoIds.Contains(v.Id))
{
continue;
}
var video = Service.GetVideo(v.Id);
if (video != null)
{
SqlStorage.Save(video);
}
else
{
C.WriteLine("video not found with id `{0}`", v.Id);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
C.WriteLine("finished all channels");
}
public void BrowseOnlyChannels(IYoutubeService service)
{
//var oldChannelsInDb = Sql.Execute<string>("select Id from Channels");
var channelId = "UCjsdYhMIQ0-fJPqEjiHYlzA"; //efe aydal
//channelId = "UCv6jcPwFujuTIwFQ11jt1Yw";
_queue.Add(new KeyValuePair <string, string>(channelId, channelId));
new Thread(WorkerThread).Start();
YoutubeChannelInfo channel;
//browse channels for popular categories
//returns pairs as following: category name and list of channels
var channels = service.BrowseChannels().ToArray();
foreach (var keyValuePair in channels)
{
foreach (var channelInfo in keyValuePair.Value)
{
channel = service.GetChannelInfo(channelInfo.Id);
SqlStorage.Save(channel);
}
}
}
public void BrowseAllVideosComments()
{
C.WriteLine("loading videos");
var videoIds = Sql
.Execute <string>(
"select top(1000) Id from Videos where Id not in (select DISTINCT(VideoId) from Comments ) order by NEWID() ")
.ToList();
C.WriteLine("loaded videos");
var count = 0;
foreach (var videoId in videoIds)
{
try
{
if (string.IsNullOrEmpty(videoId))
{
return;
}
C.WriteLine("started video with Id: " + videoId);
var commentsBatch = Service.GetRootComments(videoId);
foreach (var comments in commentsBatch)
{
try
{
count += comments.Length;
C.Write(comments.Length + "|");
foreach (var comment in comments)
{
comment.VideoId = videoId;
}
if (comments.Any(c => string.IsNullOrEmpty(c.VideoId)))
{
;
}
SqlStorage.Save(comments);
}
catch (Exception e)
{
try
{
File.AppendAllLines("error.comments", new[] { videoId });
}
catch (Exception exception)
{
C.WriteLine("\t\t" + exception.Message);
}
}
}
}
catch (WebException e)
{
if (e.Message != "The remote server returned an error: (413) Request Entity Too Large.")
{
continue;
}
}
}