Ejemplo n.º 1
0
        private void ExtendVideo(string videoId, bool useCache, int level)
        {
            Interlocked.Decrement(ref _newlyFoundVideosCount);
            if (level >= 5)
            {
                return;
            }
            try
            {
                if (useCache)
                {
                    if (_videoCache.Contains(videoId))
                    {
                        return;
                    }
                }
                C.WriteLine("working.videoId[{0}]:{1}", level, videoId);
                var video = Service.GetVideo(videoId);
                if (video == null)
                {
                    _videoCache.Add(videoId);
                    return;
                }

                if (useCache)
                {
                    if (_videoCache.Contains(videoId))
                    {
                        return;
                    }
                }
                SqlStorage.Save(video);
                _videoCache.Add(videoId);

                if (video.NextVideos != null)
                {
                    Interlocked.Add(ref _newlyFoundVideosCount, video.NextVideos?.Count ?? 0);
                    C.WriteLine("--- working.Found :" + _newlyFoundVideosCount);

                    foreach (var nv in video.NextVideos)
                    {
                        if (_videoCache.Contains(nv.Id))
                        {
                            continue;
                        }
                        var nvId      = nv.Id;
                        var levelCopy = level + 1;
                        ThreadPool.QueueUserWorkItem(x => ExtendVideo(nvId, true, levelCopy));
                    }
                }
            }
            catch (Exception e)
            {
                lock (_syncException)
                {
                    C.WriteLine("ERROR:" + videoId + " " + e.Message);
                    File.AppendAllLines("errors.txt", new[] { videoId });
                }
            }
        }
Ejemplo n.º 2
0
        public void Browse(string channelId)
        {
            //lock (sync)
            {
                if (_cache.Contains(channelId))
                {
                    return;
                }

                C.WriteLine(" queue.size::" + _queue.Count);

                C.WriteLine("browsing channel with Id: " + channelId);
                var channel = Service.GetChannelInfo(channelId);
                SqlStorage.Save(channel);
                if (_cache.Contains(channelId))
                {
                    return;
                }
                _cache.Add(channelId);

                if (channel.FriendChannels != null)
                {
                    foreach (var info in channel.FriendChannels)
                    {
                        if (!_cache.Contains(info.Id))
                        {
                            C.WriteLine("queueing related channel ... " + info.Id);
                            _queue.Add(new KeyValuePair <string, string>(info.Id, channelId));
                        }
                    }
                }
            }
        }
Ejemplo n.º 3
0
        public void AnalyzeChannelsOnDb()
        {
            //var channelIdList = Sql.Execute<string>("select Id from Channels");
            var channelIdList =
                Sql.Execute <string>(
                    "select ChannelId from Videos with (NOLOCK) where ChannelId not in (select DISTINCT(Id) from Channels)  ORDER BY NEWID()")
                .Distinct().ToArray();

            C.WriteLine("found channels " + channelIdList.Length);
            var videoIds = Sql.Execute <string>("select Id from Videos");

            foreach (var channelId in channelIdList)
            {
                try
                {
                    var channel = Service.GetChannelInfo(channelId);
                    if (channel == null)
                    {
                        continue;
                    }

                    SqlStorage.Save(channel);
                    C.WriteLine("started channel : " + channel.Name);
                    if (channel.Uploads == null)
                    {
                        continue;
                    }

                    foreach (var v in channel.Uploads)
                    {
                        if (string.IsNullOrEmpty(v.Id))
                        {
                            continue;
                        }
                        if (videoIds.Contains(v.Id))
                        {
                            continue;
                        }

                        var video = Service.GetVideo(v.Id);

                        if (video != null)
                        {
                            SqlStorage.Save(video);
                        }
                        else
                        {
                            C.WriteLine("video not found with id `{0}`", v.Id);
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                }
            }

            C.WriteLine("finished all channels");
        }
Ejemplo n.º 4
0
        public void BrowseOnlyChannels(IYoutubeService service)
        {
            //var oldChannelsInDb = Sql.Execute<string>("select Id from Channels");

            var channelId = "UCjsdYhMIQ0-fJPqEjiHYlzA"; //efe aydal

            //channelId = "UCv6jcPwFujuTIwFQ11jt1Yw";
            _queue.Add(new KeyValuePair <string, string>(channelId, channelId));
            new Thread(WorkerThread).Start();
            YoutubeChannelInfo channel;
            //browse channels for popular categories
            //returns pairs as following: category name and list of channels
            var channels = service.BrowseChannels().ToArray();

            foreach (var keyValuePair in channels)
            {
                foreach (var channelInfo in keyValuePair.Value)
                {
                    channel = service.GetChannelInfo(channelInfo.Id);
                    SqlStorage.Save(channel);
                }
            }
        }
Ejemplo n.º 5
0
        public void BrowseAllVideosComments()
        {
            C.WriteLine("loading videos");
            var videoIds = Sql
                           .Execute <string>(
                "select top(1000) Id from Videos where Id not in (select DISTINCT(VideoId) from Comments )  order by NEWID() ")
                           .ToList();

            C.WriteLine("loaded videos");

            var count = 0;

            foreach (var videoId in videoIds)
            {
                try
                {
                    if (string.IsNullOrEmpty(videoId))
                    {
                        return;
                    }
                    C.WriteLine("started video with Id: " + videoId);
                    var commentsBatch = Service.GetRootComments(videoId);
                    foreach (var comments in commentsBatch)
                    {
                        try
                        {
                            count += comments.Length;
                            C.Write(comments.Length + "|");
                            foreach (var comment in comments)
                            {
                                comment.VideoId = videoId;
                            }

                            if (comments.Any(c => string.IsNullOrEmpty(c.VideoId)))
                            {
                                ;
                            }

                            SqlStorage.Save(comments);
                        }
                        catch (Exception e)
                        {
                            try
                            {
                                File.AppendAllLines("error.comments", new[] { videoId });
                            }
                            catch (Exception exception)
                            {
                                C.WriteLine("\t\t" + exception.Message);
                            }
                        }
                    }
                }
                catch (WebException e)
                {
                    if (e.Message != "The remote server returned an error: (413) Request Entity Too Large.")
                    {
                        continue;
                    }
                }
            }