public async Task <Video> HandleVideo(VideoUrls videoUrls, string blogName, TraceWriter log)
{
CloudBlobContainer serverContainer = cloudBlobClient.GetContainerReference(blogName.ToLower());
if (!await serverContainer.ExistsAsync())
{
await serverContainer.CreateAsync();
// Set the permissions so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await serverContainer.SetPermissionsAsync(permissions);
}
using (HttpClient httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("video/*"));
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*"));
CloudBlockBlob videoBlob = await HandleVideoBlob(videoUrls.VideoUrl, serverContainer, httpClient, log);
CloudBlockBlob thumbBlob = await HandleThumbBlob(videoUrls.VideoThumbUrl, serverContainer, httpClient);
return(new Video {
Url = videoBlob.Uri.ToString(), ThumbUrl = thumbBlob.Uri.ToString(), Bytes = videoBlob.Properties.Length
});
}
}
private static async Task <VideoUrls> GetInstagramVideo(string url)
{
using (HttpClient httpClient = new HttpClient())
{
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(content);
List <HtmlNode> metaNodes = htmlDoc.DocumentNode.Descendants("meta").ToList();
VideoUrls videoUrls = new VideoUrls();
foreach (HtmlNode node in metaNodes)
{
if (node.Attributes["property"] != null && node.Attributes["property"].Value.Equals("og:image"))
{
videoUrls.VideoThumbUrl = node.Attributes["content"].Value;
}
else if (node.Attributes["property"] != null && node.Attributes["property"].Value.Equals("og:video"))
{
videoUrls.VideoUrl = node.Attributes["content"].Value;
}
}
if (videoUrls.VideoUrl != null && videoUrls.VideoThumbUrl != null)
{
return(videoUrls);
}
}
}
return(null);
}
private void ExecuteAction()
{
//Since CefSharp Evaluate "document.getElementByClasName('a_class_name')" return null so we need to use Regular Expression
foreach (var item in URLCollection)
{
Executer.Load(item);
Thread.Sleep(10000);
string source = Executer.GetSourceAsync().Result;
Regex regex = new Regex(@"<script type=""text/javascript"">var(.*?)</script>", RegexOptions.Singleline);
string videoElementContainer = regex.Match(source).Value;
//Google APIs
if (videoElementContainer.IndexOf("googleapis") != -1)
{
VideoUrls.Add(new VideoUrlsInformation(item, "Found Google APIs (this is old server 1 which is down in mid 2017, rip a really good server to watch anime :( ) all video (all mean video in Anime47.com) in Google APIs already be removed in mid 2017 so we can't get video from this server, The process will stop, Would you like to download from Openload.co? "));
OnExecute_Complete();
return;
}
//default server which is //video.xx.fbcdn.net
Regex getVideoURL = new Regex(@"//(.*?)'", RegexOptions.Singleline);
string VideoURL = getVideoURL.Match(videoElementContainer).Value.Replace("'", "");
//Zing Tv
if (videoElementContainer.IndexOf(".zadn.vn") != -1)
{
getVideoURL = new Regex(@"//(.*?)""", RegexOptions.Singleline);
VideoURL = getVideoURL.Match(videoElementContainer).Value.Replace("\"", "");
}
VideoUrls.Add(new VideoUrlsInformation(item, VideoURL));
}
OnExecute_Complete();
}
public List <VideoUrls> GetVideoUrls(HtmlDocument htmlDocument, TraceWriter log)
{
List <HtmlNode> videoNodes = htmlDocument.DocumentNode.Descendants("video").ToList();
List <VideoUrls> videoUrlsList = new List <VideoUrls>();
foreach (HtmlNode videoNode in videoNodes)
{
VideoUrls videoUrls = new VideoUrls();
if (videoNode.Attributes["poster"] != null && !string.IsNullOrEmpty(videoNode.Attributes["poster"].Value))
{
videoUrls.VideoThumbUrl = videoNode.Attributes["poster"].Value;
}
if (videoNode.Attributes["src"] != null && !string.IsNullOrEmpty(videoNode.Attributes["src"].Value))
{
videoUrls.VideoUrl = videoNode.Attributes["src"].Value;
}
else
{
HtmlNode sourceNode = videoNode.Descendants("source").FirstOrDefault();
if (sourceNode?.Attributes["src"] != null && !string.IsNullOrEmpty(sourceNode.Attributes["src"].Value))
{
videoUrls.VideoUrl = sourceNode.Attributes["src"].Value;
}
}
if (videoUrls.VideoUrl != null && videoUrls.VideoThumbUrl != null)
{
videoUrlsList.Add(videoUrls);
}
else
{
if (videoUrls.VideoUrl == null)
{
log.Warning("Missing video url for video thumb url: " + videoUrls.VideoThumbUrl);
}
else
{
log.Warning("Missing thumb url for video url: " + videoUrls.VideoUrl);
}
}
}
return(videoUrlsList);
}
public async Task ProcessPosts(IEnumerable <Post> posts, TraceWriter log, string likerBlogname = null)
{
foreach (Post post in posts)
{
SanitizePostPhotos(post); // sometimes post.Photos has Alt_sizes with length 0, needs to be sanitized
PostEntity postEntityInTable = postsTableAdapter.GetPost(post.Blog_name, post.Id.ToString());
PostEntity postEntityFromTumblr = new PostEntity(post);
if (!postsTableAdapter.InsertPost(postEntityFromTumblr))
{
break;
}
if (likerBlogname != null && post.Liked_Timestamp.HasValue)
{
likeIndexTableAdapter.InsertLikeIndex(likerBlogname, post.Liked_Timestamp.ToString(), post.Blog_name, post.Id.ToString(), post.Reblog_key);
}
log.Info("Post " + post.Blog_name + "/" + post.Id + " inserted to table");
PhotosToDownload photosToDownloadMessage = null;
if (postEntityFromTumblr.PhotosJson != null)
{
if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
{
photosToDownloadMessage = new PhotosToDownload(post)
{
Photos = post.Photos
};
}
else
{
log.Info("Photos already downloaded");
}
}
List <VideoUrls> videoUrlsList = new List <VideoUrls>();
if (post.Content != null && post.Content.Length > 0)
{
List <Photo> photos = new List <Photo>(post.Content.Length);
foreach (Content content in post.Content)
{
if (content.Type == "image")
{
Photo photo = ConvertContentToPhoto(content);
photos.Add(photo);
}
else if (content.Type == "video" && content.Url != null && content.Poster != null)
{
VideoUrls videoUrls = new VideoUrls
{
VideoUrl = content.Url,
VideoThumbUrl = content.Poster.OrderBy(x => x.Width).LastOrDefault()?.Url
};
videoUrlsList.Add(videoUrls);
}
}
if (photos.Count > 0)
{
UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
}
}
if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
{
if (!string.IsNullOrEmpty(post.Video_url))
{
VideoUrls videoUrls = new VideoUrls
{
VideoUrl = post.Video_url,
VideoThumbUrl = post.Thumbnail_url
};
videoUrlsList.Add(videoUrls);
}
if (post.Player != null && post.Player.Length > 0 && post.Video_type.Equals("instagram", StringComparison.OrdinalIgnoreCase))
{
Player largestPlayer = post.Player.OrderBy(x => x.Width).Last();
HtmlDocument playerHtmlDoc = new HtmlDocument();
playerHtmlDoc.LoadHtml(largestPlayer.Embed_code);
HtmlNode blockquoteNode = playerHtmlDoc.DocumentNode.Descendants("blockquote")
.FirstOrDefault(x => !string.IsNullOrEmpty(x.Attributes["data-instgrm-permalink"].Value));
if (blockquoteNode != null)
{
string url = blockquoteNode.Attributes["data-instgrm-permalink"].Value;
VideoUrls videoUrls = await GetInstagramVideo(url);
if (videoUrls != null)
{
videoUrlsList.Add(videoUrls);
}
}
}
}
if (!string.IsNullOrEmpty(post.Body))
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(post.Body);
if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
{
List <Photo> photos = ExctractPhotosFromHtml(htmlDoc);
if (photos.Count > 0)
{
photosToDownloadMessage = UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
}
}
if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
{
List <VideoUrls> videoUrlsListFromBody = GetVideoUrls(htmlDoc, log);
videoUrlsList.AddRange(videoUrlsListFromBody);
}
}
if (photosToDownloadMessage != null)
{
queueAdapter.SendPhotosToDownload(photosToDownloadMessage);
log.Info("PhotosToDownload message published");
}
if (videoUrlsList.Count > 0)
{
VideosToDownload videosToDownload = new VideosToDownload(post)
{
VideoUrls = videoUrlsList.ToArray()
};
queueAdapter.SendVideosToDownload(videosToDownload);
log.Info("VideosToDownload message published");
}
}
}
