public async Task <Crawled> AddCrawledAsync(Crawled crawled)
{
await dbContext.Crawled.AddAsync(crawled);
await dbContext.SaveChangesAsync();
return(crawled);
}
public static VideoDTO CrawledToVideoDTO(Crawled x)
{
VideoDTO videoDTO = new VideoDTO();
videoDTO.Id = x.Id;
videoDTO.ChannelId = x.ChannelId;
videoDTO.ChannelTitle = x.ChannelTitle;
videoDTO.CreatedAt = x.CreatedAt;
videoDTO.Description = x.Description;
videoDTO.FileName = x.FileName;
videoDTO.FullPath = x.FullPath;
videoDTO.Order = x.Order;
videoDTO.Path = x.Path;
videoDTO.PublishedAt = x.PublishedAt;
videoDTO.ThumbnailDefault = x.ThumbnailDefault;
videoDTO.ThumbnailHigh = x.ThumbnailHigh;
videoDTO.ThumbnailMedium = x.ThumbnailMedium;
videoDTO.Title = x.Title;
videoDTO.UpdatedAt = x.UpdatedAt;
videoDTO.VideoId = x.VideoId;
return(videoDTO);
}
public async Task <ResponseDTO <string> > Get(string youtubeUrlId = "", [FromQuery] string inputStr = "undefined", [FromQuery] bool bMakeItPrivate = false)
{
var crawledVideo = youtubeDownloadManager.getCrawledByYoutubeVideoId(youtubeUrlId);
if (crawledVideo != null)
{
return(new ResponseDTO <string>
{
success = false,
data = null,
responseMessage = "alrdy Crawled this video"
});
}
string downloadPath = Config.PhysicalFilePath;
var youtube = new YoutubeClient();
string youtubeLink = $"https://www.youtube.com/watch?v={youtubeUrlId}";
var videoInfo = await youtube.Videos.GetAsync(youtubeLink);
string publishedAt = videoInfo.UploadDate.ToString();
string title = CleanString.RemoveEmojisSChars(videoInfo.Title);
string author = CleanString.RemoveEmojisSChars(videoInfo.Author);
string description = CleanString.RemoveEmojisSChars(videoInfo.Description);
string channelId = videoInfo.ChannelId;
string thumbnailDefault = videoInfo.Thumbnails.StandardResUrl;
string thumbnailMedium = videoInfo.Thumbnails.MediumResUrl;
string thumbnailHigh = videoInfo.Thumbnails.HighResUrl;
var duration = videoInfo.Duration;
var streamManifest = await youtube.Videos.Streams.GetManifestAsync(youtubeUrlId);
var videoStreamInfo = streamManifest.GetVideoOnly().FirstOrDefault(s => s.VideoQualityLabel == "1080p60")
?? streamManifest.GetVideo().WithHighestVideoQuality();
if (System.IO.File.Exists($"{Config.PhysicalFilePath}/{inputStr.Trim()}/{title}.{videoStreamInfo.Container}") ||
System.IO.File.Exists($"{Config.PhysicalFilePath}/{inputStr.Trim()}/{title}.{videoStreamInfo.Container}..stream-1.tmp") ||
System.IO.File.Exists($"{Config.PhysicalFilePath}/{inputStr.Trim()}/{title}.{videoStreamInfo.Container}..stream-2.tmp")
)
{
return(new ResponseDTO <string>
{
success = false,
data = null,
responseMessage = "alrdy downloading"
});
}
var audioStreamInfo = streamManifest.GetAudio().WithHighestBitrate();
var streamInfos = new IStreamInfo[] { audioStreamInfo, videoStreamInfo };
//var streamInfo = streamManifest.GetMuxed().WithHighestVideoQuality();
if (streamInfos != null)
{
if (!System.IO.Directory.Exists($"{downloadPath}/{inputStr.Trim()}"))
{
System.IO.Directory.CreateDirectory($"{downloadPath}/{inputStr.Trim()}");
}
// Get the actual stream
//var stream = await youtube.Videos.Streams.GetAsync(streamInfos);
// Download the stream to file
//await youtube.Videos.Streams.DownloadAsync(streamInfos, $"{downloadPath}/{title}.{streamInfo.Container}");
string timeStr = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
string path = $"{inputStr.Trim()}";
string fileName = $"{timeStr}_{title}.{videoStreamInfo.Container}";
string fullPath = $"{path}/{fileName}";
await youtube.Videos.DownloadAsync(streamInfos, new ConversionRequestBuilder($"{downloadPath}/{fullPath}").Build());
try
{
Crawled crawled = new Crawled
{
ChannelId = channelId,
CreatedAt = DateTime.UtcNow,
Description = description,
FileName = fileName,
FullPath = fullPath,
Path = path,
PublishedAt = publishedAt,
ThumbnailDefault = thumbnailDefault,
ThumbnailHigh = thumbnailHigh,
ThumbnailMedium = thumbnailMedium,
Title = title,
UpdatedAt = DateTime.UtcNow,
VideoId = youtubeUrlId,
Order = (await youtubeDownloadManager.getCrawledMaxOrderAsync()) + 1
};
using (TransactionScope scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
crawled = await youtubeDownloadManager.AddCrawledAsync(crawled);
if (bMakeItPrivate)
{
await youtubeDownloadManager.AddPrivateCrawledAsync(new PrivateCrawled
{
CrawledId = crawled.Id,
Order = await youtubeDownloadManager.getPrivateCrawledMaxOrderAsync() + 1
});
}
else if (!bMakeItPrivate)
{
await youtubeDownloadManager.AddPublicCrawledAsync(new PublicCrawled
{
CrawledId = crawled.Id,
Order = await youtubeDownloadManager.getPublicCrawledMaxOrderAsync() + 1
});
}
scope.Complete();
}
}
catch (Exception e)
{
Console.WriteLine($"!!! {e.Message}");
Console.WriteLine($"!!! {e.InnerException}");
return(new ResponseDTO <string>
{
success = false,
data = null,
responseMessage = e.Message
});
}
}
return(new ResponseDTO <string>
{
success = true,
data = youtubeUrlId,
});
}
public async Task <ResponseDTO <string> > SaveVideoToDisk(string youtubeUrlId = null, [FromQuery] string inputStr = "undefined", [FromQuery] bool bMakeItPrivate = false)
{
inputStr = CleanString.RemoveEmojisSChars(inputStr);
string videoId = youtubeUrlId;
string url = $"https://www.youtube.com/watch?v={videoId}";
var crawledVideo = youtubeDownloadManager.getCrawledByYoutubeVideoId(videoId);
if (crawledVideo != null)
{
return(new ResponseDTO <string>
{
success = false,
data = null,
responseMessage = "alrdy Crawled this video"
});
}
if (!System.IO.Directory.Exists($"{Config.PhysicalFilePath}/{inputStr}"))
{
System.IO.Directory.CreateDirectory($"{Config.PhysicalFilePath}/{inputStr}");
}
var ytdl = new YoutubeDL();
// set the path of the youtube-dl and FFmpeg if they're not in PATH or current directory
ytdl.YoutubeDLPath = $"H:/MyProjects/youtube-dl.exe";
ytdl.FFmpegPath = $"C:/ffmpeg/bin/ffmpeg.exe";
// optional: set a different download folder
ytdl.OutputFolder = $"{Config.PhysicalFilePath}/{inputStr}";
ytdl.RestrictFilenames = true;
// download a video
var data = await ytdl.RunVideoDataFetch(url : url);
string title = data.Data.Title;
string publishedAt = data.Data.UploadDate.ToString();
string author = CleanString.RemoveEmojisSChars(data.Data.Uploader);
string description = data.Data.Description;
string channelId = data.Data.ChannelID;
string thumbnailDefault = data.Data.Thumbnail;
var res = await ytdl.RunVideoDownload(url : url);
// the path of the downloaded file
string path = res.Data;
var splitedPath = path.Split("\\");
string folder = $"{inputStr}";
string fileName = splitedPath[splitedPath.Length - 1];
string fullPath = $"{inputStr}/{fileName}";
string timeStr = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
string newFileName = timeStr + fileName;
string newFullPath = $"{folder}/{newFileName}";
System.IO.File.Move($"{Config.PhysicalFilePath}/{fullPath}", $"{Config.PhysicalFilePath}/{newFullPath}");
try
{
Crawled crawled = new Crawled
{
ChannelId = channelId,
CreatedAt = DateTime.UtcNow,
Description = description,
FileName = newFileName,
FullPath = newFullPath,
Path = folder,
PublishedAt = publishedAt,
ThumbnailDefault = thumbnailDefault,
Title = title,
UpdatedAt = DateTime.UtcNow,
VideoId = videoId,
Order = (await youtubeDownloadManager.getCrawledMaxOrderAsync()) + 1
};
using (TransactionScope scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
crawled = await youtubeDownloadManager.AddCrawledAsync(crawled);
if (bMakeItPrivate)
{
await youtubeDownloadManager.AddPrivateCrawledAsync(new PrivateCrawled
{
CrawledId = crawled.Id,
Order = await youtubeDownloadManager.getPrivateCrawledMaxOrderAsync() + 1
});
}
else if (!bMakeItPrivate)
{
await youtubeDownloadManager.AddPublicCrawledAsync(new PublicCrawled
{
CrawledId = crawled.Id,
Order = await youtubeDownloadManager.getPublicCrawledMaxOrderAsync() + 1
});
}
scope.Complete();
}
}
catch (Exception e)
{
Console.WriteLine($"!!! {e.Message}");
Console.WriteLine($"!!! {e.InnerException}");
return(new ResponseDTO <string>
{
success = false,
data = null,
responseMessage = e.Message
});
}
return(new ResponseDTO <string>
{
success = true,
data = videoId,
});
}
