public void complete_setup(String __path, SetupManager form)
{
var Helper = new DownloadHelpers();
var client = Helper.DownloadFfmpeg(__path);
form.updateSetupLabel("Downloading FFmpeg");
this.client = client;
client.DownloadProgressChanged += (t, k) =>
{
form.updateProgress(k.ProgressPercentage);
};
client.DownloadFileCompleted += (k, t) =>
{
if (t.Error != null)
{
MessageBox.Show(t.Error.ToString());
}
;
if (t.Cancelled)
{
return;
}
form.updateSetupLabel("Extracting FFmpeg");
ZipHelpers.ExtractFfmpeg($@"{__path}/ffmpeg", $@"{__path}/ffmpeg-zip.zip");
form.onComplete();
};
}
public bool Logic()
{
var logs = new List <Log>();
try
{
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, "Starting Downloader logic", null));
var showRepo = new ShowRepository(ConnectionString);
var episodeRepo = new EpisodeRepository(ConnectionString);
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Getting shows from database", null));
var showsInDatabase = showRepo.Select().Where(x => x.Download == true && x.Enabled == true).ToList();
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Found {showsInDatabase.Count} shows in database", null));
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Getting episodes from database", null));
var episodesInDatabase = episodeRepo.Select();
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Found {episodesInDatabase.Count} in database", null));
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, $"Looping through shows that are enabled to download", null));
foreach (var show in showsInDatabase)
{
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Checking episodes scraped", null));
var episodesByShow = episodesInDatabase.Where(x => x.ShowId == show.Id && x.Status == EpisodeStatus.SCRAPED).OrderBy(y => y.Filename).ToList();
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Found {episodesByShow.Count} to download", null));
foreach (var episode in episodesByShow)
{
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, $"Downloading {episode.Filename}", null));
var path = System.IO.Path.Combine(Path, show.Title);
episode.Status = DownloadHelpers.DownloadFile(episode.Url, path, episode.Filename) ? EpisodeStatus.DOWNLOADED : EpisodeStatus.FAILED;
var downloadMessage = episode.Status == EpisodeStatus.DOWNLOADED ? "successfully" : "failed";
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, $"Downladed {downloadMessage}", null));
episodeRepo.Update(episode);
logs.Add(new Log(Core.Enums.LogHelpers.LogType.DEBUG, $"Updated show", null));
}
}
}
catch (Exception e)
{
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, e.Message, e.StackTrace));
}
logs.Add(new Log(Core.Enums.LogHelpers.LogType.INFO, "Finished Downloader logic", null));
LogRepository.Create(logs);
return(true);
}
public static string DownloadYtDlp()
{
DirectoryInfo executablesFolder = Directory.CreateDirectory(GlobalConfig.GetGlobalConfig("contentRootPath") + "executables");
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD))
{
DownloadHelpers.DownloadFile("https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp",
executablesFolder + "/yt-dlp");
try {
var processInfo = new ProcessStartInfo("chmod", $"+x {executablesFolder}/yt-dlp");
processInfo.CreateNoWindow = true;
processInfo.UseShellExecute = false;
processInfo.RedirectStandardError = true;
processInfo.RedirectStandardOutput = true;
var process = Process.Start(processInfo);
process.WaitForExit();
YtDlpTestLogic.TestYtDlpPath(executablesFolder + "/yt-dlp");
} catch (Win32Exception) {
throw new Win32Exception("Could not chmod yt-dlp file.");
}
GlobalConfig.SetGlobalConfig("yt-dlp", executablesFolder + "/yt-dlp");
GlobalConfig.SetGlobalConfig("yt-DlpVersion", GetLatestYtDlpVersion().ToString());
return(executablesFolder + "/yt-dlp");
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
DownloadHelpers.DownloadFile("https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe",
executablesFolder + "/yt-dlp.exe");
try {
YtDlpTestLogic.TestYtDlpPath(executablesFolder + "/yt-dlp.exe");
} catch (Win32Exception) {
return(null);
}
GlobalConfig.SetGlobalConfig("yt-dlp", executablesFolder + "/yt-dlp.exe");
GlobalConfig.SetGlobalConfig("yt-DlpVersion", GetLatestYtDlpVersion().ToString());
return(executablesFolder + "/yt-dlp.exe");
}
throw new NotSupportedException("OS not supported.");
}
static async Task Main(string[] args)
{
// "scripts/produce.sh".BashRunScriptFileSync();
// "scripts/produce.sh".BashRunScriptFileAsync();
//Console.WriteLine($"Did we create directory: {DownloadHelpers.TryCreateDirectory("/Users/ispoa/Downloads/EpohScraper/NonExistent")}");
// Download File from internet
//var _downloadTasks = DownloadHelpers.DownloadFiles(new string[] {
// //"http://*****:*****@"""01 Look Ahead.mp3, 02 T - Shirt.mp3, 03 Move That Dope.mp3, 04 My Momma(Ft.Wiz Khalifa).mp3, 05 Honest.mp3, 06 I Won.mp3, 07 Never Satisfied.mp3, 08 I Be U.mp3, 09 Covered N Money.mp3,
12 Blood Sweat Tears.mp3, 13 Big Rube Speaks.mp3, 14 Side Effects.mp3, 15 Ill Be Yours.mp3, 16 How Can I Not(Ft.Young Scooter).mp3, 17 Sh!t.mp3,
18 Karate Chop Ft.Lil Wayne(Remix).mp3""";
var media = new MediaEntity("Future", "Honest");
foreach (string song in songs.Trim().Split(new char[] { ',' }))
{
// TODO (oneeyedsunday) handle backpressure
media.AddDownloadUrl("http://localhost:3000/?artist=Future&album=Honest&song=" + HttpUtility.UrlEncode(song.Trim()));
}
CancellationTokenSource cancellationToken = new CancellationTokenSource();
var _downloadTasks = DownloadHelpers.DownloadAlbum(media, cancellationToken.Token);
_downloadTasks.GetAwaiter().OnCompleted(() =>
{
Console.WriteLine("[+] Finished downloading");
});
await _downloadTasks;
}
public Streamer CreateStreamerLogic(Streamer body)
{
Streamer returnStreamer;
using (var context = new MainDataContext()) {
Streamer streamer = context.Streamers.FirstOrDefault(item => item.streamerId == body.streamerId);
if (streamer == null)
{
// if streamer does not exist in database and want to add
_logger.Info("Adding new streamer...");
string etag = "";
if (GlobalConfig.GetGlobalConfig("contentRootPath") != null)
{
CreateFolder($"{GlobalConfig.GetGlobalConfig("contentRootPath")}/streamers/{body.streamerId}/");
if (!string.IsNullOrEmpty(body.thumbnailLocation))
{
etag = DownloadHelpers.DownloadFile(body.thumbnailLocation,
$"{GlobalConfig.GetGlobalConfig("contentRootPath")}/streamers/{body.streamerId}/thumbnail.png");
}
}
streamer = new Streamer {
streamerId = body.streamerId,
displayName = body.displayName,
username = body.username,
isLive = body.isLive ?? false,
quality = body.quality == "{\"resolution\":0,\"fps\":0}" ? null : body.quality,
getLive = body.getLive ?? false,
thumbnailLocation = $"streamers/{body.streamerId}/thumbnail.png",
thumbnailETag = etag
};
context.Streamers.Add(streamer);
returnStreamer = streamer;
} /*else if (streamer != null) {
* // if streamer exists then update
* Console.WriteLine("Updating streamer...");
* streamer.streamerId = body.streamerId;
* streamer.displayName = body.displayName;
* streamer.username = body.username;
* streamer.description = body.description;
* streamer.viewCount = body.viewCount;
* streamer.thumbnailLocation = $"streamers/{body.streamerId}/thumbnail.png";
* returnStreamer = streamer;
*
* IList<Parameter> headers = downloadHelpers.GetHeaders(body.thumbnailLocation);
* for (var x = 0; x < headers.Count; x++) {
* if (headers[x].Name == "ETag") {
* var etag = headers[x].Value;
* if (etag != null) {
* if (streamer.thumbnailETag != etag.ToString().Replace("\"", "")) {
* if (contentRootPath != null)
* Console.WriteLine("Detected new thumbnail image, downloading...");
* streamer.thumbnailETag = downloadHelpers.DownloadFile(body.thumbnailLocation,
* $"{contentRootPath.value}/streamers/{body.streamerId}/thumbnail.png");
* }
* }
* }
* }
* }*/
else
{
//something strange has happened
returnStreamer = new Streamer();
}
//if (isNew) {
//StartupJobs startupJobs = new StartupJobs();
List <Streamer> streamers = new List <Streamer> {
streamer
}; //lazy
JobHelpers.NormalJob <CheckForStreamerLiveStatusJob>("CreateStreamerUpdateLiveStatusJob", "CreateStreamerUpdateLiveStatusTrigger", QuartzSchedulers.PrimaryScheduler());
IJobDetail job = JobBuilder.Create <UpdateStreamerDetailsJob>()
.WithIdentity("UpdateStreamerDetailsJob")
.Build();
job.JobDataMap.Put("listOfStreamers", streamers);
var schedulerFactory = new StdSchedulerFactory(QuartzSchedulers.PrimaryScheduler());
IScheduler scheduler = schedulerFactory.GetScheduler().Result;
scheduler.Start();
ISimpleTrigger trigger = (ISimpleTrigger)TriggerBuilder.Create()
.WithIdentity("UpdateStreamerDetailsTrigger")
.StartNow()
.Build();
scheduler.ScheduleJob(job, trigger);
//}
context.SaveChanges();
}
return(returnStreamer);
}
public Streamer UpdateStreamer(Streamer body, int?id)
{
Streamer returnStreamer;
using (var context = new MainDataContext()) {
Streamer streamer = id == null?context.Streamers.FirstOrDefault(item => item.streamerId == body.streamerId) : context.Streamers.FirstOrDefault(item => item.id == id);
_logger.Info("Updating streamer...");
if (body.streamerId != 0)
{
streamer.streamerId = body.streamerId;
}
if (body.displayName != null)
{
streamer.displayName = body.displayName;
}
if (body.username != null)
{
streamer.username = body.username;
}
if (body.description != null)
{
streamer.description = body.description;
}
if (body.isLive != null)
{
streamer.isLive = body.isLive;
}
if (body.quality != null)
{
if (body.quality == "{\"resolution\":0,\"fps\":0}")
{
streamer.quality = null;
}
else
{
streamer.quality = body.quality;
}
}
if (body.getLive != null)
{
streamer.getLive = body.getLive;
}
if (body.viewCount != null)
{
streamer.viewCount = body.viewCount;
}
returnStreamer = streamer;
if (streamer != null)
{
context.Update(streamer);
}
if (body.thumbnailLocation != null)
{
IList <Parameter> headers = DownloadHelpers.GetHeaders(body.thumbnailLocation);
for (var x = 0; x < headers.Count; x++)
{
if (headers[x].Name == "ETag")
{
var etag = headers[x].Value;
if (etag != null)
{
if (streamer.thumbnailETag != etag.ToString().Replace("\"", ""))
{
_logger.Info("Detected new thumbnail image, downloading...");
streamer.thumbnailETag = DownloadHelpers.DownloadFile(body.thumbnailLocation,
$"{GlobalConfig.GetGlobalConfig("contentRootPath")}/streamers/{body.streamerId}/thumbnail.png");
}
}
}
}
}
context.SaveChanges();
}
return(returnStreamer);
}
public bool PrepareDownload(StreamExtended stream)
{
string streamUrl;
streamUrl = "https://www.twitch.tv/videos/" + stream.streamId;
YoutubeDlVideoJson.YoutubeDlVideoInfo youtubeDlVideoInfo =
StreamHelpers.GetDownloadQualityUrl(streamUrl, stream.streamerId);
string streamDirectory = $"{GlobalConfig.GetGlobalConfig("contentRootPath")}streamers/{stream.streamerId}/vods/{stream.streamId}";
Directory.CreateDirectory(streamDirectory);
if (!string.IsNullOrEmpty(stream.thumbnailLocation))
{
//todo handle missing thumbnail, maybe use youtubedl generated thumbnail instead
DownloadHelpers.DownloadFile(
stream.thumbnailLocation.Replace("%{width}", "320").Replace("%{height}", "180"),
$"{streamDirectory}/thumbnail.jpg");
}
string title = String.IsNullOrEmpty(stream.title) ? "vod" : stream.title;
string outputPath = $"{streamDirectory}/{title}.{stream.streamId}";
string dbOutputPath = $"streamers/{stream.streamerId}/vods/{stream.streamId}/{title}.{stream.streamId}.mp4";
//TODO more should be queued, not done immediately
IJobDetail job;
string triggerIdentity;
job = JobBuilder.Create <DownloadStreamJob>()
.WithIdentity("StreamDownload" + stream.streamId)
.UsingJobData("title", title)
.UsingJobData("streamDirectory", streamDirectory)
.UsingJobData("formatId", youtubeDlVideoInfo.formatId)
.UsingJobData("url", streamUrl)
.UsingJobData("isLive", false)
.UsingJobData("youtubeDlVideoInfoDuration", youtubeDlVideoInfo.duration)
.UsingJobData("retry", true)
.RequestRecovery()
.Build();
job.JobDataMap.Put("stream", stream);
triggerIdentity = $"StreamDownload{stream.streamId}";
/*string jobId = BackgroundJob.Enqueue(() =>
* DownloadStream(stream, title, streamDirectory, youtubeDlVideoInfo.url, CancellationToken.None,
* isLive, youtubeDlVideoInfo.duration));*/
Stream? dbStream;
bool downloadChat = false;
IJobDetail chatDownloadJob = new JobDetailImpl();
using (var context = new MainDataContext()) {
dbStream = context.Streams.FirstOrDefault(item => item.streamId == stream.streamId);
if (dbStream != null)
{
dbStream.streamId = stream.streamId;
dbStream.streamerId = stream.streamerId;
dbStream.quality = youtubeDlVideoInfo.quality;
dbStream.url = youtubeDlVideoInfo.url;
dbStream.title = stream.title;
dbStream.createdAt = stream.createdAt;
dbStream.location = $"streamers/{stream.streamerId}/vods/{stream.streamId}/";
dbStream.fileName = $"{title}.{stream.streamId}.mp4";
dbStream.duration = youtubeDlVideoInfo.duration;
dbStream.downloading = true;
dbStream.downloadJobId = job.Key.ToString();
}
else
{
downloadChat = true;
chatDownloadJob = JobBuilder.Create <ChatDownloadJob>()
.WithIdentity("DownloadChat" + stream.streamId)
.UsingJobData("streamId", stream.streamId)
.UsingJobData("retry", true)
.RequestRecovery()
.Build();
dbStream = new Stream {
streamId = stream.streamId,
streamerId = stream.streamerId,
quality = youtubeDlVideoInfo.quality,
title = stream.title,
url = youtubeDlVideoInfo.url,
createdAt = stream.createdAt,
location = $"streamers/{stream.streamerId}/vods/{stream.streamId}/",
fileName = $"{title}.{stream.streamId}.mp4",
duration = youtubeDlVideoInfo.duration,
downloading = true,
chatDownloading = true,
downloadJobId = job.Key.ToString(),
chatDownloadJobId = chatDownloadJob.Key.ToString()
};
// only download chat if this is a new vod
context.Add(dbStream);
}
context.SaveChanges();
}
//var chatSchedulerFactory = new StdSchedulerFactory(QuartzSchedulers.SingleThreadScheduler());
var vodSchedulerFactory = new StdSchedulerFactory(QuartzSchedulers.PrimaryScheduler());
//IScheduler chatScheduler = chatSchedulerFactory.GetScheduler().Result;
IScheduler vodScheduler = vodSchedulerFactory.GetScheduler().Result;
//chatScheduler.Start();
vodScheduler.Start();
ISimpleTrigger trigger = (ISimpleTrigger)TriggerBuilder.Create()
.WithIdentity(triggerIdentity)
.StartNow()
.Build();
vodScheduler.ScheduleJob(job, trigger);
if (downloadChat)
{
ISimpleTrigger chatDownloadTrigger = (ISimpleTrigger)TriggerBuilder.Create()
.WithIdentity("DownloadChat" + stream.streamId)
.StartNow()
.Build();
vodScheduler.ScheduleJob(chatDownloadJob, chatDownloadTrigger);
}
//_hubContext.Clients.All.SendAsync("ReceiveMessage", CheckForDownloadingStreams());
return(true);
}