static void Main(string[] args)
{
//DoVideo(
// "https://www.youtube.com/watch?v=SgQwBSHJNp0"
// //"https://www.youtube.com/watch?v=h-8UCEigYTI"
// );
// https://sites.google.com/a/jsc-solutions.net/work/knowledge-base/15-dualvr/20150609/360
//DoVideo(
//// //"https://www.youtube.com/watch?v=ZABusb0bsnw"
//// "https://www.youtube.com/watch?v=gRUk3po8TcA"
//"https://www.youtube.com/watch?v=LgQIqxyjwYA"
//);
#if REMOTE
#region WNetRestoreSingleConnection
try
{
var ee = Directory.GetFileSystemEntries("r:\\");
}
catch
{
// \\192.168.43.12\x$
// [Window Title]
// Location is not available
// [Content]
//R:\ is unavailable.If the location is on this PC, make sure the device or drive is connected or the disc is inserted, and then try again.If the location is on a network, make sure you’re connected to the network or Internet, and then try again.If the location still can’t be found, it might have been moved or deleted.
//[OK]
// ---------------------------
//Error
//-------------------------- -
//This network connection does not exist.
//-------------------------- -
//OK
//-------------------------- -
IntPtr hWnd = new IntPtr(0);
int res = WNetRestoreSingleConnection(hWnd, "r:", true);
}
#endregion
#endif
// or what if debugger starts asking for developer license and clicking ok kills to downloads in progress?
// what if device looses power.
// how are we to know or resume?
// X:\jsc.svn\examples\merge\Test\TestJObjectParse\TestJObjectParse\Program.cs
// https://sites.google.com/a/jsc-solutions.net/backlog/knowledge-base/2015/201501/20150115/youtubeextractor
// X:\jsc.svn\examples\merge\Test\TestYouTubeExtractor\TestYouTubeExtractor\Program.cs
// x:\jsc.svn\market\synergy\github\youtubeextractor\external\exampleapplication\program.cs
//var p = 1;
for (int p = 1; p < 96; p++)
foreach (var src in new[] {
//$"http://consciousresonance.net/?page_id=1587&paged={p}",
//$"https://faustuscrow.wordpress.com/page/{p}/",
//$"https://hiddenlighthouse.wordpress.com/page/{p}/",
//$"https://zproxy.wordpress.com/page/{p}/"
"https://zproxy.wordpress.com/page/" + p + "/"
})
{
Console.WriteLine("DownloadString ... " + new { p, src });
// Additional information: The underlying connection was closed: An unexpected error occurred on a send.
// Additional information: The operation has timed out.
// Additional information: The underlying connection was closed: The connection was closed unexpectedly.
// Additional information: The request was aborted: Could not create SSL/TLS secure channel.
// xml tidy?
var page0 = new WebClient().DownloadStringOrRetry(src);
Console.WriteLine("DownloadString ... done " + new { p });
// http://stackoverflow.com/questions/281682/reference-to-undeclared-entity-exception-while-working-with-xml
// Additional information: Reference to undeclared entity 'raquo'. Line 11, position 73.
// Additional information: The 'p' start tag on line 105 position 2 does not match the end tag of 'div'.Line 107, position 10.
// http://stackoverflow.com/questions/15926142/regular-expression-for-finding-href-value-of-a-a-link
// Command: Checkout from https://htmlagilitypack.svn.codeplex.com/svn/trunk, revision HEAD, Fully recursive, Externals included
//// could it be used within a service worker?
//var doc = new HtmlAgilityPack.HtmlDocument();
//doc.LoadHtml(page0);
//var hrefList = doc.DocumentNode.SelectNodes("//a")
// .Select(xp => xp.GetAttributeValue("href", "not found"))
// .ToList();
////var xpage0 = XElement.Parse(
// System.Net.WebUtility.HtmlDecode(page0)
// );
// http://htmlagilitypack.codeplex.com/
//Console.WriteLine("DownloadString ... done " + new { p, hrefList.Count });
//p++;
// https://www.youtube.com/embed/FhEYvOYceNs?
while (!string.IsNullOrEmpty(page0))
{
// <iframe src="//www.youtube.com/embed/umfjGNlxWcw"
// <span class='embed-youtube' style='text-align:center; display: block;'><iframe class='youtube-player' type='text/html' width='640' height='390' src='https://www.youtube.com/embed/8vwzVVJ9lvg?version=3&rel=1&fs=1&showsearch=0&showinfo=1&iv_load_policy=1&wmode=transparent' frameborder='0' allowfullscreen='true'></iframe></span>
var prefix = "//www.youtube.com/embed/";
//var prefix = "https://www.youtube.com/embed/";
var embed = page0.SkipUntilOrEmpty(prefix);
var id = embed.TakeUntilIfAny("\"").TakeUntilIfAny("?");
var link = prefix + id;
page0 = embed.SkipUntilOrEmpty("?");
Console.WriteLine();
try
{
// a running applicaion should know when it can reload itself
// when all running tasks are complete and no new tasks are to be taken.
var videoUrl = link;
bool isYoutubeUrl = DownloadUrlResolver.TryNormalizeYoutubeUrl(videoUrl, out videoUrl);
//Console.WriteLine(new { sw.ElapsedMilliseconds, px, videoUrl });
// wont help
//var y = DownloadUrlResolver.GetDownloadUrls(link);
//var j = DownloadUrlResolver.LoadJson(videoUrl);
var c = new WebClient().DownloadString(videoUrl);
// "Kryon - Timing o..." The YouTube account associated with this video has been terminated due to multiple third-party notifications of copyright infringement.
// <link itemprop="url" href="http://www.youtube.com/user/melania1172">
// { videoUrl = http://youtube.com/watch?v=li0E4_7ap3g, ch_name = , userurl = https://youtube.com/user/ }
//{ url = http://youtube.com/watch?v=li0E4_7ap3g }
//{ err = YoutubeExtractor.YoutubeParseException: Could not parse the Youtube page for URL http://youtube.com/watch?v=li0E4_7ap3g
// <h1 id="unavailable-message" class="message">
// 'IS_UNAVAILABLE_PAGE': false,
var unavailable =
!c.Contains("'IS_UNAVAILABLE_PAGE': false") ?
c.SkipUntilOrEmpty("<h1 id=\"unavailable-message\" class=\"message\">").TakeUntilOrEmpty("<").Trim() : "";
if (unavailable != "")
{
// 180?
countunavailable++;
Console.Title = new { countunavailable }.ToString();
Console.WriteLine(new { videoUrl, unavailable });
//Thread.Sleep(3000);
continue;
}
var ch = c.SkipUntilOrEmpty(" <div class=\"yt-user-info\">").SkipUntilOrEmpty("<a href=\"/channel/");
var ch_id = ch.TakeUntilOrEmpty("\"");
var ch_name = ch.SkipUntilOrEmpty(">").TakeUntilOrEmpty("<");
// https://www.youtube.com/channel/UCP-Q2vpvpQmdShz-ASBj2fA/videos
// ! originally there were users, now there are thos gplus accounts?
//var usertoken = c.SkipUntilOrEmpty("<link itemprop=\"url\" href=\"http://www.youtube.com/user/");
//var userid = usertoken.TakeUntilOrEmpty("\"");
////var ch_name = ch.SkipUntilOrEmpty(">").TakeUntilOrEmpty("<");
//var userurl = "https://youtube.com/user/" + userid;
Console.WriteLine(new { src, link, ch_name, ch_id });
//Console.WriteLine(new { page0, link });
// Our test youtube link
//const string link = "https://www.youtube.com/watch?v=BJ9v4ckXyrU";
//Debugger.Break();
// rewrite broke JObject Parse.
// Additional information: Bad JSON escape sequence: \5.Path 'args.afv_ad_tag_restricted_to_instream', line 1, position 3029.
// https://sites.google.com/a/jsc-solutions.net/backlog/knowledge-base/2015/201510/20151022
var ytconfig = c.SkipUntilOrEmpty("<script>var ytplayer = ytplayer || {};ytplayer.config =").TakeUntilOrEmpty(";ytplayer.load =");
dynamic ytconfigJSON = Newtonsoft.Json.JsonConvert.DeserializeObject(ytconfig);
var ytconfigJSON_args = ytconfigJSON.args;
// not available for 8K 360 3D ?
string ytconfigJSON_args_adaptive_fmts = ytconfigJSON.args.adaptive_fmts;
//if (ytconfigJSON_args_adaptive_fmts == null)
// Debugger.Break();
string adaptive_fmts = Uri.UnescapeDataString(ytconfigJSON_args_adaptive_fmts ?? "");
// projection_type=3
// + ((dynamic)((Newtonsoft.Json.Linq.JObject)(ytconfigJSON))).args
// https://sites.google.com/a/jsc-solutions.net/work/knowledge-base/15-dualvr/20151106/spherical3d
var Spherical3D = adaptive_fmts.Contains("projection_type=3");
var Spherical = adaptive_fmts.Contains("projection_type=2");
if (!Spherical)
if (!Spherical3D)
{
var get_video_info0 = new WebClient().DownloadString("https://www.youtube.com/get_video_info?html5=1&video_id=" + id);
var get_video_info1 = Uri.UnescapeDataString(get_video_info0);
var statusfail = get_video_info1.Contains("status=fail");
if (statusfail)
{
}
else
{
// url_encoded_fmt_stream_map=type=video
Spherical3D = get_video_info1.Contains("projection_type=3");
Spherical = get_video_info1.Contains("projection_type=2");
}
}
// "yt:projectionType"), t = Ss(b, "yt:stereoLayout"), u = "equirectangular" == n, x, z;
//u && "layout_top_bottom" ==
//t ? x = 3 : u && !n ? x = 2 : "layout_left_right" ==
// jsc rewriter breaks it?
IEnumerable<VideoInfo> videoInfos = DownloadUrlResolver.GetDownloadUrls(link);
// Additional information: The remote name could not be resolved: 'youtube.com'
//DownloadAudio(videoInfos);
DownloadVideo(ch_name,
Spherical3D ? projection.x360TB :
Spherical ? projection.x360 :
projection.x2D
, link, videoInfos);
//{
// err = System.IO.IOException: Unable to read data from the transport connection: An established connection was aborted by the software in your host machine. --->System.Net.Sockets.SocketException: An established connection was aborted by the software in your host machine
// at System.Net.Sockets.Socket.Receive(Byte[] buffer, Int32 offset, Int32 size, SocketFlags socketFlags)
// at System.Net.Sockets.NetworkStream.Read(Byte[] buffer, Int32 offset, Int32 size)
// -- - End of inner exception stack trace-- -
// at System.Net.ConnectStream.Read(Byte[] buffer, Int32 offset, Int32 size)
// at YoutubeExtractor.VideoDownloader.Execute()
}
catch (Exception err)
{
//ScriptCoreLib.Desktop.TaskbarProgress.SetMainWindowError();
// https://discutils.codeplex.com/
// Message = "Result cannot be called on a failed Match."
Console.WriteLine(new { err });
Thread.Sleep(3000);
//ScriptCoreLib.Desktop.TaskbarProgress.SetMainWindowNoProgress();
}
//goto next;
}
}
Debugger.Break();
}
static void Main(string[] args)
{
// if a network has a set of mp4 files
// which are tagged to yt
// can we get their channel info?
// X:\jsc.svn\examples\merge\Test\TestYouTubeExtractor\TestYouTubeExtractor\Program.cs
var p = "r:/media";
// should any calculated data be stored into a local sqlite for any restarts?
// http://blogs.msdn.com/b/csharpfaq/archive/2012/01/23/using-async-for-file-access-alan-berman.aspx
var sw = Stopwatch.StartNew();
Console.WriteLine(new { sw.ElapsedMilliseconds, p });
// can we async/await ourselves onto the other device?
// closer to the data?
foreach (var px in Directory.EnumerateFiles(p, "*.mp4"))
{
Console.WriteLine(new { sw.ElapsedMilliseconds, px } + " before Create");
// whats the link?
// Show Details Severity Code Description Project File Line
//Error Error signing output with public key from file 'taglib-sharp.snk' -- File not found.taglib-sharp CSC
// takes a while
// how would we know how much IO
// is being moved in this api?
// would a jsc nuget allow to augment such performance counters?
TagLib.File videoFile = TagLib.File.Create(px);
Console.WriteLine(new { sw.ElapsedMilliseconds, px } + " after Create");
//TagLib.Mpeg4.AppleTag customTag = (TagLib.Mpeg4.Comm)videoFile.GetTag(TagLib.TagTypes.Apple);
TagLib.Mpeg4.AppleTag customTag = (TagLib.Mpeg4.AppleTag)videoFile.GetTag(TagLib.TagTypes.Apple);
//customTag.SetDashBox("Producer", "Producer1",link);
//customTag.Comment = link;
var link = customTag.Album;
var videoUrl = link;
bool isYoutubeUrl = DownloadUrlResolver.TryNormalizeYoutubeUrl(videoUrl, out videoUrl);
Console.WriteLine(new { sw.ElapsedMilliseconds, px, videoUrl });
// wont help
//var y = DownloadUrlResolver.GetDownloadUrls(link);
//var j = DownloadUrlResolver.LoadJson(videoUrl);
var c = new WebClient().DownloadString(videoUrl);
var ch = c.SkipUntilOrEmpty("<a href=\"/channel/");
var ch_id = ch.TakeUntilOrEmpty("\"");
var ch_name = ch.SkipUntilOrEmpty(">").TakeUntilOrEmpty("<");
Console.WriteLine(new { sw.ElapsedMilliseconds, ch_name, ch_id });
// { ElapsedMilliseconds = 301714, ch_name = TheScariestMovieEver, ch_id = UCo8fiE2-s0SZu6Onb8lNLMQ }
// <a href="/channel/UCo8fiE2-s0SZu6Onb8lNLMQ" class=" yt-uix-sessionlink spf-link g-hovercard" data-ytid="UCo8fiE2-s0SZu6Onb8lNLMQ" data-sessionlink="ei=Ihu5VObLE4_2ygP--4HABQ" data-name="">TheScariestMovieEver</a>
// [95] = {[uid, o8fiE2-s0SZu6Onb8lNLMQ]}
// ElapsedMilliseconds = 0, p = r:/ media }
// ElapsedMilliseconds = 4219, px = r:/ media\'Battle of the Sexes' is Over!EVERYONE LOST!!.mp4 }
// before Create
//ElapsedMilliseconds = 171092, px = r:/media\'Battle of the Sexes' is Over! EVERYONE LOST!!.mp4
// }
// after Create
//ElapsedMilliseconds = 185783, px = r:/media\'Battle of the Sexes' is Over! EVERYONE LOST!!.mp4, videoUrl = http://youtube.com/watch?v=hsVdNrZZM2Q }
//url = http://youtube.com/watch?v=hsVdNrZZM2Q }
// https://www.youtube.com/channel/UCo8fiE2-s0SZu6Onb8lNLMQ
}
}
// CALLED BY?
public static void DoVideo(string link)
{
//Debugger.Break();
try
{
// a running applicaion should know when it can reload itself
// when all running tasks are complete and no new tasks are to be taken.
var videoUrl = link;
var prefix2 = "//www.youtube.com/watch?v=";
var prefix = "//www.youtube.com/embed/";
//var prefix = "https://www.youtube.com/embed/";
var embed = link.SkipUntilOrNull(prefix) ?? link.SkipUntilOrNull(prefix2);
var id = embed.TakeUntilIfAny("\"").TakeUntilIfAny("?");
bool isYoutubeUrl = DownloadUrlResolver.TryNormalizeYoutubeUrl(videoUrl, out videoUrl);
//Console.WriteLine(new { sw.ElapsedMilliseconds, px, videoUrl });
// wont help
//var y = DownloadUrlResolver.GetDownloadUrls(link);
//var j = DownloadUrlResolver.LoadJson(videoUrl);
var c = new WebClient().DownloadString(videoUrl);
// "Kryon - Timing o..." The YouTube account associated with this video has been terminated due to multiple third-party notifications of copyright infringement.
// <link itemprop="url" href="http://www.youtube.com/user/melania1172">
// { videoUrl = http://youtube.com/watch?v=li0E4_7ap3g, ch_name = , userurl = https://youtube.com/user/ }
//{ url = http://youtube.com/watch?v=li0E4_7ap3g }
//{ err = YoutubeExtractor.YoutubeParseException: Could not parse the Youtube page for URL http://youtube.com/watch?v=li0E4_7ap3g
// <h1 id="unavailable-message" class="message">
// 'IS_UNAVAILABLE_PAGE': false,
var unavailable =
!c.Contains("'IS_UNAVAILABLE_PAGE': false") ?
c.SkipUntilOrEmpty("<h1 id=\"unavailable-message\" class=\"message\">").TakeUntilOrEmpty("<").Trim() : "";
if (unavailable != "")
{
Console.WriteLine(new { videoUrl, unavailable });
Thread.Sleep(3000);
return;
}
var ch = c.SkipUntilOrEmpty(" <div class=\"yt-user-info\">").SkipUntilOrEmpty("<a href=\"/channel/");
var ch_id = ch.TakeUntilOrEmpty("\"");
var ch_name = ch.SkipUntilOrEmpty(">").TakeUntilOrEmpty("<");
// https://www.youtube.com/channel/UCP-Q2vpvpQmdShz-ASBj2fA/videos
// ! originally there were users, now there are thos gplus accounts?
//var usertoken = c.SkipUntilOrEmpty("<link itemprop=\"url\" href=\"http://www.youtube.com/user/");
//var userid = usertoken.TakeUntilOrEmpty("\"");
////var ch_name = ch.SkipUntilOrEmpty(">").TakeUntilOrEmpty("<");
//var userurl = "https://youtube.com/user/" + userid;
Console.WriteLine(new { link, ch_name, ch_id });
//Console.WriteLine(new { page0, link });
// Our test youtube link
//const string link = "https://www.youtube.com/watch?v=BJ9v4ckXyrU";
//Debugger.Break();
// rewrite broke JObject Parse.
// Additional information: Bad JSON escape sequence: \5.Path 'args.afv_ad_tag_restricted_to_instream', line 1, position 3029.
// <script>var ytplayer = ytplayer || {};ytplayer.config =
var ytconfig = c.SkipUntilOrEmpty("<script>var ytplayer = ytplayer || {};ytplayer.config =").TakeUntilOrEmpty(";ytplayer.load =");
dynamic ytconfigJSON = Newtonsoft.Json.JsonConvert.DeserializeObject(ytconfig);
var ytconfigJSON_args = ytconfigJSON.args;
string ytconfigJSON_args_adaptive_fmts = ytconfigJSON.args.adaptive_fmts;
string adaptive_fmts = Uri.UnescapeDataString(ytconfigJSON_args_adaptive_fmts);
// projection_type=3
// + ((dynamic)((Newtonsoft.Json.Linq.JObject)(ytconfigJSON))).args
//var get_video_info = new WebClient().DownloadString("https://www.youtube.com/get_video_info?html5=1&video_id=" + id);
//var statusfail = get_video_info.Contains("status=fail");
//if (statusfail)
// return;
// https://sites.google.com/a/jsc-solutions.net/work/knowledge-base/15-dualvr/20151106/spherical3d
var Spherical3D = adaptive_fmts.Contains("projection_type=3");
var Spherical = adaptive_fmts.Contains("projection_type=2");
// "yt:projectionType"), t = Ss(b, "yt:stereoLayout"), u = "equirectangular" == n, x, z;
//u && "layout_top_bottom" ==
//t ? x = 3 : u && !n ? x = 2 : "layout_left_right" ==
// jsc rewriter breaks it?
IEnumerable<VideoInfo> videoInfos = DownloadUrlResolver.GetDownloadUrls(link, decryptSignature: false);
// Additional information: The remote name could not be resolved: 'youtube.com'
//DownloadAudio(videoInfos);
DownloadVideo(ch_name,
Spherical3D ? projection.x360TB :
Spherical ? projection.x360 :
projection.x2D
, link, videoInfos);
//{
// err = System.IO.IOException: Unable to read data from the transport connection: An established connection was aborted by the software in your host machine. --->System.Net.Sockets.SocketException: An established connection was aborted by the software in your host machine
// at System.Net.Sockets.Socket.Receive(Byte[] buffer, Int32 offset, Int32 size, SocketFlags socketFlags)
// at System.Net.Sockets.NetworkStream.Read(Byte[] buffer, Int32 offset, Int32 size)
// -- - End of inner exception stack trace-- -
// at System.Net.ConnectStream.Read(Byte[] buffer, Int32 offset, Int32 size)
// at YoutubeExtractor.VideoDownloader.Execute()
}
catch (Exception err)
{
//ScriptCoreLib.Desktop.TaskbarProgress.SetMainWindowError();
// https://discutils.codeplex.com/
// Message = "Result cannot be called on a failed Match."
Console.WriteLine(new { err });
Thread.Sleep(3000);
//ScriptCoreLib.Desktop.TaskbarProgress.SetMainWindowNoProgress();
}
}