/// <summary>
/// Descrambles data of all streams at the given args position and breaks them down into individual parameters.
/// Result is written into back into the given position as a parsed ObscuredContainer and returned
/// </summary>
public static ObscuredContainer ApplyDescrambler(ObscuredContainer args, string streamKey)
{
// Break up individual streams
string[] streamBundle = args.GetValue <string>(streamKey).Split(',');
ObscuredContainer streamContainer = new ObscuredContainer();
foreach (string streamRaw in streamBundle)
{ // Descramble and index stream data
NameValueCollection streamParsed = HttpUtility.ParseQueryString(streamRaw);
ObscuredContainer streamInfo = ObscuredContainer.FromDictionary(
streamParsed.AllKeys.ToDictionary(k => k, k => (object)HttpUtility.UrlDecode(streamParsed[k]))
);
streamContainer.Add(streamRaw, streamInfo);
}
// Write descrambled data back into args
args[streamKey] = streamContainer;
/*YouTube.Log(string.Format (
* "Applying Descrambler: \n \t " +
* string.Join(" \n \t ", streamContainer.Select(s =>
* string.Join("; ", ((ObscuredContainer)s.Value).Select(p => (string)p.Key + "=" + (string)p.Value))
* ))
* ));*/
return(streamContainer);
}
/// <summary>
/// Read the YouTube player configuration (args and assets) from the JSON data embedded into the HTML page.
/// It serves as the primary source of obtaining the stream manifest data.
/// </summary>
internal static YTPlayerConfig getYTPlayerConfig(string watchHTML)
{
string configRaw = Helpers.DoRegex(extractYTPlayerConfig, watchHTML, 1);
if (string.IsNullOrEmpty(configRaw))
{
CSTube.Log("ERROR: Video is unavailable!");
return(new YTPlayerConfig());
}
// Get config as JSON structure
JObject obj = Helpers.TryParseJObject(configRaw);
JToken argsToken = obj.GetValue("args");
JToken assetsToken = obj.GetValue("assets");
// Create config and read it from JSON
YTPlayerConfig config = new YTPlayerConfig();
config.args = ObscuredContainer.FromJSONRecursive(argsToken, 10);
config.assets = ObscuredContainer.FromJSONRecursive(assetsToken, 10);
if (config.args == null || config.assets == null)
{
CSTube.Log("ERROR: Player Config JSON is invalid!");
}
return(config);
}
/// <summary>
/// Apply the decrypted signature to the stream manifest.
/// </summary>
public static void ApplySignature(ObscuredContainer streamContainer, string js)
{
int numSignaturesFound = 0;
foreach (KeyValuePair <string, object> s in streamContainer)
{ // Iterate over each stream and sign the URLs
ObscuredContainer stream = (ObscuredContainer)s.Value;
string URL = stream.GetValue <string>("url");
if (URL.Contains("signature="))
{ // Sometimes, signature is provided directly by YT, so we can skip the whole signature descrambling
numSignaturesFound++;
continue;
}
// Get, decode and save signature
string cipheredSignature = stream.GetValue <string>("s");
string signature = Cipher.DecodeSignature(js, cipheredSignature);
stream["url"] = URL + "&signature=" + signature;
/*YouTube.Log(string.Format (
* "Descrambled Signature for ITag={0} \n \t s={1} \n \t signature={2}",
* stream.GetValue<string>("itag"),
* cipheredSignature,
* signature));*/
}
if (numSignaturesFound > 0)
{
CSTube.Log(string.Format("{0} out of {1} URLs already contained a signature!", numSignaturesFound, streamContainer.Count));
}
}
/// <summary>
/// Retrieves an arbitrary attribute from the specified container.
/// Path separated by / and \ expects respective container tree.
/// Returns default(T) on failure to retrive the attribute.
/// </summary>
public static T TryGetAttribute <T>(ObscuredContainer container, string path)
{
string[] cmd = path.Split('\\', '/');
ObscuredContainer cur = container;
for (int i = 0; i < cmd.Length - 1; i++)
{ // Follows the specified path down the container hierarchy if possible
cur = cur.GetValue <ObscuredContainer>(cmd[i]);
if (cur == null)
{
return(default(T));
}
}
return(cur.GetValue <T>(cmd[cmd.Length - 1]));
}
/// <summary>
/// Read value of the give attribute collection into the class members
/// Affects type, URL, ITag and Abr
/// </summary>
private void ReadAttributeValues(ObscuredContainer attributes)
{
if (attributes.ContainsKey("type"))
{
type = attributes.GetValue <string>("type");
}
if (attributes.ContainsKey("url"))
{
URL = attributes.GetValue <string>("url");
}
if (attributes.ContainsKey("itag"))
{
ITag = attributes.GetValue <string>("itag");
}
if (attributes.ContainsKey("abr"))
{
Abr = attributes.GetValue <string>("abr");
}
}
/// <summary>
/// Decodes stream data found in the specified raw container into an object.
/// </summary>
internal Stream(ObscuredContainer streamContainer)
{
stream = streamContainer;
// Read parameters ITag, abr and URL
ReadAttributeValues(stream);
// Read format information from tables using ITag
format = ITags.getFormatProfile(ITag);
// 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', 'vp8, vorbis'
Tuple <string, string> typeSplit = Extract.splitType(type);
// Seperate type info
mimeType = typeSplit.Item1;
string[] typeInfo = mimeType.Split('/');
type = typeInfo[0];
subType = typeInfo[1];
// Read and parse codecs (seperate audio and video)
codecs = typeSplit.Item2.Split(',').Select(c => c.Trim()).ToList();
videoCodec = isProgressive ? codecs[0] : (type == "video" ? codecs[0] : null);
audioCodec = isProgressive ? codecs[1] : (type == "audio" ? codecs[0] : null);
}
/// <summary>
/// Tries to convert the tree of the given JToken into a ObscuredContainer tree.
/// Values are either own ObscuredContainer or string values, building a tree.
///
/// Usually assumes jToken to be a JObject or JArray.
/// If it is not, tries to find an encapsulated structure (quoted & escaped) and parses that.
/// If no such structure was found, returns null!
/// </summary>
public static ObscuredContainer FromJSONRecursive(JToken jToken, int depth)
{
if (jToken == null || depth <= 0 || !jToken.HasValues)
{
return(null);
}
if (string.IsNullOrWhiteSpace(jToken.ToString(Newtonsoft.Json.Formatting.None)))
{
return(null);
}
if (jToken.Type == JTokenType.Object)
{ // Convert the JObject
JObject jObject = (JObject)jToken;
ObscuredContainer objectContainer = new ObscuredContainer();
foreach (KeyValuePair <string, JToken> cToken in jObject)
{ // Interpret the child tokens either as string values or as own ObscureContainers
string value = cToken.Value.ToString(Newtonsoft.Json.Formatting.None).Trim(' ', '\"');
object childObject = FromJSONRecursive(cToken.Value, depth - 1);
objectContainer.Add(cToken.Key, childObject ?? value);
}
return(objectContainer);
}
if (jToken.Type == JTokenType.Array)
{ // Convert all JObject children of the JArray
JArray jArray = (JArray)jToken;
if (jArray.First.Type != JTokenType.Object)
{
return(null);
}
int counter = 0;
ObscuredContainer arrayContainer = new ObscuredContainer();
foreach (JObject cArrObject in jArray.Children <JObject>())
{ // Parse array children as own containers and encapsulate them
ObscuredContainer cArrContainer = FromJSONRecursive(cArrObject, depth - 1);
arrayContainer.Add((counter++).ToString(), cArrContainer);
}
return(arrayContainer);
}
// Token is NOT a JObject -> Try to find an encapsulated structure in the token
string rawValue = jToken.ToString(Newtonsoft.Json.Formatting.None).Trim(' ', '\"');
string safeValue = Regex.Unescape(rawValue);
// Check if it is an embedded JObject
if (safeValue.StartsWith("{") && safeValue.EndsWith("}") && safeValue.Length > 2)
{
return(FromJSONRecursive(Helpers.TryParseJObject(safeValue), depth - 1));
}
// Check if it is an embedded JArray
if (safeValue.StartsWith("[") && safeValue.EndsWith("]"))
{
return(FromJSONRecursive(Helpers.TryParseJArray(safeValue), depth - 1));
}
return(null);
}
/// <summary>
/// Construct a Caption and reads meta-data (no HTTP fetch)
/// </summary>
public Caption(ObscuredContainer captionTrack)
{
url = captionTrack.GetValue <string>("baseUrl");
name = captionTrack.GetValue <ObscuredContainer>("name").GetValue <string>("simpleText");
code = captionTrack.GetValue <string>("languageCode");
}
/// <summary>
/// Fetches the information about this video (including available streams and captions).
/// Conains two synchronous HTTP fetches and several descrambling and signing algorithms.
/// </summary>
public async Task FetchInformation()
{
CSTube.Log("Fetching information of video " + videoID);
// Page HTML
watchHTML = await Helpers.ReadHTML(watchURL);
if (string.IsNullOrEmpty(watchHTML))
{
return;
}
// PlayerConfig JSON
playerConfig = Extract.getYTPlayerConfig(watchHTML);
if (playerConfig.args == null || playerConfig.assets == null)
{
return;
}
// JavaScript Source
string jsURL = Extract.getJSURL(playerConfig);
jsSRC = await Helpers.ReadHTML(jsURL);
// VideoInfo Raw Data
//string videoInfoURL = Extract.getVideoInfoURL(videoID, watchURL, watchHTML);
//videoInfoRAW = Helpers.ReadHTML(videoInfoURL);
CSTube.Log("Finished downloading information! Continuing to parse!");
// Parse Raw video info data
//System.Collections.Specialized.NameValueCollection p = HttpUtility.ParseQueryString(videoInfoRAW);
//videoInfo = ObscuredContainer.FromDictionary(p.AllKeys.ToDictionary(k => k, k => (object)p[k]));
// Get all stream formats this video has (progressive and adaptive)
List <string> streamMaps = new List <string>();
streamMaps.Add("url_encoded_fmt_stream_map");
if (playerConfig.args.ContainsKey("adaptive_fmts"))
{
streamMaps.Add("adaptive_fmts");
}
foreach (string streamFormat in streamMaps)
{
// Descramble stream data in player args
ObscuredContainer streamBundle = Mixins.ApplyDescrambler(playerConfig.args, streamFormat);
// If required, apply signature to the stream URLs
Mixins.ApplySignature(streamBundle, jsSRC);
// Write stream data into Stream objects
foreach (object streamData in streamBundle.Values)
{
formatStreams.Add(new Stream((ObscuredContainer)streamData));
}
}
// Try to read out captionTracks if existant
ObscuredContainer captionTrackBundle =
Helpers.TryGetAttribute <ObscuredContainer>(playerConfig.args,
"player_response/captions/playerCaptionsTracklistRenderer/captionTracks");
if (captionTrackBundle != null)
{ // Write caption tracks into Caption objects
foreach (object captionTrack in captionTrackBundle.Values)
{
captionTracks.Add(new Caption((ObscuredContainer)captionTrack));
}
}
videoDataAvailable = true;
// Log success!
CSTube.Log(string.Format(
"Finished parsing video data! Found {0} video streams and {1} caption tracks for video '{2}'!",
formatStreams.Count, captionTracks.Count, title
));
CSTube.Log(string.Format("Video Streams: \n \t " +
string.Join(" \n \t ", formatStreams.Select(s => s.ToString()))
));
if (captionTracks.Count > 0)
{
CSTube.Log(string.Format("Caption Tracks: \n \t " +
string.Join(" \n \t ", captionTracks.Select(s => s.ToString()))
));
}
}
