From 31244cbcc89fa816e38afad1b4962fbe46497326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Tue, 30 Aug 2022 14:20:08 +0000 Subject: [PATCH 1/2] replicate headers and params made by yt apps --- src/invidious/yt_backend/connection_pool.cr | 14 +-- src/invidious/yt_backend/youtube_api.cr | 110 ++++++++++++++++---- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 3feb9233..23e98ae3 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -7,17 +7,19 @@ {% end %} def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" - request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["accept-language"] ||= "en-us,en;q=0.5" + if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" + end + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" return if request.resource.starts_with? "/sorry/index" request.headers["x-youtube-client-name"] ||= "1" request.headers["x-youtube-client-version"] ||= "2.20200609" # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" end end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 30d7613b..c014dc0e 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -7,9 +7,12 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_APP_VERSION = "17.29.35" - private ANDROID_SDK_VERSION = 30_i64 - private IOS_APP_VERSION = "17.30.1" + private ANDROID_APP_VERSION = "17.33.42" + private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US)" + private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_VERSION = "12" + private IOS_APP_VERSION = "17.33.2" + private WINDOWS_VERSION = "10.0" # Enumerate used to select one of the clients supported by the API enum ClientType @@ -33,27 +36,39 @@ module YoutubeAPI # List of hard-coded values used by the different clients HARDCODED_CLIENTS = { ClientType::Web => { - name: "WEB", - version: "2.20220804.07.00", - api_key: DEFAULT_API_KEY, - screen: "WATCH_FULL_SCREEN", + name: "WEB", + version: "2.20220804.07.00", + api_key: DEFAULT_API_KEY, + screen: "WATCH_FULL_SCREEN", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", # 56 - version: "1.20220803.01.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB_EMBEDDED_PLAYER", # 56 + version: "1.20220803.01.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebMobile => { - name: "MWEB", - version: "2.20220805.01.00", - api_key: DEFAULT_API_KEY, + name: "MWEB", + version: "2.20220805.01.00", + api_key: DEFAULT_API_KEY, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::WebScreenEmbed => { - name: "WEB", - version: "2.20220804.00.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB", + version: "2.20220804.00.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, # Android @@ -63,6 +78,10 @@ module YoutubeAPI version: ANDROID_APP_VERSION, api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::AndroidEmbeddedPlayer => { name: "ANDROID_EMBEDDED_PLAYER", # 55 @@ -75,6 +94,10 @@ module YoutubeAPI api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, # IOS @@ -179,6 +202,22 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:android_sdk_version]? end + def user_agent : String? + HARDCODED_CLIENTS[@client_type][:user_agent]? + end + + def os_name : String? + HARDCODED_CLIENTS[@client_type][:os_name]? + end + + def os_version : String? + HARDCODED_CLIENTS[@client_type][:os_version]? + end + + def platform : String? + HARDCODED_CLIENTS[@client_type][:platform]? + end + # Convert to string, for logging purposes def to_s return { @@ -226,6 +265,18 @@ module YoutubeAPI client_context["client"]["androidSdkVersion"] = android_sdk_version end + if os_name = client_config.os_name + client_context["client"]["osName"] = os_name + end + + if os_version = client_config.os_version + client_context["client"]["osVersion"] = os_version + end + + if platform = client_config.platform + client_context["client"]["platform"] = platform + end + return client_context end @@ -361,8 +412,18 @@ module YoutubeAPI ) # JSON Request data, required by the API data = { - "videoId" => video_id, - "context" => self.make_context(client_config), + "contentCheckOk" => true, + "videoId" => video_id, + "context" => self.make_context(client_config), + "racyCheckOk" => true, + "user" => { + "lockedSafetyMode" => false, + }, + "playbackContext" => { + "contentPlaybackContext" => { + "html5Preference": "HTML5_PREF_WANTS", + }, + }, } # Append the additional parameters if those were provided @@ -460,10 +521,15 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", + "x-goog-api-format-version" => "2", } + if user_agent = client_config.user_agent + headers["User-Agent"] = user_agent + end + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") From c658fd27cced47c438eb148f1a1aedf482be8f46 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Fri, 2 Sep 2022 21:18:56 +0200 Subject: [PATCH 2/2] better spoof requests --- src/invidious/yt_backend/connection_pool.cr | 3 - src/invidious/yt_backend/youtube_api.cr | 103 +++++++++++++++----- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 23e98ae3..46e5bf85 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -13,9 +13,6 @@ def add_yt_headers(request) request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "2.20200609" # Preserve original cookies and add new YT consent cookie for EU servers request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index c014dc0e..02327025 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -8,11 +8,16 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" private ANDROID_APP_VERSION = "17.33.42" - private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US)" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 + private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" private IOS_APP_VERSION = "17.33.2" - private WINDOWS_VERSION = "10.0" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 + private IOS_USER_AGENT = "com.google.ios.youtube/17.33.2 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 + private IOS_VERSION = "15.6.0.19G71" + private WINDOWS_VERSION = "10.0" # Enumerate used to select one of the clients supported by the API enum ClientType @@ -37,6 +42,7 @@ module YoutubeAPI HARDCODED_CLIENTS = { ClientType::Web => { name: "WEB", + name_proto: "1", version: "2.20220804.07.00", api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", @@ -45,7 +51,8 @@ module YoutubeAPI platform: "DESKTOP", }, ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", # 56 + name: "WEB_EMBEDDED_PLAYER", + name_proto: "56", version: "1.20220803.01.00", api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -55,6 +62,7 @@ module YoutubeAPI }, ClientType::WebMobile => { name: "MWEB", + name_proto: "2", version: "2.20220805.01.00", api_key: DEFAULT_API_KEY, os_name: "Android", @@ -63,6 +71,7 @@ module YoutubeAPI }, ClientType::WebScreenEmbed => { name: "WEB", + name_proto: "1", version: "2.20220804.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -75,6 +84,7 @@ module YoutubeAPI ClientType::Android => { name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, @@ -84,12 +94,14 @@ module YoutubeAPI platform: "MOBILE", }, ClientType::AndroidEmbeddedPlayer => { - name: "ANDROID_EMBEDDED_PLAYER", # 55 - version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "ANDROID_EMBEDDED_PLAYER", + name_proto: "55", + version: ANDROID_APP_VERSION, + api_key: DEFAULT_API_KEY, }, ClientType::AndroidScreenEmbed => { - name: "ANDROID", # 3 + name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -103,33 +115,56 @@ module YoutubeAPI # IOS ClientType::IOS => { - name: "IOS", # 5 - version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + name: "IOS", + name_proto: "5", + version: IOS_APP_VERSION, + api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSEmbedded => { - name: "IOS_MESSAGES_EXTENSION", # 66 - version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "IOS_MESSAGES_EXTENSION", + name_proto: "66", + version: IOS_APP_VERSION, + api_key: DEFAULT_API_KEY, + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSMusic => { - name: "IOS_MUSIC", # 26 - version: "4.32", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + name: "IOS_MUSIC", + name_proto: "26", + version: "5.21", + api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, # TV app ClientType::TvHtml5 => { - name: "TVHTML5", # 7 - version: "7.20220325", - api_key: DEFAULT_API_KEY, + name: "TVHTML5", + name_proto: "7", + version: "7.20220325", + api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { - name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", # 85 - version: "2.0", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + name_proto: "85", + version: "2.0", + api_key: DEFAULT_API_KEY, + screen: "EMBED", }, } @@ -183,6 +218,10 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:name] end + def name_proto : String + HARDCODED_CLIENTS[@client_type][:name_proto] + end + # :ditto: def version : String HARDCODED_CLIENTS[@client_type][:version] @@ -210,6 +249,14 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:os_name]? end + def device_make : String? + HARDCODED_CLIENTS[@client_type][:device_make]? + end + + def device_model : String? + HARDCODED_CLIENTS[@client_type][:device_model]? + end + def os_version : String? HARDCODED_CLIENTS[@client_type][:os_version]? end @@ -265,6 +312,14 @@ module YoutubeAPI client_context["client"]["androidSdkVersion"] = android_sdk_version end + if device_make = client_config.device_make + client_context["client"]["deviceMake"] = device_make + end + + if device_model = client_config.device_model + client_context["client"]["deviceModel"] = device_model + end + if os_name = client_config.os_name client_context["client"]["osName"] = os_name end @@ -524,6 +579,8 @@ module YoutubeAPI "Content-Type" => "application/json; charset=UTF-8", "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", + "x-youtube-client-name" => client_config.name_proto, + "x-youtube-client-version" => client_config.version, } if user_agent = client_config.user_agent