From 1bd3704890538c133a899c0d6f6058be1a5522af Mon Sep 17 00:00:00 2001 From: deniscerri <64997243+deniscerri@users.noreply.github.com> Date: Sun, 6 Apr 2025 11:18:20 +0200 Subject: [PATCH] more fixes --- .../ui/adapter/YoutubePlayerClientAdapter.kt | 10 +- .../ytdl/ui/downloads/HistoryFragment.kt | 6 +- .../ytdl/util/extractors/YTDLPUtil.kt | 6 + .../potoken/NewPipePoTokenGenerator.kt | 2 +- .../newpipe/potoken/PoTokenWebView.kt | 305 ++++++++---------- .../main/res/layout/player_client_item.xml | 21 -- 6 files changed, 159 insertions(+), 191 deletions(-) diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt b/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt index 35e8b56f..0ad467a6 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt @@ -83,10 +83,12 @@ class YoutubePlayerClientAdapter(onItemClickListener: OnItemClickListener, activ if (item.urlRegex.isNotEmpty()) { val text = "URL Regex: " + item.urlRegex.joinToString(", ") - content.findViewById(R.id.urlRegex).apply { - isVisible = true - setText(text) - } + val tmp = activity.layoutInflater.inflate(R.layout.textview_chip, content, false) as TextView + tmp.maxWidth = 500 + tmp.maxLines = 1 + tmp.ellipsize = TextUtils.TruncateAt.END + tmp.text = text + content.addView(tmp) } title.alpha = if (item.enabled) 1f else 0.3f diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/downloads/HistoryFragment.kt b/app/src/main/java/com/deniscerri/ytdl/ui/downloads/HistoryFragment.kt index 061c5a97..39c92938 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/downloads/HistoryFragment.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/downloads/HistoryFragment.kt @@ -136,13 +136,13 @@ class HistoryFragment : Fragment(), HistoryPaginatedAdapter.OnItemClickListener{ noResults.isVisible = false historyViewModel = ViewModelProvider(this)[HistoryViewModel::class.java] recyclerView.adapter = historyAdapter + lifecycleScope.launch { historyViewModel.paginatedItems.collectLatest { - withContext(Dispatchers.IO){ - historyAdapter.submitData(it) - } + historyAdapter.submitData(it) } } + lifecycleScope.launch { historyViewModel.websites.collectLatest { if(it.isEmpty()) { diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt index ab6b6602..9d0371d3 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt @@ -1253,6 +1253,12 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co request.merge(metadataCommands) if (downloadItem.extraCommands.isNotBlank() && downloadItem.type != DownloadViewModel.Type.command){ + // check for cache dir as extra command and add it as an actual option to prevent --no-cache-dir in youtubedl_android + val cacheDirArg = """(--cache-dir (".*"))""".toRegex().find(downloadItem.extraCommands) + if (cacheDirArg != null) { + ytDlRequest.addOption("--cache-dir", cacheDirArg.groupValues.last().replace("\"", "")) + downloadItem.extraCommands.replace(cacheDirArg.value, "") + } request.addOption(downloadItem.extraCommands) } diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt index 0cacc5be..461f7faf 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt @@ -45,7 +45,7 @@ class NewPipePoTokenGenerator : PoTokenProvider { val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) = synchronized(WebPoTokenGenLock) { - val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired + val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired() if (shouldRecreate) { val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt index 0cabfdf2..ec3ff51d 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt @@ -2,6 +2,8 @@ package com.deniscerri.ytdl.util.extractors.newpipe.potoken import android.content.Context import android.os.Build +import android.os.Handler +import android.os.Looper import android.util.Log import android.webkit.ConsoleMessage import android.webkit.JavascriptInterface @@ -21,6 +23,8 @@ import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseC import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseIntegrityTokenData import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.stringToU8 import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.u8ToBase64 +import com.google.gson.Gson +import kotlinx.coroutines.CoroutineScope import okhttp3.Headers.Companion.toHeaders import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody @@ -34,48 +38,29 @@ import kotlin.coroutines.resumeWithException class PoTokenWebView private constructor( context: Context, // to be used exactly once only during initialization! - private val continuation: Continuation, + private val generatorContinuation: Continuation, ) { private val webView = WebView(context) - private val scope = MainScope() - private val poTokenContinuations = Collections.synchronizedMap(ArrayMap>()) - private val exceptionHandler = CoroutineExceptionHandler { _, t -> - onInitializationErrorCloseAndCancel(t) + private val poTokenContinuations = mutableMapOf>() + private val exceptionHandler = CoroutineExceptionHandler { context, exception -> + onInitializationError(exception) } private lateinit var expirationInstant: Instant //region Initialization init { - val webViewSettings = webView.settings - //noinspection SetJavaScriptEnabled we want to use JavaScript! - webViewSettings.javaScriptEnabled = true - if (Build.VERSION.SDK_INT >= 26) { - webViewSettings.safeBrowsingEnabled = false + webView.settings.apply { + //noinspection SetJavaScriptEnabled we want to use JavaScript! + javaScriptEnabled = true + if (Build.VERSION.SDK_INT >= 26) { + safeBrowsingEnabled = false + } + userAgentString = USER_AGENT + blockNetworkLoads = true // the WebView does not need internet access } - webViewSettings.userAgentString = USER_AGENT - webViewSettings.blockNetworkLoads = true // the WebView does not need internet access // so that we can run async functions and get back the result webView.addJavascriptInterface(this, JS_INTERFACE) - - webView.webChromeClient = object : WebChromeClient() { - override fun onConsoleMessage(m: ConsoleMessage): Boolean { - if (m.message().contains("Uncaught")) { - // There should not be any uncaught errors while executing the code, because - // everything that can fail is guarded by try-catch. Therefore, this likely - // indicates that there was a syntax error in the code, i.e. the WebView only - // supports a really old version of JS. - - val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" - val exception = Exception(fmt) - Log.e(TAG, "This WebView implementation is broken: $fmt") - - onInitializationErrorCloseAndCancel(exception) - popAllPoTokenContinuations().forEach { (_, cont) -> cont.resumeWithException(exception) } - } - return super.onConsoleMessage(m) - } - } } /** @@ -83,17 +68,30 @@ class PoTokenWebView private constructor( * initialization. This will asynchronously go through all the steps needed to load BotGuard, * run it, and obtain an `integrityToken`. */ - private fun loadHtmlAndObtainBotguard() { - Log.d(TAG, "loadHtmlAndObtainBotguard() called") + private fun loadHtmlAndObtainBotguard(context: Context) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "loadHtmlAndObtainBotguard() called") + } - scope.launch(exceptionHandler) { - val html = withContext(Dispatchers.IO) { - webView.context.assets.open("po_token.html").bufferedReader().use { it.readText() } + CoroutineScope(Dispatchers.IO).launch(exceptionHandler) { + try { + val html = context.assets.open("po_token.html").bufferedReader().use { it.readText() } + withContext(Dispatchers.Main) { + webView.loadDataWithBaseURL( + "https://www.youtube.com", + html.replaceFirst( + "", + // calls downloadAndRunBotguard() when the page has finished loading + "\n$JS_INTERFACE.downloadAndRunBotguard()" + ), + "text/html", + "utf-8", + null, + ) + } + } catch (e: Exception) { + onInitializationError(e) } - - // calls downloadAndRunBotguard() when the page has finished loading - val data = html.replaceFirst("", "\n$JS_INTERFACE.downloadAndRunBotguard()") - webView.loadDataWithBaseURL("https://www.youtube.com", data, "text/html", "utf-8", null) } } @@ -103,27 +101,32 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun downloadAndRunBotguard() { - Log.d(TAG, "downloadAndRunBotguard() called") + if (BuildConfig.DEBUG) { + Log.d(TAG, "downloadAndRunBotguard() called") + } - makeBotguardServiceRequest( - "https://www.youtube.com/api/jnn/v1/Create", - "[ \"$REQUEST_KEY\" ]", - ) { responseBody -> - val parsedChallengeData = parseChallengeData(responseBody) - webView.evaluateJavascript( - """try { - data = $parsedChallengeData - runBotGuard(data).then(function (result) { - this.webPoSignalOutput = result.webPoSignalOutput - $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) - }, function (error) { - $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) - }) - } catch (error) { - $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) - }""", - null + CoroutineScope(Dispatchers.IO).launch(exceptionHandler) { + val responseBody = makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/Create", + listOf(REQUEST_KEY) ) + val parsedChallengeData = parseChallengeData(responseBody) + withContext(Dispatchers.Main) { + webView.evaluateJavascript( + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { + this.webPoSignalOutput = result.webPoSignalOutput + $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) + }, function (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }) + } catch (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }""", + null + ) + } } } @@ -136,7 +139,7 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.e(TAG, "Initialization error from JavaScript: $error") } - onInitializationErrorCloseAndCancel(Exception(error)) + onInitializationError(Exception(error)) } /** @@ -145,20 +148,25 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onRunBotguardResult(botguardResponse: String) { - Log.d(TAG, "botguardResponse: $botguardResponse") - makeBotguardServiceRequest( - "https://www.youtube.com/api/jnn/v1/GenerateIT", - "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", - ) { responseBody -> - Log.d(TAG, "GenerateIT response: $responseBody") - val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) + CoroutineScope(Dispatchers.IO).launch(exceptionHandler) { + val response = makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/GenerateIT", + listOf(REQUEST_KEY, botguardResponse) + ) + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(response) // leave 10 minutes of margin just to be sure - expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds).minus(10, ChronoUnit.MINUTES) + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) - webView.evaluateJavascript("this.integrityToken = $integrityToken") { - Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") - continuation.resume(this) + withContext(Dispatchers.Main) { + webView.evaluateJavascript( + "this.integrityToken = $integrityToken" + ) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + } + generatorContinuation.resume(this@PoTokenWebView) + } } } } @@ -166,22 +174,29 @@ class PoTokenWebView private constructor( //region Obtaining poTokens suspend fun generatePoToken(identifier: String): String { - return withContext(Dispatchers.Main) { - suspendCancellableCoroutine { cont -> - Log.d(TAG, "generatePoToken() called with identifier $identifier") - addPoTokenEmitter(identifier, cont) + if (BuildConfig.DEBUG) { + Log.d(TAG, "generatePoToken() called with identifier $identifier") + } + return suspendCancellableCoroutine { continuation -> + poTokenContinuations[identifier] = continuation + val u8Identifier = stringToU8(identifier) + + Handler(Looper.getMainLooper()).post { webView.evaluateJavascript( """try { identifier = "$identifier" - u8Identifier = ${stringToU8(identifier)} + u8Identifier = $u8Identifier poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier) - poTokenU8String = poTokenU8.join(",") + poTokenU8String = "" + for (i = 0; i < poTokenU8.length; i++) { + if (i != 0) poTokenU8String += "," + poTokenU8String += poTokenU8[i] + } $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String) } catch (error) { $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack) }""", - null - ) + ) {} } } } @@ -195,7 +210,7 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.e(TAG, "obtainPoToken error from JavaScript: $error") } - popPoTokenContinuation(identifier)?.resumeWithException(Exception(error)) + poTokenContinuations.remove(identifier)?.resumeWithException(Exception(error)) } /** @@ -204,118 +219,84 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onObtainPoTokenResult(identifier: String, poTokenU8: String) { - Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + } val poToken = try { u8ToBase64(poTokenU8) } catch (t: Throwable) { - popPoTokenContinuation(identifier)?.resumeWithException(t) + poTokenContinuations.remove(identifier)?.resumeWithException(t) return } - Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") - popPoTokenContinuation(identifier)?.resume(poToken) + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") + } + poTokenContinuations.remove(identifier)?.resume(poToken) } - val isExpired: Boolean - get() = Instant.now().isAfter(expirationInstant) - //endregion - - //region Handling multiple emitters - /** - * Adds the ([identifier], [continuation]) pair to the [poTokenContinuations] list. This makes - * it so that multiple poToken requests can be generated in parallel, and the results will be - * notified to the right continuations. - */ - private fun addPoTokenEmitter(identifier: String, continuation: Continuation) { - poTokenContinuations[identifier] = continuation - } - - /** - * Extracts and removes from the [poTokenContinuations] list a [Continuation] based on its - * [identifier]. The continuation is supposed to be used immediately after to either signal a - * success or an error. - */ - private fun popPoTokenContinuation(identifier: String): Continuation? { - return poTokenContinuations.remove(identifier) - } - - /** - * Clears [poTokenContinuations] and returns its previous contents. The continuations are supposed - * to be used immediately after to either signal a success or an error. - */ - private fun popAllPoTokenContinuations(): Map> { - val result = poTokenContinuations.toMap() - poTokenContinuations.clear() - return result + fun isExpired(): Boolean { + return Instant.now().isAfter(expirationInstant) } //endregion //region Utils /** - * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls - * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response - * does not have HTTP code 200, therefore this is supposed to be used only during - * initialization. Calls [handleResponseBody] with the response body if the response is - * successful. The request is performed in the background and a disposable is added to - * [disposables]. + * Makes a POST request to [url] with the given [data] by setting the correct headers. + * This is supposed to be used only during initialization. Returns the response body + * as a String if the response is successful. */ - private fun makeBotguardServiceRequest( - url: String, - data: String, - handleResponseBody: (String) -> Unit, - ) { - scope.launch(exceptionHandler) { - val requestBuilder = okhttp3.Request.Builder() - .post(data.toRequestBody()) - .headers(mapOf( - "User-Agent" to USER_AGENT, - "Accept" to "application/json", - "Content-Type" to "application/json+protobuf", - "x-goog-api-key" to GOOGLE_API_KEY, - "x-user-agent" to "grpc-web-javascript/0.1", - ).toHeaders()) - .url(url) - val response = withContext(Dispatchers.IO) { - httpClient.newCall(requestBuilder.build()).execute() - } - val httpCode = response.code - if (httpCode != 200) { - onInitializationErrorCloseAndCancel(Exception("Invalid response code: $httpCode")) - } else { - val body = withContext(Dispatchers.IO) { - response.body.string() - } - handleResponseBody(body) + private suspend fun makeBotguardServiceRequest(url: String, data: List): String = withContext(Dispatchers.IO) { + val requestBuilder = okhttp3.Request.Builder() + .post(Gson().toJson(data).toRequestBody()) + .headers(mapOf( + "User-Agent" to USER_AGENT, + "Accept" to "application/json", + "Content-Type" to "application/json+protobuf", + "x-goog-api-key" to GOOGLE_API_KEY, + "x-user-agent" to "grpc-web-javascript/0.1", + ).toHeaders()) + .url(url) + val response = withContext(Dispatchers.IO) { + httpClient.newCall(requestBuilder.build()).execute() + } + val httpCode = response.code + if (httpCode != 200) { + throw Exception("Invalid response code: $httpCode") + } else { + val body = withContext(Dispatchers.IO) { + response.body.string() } + body } } /** * Handles any error happening during initialization, releasing resources and sending the error - * to [continuation]. + * to [generatorContinuation]. */ - private fun onInitializationErrorCloseAndCancel(error: Throwable) { - close() - continuation.resumeWithException(error) + private fun onInitializationError(error: Throwable) { + CoroutineScope(Dispatchers.Main).launch { + close() + generatorContinuation.resumeWithException(error) + } } /** * Releases all [webView] resources. */ @MainThread - fun close() { - scope.cancel() - - webView.clearHistory() + fun close() = with(webView) { + clearHistory() // clears RAM cache and disk cache (globally for all WebViews) - webView.clearCache(true) + clearCache(true) // ensures that the WebView isn't doing anything when destroying it - webView.loadUrl("about:blank") + loadUrl("about:blank") - webView.onPause() - webView.removeAllViews() - webView.destroy() + onPause() + removeAllViews() + destroy() } //endregion @@ -336,7 +317,7 @@ class PoTokenWebView private constructor( return withContext(Dispatchers.Main) { suspendCancellableCoroutine { cont -> val potWv = PoTokenWebView(context, cont) - potWv.loadHtmlAndObtainBotguard() + potWv.loadHtmlAndObtainBotguard(context) } } } diff --git a/app/src/main/res/layout/player_client_item.xml b/app/src/main/res/layout/player_client_item.xml index de13f7c8..5caa09b8 100644 --- a/app/src/main/res/layout/player_client_item.xml +++ b/app/src/main/res/layout/player_client_item.xml @@ -61,27 +61,6 @@ android:layout_width="0dp" android:layout_height="wrap_content"> - -