more fixes

This commit is contained in:
deniscerri 2025-04-06 11:18:20 +02:00
parent 5b31612462
commit 1bd3704890
No known key found for this signature in database
GPG key ID: 95C43D517D830350
6 changed files with 159 additions and 191 deletions

View file

@ -83,10 +83,12 @@ class YoutubePlayerClientAdapter(onItemClickListener: OnItemClickListener, activ
if (item.urlRegex.isNotEmpty()) {
val text = "URL Regex: " + item.urlRegex.joinToString(", ")
content.findViewById<TextView>(R.id.urlRegex).apply {
isVisible = true
setText(text)
}
val tmp = activity.layoutInflater.inflate(R.layout.textview_chip, content, false) as TextView
tmp.maxWidth = 500
tmp.maxLines = 1
tmp.ellipsize = TextUtils.TruncateAt.END
tmp.text = text
content.addView(tmp)
}
title.alpha = if (item.enabled) 1f else 0.3f

View file

@ -136,13 +136,13 @@ class HistoryFragment : Fragment(), HistoryPaginatedAdapter.OnItemClickListener{
noResults.isVisible = false
historyViewModel = ViewModelProvider(this)[HistoryViewModel::class.java]
recyclerView.adapter = historyAdapter
lifecycleScope.launch {
historyViewModel.paginatedItems.collectLatest {
withContext(Dispatchers.IO){
historyAdapter.submitData(it)
}
historyAdapter.submitData(it)
}
}
lifecycleScope.launch {
historyViewModel.websites.collectLatest {
if(it.isEmpty()) {

View file

@ -1253,6 +1253,12 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
request.merge(metadataCommands)
if (downloadItem.extraCommands.isNotBlank() && downloadItem.type != DownloadViewModel.Type.command){
// check for cache dir as extra command and add it as an actual option to prevent --no-cache-dir in youtubedl_android
val cacheDirArg = """(--cache-dir (".*"))""".toRegex().find(downloadItem.extraCommands)
if (cacheDirArg != null) {
ytDlRequest.addOption("--cache-dir", cacheDirArg.groupValues.last().replace("\"", ""))
downloadItem.extraCommands.replace(cacheDirArg.value, "")
}
request.addOption(downloadItem.extraCommands)
}

View file

@ -45,7 +45,7 @@ class NewPipePoTokenGenerator : PoTokenProvider {
val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) =
synchronized(WebPoTokenGenLock) {
val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired
val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired()
if (shouldRecreate) {
val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient()

View file

@ -2,6 +2,8 @@ package com.deniscerri.ytdl.util.extractors.newpipe.potoken
import android.content.Context
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.webkit.ConsoleMessage
import android.webkit.JavascriptInterface
@ -21,6 +23,8 @@ import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseC
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseIntegrityTokenData
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.stringToU8
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.u8ToBase64
import com.google.gson.Gson
import kotlinx.coroutines.CoroutineScope
import okhttp3.Headers.Companion.toHeaders
import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
@ -34,48 +38,29 @@ import kotlin.coroutines.resumeWithException
class PoTokenWebView private constructor(
context: Context,
// to be used exactly once only during initialization!
private val continuation: Continuation<PoTokenWebView>,
private val generatorContinuation: Continuation<PoTokenWebView>,
) {
private val webView = WebView(context)
private val scope = MainScope()
private val poTokenContinuations = Collections.synchronizedMap(ArrayMap<String, Continuation<String>>())
private val exceptionHandler = CoroutineExceptionHandler { _, t ->
onInitializationErrorCloseAndCancel(t)
private val poTokenContinuations = mutableMapOf<String, Continuation<String>>()
private val exceptionHandler = CoroutineExceptionHandler { context, exception ->
onInitializationError(exception)
}
private lateinit var expirationInstant: Instant
//region Initialization
init {
val webViewSettings = webView.settings
//noinspection SetJavaScriptEnabled we want to use JavaScript!
webViewSettings.javaScriptEnabled = true
if (Build.VERSION.SDK_INT >= 26) {
webViewSettings.safeBrowsingEnabled = false
webView.settings.apply {
//noinspection SetJavaScriptEnabled we want to use JavaScript!
javaScriptEnabled = true
if (Build.VERSION.SDK_INT >= 26) {
safeBrowsingEnabled = false
}
userAgentString = USER_AGENT
blockNetworkLoads = true // the WebView does not need internet access
}
webViewSettings.userAgentString = USER_AGENT
webViewSettings.blockNetworkLoads = true // the WebView does not need internet access
// so that we can run async functions and get back the result
webView.addJavascriptInterface(this, JS_INTERFACE)
webView.webChromeClient = object : WebChromeClient() {
override fun onConsoleMessage(m: ConsoleMessage): Boolean {
if (m.message().contains("Uncaught")) {
// There should not be any uncaught errors while executing the code, because
// everything that can fail is guarded by try-catch. Therefore, this likely
// indicates that there was a syntax error in the code, i.e. the WebView only
// supports a really old version of JS.
val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})"
val exception = Exception(fmt)
Log.e(TAG, "This WebView implementation is broken: $fmt")
onInitializationErrorCloseAndCancel(exception)
popAllPoTokenContinuations().forEach { (_, cont) -> cont.resumeWithException(exception) }
}
return super.onConsoleMessage(m)
}
}
}
/**
@ -83,17 +68,30 @@ class PoTokenWebView private constructor(
* initialization. This will asynchronously go through all the steps needed to load BotGuard,
* run it, and obtain an `integrityToken`.
*/
private fun loadHtmlAndObtainBotguard() {
Log.d(TAG, "loadHtmlAndObtainBotguard() called")
private fun loadHtmlAndObtainBotguard(context: Context) {
if (BuildConfig.DEBUG) {
Log.d(TAG, "loadHtmlAndObtainBotguard() called")
}
scope.launch(exceptionHandler) {
val html = withContext(Dispatchers.IO) {
webView.context.assets.open("po_token.html").bufferedReader().use { it.readText() }
CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
try {
val html = context.assets.open("po_token.html").bufferedReader().use { it.readText() }
withContext(Dispatchers.Main) {
webView.loadDataWithBaseURL(
"https://www.youtube.com",
html.replaceFirst(
"</script>",
// calls downloadAndRunBotguard() when the page has finished loading
"\n$JS_INTERFACE.downloadAndRunBotguard()</script>"
),
"text/html",
"utf-8",
null,
)
}
} catch (e: Exception) {
onInitializationError(e)
}
// calls downloadAndRunBotguard() when the page has finished loading
val data = html.replaceFirst("</script>", "\n$JS_INTERFACE.downloadAndRunBotguard()</script>")
webView.loadDataWithBaseURL("https://www.youtube.com", data, "text/html", "utf-8", null)
}
}
@ -103,27 +101,32 @@ class PoTokenWebView private constructor(
*/
@JavascriptInterface
fun downloadAndRunBotguard() {
Log.d(TAG, "downloadAndRunBotguard() called")
if (BuildConfig.DEBUG) {
Log.d(TAG, "downloadAndRunBotguard() called")
}
makeBotguardServiceRequest(
"https://www.youtube.com/api/jnn/v1/Create",
"[ \"$REQUEST_KEY\" ]",
) { responseBody ->
val parsedChallengeData = parseChallengeData(responseBody)
webView.evaluateJavascript(
"""try {
data = $parsedChallengeData
runBotGuard(data).then(function (result) {
this.webPoSignalOutput = result.webPoSignalOutput
$JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
}, function (error) {
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
})
} catch (error) {
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
}""",
null
CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
val responseBody = makeBotguardServiceRequest(
"https://www.youtube.com/api/jnn/v1/Create",
listOf(REQUEST_KEY)
)
val parsedChallengeData = parseChallengeData(responseBody)
withContext(Dispatchers.Main) {
webView.evaluateJavascript(
"""try {
data = $parsedChallengeData
runBotGuard(data).then(function (result) {
this.webPoSignalOutput = result.webPoSignalOutput
$JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
}, function (error) {
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
})
} catch (error) {
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
}""",
null
)
}
}
}
@ -136,7 +139,7 @@ class PoTokenWebView private constructor(
if (BuildConfig.DEBUG) {
Log.e(TAG, "Initialization error from JavaScript: $error")
}
onInitializationErrorCloseAndCancel(Exception(error))
onInitializationError(Exception(error))
}
/**
@ -145,20 +148,25 @@ class PoTokenWebView private constructor(
*/
@JavascriptInterface
fun onRunBotguardResult(botguardResponse: String) {
Log.d(TAG, "botguardResponse: $botguardResponse")
makeBotguardServiceRequest(
"https://www.youtube.com/api/jnn/v1/GenerateIT",
"[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
) { responseBody ->
Log.d(TAG, "GenerateIT response: $responseBody")
val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody)
CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
val response = makeBotguardServiceRequest(
"https://www.youtube.com/api/jnn/v1/GenerateIT",
listOf(REQUEST_KEY, botguardResponse)
)
val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(response)
// leave 10 minutes of margin just to be sure
expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds).minus(10, ChronoUnit.MINUTES)
expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
webView.evaluateJavascript("this.integrityToken = $integrityToken") {
Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
continuation.resume(this)
withContext(Dispatchers.Main) {
webView.evaluateJavascript(
"this.integrityToken = $integrityToken"
) {
if (BuildConfig.DEBUG) {
Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
}
generatorContinuation.resume(this@PoTokenWebView)
}
}
}
}
@ -166,22 +174,29 @@ class PoTokenWebView private constructor(
//region Obtaining poTokens
suspend fun generatePoToken(identifier: String): String {
return withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
Log.d(TAG, "generatePoToken() called with identifier $identifier")
addPoTokenEmitter(identifier, cont)
if (BuildConfig.DEBUG) {
Log.d(TAG, "generatePoToken() called with identifier $identifier")
}
return suspendCancellableCoroutine { continuation ->
poTokenContinuations[identifier] = continuation
val u8Identifier = stringToU8(identifier)
Handler(Looper.getMainLooper()).post {
webView.evaluateJavascript(
"""try {
identifier = "$identifier"
u8Identifier = ${stringToU8(identifier)}
u8Identifier = $u8Identifier
poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier)
poTokenU8String = poTokenU8.join(",")
poTokenU8String = ""
for (i = 0; i < poTokenU8.length; i++) {
if (i != 0) poTokenU8String += ","
poTokenU8String += poTokenU8[i]
}
$JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String)
} catch (error) {
$JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack)
}""",
null
)
) {}
}
}
}
@ -195,7 +210,7 @@ class PoTokenWebView private constructor(
if (BuildConfig.DEBUG) {
Log.e(TAG, "obtainPoToken error from JavaScript: $error")
}
popPoTokenContinuation(identifier)?.resumeWithException(Exception(error))
poTokenContinuations.remove(identifier)?.resumeWithException(Exception(error))
}
/**
@ -204,118 +219,84 @@ class PoTokenWebView private constructor(
*/
@JavascriptInterface
fun onObtainPoTokenResult(identifier: String, poTokenU8: String) {
Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
if (BuildConfig.DEBUG) {
Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
}
val poToken = try {
u8ToBase64(poTokenU8)
} catch (t: Throwable) {
popPoTokenContinuation(identifier)?.resumeWithException(t)
poTokenContinuations.remove(identifier)?.resumeWithException(t)
return
}
Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
popPoTokenContinuation(identifier)?.resume(poToken)
if (BuildConfig.DEBUG) {
Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
}
poTokenContinuations.remove(identifier)?.resume(poToken)
}
val isExpired: Boolean
get() = Instant.now().isAfter(expirationInstant)
//endregion
//region Handling multiple emitters
/**
* Adds the ([identifier], [continuation]) pair to the [poTokenContinuations] list. This makes
* it so that multiple poToken requests can be generated in parallel, and the results will be
* notified to the right continuations.
*/
private fun addPoTokenEmitter(identifier: String, continuation: Continuation<String>) {
poTokenContinuations[identifier] = continuation
}
/**
* Extracts and removes from the [poTokenContinuations] list a [Continuation] based on its
* [identifier]. The continuation is supposed to be used immediately after to either signal a
* success or an error.
*/
private fun popPoTokenContinuation(identifier: String): Continuation<String>? {
return poTokenContinuations.remove(identifier)
}
/**
* Clears [poTokenContinuations] and returns its previous contents. The continuations are supposed
* to be used immediately after to either signal a success or an error.
*/
private fun popAllPoTokenContinuations(): Map<String, Continuation<String>> {
val result = poTokenContinuations.toMap()
poTokenContinuations.clear()
return result
fun isExpired(): Boolean {
return Instant.now().isAfter(expirationInstant)
}
//endregion
//region Utils
/**
* Makes a POST request to [url] with the given [data] by setting the correct headers. Calls
* [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response
* does not have HTTP code 200, therefore this is supposed to be used only during
* initialization. Calls [handleResponseBody] with the response body if the response is
* successful. The request is performed in the background and a disposable is added to
* [disposables].
* Makes a POST request to [url] with the given [data] by setting the correct headers.
* This is supposed to be used only during initialization. Returns the response body
* as a String if the response is successful.
*/
private fun makeBotguardServiceRequest(
url: String,
data: String,
handleResponseBody: (String) -> Unit,
) {
scope.launch(exceptionHandler) {
val requestBuilder = okhttp3.Request.Builder()
.post(data.toRequestBody())
.headers(mapOf(
"User-Agent" to USER_AGENT,
"Accept" to "application/json",
"Content-Type" to "application/json+protobuf",
"x-goog-api-key" to GOOGLE_API_KEY,
"x-user-agent" to "grpc-web-javascript/0.1",
).toHeaders())
.url(url)
val response = withContext(Dispatchers.IO) {
httpClient.newCall(requestBuilder.build()).execute()
}
val httpCode = response.code
if (httpCode != 200) {
onInitializationErrorCloseAndCancel(Exception("Invalid response code: $httpCode"))
} else {
val body = withContext(Dispatchers.IO) {
response.body.string()
}
handleResponseBody(body)
private suspend fun makeBotguardServiceRequest(url: String, data: List<String>): String = withContext(Dispatchers.IO) {
val requestBuilder = okhttp3.Request.Builder()
.post(Gson().toJson(data).toRequestBody())
.headers(mapOf(
"User-Agent" to USER_AGENT,
"Accept" to "application/json",
"Content-Type" to "application/json+protobuf",
"x-goog-api-key" to GOOGLE_API_KEY,
"x-user-agent" to "grpc-web-javascript/0.1",
).toHeaders())
.url(url)
val response = withContext(Dispatchers.IO) {
httpClient.newCall(requestBuilder.build()).execute()
}
val httpCode = response.code
if (httpCode != 200) {
throw Exception("Invalid response code: $httpCode")
} else {
val body = withContext(Dispatchers.IO) {
response.body.string()
}
body
}
}
/**
* Handles any error happening during initialization, releasing resources and sending the error
* to [continuation].
* to [generatorContinuation].
*/
private fun onInitializationErrorCloseAndCancel(error: Throwable) {
close()
continuation.resumeWithException(error)
private fun onInitializationError(error: Throwable) {
CoroutineScope(Dispatchers.Main).launch {
close()
generatorContinuation.resumeWithException(error)
}
}
/**
* Releases all [webView] resources.
*/
@MainThread
fun close() {
scope.cancel()
webView.clearHistory()
fun close() = with(webView) {
clearHistory()
// clears RAM cache and disk cache (globally for all WebViews)
webView.clearCache(true)
clearCache(true)
// ensures that the WebView isn't doing anything when destroying it
webView.loadUrl("about:blank")
loadUrl("about:blank")
webView.onPause()
webView.removeAllViews()
webView.destroy()
onPause()
removeAllViews()
destroy()
}
//endregion
@ -336,7 +317,7 @@ class PoTokenWebView private constructor(
return withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
val potWv = PoTokenWebView(context, cont)
potWv.loadHtmlAndObtainBotguard()
potWv.loadHtmlAndObtainBotguard(context)
}
}
}

View file

@ -61,27 +61,6 @@
android:layout_width="0dp"
android:layout_height="wrap_content">
<TextView
android:id="@+id/urlRegex"
style="@style/Widget.Material3.FloatingActionButton.Large.Secondary"
android:layout_width="wrap_content"
android:visibility="gone"
android:layout_height="wrap_content"
android:text="URL Regex"
android:background="@drawable/rounded_corner"
android:backgroundTint="?attr/colorAccent"
android:clickable="false"
android:gravity="center"
android:minWidth="30dp"
android:paddingHorizontal="5dp"
android:textSize="12sp"
android:textStyle="bold"
app:cornerRadius="10dp"
app:layout_constraintTop_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
tools:ignore="HardcodedText" />
</com.google.android.material.chip.ChipGroup>