more fixes

app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt

@@ -83,10 +83,12 @@ class YoutubePlayerClientAdapter(onItemClickListener: OnItemClickListener, activ
 
         if (item.urlRegex.isNotEmpty()) {
             val text = "URL Regex: " + item.urlRegex.joinToString(", ")
-            content.findViewById<TextView>(R.id.urlRegex).apply {
+            val tmp =  activity.layoutInflater.inflate(R.layout.textview_chip, content, false) as TextView
-                isVisible = true
+            tmp.maxWidth = 500
-                setText(text)
+            tmp.maxLines = 1
-            }
+            tmp.ellipsize = TextUtils.TruncateAt.END
+            tmp.text = text
+            content.addView(tmp)
         }
 
         title.alpha = if (item.enabled) 1f else 0.3f

app/src/main/java/com/deniscerri/ytdl/ui/downloads/HistoryFragment.kt

@@ -136,13 +136,13 @@ class HistoryFragment : Fragment(), HistoryPaginatedAdapter.OnItemClickListener{
         noResults.isVisible = false
         historyViewModel = ViewModelProvider(this)[HistoryViewModel::class.java]
         recyclerView.adapter = historyAdapter
+
         lifecycleScope.launch {
             historyViewModel.paginatedItems.collectLatest {
-                withContext(Dispatchers.IO){
+                historyAdapter.submitData(it)
-                    historyAdapter.submitData(it)
-                }
             }
         }
+
         lifecycleScope.launch {
             historyViewModel.websites.collectLatest {
                 if(it.isEmpty()) {

app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt

@@ -1253,6 +1253,12 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
         request.merge(metadataCommands)
 
         if (downloadItem.extraCommands.isNotBlank() && downloadItem.type != DownloadViewModel.Type.command){
+            // check for cache dir as extra command and add it as an actual option to prevent --no-cache-dir in youtubedl_android
+            val cacheDirArg = """(--cache-dir (".*"))""".toRegex().find(downloadItem.extraCommands)
+            if (cacheDirArg != null) {
+                ytDlRequest.addOption("--cache-dir", cacheDirArg.groupValues.last().replace("\"", ""))
+                downloadItem.extraCommands.replace(cacheDirArg.value, "")
+            }
             request.addOption(downloadItem.extraCommands)
         }
 

app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt

@@ -45,7 +45,7 @@ class NewPipePoTokenGenerator : PoTokenProvider {
 
         val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) =
             synchronized(WebPoTokenGenLock) {
-                val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired
+                val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired()
 
                 if (shouldRecreate) {
                     val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient()

app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt

@@ -2,6 +2,8 @@ package com.deniscerri.ytdl.util.extractors.newpipe.potoken
 
 import android.content.Context
 import android.os.Build
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
 import android.webkit.ConsoleMessage
 import android.webkit.JavascriptInterface
@@ -21,6 +23,8 @@ import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseC
 import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseIntegrityTokenData
 import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.stringToU8
 import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.u8ToBase64
+import com.google.gson.Gson
+import kotlinx.coroutines.CoroutineScope
 import okhttp3.Headers.Companion.toHeaders
 import okhttp3.OkHttpClient
 import okhttp3.RequestBody.Companion.toRequestBody
@@ -34,48 +38,29 @@ import kotlin.coroutines.resumeWithException
 class PoTokenWebView private constructor(
     context: Context,
     // to be used exactly once only during initialization!
-    private val continuation: Continuation<PoTokenWebView>,
+    private val generatorContinuation: Continuation<PoTokenWebView>,
 ) {
     private val webView = WebView(context)
-    private val scope = MainScope()
+    private val poTokenContinuations = mutableMapOf<String, Continuation<String>>()
-    private val poTokenContinuations = Collections.synchronizedMap(ArrayMap<String, Continuation<String>>())
+    private val exceptionHandler = CoroutineExceptionHandler { context, exception ->
-    private val exceptionHandler = CoroutineExceptionHandler { _, t ->
+        onInitializationError(exception)
-        onInitializationErrorCloseAndCancel(t)
     }
     private lateinit var expirationInstant: Instant
 
     //region Initialization
     init {
-        val webViewSettings = webView.settings
+        webView.settings.apply {
-        //noinspection SetJavaScriptEnabled we want to use JavaScript!
+            //noinspection SetJavaScriptEnabled we want to use JavaScript!
-        webViewSettings.javaScriptEnabled = true
+            javaScriptEnabled = true
-        if (Build.VERSION.SDK_INT >= 26) {
+            if (Build.VERSION.SDK_INT >= 26) {
-            webViewSettings.safeBrowsingEnabled = false
+                safeBrowsingEnabled = false
+            }
+            userAgentString = USER_AGENT
+            blockNetworkLoads = true // the WebView does not need internet access
         }
-        webViewSettings.userAgentString = USER_AGENT
-        webViewSettings.blockNetworkLoads = true // the WebView does not need internet access
 
         // so that we can run async functions and get back the result
         webView.addJavascriptInterface(this, JS_INTERFACE)
-
-        webView.webChromeClient = object : WebChromeClient() {
-            override fun onConsoleMessage(m: ConsoleMessage): Boolean {
-                if (m.message().contains("Uncaught")) {
-                    // There should not be any uncaught errors while executing the code, because
-                    // everything that can fail is guarded by try-catch. Therefore, this likely
-                    // indicates that there was a syntax error in the code, i.e. the WebView only
-                    // supports a really old version of JS.
-
-                    val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})"
-                    val exception = Exception(fmt)
-                    Log.e(TAG, "This WebView implementation is broken: $fmt")
-
-                    onInitializationErrorCloseAndCancel(exception)
-                    popAllPoTokenContinuations().forEach { (_, cont) -> cont.resumeWithException(exception) }
-                }
-                return super.onConsoleMessage(m)
-            }
-        }
     }
 
     /**
@@ -83,17 +68,30 @@ class PoTokenWebView private constructor(
      * initialization. This will asynchronously go through all the steps needed to load BotGuard,
      * run it, and obtain an `integrityToken`.
      */
-    private fun loadHtmlAndObtainBotguard() {
+    private fun loadHtmlAndObtainBotguard(context: Context) {
-        Log.d(TAG, "loadHtmlAndObtainBotguard() called")
+        if (BuildConfig.DEBUG) {
+            Log.d(TAG, "loadHtmlAndObtainBotguard() called")
+        }
 
-        scope.launch(exceptionHandler) {
+        CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
-            val html = withContext(Dispatchers.IO) {
+            try {
-                webView.context.assets.open("po_token.html").bufferedReader().use { it.readText() }
+                val html = context.assets.open("po_token.html").bufferedReader().use { it.readText() }
+                withContext(Dispatchers.Main) {
+                    webView.loadDataWithBaseURL(
+                        "https://www.youtube.com",
+                        html.replaceFirst(
+                            "</script>",
+                            // calls downloadAndRunBotguard() when the page has finished loading
+                            "\n$JS_INTERFACE.downloadAndRunBotguard()</script>"
+                        ),
+                        "text/html",
+                        "utf-8",
+                        null,
+                    )
+                }
+            } catch (e: Exception) {
+                onInitializationError(e)
             }
-
-            // calls downloadAndRunBotguard() when the page has finished loading
-            val data = html.replaceFirst("</script>", "\n$JS_INTERFACE.downloadAndRunBotguard()</script>")
-            webView.loadDataWithBaseURL("https://www.youtube.com", data, "text/html", "utf-8", null)
         }
     }
 
@@ -103,27 +101,32 @@ class PoTokenWebView private constructor(
      */
     @JavascriptInterface
     fun downloadAndRunBotguard() {
-        Log.d(TAG, "downloadAndRunBotguard() called")
+        if (BuildConfig.DEBUG) {
+            Log.d(TAG, "downloadAndRunBotguard() called")
+        }
 
-        makeBotguardServiceRequest(
+        CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
-            "https://www.youtube.com/api/jnn/v1/Create",
+            val responseBody = makeBotguardServiceRequest(
-            "[ \"$REQUEST_KEY\" ]",
+                "https://www.youtube.com/api/jnn/v1/Create",
-        ) { responseBody ->
+                listOf(REQUEST_KEY)
-            val parsedChallengeData = parseChallengeData(responseBody)
-            webView.evaluateJavascript(
-                """try {
-                    data = $parsedChallengeData
-                    runBotGuard(data).then(function (result) {
-                        this.webPoSignalOutput = result.webPoSignalOutput
-                        $JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
-                    }, function (error) {
-                        $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
-                    })
-                } catch (error) {
-                    $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
-                }""",
-                null
             )
+            val parsedChallengeData = parseChallengeData(responseBody)
+            withContext(Dispatchers.Main) {
+                webView.evaluateJavascript(
+                    """try {
+                             data = $parsedChallengeData
+                             runBotGuard(data).then(function (result) {
+                                 this.webPoSignalOutput = result.webPoSignalOutput
+                                 $JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
+                             }, function (error) {
+                                 $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
+                             })
+                         } catch (error) {
+                             $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
+                         }""",
+                    null
+                )
+            }
         }
     }
 
@@ -136,7 +139,7 @@ class PoTokenWebView private constructor(
         if (BuildConfig.DEBUG) {
             Log.e(TAG, "Initialization error from JavaScript: $error")
         }
-        onInitializationErrorCloseAndCancel(Exception(error))
+        onInitializationError(Exception(error))
     }
 
     /**
@@ -145,20 +148,25 @@ class PoTokenWebView private constructor(
      */
     @JavascriptInterface
     fun onRunBotguardResult(botguardResponse: String) {
-        Log.d(TAG, "botguardResponse: $botguardResponse")
+        CoroutineScope(Dispatchers.IO).launch(exceptionHandler) {
-        makeBotguardServiceRequest(
+            val response = makeBotguardServiceRequest(
-            "https://www.youtube.com/api/jnn/v1/GenerateIT",
+                "https://www.youtube.com/api/jnn/v1/GenerateIT",
-            "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
+                listOf(REQUEST_KEY, botguardResponse)
-        ) { responseBody ->
+            )
-            Log.d(TAG, "GenerateIT response: $responseBody")
+            val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(response)
-            val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody)
 
             // leave 10 minutes of margin just to be sure
-            expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds).minus(10, ChronoUnit.MINUTES)
+            expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
 
-            webView.evaluateJavascript("this.integrityToken = $integrityToken") {
+            withContext(Dispatchers.Main) {
-                Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
+                webView.evaluateJavascript(
-                continuation.resume(this)
+                    "this.integrityToken = $integrityToken"
+                ) {
+                    if (BuildConfig.DEBUG) {
+                        Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
+                    }
+                    generatorContinuation.resume(this@PoTokenWebView)
+                }
             }
         }
     }
@@ -166,22 +174,29 @@ class PoTokenWebView private constructor(
 
     //region Obtaining poTokens
     suspend fun generatePoToken(identifier: String): String {
-        return withContext(Dispatchers.Main) {
+        if (BuildConfig.DEBUG) {
-            suspendCancellableCoroutine { cont ->
+            Log.d(TAG, "generatePoToken() called with identifier $identifier")
-                Log.d(TAG, "generatePoToken() called with identifier $identifier")
+        }
-                addPoTokenEmitter(identifier, cont)
+        return suspendCancellableCoroutine { continuation ->
+            poTokenContinuations[identifier] = continuation
+            val u8Identifier = stringToU8(identifier)
+
+            Handler(Looper.getMainLooper()).post {
                 webView.evaluateJavascript(
                     """try {
                         identifier = "$identifier"
-                        u8Identifier = ${stringToU8(identifier)}
+                        u8Identifier = $u8Identifier
                         poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier)
-                        poTokenU8String = poTokenU8.join(",")
+                        poTokenU8String = ""
+                        for (i = 0; i < poTokenU8.length; i++) {
+                            if (i != 0) poTokenU8String += ","
+                            poTokenU8String += poTokenU8[i]
+                        }
                         $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String)
                     } catch (error) {
                         $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack)
                     }""",
-                    null
+                ) {}
-                )
             }
         }
     }
@@ -195,7 +210,7 @@ class PoTokenWebView private constructor(
         if (BuildConfig.DEBUG) {
             Log.e(TAG, "obtainPoToken error from JavaScript: $error")
         }
-        popPoTokenContinuation(identifier)?.resumeWithException(Exception(error))
+        poTokenContinuations.remove(identifier)?.resumeWithException(Exception(error))
     }
 
     /**
@@ -204,118 +219,84 @@ class PoTokenWebView private constructor(
      */
     @JavascriptInterface
     fun onObtainPoTokenResult(identifier: String, poTokenU8: String) {
-        Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
+        if (BuildConfig.DEBUG) {
+            Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
+        }
         val poToken = try {
             u8ToBase64(poTokenU8)
         } catch (t: Throwable) {
-            popPoTokenContinuation(identifier)?.resumeWithException(t)
+            poTokenContinuations.remove(identifier)?.resumeWithException(t)
             return
         }
 
-        Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
+        if (BuildConfig.DEBUG) {
-        popPoTokenContinuation(identifier)?.resume(poToken)
+            Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
+        }
+        poTokenContinuations.remove(identifier)?.resume(poToken)
     }
 
-    val isExpired: Boolean
+    fun isExpired(): Boolean {
-        get() = Instant.now().isAfter(expirationInstant)
+        return Instant.now().isAfter(expirationInstant)
-    //endregion
-
-    //region Handling multiple emitters
-    /**
-     * Adds the ([identifier], [continuation]) pair to the [poTokenContinuations] list. This makes
-     * it so that multiple poToken requests can be generated in parallel, and the results will be
-     * notified to the right continuations.
-     */
-    private fun addPoTokenEmitter(identifier: String, continuation: Continuation<String>) {
-        poTokenContinuations[identifier] = continuation
-    }
-
-    /**
-     * Extracts and removes from the [poTokenContinuations] list a [Continuation] based on its
-     * [identifier]. The continuation is supposed to be used immediately after to either signal a
-     * success or an error.
-     */
-    private fun popPoTokenContinuation(identifier: String): Continuation<String>? {
-        return poTokenContinuations.remove(identifier)
-    }
-
-    /**
-     * Clears [poTokenContinuations] and returns its previous contents. The continuations are supposed
-     * to be used immediately after to either signal a success or an error.
-     */
-    private fun popAllPoTokenContinuations(): Map<String, Continuation<String>> {
-        val result = poTokenContinuations.toMap()
-        poTokenContinuations.clear()
-        return result
     }
     //endregion
 
     //region Utils
     /**
-     * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls
+     * Makes a POST request to [url] with the given [data] by setting the correct headers.
-     * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response
+     * This is supposed to be used only during initialization. Returns the  response body
-     * does not have HTTP code 200, therefore this is supposed to be used only during
+     * as a String if the response is successful.
-     * initialization. Calls [handleResponseBody] with the response body if the response is
-     * successful. The request is performed in the background and a disposable is added to
-     * [disposables].
      */
-    private fun makeBotguardServiceRequest(
+    private suspend fun makeBotguardServiceRequest(url: String, data: List<String>): String = withContext(Dispatchers.IO) {
-        url: String,
+        val requestBuilder = okhttp3.Request.Builder()
-        data: String,
+            .post(Gson().toJson(data).toRequestBody())
-        handleResponseBody: (String) -> Unit,
+            .headers(mapOf(
-    ) {
+                "User-Agent" to USER_AGENT,
-        scope.launch(exceptionHandler) {
+                "Accept" to "application/json",
-            val requestBuilder = okhttp3.Request.Builder()
+                "Content-Type" to "application/json+protobuf",
-                .post(data.toRequestBody())
+                "x-goog-api-key" to GOOGLE_API_KEY,
-                .headers(mapOf(
+                "x-user-agent" to "grpc-web-javascript/0.1",
-                    "User-Agent" to USER_AGENT,
+            ).toHeaders())
-                    "Accept" to "application/json",
+            .url(url)
-                    "Content-Type" to "application/json+protobuf",
+        val response = withContext(Dispatchers.IO) {
-                    "x-goog-api-key" to GOOGLE_API_KEY,
+            httpClient.newCall(requestBuilder.build()).execute()
-                    "x-user-agent" to "grpc-web-javascript/0.1",
+        }
-                ).toHeaders())
+        val httpCode = response.code
-                .url(url)
+        if (httpCode != 200) {
-            val response = withContext(Dispatchers.IO) {
+            throw Exception("Invalid response code: $httpCode")
-                httpClient.newCall(requestBuilder.build()).execute()
+        } else {
-            }
+            val body = withContext(Dispatchers.IO) {
-            val httpCode = response.code
+                response.body.string()
-            if (httpCode != 200) {
-                onInitializationErrorCloseAndCancel(Exception("Invalid response code: $httpCode"))
-            } else {
-                val body = withContext(Dispatchers.IO) {
-                    response.body.string()
-                }
-                handleResponseBody(body)
             }
+            body
         }
     }
 
     /**
      * Handles any error happening during initialization, releasing resources and sending the error
-     * to [continuation].
+     * to [generatorContinuation].
      */
-    private fun onInitializationErrorCloseAndCancel(error: Throwable) {
+    private fun onInitializationError(error: Throwable) {
-        close()
+        CoroutineScope(Dispatchers.Main).launch {
-        continuation.resumeWithException(error)
+            close()
+            generatorContinuation.resumeWithException(error)
+        }
     }
 
     /**
      * Releases all [webView] resources.
      */
     @MainThread
-    fun close() {
+    fun close() = with(webView) {
-        scope.cancel()
+        clearHistory()
-
-        webView.clearHistory()
         // clears RAM cache and disk cache (globally for all WebViews)
-        webView.clearCache(true)
+        clearCache(true)
 
         // ensures that the WebView isn't doing anything when destroying it
-        webView.loadUrl("about:blank")
+        loadUrl("about:blank")
 
-        webView.onPause()
+        onPause()
-        webView.removeAllViews()
+        removeAllViews()
-        webView.destroy()
+        destroy()
     }
     //endregion
 
@@ -336,7 +317,7 @@ class PoTokenWebView private constructor(
             return withContext(Dispatchers.Main) {
                 suspendCancellableCoroutine { cont ->
                     val potWv = PoTokenWebView(context, cont)
-                    potWv.loadHtmlAndObtainBotguard()
+                    potWv.loadHtmlAndObtainBotguard(context)
                 }
             }
         }

app/src/main/res/layout/player_client_item.xml

@@ -61,27 +61,6 @@
             android:layout_width="0dp"
             android:layout_height="wrap_content">
 
-            <TextView
-                android:id="@+id/urlRegex"
-                style="@style/Widget.Material3.FloatingActionButton.Large.Secondary"
-                android:layout_width="wrap_content"
-                android:visibility="gone"
-                android:layout_height="wrap_content"
-                android:text="URL Regex"
-                android:background="@drawable/rounded_corner"
-                android:backgroundTint="?attr/colorAccent"
-                android:clickable="false"
-                android:gravity="center"
-                android:minWidth="30dp"
-                android:paddingHorizontal="5dp"
-                android:textSize="12sp"
-                android:textStyle="bold"
-                app:cornerRadius="10dp"
-                app:layout_constraintTop_toBottomOf="parent"
-                app:layout_constraintStart_toStartOf="parent"
-                app:layout_constraintEnd_toEndOf="parent"
-                tools:ignore="HardcodedText" />
-
         </com.google.android.material.chip.ChipGroup>