add --lazy-playlist support & subs po token generated

app/src/main/java/com/deniscerri/ytdl/database/repository/ResultRepository.kt

@@ -18,8 +18,11 @@ import com.deniscerri.ytdl.util.extractors.GoogleApiUtil
 import com.deniscerri.ytdl.util.extractors.newpipe.NewPipeUtil
 import com.deniscerri.ytdl.util.extractors.YTDLPUtil
 import com.deniscerri.ytdl.util.extractors.YoutubeApiUtil
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 
 class ResultRepository(private val resultDao: ResultDao, private val commandTemplateDao: CommandTemplateDao, private val context: Context) {
@@ -68,7 +71,7 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
             "custom" -> {
                 val customURL = sharedPreferences.getString("custom_home_recommendation_url", "")
                 if (customURL.isNullOrBlank()) arrayListOf()
-                else ytdlpUtil.getFromYTDL(customURL)
+                else ytdlpUtil.getFromYTDL(customURL, resultsGenerated = {})
             }
             else -> arrayListOf()
         }
@@ -96,7 +99,7 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
             .getOrDefault(Pair(listOf(""), null))
     }
 
-    suspend fun search(inputQuery: String, resetResults: Boolean, addToResults: Boolean) : ArrayList<ResultItem>{
+    suspend fun search(inputQuery: String, resetResults: Boolean, addToResults: Boolean) : List<ResultItem>{
         if (resetResults) deleteAll()
         val res = when(sharedPreferences.getString("search_engine", "ytsearch")) {
             "ytsearch" -> newPipeUtil.search(inputQuery)
@@ -108,7 +111,7 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
             res.getOrNull()!!
         }else{
             //fallback to yt-dlp
-            ytdlpUtil.getFromYTDL(inputQuery)
+            ytdlpUtil.getFromYTDL(inputQuery, resultsGenerated = {})
         }
 
         itemCount.value = items.size
@@ -145,7 +148,7 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
         val response = if (newpipeExtractorResult.isSuccess){
             newpipeExtractorResult.getOrElse { items }
         }else{
-            val res = ytdlpUtil.getFromYTDL(inputQuery)
+            val res = ytdlpUtil.getFromYTDL(inputQuery, resultsGenerated = {})
             if (addToResults) {
                 val ids = resultDao.insertMultiple(res)
                 ids.forEachIndexed { index, id ->
@@ -170,7 +173,7 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
         val res = if (newpipeExtractorResult.isSuccess) {
             newpipeExtractorResult.getOrNull()!!
         }else{
-            ytdlpUtil.getFromYTDL(inputQuery)
+            ytdlpUtil.getFromYTDL(inputQuery, resultsGenerated = {})
         }
 
         if (resetResults) {
@@ -209,22 +212,30 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
             Result.failure(Throwable())
         }
 
-        val result = if (ytExtractorResult.isSuccess){
+        val finalResults = mutableListOf<ResultItem>()
-            ytExtractorResult.getOrElse { items }
+        if (ytExtractorResult.isSuccess) {
-        }else{
+            ytExtractorResult.getOrElse { items }.apply {
-            val res = ytdlpUtil.getFromYTDL(inputQuery)
+                finalResults.addAll(this)
-            if (addToResults) {
+                itemCount.value = this.size
-                val ids = resultDao.insertMultiple(res)
+            }
-                ids.forEachIndexed { index, id ->
+        }else {
-                    res[index].id = id
+            var itemCounts = 0
-                }
+            ytdlpUtil.getFromYTDL(inputQuery) {
+                if (addToResults) {
+                    runBlocking {
+                        val ids = resultDao.insertMultiple(it)
+                        ids.forEachIndexed { index, id ->
+                            it[index].id = id
+                        }
+                    }
+                }
+                finalResults.addAll(it)
+                itemCounts+=it.size
+                itemCount.value = itemCounts
             }
-
-            res
         }
 
-        itemCount.value = result.size
+        return finalResults
-        return result
     }
 
     private suspend fun getYoutubeChannel(url: String, resetResults: Boolean, addToResults: Boolean) : List<ResultItem>{
@@ -246,38 +257,58 @@ class ResultRepository(private val resultDao: ResultDao, private val commandTemp
             Result.failure(Throwable())
         }
 
-        val response = if (ytExtractorResult.isSuccess){
+        val finalResults = mutableListOf<ResultItem>()
-            ytExtractorResult.getOrElse { items }
+        if (ytExtractorResult.isSuccess) {
-        }else{
+            ytExtractorResult.getOrElse { items }.apply {
-            val res = ytdlpUtil.getFromYTDL(url)
+                finalResults.addAll(this)
-            val ids = resultDao.insertMultiple(res)
+                itemCount.value = this.size
-            ids.forEachIndexed { index, id ->
+            }
-                res[index].id = id
+        }else {
+            var itemCounts = 0
+            ytdlpUtil.getFromYTDL(url) {
+                if (addToResults) {
+                    runBlocking {
+                        val ids = resultDao.insertMultiple(it)
+                        ids.forEachIndexed { index, id ->
+                            it[index].id = id
+                        }
+                    }
+                }
+                finalResults.addAll(it)
+                itemCounts+=it.size
+                itemCount.value = itemCounts
             }
-            res
         }
 
-        itemCount.value = response.size
+        return finalResults
-        return response
     }
 
-    private suspend fun getFromYTDLP(inputQuery: String, resetResults: Boolean, addToResults: Boolean, singleItem: Boolean = false) : ArrayList<ResultItem> {
+    private suspend fun getFromYTDLP(inputQuery: String, resetResults: Boolean, addToResults: Boolean, singleItem: Boolean = false) : List<ResultItem> {
-        val items = ytdlpUtil.getFromYTDL(inputQuery, singleItem)
         if (resetResults) {
             deleteAll()
-            itemCount.value = items.size
-        }else{
-            items.filter { it.playlistTitle.isBlank() }.forEach { it.playlistTitle = YTDLNIS_SEARCH }
         }
 
-        if (addToResults){
+        var itemsCount = 0
-            val ids = resultDao.insertMultiple(items.toList())
+        val itemsToReturn = mutableListOf<ResultItem>()
-            ids.forEachIndexed { index, id ->
+
-                items[index].id = id
+        ytdlpUtil.getFromYTDL(inputQuery, singleItem) { results ->
+            if (resetResults) {
+                itemsCount += results.size
+                itemCount.value = itemsCount
             }
+            results.filter { it.playlistTitle.isBlank() }.forEach { it.playlistTitle = YTDLNIS_SEARCH }
+            if (addToResults) {
+                runBlocking {
+                    val ids = resultDao.insertMultiple(results)
+                    ids.forEachIndexed { index, id ->
+                        results[index].id = id
+                    }
+                }
+            }
+            itemsToReturn.addAll(results)
         }
 
-        return items
+        return itemsToReturn
     }
 
     fun getFormats(url: String, source : String? = null) : List<Format> {

app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/generateyoutubepotokens/GenerateYoutubePoTokensFragment.kt

@@ -147,6 +147,7 @@ class GenerateYoutubePoTokensFragment : Fragment() {
             if (result.resultCode == Activity.RESULT_OK) {
                 val streamingDataPoTokenResult = result.data!!.getStringExtra("streaming_potoken") ?: ""
                 val playerRequestPoTokenResult = result.data!!.getStringExtra("player_potoken") ?: ""
+                val subsRequestPoTokenResult = result.data!!.getStringExtra("subs_potoken") ?: ""
                 val visitorDataResult = result.data!!.getStringExtra("visitor_data") ?: ""
 
 
@@ -154,6 +155,7 @@ class GenerateYoutubePoTokensFragment : Fragment() {
                 conf.poTokens.clear()
                 conf.poTokens.add(YoutubePoTokenItem("gvs", streamingDataPoTokenResult))
                 conf.poTokens.add(YoutubePoTokenItem("player", playerRequestPoTokenResult))
+                conf.poTokens.add(YoutubePoTokenItem("subs", subsRequestPoTokenResult))
                 conf.visitorData = visitorDataResult
 
                 configuration.add(conf)

app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/generateyoutubepotokens/webview/PoTokenWebViewLoginActivity.kt

@@ -120,6 +120,7 @@ class PoTokenWebViewLoginActivity : BaseActivity() {
                         val intent = Intent()
                         intent.putExtra("visitor_data", res.visitorData)
                         intent.putExtra("player_potoken", res.playerRequestPoToken)
+                        intent.putExtra("subs_potoken", res.playerRequestPoToken)
                         intent.putExtra("streaming_potoken", res.streamingDataPoToken)
                         setResult(RESULT_OK, intent)
                     }

app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt

@@ -104,7 +104,7 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
     }
 
     @SuppressLint("RestrictedApi")
-    fun getFromYTDL(query: String, singleItem: Boolean = false): ArrayList<ResultItem> {
+    fun getFromYTDL(query: String, singleItem: Boolean = false, resultsGenerated: (results: List<ResultItem>) -> Unit): List<ResultItem> {
         val searchEngine = sharedPreferences.getString("search_engine", "ytsearch")
 
         val request : YoutubeDLRequest
@@ -134,21 +134,31 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
         }
 
         request.addOption("--flat-playlist")
+        request.addOption("--lazy-playlist")
         request.addOption(if (singleItem) "-J" else "-j")
         request.applyDefaultOptionsForFetchingData(if (query.isURL()) query else null)
 
-        println(parseYTDLRequestString(request))
+        val finalResults = mutableListOf<ResultItem>()
-        val youtubeDLResponse = YoutubeDL.getInstance().execute(request)
+        var postedProgress = false
-        val results: List<String?> = try {
+        YoutubeDL.getInstance().execute(request) { progress, _, line ->
-            val lineSeparator = System.getProperty("line.separator")
+            runCatching {
-            youtubeDLResponse.out.split(lineSeparator!!)
+                val generatedResults = parseYTDLPListResults(listOf(line))
-        } catch (e: Exception) {
+                if (generatedResults.isNotEmpty()) {
-            listOf(youtubeDLResponse.out)
+                    finalResults.addAll(generatedResults)
-        }.filter { it.isNotBlank() }.apply {
+                    if (!postedProgress) {
-            if (this.isEmpty()) throw Exception("Command Used: \n yt-dlp ${parseYTDLRequestString(request)}")
+                        if (finalResults.size > 1) {
+                            resultsGenerated(finalResults)
+                            postedProgress = true
+                        }
+                    }else{
+                        resultsGenerated(generatedResults)
+                    }
+                }
+            }
         }
 
-        return parseYTDLPListResults(results, query)
+        if (!postedProgress) resultsGenerated(finalResults)
+        return finalResults
     }
 
     private fun parseYTDLPListResults(results: List<String?>, query: String = "") : ArrayList<ResultItem> {