Merge pull request #40 from Udhayarajan/patch-3

Minor fixes

build.gradle.kts

@@ -28,7 +28,7 @@ plugins {
 }
 
 group = "io.github.udhayarajan"
-version = "5.6.10"
+version = "5.6.11"
 // Version Naming incremented if "<NEW_FEATURE_ADDED>.<WORKED_ON_BUG>.<BETA_VERSION_COUNT_OR_PRE_RELEASE>"
 // Priority on incrementing Feature > BugFix > Beta
 

src/commonMain/kotlin/com/mugames/vidsnapkit/M3u8.kt

@@ -35,11 +35,16 @@ class M3u8(private val url: String, private val onProgressCallback: ProgressCall
 
     var info: JSONObject? = null
 
-    val httpRequestService = HttpRequestService.create()
+    val httpRequestService by lazy {
+        HttpRequestService.create()
+    }
 
-    private val localFormats = Formats()
+    private val localFormats by lazy {
+        Formats()
+    }
 
     private fun nonFatalError(msg: String) {
+        httpRequestService.close()
         onProgressCallback.onProgress(Result.Failed(Error.NonFatalError(msg)))
     }
 

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt

@@ -95,7 +95,7 @@ class DailyMotion(url: String) : Extractor(url) {
                 }
             }
         } ?: run {
-            onProgress(Result.Failed(Error.InvalidUrl))
+            failed(Error.InvalidUrl)
         }
     }
 

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt

@@ -98,7 +98,9 @@ abstract class Extractor(
     protected lateinit var onProgress: (Result) -> Unit
 
     protected var headers: Hashtable<String, String> = Hashtable()
-    private val store = AcceptAllCookiesStorage()
+    private val store by lazy {
+        AcceptAllCookiesStorage()
+    }
 
     protected var httpRequestService = run {
         val str = if (inputUrl.contains(Regex("/reels/audio/|tiktok"))) store else null
@@ -215,6 +217,7 @@ abstract class Extractor(
             filteredFormats.forEach {
                 it.cookies.addAll(store.get(Url(inputUrl)))
             }
+            httpRequestService.close()
             onProgress(Result.Success(filteredFormats))
         }
     }
@@ -250,18 +253,27 @@ abstract class Extractor(
     }
 
     protected fun clientRequestError(msg: String = "error making request") {
+        httpRequestService.close()
         onProgress(Result.Failed(Error.NonFatalError(msg)))
     }
 
+    public fun failed(error: Error) {
+        httpRequestService.close()
+        onProgress(Result.Failed(error))
+    }
+
     protected fun loginRequired() {
+        httpRequestService.close()
         onProgress(Result.Failed(Error.LoginRequired))
     }
 
     protected fun internalError(msg: String, e: Exception? = null) {
+        httpRequestService.close()
         onProgress(Result.Failed(Error.InternalError(msg, e)))
     }
 
     protected fun missingLogic() {
+        httpRequestService.close()
         onProgress(Result.Failed(Error.MethodMissingLogic))
     }
 

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt

@@ -105,7 +105,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
             extractInfo()
         } catch (e: JSONException) {
             e.printStackTrace()
-            onProgress(Result.Failed(Error.InternalError("Something went wrong", e)))
+            internalError("Something went wrong", e)
         } catch (e: Exception) {
             logger.warn("$TAG+ analyze: ", e)
             throw e
@@ -168,7 +168,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
                 return
             }
             if (webPage.contains("You must log in to continue")) {
-                onProgress(Result.Failed(Error.LoginRequired))
+                loginRequired()
                 return
             }
         }
@@ -226,7 +226,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
         } ?: apply {
             val uuid = "fb." + UUID.randomUUID().toString() + ".html"
             File(uuid).writeText(webPage)
-            onProgress(Result.Failed(Error.NonFatalError("Sorry! we can't see the page, refer=$uuid")))
+            clientRequestError("Sorry! we can't see the page, refer=$uuid")
         }
     }
 
@@ -387,7 +387,9 @@ class Facebook internal constructor(url: String) : Extractor(url) {
             return extractFromCreationStory(media)
         }
         val scopedFormats = localFormats.copy(
-            title = "", videoData = mutableListOf(), audioData = mutableListOf(), imageData = mutableListOf()
+            videoData = mutableListOf(),
+            audioData = mutableListOf(),
+            imageData = mutableListOf()
         )
         if (media.getNullableJSONObject("video_grid_renderer") != null) {
             return getVideoFromVideoGridRenderer(media)
@@ -402,10 +404,14 @@ class Facebook internal constructor(url: String) : Extractor(url) {
                 url = thumbnailUrl
             )
         )
-        scopedFormats.title = media.getNullableString("name") ?: media.getNullableJSONObject("savable_description")
-            ?.getNullableString("text") ?: media.getNullableJSONObject("title")?.getString("text")?.ifEmpty {
-            "Facebook_Video"
-        } ?: "Facebook_Video"
+        val title = media.getNullableString("name") ?: media.getNullableJSONObject("savable_description")
+            ?.getNullableString("text") ?: media.getNullableJSONObject("title")?.getString("text")
+        title?.let {
+            if (scopedFormats.title.isEmpty() || it != scopedFormats.title)
+                scopedFormats.title = title
+        } ?: run {
+            if (scopedFormats.title.isEmpty()) scopedFormats.title = "Facebook_Video"
+        }
 
         val dashXml = media.getNullableString("dash_manifest")
         dashXml?.let {
@@ -463,6 +469,9 @@ class Facebook internal constructor(url: String) : Extractor(url) {
         val playbackVideo =
             media.getNullableJSONObject("creation_story")?.getNullableJSONObject("short_form_video_context")
                 ?.getNullableJSONObject("playback_video")
+        localFormats.title = media.getNullableJSONObject("creation_story")
+            ?.getNullableJSONObject("message")
+            ?.getNullableString("text") ?: ""
         return if (playbackVideo != null) parseGraphqlVideo(playbackVideo)
         else parseGraphqlVideo(media, false)
     }

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt

@@ -46,6 +46,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         const val GRAPHQL_URL =
             "https://www.instagram.com/graphql/query/?query_hash=%s&variables={\"shortcode\":\"%s\"}&__a=1&__d=dis"
         const val DEFAULT_QUERY_HASH = "b3055c01b4b222b8a47dc12b090e4e64"
+        const val DEFAULT_APP_ID = "936619743392459"
         const val AUDIO_API = "https://www.instagram.com/api/v1/clips/music/"
         private val logger = LoggerFactory.getLogger(Instagram::class.java)
     }
@@ -94,6 +95,22 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    private fun getAppID(page: String?): String {
+        if (page == null)
+            return DEFAULT_APP_ID
+        val appIdRegex = listOf(
+            "\"app_id\":\"(\\d.*?)\"".toRegex(),
+            "\"appId\":\"(\\d.*?)\"".toRegex(),
+            "\"APP_ID\":\"(\\d.*?)\"".toRegex(),
+            "\"X-IG-App-ID\":\"(.*?)\"".toRegex()
+        )
+        for (regex in appIdRegex) {
+            val matcher = regex.find(page)
+            return matcher?.groups?.get(1)?.value ?: DEFAULT_APP_ID
+        }
+        return DEFAULT_APP_ID
+    }
+
     private fun getMediaId(page: String? = null): String? {
         if (page == null) {
             val matcher = Pattern.compile("/([0-9]{19})(?:/|)").matcher(inputUrl)
@@ -162,12 +179,12 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 return null
             }
             if (!isAccessible(response, it)) {
-                onProgress(Result.Failed(Error.InvalidCookies))
+                loginRequired()
                 return null
             }
             return response.getJSONObject("graphql")?.getJSONObject("user")?.getString("id")
         } ?: run {
-            onProgress(Result.Failed(Error.InvalidUrl))
+            failed(Error.InvalidUrl)
         }
         return null
     } ?: run {
@@ -254,24 +271,12 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 loginRequired()
                 return
             }
-            var appId = "936619743392459"
 
-            withTimeoutOrNull(2000) {
-                httpRequestService.getResponse(inputUrl, headers)?.let {
-                    val appIdRegex = listOf(
-                        "\"app_id\":\"(\\d.*?)\"".toRegex(),
-                        "\"appId\":\"(\\d.*?)\"".toRegex(),
-                        "\"APP_ID\":\"(\\d.*?)\"".toRegex(),
-                        "\"X-IG-App-ID\":\"(.*?)\"".toRegex()
-                    )
-                    for (regex in appIdRegex) {
-                        val matcher = regex.find(it)
-                        matcher?.groups?.get(1)?.let {
-                            appId = it.value
-                        }
-                    }
-                }
-            }
+            val appID = withTimeoutOrNull(1500) {
+                getAppID(httpRequestService.getResponse(inputUrl, headers))
+            } ?: DEFAULT_APP_ID
+
+            headers["X-Ig-App-Id"] = appID
 
             val tempHeader = headers.clone() as Hashtable<String, String>
             pre.headers.getAll("set-cookie")?.forEach {
@@ -281,7 +286,6 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 }
             }
             tempHeader.remove("User-Agent")
-            tempHeader["X-Ig-App-Id"] = appId
             val res = httpRequestService.postRequest(AUDIO_API, tempHeader, audioPayload)
             val metadata = res?.toJSONObject()?.getJSONObject("metadata")
             metadata?.run {
@@ -328,29 +332,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
             return
         }
         if (res == "429" && isCookieValid()) {
-            val mediaID = shortcodeToMediaID(getShortcode())
-            mediaID?.let {
-                val items =
-                    httpRequestService
-                        .getResponse(
-                            POST_API.format(shortcodeToMediaID(getShortcode())),
-                            headers
-                        )?.let {
-                            it.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run {
-                                loginRequired()
-                                return
-                            }
-                        } ?: run {
-                        loginRequired()
-                        return
-                    }
-                extractFromItems(items)
-                return
-            } ?: run {
-                logger.error("unable to find mediaID for url $inputUrl")
-                loginRequired()
-                return
-            }
+            shortcodeExtraction()
         }
         extractFromItems(
             res.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run {
@@ -360,6 +342,33 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         )
     }
 
+    // Works only with valid cookies
+    private suspend fun shortcodeExtraction() {
+        val mediaID = shortcodeToMediaID(getShortcode())
+        mediaID?.let {
+            val items =
+                httpRequestService
+                    .getResponse(
+                        POST_API.format(shortcodeToMediaID(getShortcode())),
+                        headers
+                    )?.let {
+                        it.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run {
+                            loginRequired()
+                            return
+                        }
+                    } ?: run {
+                    loginRequired()
+                    return
+                }
+            extractFromItems(items)
+            return
+        } ?: run {
+            logger.error("unable to find mediaID for url $inputUrl")
+            loginRequired()
+            return
+        }
+    }
+
     private suspend fun extractHighlights(highlightsId: String, isStory: Boolean = false) {
         val highlights = httpRequestService.getResponse(
             HIGHLIGHTS_API.format(if (!isStory) "highlight%3A$highlightsId" else highlightsId),
@@ -455,15 +464,15 @@ class Instagram internal constructor(url: String) : Extractor(url) {
             if (isObjectPresentInEntryData("LoginAndSignupPage")) {
                 loginRequired()
             } else if (isObjectPresentInEntryData("HttpErrorPage")) {
-                onProgress(Result.Failed(Error.Instagram404Error(cookies != null)))
+                failed(Error.Instagram404Error(cookies != null))
             } else {
                 val user0 = jsonObject.getJSONObject("entry_data").getNullableJSONArray("ProfilePage")?.getJSONObject(0)
                     ?: run {
                         newApiRequest()
                         return
                     }
-                if (!isAccessible(user0)) onProgress(Result.Failed(Error.InvalidCookies))
-                else onProgress(Result.Failed(Error.InternalError("can't find problem")))
+                if (!isAccessible(user0)) loginRequired()
+                else internalError("can't find problem")
             }
         }
     }
@@ -655,7 +664,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
             media?.let { mediaIt ->
                 setInfo(mediaIt)
             } ?: run {
-                onProgress(Result.Failed(Error.InternalError("MediaNotFound")))
+                internalError("MediaNotFound")
             }
         } ?: run {
             extractFromItems(jsonObject.getJSONArray("items"))
@@ -666,6 +675,8 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         val queryHash = withTimeoutOrNull(2000) {
             getQueryHashFromAllJSInPage(page)
         } ?: DEFAULT_QUERY_HASH
+        val appID = getAppID(page)
+        headers["X-Ig-App-Id"] = appID
         val res = httpRequestService.getResponse(GRAPHQL_URL.format(queryHash, getShortcode()), headers)
         logger.info("graphQL response = $res")
         val shortcodeMedia =
@@ -742,7 +753,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
             }
         }
         if (!isPostUrl() && videoFormats.isEmpty()) {
-            onProgress(Result.Failed(Error.NonFatalError(NO_VIDEO_STATUS_AVAILABLE)))
+            clientRequestError(NO_VIDEO_STATUS_AVAILABLE)
         } else finalize()
     }
 

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt

@@ -54,7 +54,7 @@ class Likee internal constructor(url: String) : Extractor(url) {
         responseData.getJSONArray("videoList")?.let {
             extractVideoList(it)
         } ?: run {
-            onProgress(Result.Failed(Error.MethodMissingLogic))
+            missingLogic()
         }
     }
 

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt

@@ -106,10 +106,10 @@ class LinkedIn internal constructor(url: String) : Extractor(url) {
             }
         }
         if (page.contains("Please enter your email address", true)) {
-            onProgress(Result.Failed(Error.LoginRequired))
+            loginRequired()
             return
         }
-        onProgress(Result.Failed(Error.MethodMissingLogic))
+        missingLogic()
     }
 
     private suspend fun extractFromIncluded(included: JSONArray) {

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt

@@ -18,9 +18,7 @@
 package com.mugames.vidsnapkit.extractor
 
 import com.mugames.vidsnapkit.MimeType
-import com.mugames.vidsnapkit.dataholders.Error
 import com.mugames.vidsnapkit.dataholders.Formats
-import com.mugames.vidsnapkit.dataholders.Result
 import com.mugames.vidsnapkit.dataholders.VideoResource
 import com.mugames.vidsnapkit.getNullableString
 import com.mugames.vidsnapkit.toJSONObject
@@ -80,7 +78,7 @@ class Periscope internal constructor(url: String) : Extractor(url) {
                     if (videoUrl.isNullOrEmpty() || videUrls.contains(videoUrl)) continue
                     localFormats.videoData.add(VideoResource(videoUrl, MimeType.VIDEO_MP4))
                     if (formatId != "rtmp") {
-                        onProgress(Result.Failed(Error.MethodMissingLogic))
+                        missingLogic()
                         break
                     }
                 }

src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt

@@ -52,7 +52,8 @@ class ShareChat internal constructor(url: String) : Extractor(url) {
             Pattern.compile("""<script data-rh="true" type="application\/ld\+json">(\{"@context":"http:\/\/schema\.org","@type":"(?:Image|Video)Object".*?\})<\/script>""")
                 .matcher(response)
         if (!matcher.find()) {
-            onProgress(Result.Failed(Error.InternalError("Unable detect the contentUrl for $inputUrl")))
+            internalError("Unable detect the contentUrl for $inputUrl")
+            return
         }
         onProgress(Result.Progress(ProgressState.Middle))
         val responseObject = JSONObject(matcher.group(1)!!)