Add support for nitter.net urls

saket · Nov 19, 2023 · 9c811d9 · 9c811d9
1 parent bf6436a
commit 9c811d9
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 2 deletions.
diff --git a/unfurl/src/main/kotlin/me/saket/unfurl/extension/HtmlTagsBasedUnfurler.kt b/unfurl/src/main/kotlin/me/saket/unfurl/extension/HtmlTagsBasedUnfurler.kt
@@ -17,12 +17,14 @@ open class HtmlTagsBasedUnfurler : UnfurlerExtension {
 
   private fun UnfurlerScope.downloadHtml(url: HttpUrl): Document? {
     val request: Request = Request.Builder()
-      // Some websites will deny empty/unknown user agents,
-      // probably in an attempt to prevent scrapers?
+      // Some websites will deny empty/unknown user agents, probably in an
+      // attempt to prevent scrapers? Some goes for the following headers.
       .header(
         "User-Agent",
         "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Mobile Safari/537.36"
       )
+      .header("Accept", "text/html")
+      .header("Accept-Language", "en-US,en;q=0.5")
       .url(url)
       .build()
 

diff --git a/unfurl/src/test/kotlin/me/saket/unfurl/UnfurlerTest.kt b/unfurl/src/test/kotlin/me/saket/unfurl/UnfurlerTest.kt
@@ -46,6 +46,19 @@ class UnfurlerTest {
     )
   }
 
+  @Test fun `websites that deny requests without content type and language headers`() {
+    val result = unfurler.unfurl("https://nitter.net/saketme/status/1716330453311877183")
+    assertThat(result).isEqualTo(
+      UnfurlResult(
+        url = "https://nitter.net/saketme/status/1716330453311877183".toHttpUrl(),
+        title = "saket@androiddev.social (@saketme)",
+        description = "When the sole developer of a project starts using \"we\" instead of \"I\" in their code comments.",
+        favicon = "https://nitter.net/apple-touch-icon.png".toHttpUrl(),
+        thumbnail = "https://nitter.net/pic/media%2FF9GhXLmXYAAXIcb.png".toHttpUrl(),
+      )
+    )
+  }
+
   @Test fun `follow redirects`() {
     server.enqueue(
       MockResponse()