Replace tokens with guest accounts, swap endpoints

nitter.nimble

@@ -23,7 +23,7 @@ requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#ca5989a"
 requires "flatty#e668085"
 requires "jsony#ea811be"
-
+requires "oauth#b8c163b"
 
 # Tasks
 

src/api.nim

@@ -33,23 +33,6 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
     js = await fetch(url ? params, apiId)
   result = parseGraphTimeline(js, "user", after)
 
-# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
-#   if id.len == 0: return
-#   let
-#     ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-#     url = oldUserTweets / (id & ".json") ? ps
-#   result = parseTimeline(await fetch(url, Api.timeline), after)
-
-proc getUserTimeline*(id: string; after=""): Future[Profile] {.async.} =
-  var ps = genParams({"id": id})
-  if after.len > 0:
-    ps.add ("down_cursor", after)
-
-  let
-    url = legacyUserTweets ? ps
-    js = await fetch(url, Api.userTimeline)
-  result = parseUserTimeline(js, after)
-
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
@@ -112,10 +95,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
   if after.len > 0:
     result.replies = await getReplies(id, after)
 
-proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
+proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   let q = genQueryParam(query)
   if q.len == 0 or q == emptyQuery:
-    return Profile(tweets: Timeline(query: query, beginning: true))
+    return Timeline(query: query, beginning: true)
 
   var
     variables = %*{
@@ -129,44 +112,29 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
   if after.len > 0:
     variables["cursor"] = % after
   let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after))
-  result.tweets.query = query
-
-proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
-  var q = genQueryParam(query)
-
-  if q.len == 0 or q == emptyQuery:
-    return Timeline(query: query, beginning: true)
-
-  if after.len > 0:
-    q &= " max_id:" & after
-
-  let url = tweetSearch ? genParams({
-    "q": q ,
-    "modules": "status",
-    "result_type": "recent",
-  })
-
-  result = parseTweetSearch(await fetch(url, Api.search), after)
+  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
   result.query = query
 
-proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
   if query.text.len == 0:
     return Result[User](query: query, beginning: true)
 
-  var url = userSearch ? {
-    "q": query.text,
-    "skip_status": "1",
-    "count": "20",
-    "page": page
-  }
+  var
+    variables = %*{
+      "rawQuery": query.text,
+      "count": 20,
+      "product": "People",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+    result.beginning = false
 
-  result = parseUsers(await fetchRaw(url, Api.userSearch))
+  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphSearch[User](await fetch(url, Api.search), after)
   result.query = query
-  if page.len == 0:
-    result.bottom = "2"
-  elif page.allCharsInSet(Digits):
-    result.bottom = $(parseInt(page) + 1)
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return

src/apiutils.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, strutils, uri
-import jsony, packedjson, zippy
+import httpclient, asyncdispatch, options, strutils, uri, times, math
+import jsony, packedjson, zippy, oauth1
 import types, tokens, consts, parserutils, http_pool
 import experimental/types/common
 
@@ -29,12 +29,30 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     else:
       result &= ("cursor", cursor)
 
-proc genHeaders*(token: Token = nil): HttpHeaders =
+proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
+  let
+    encodedUrl = url.replace(",", "%2C").replace("+", "%20")
+    params = OAuth1Parameters(
+      consumerKey: consumerKey,
+      signatureMethod: "HMAC-SHA1",
+      timestamp: $int(round(epochTime())),
+      nonce: "0",
+      isIncludeVersionToHeader: true,
+      token: oauthToken
+    )
+    signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
+
+  params.signature = percentEncode(signature)
+
+  return getOauth1RequestHeader(params)["authorization"]
+
+proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
+  let header = getOauthHeader(url, oauthToken, oauthTokenSecret)
+
   result = newHttpHeaders({
     "connection": "keep-alive",
-    "authorization": auth,
+    "authorization": header,
     "content-type": "application/json",
-    "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
     "authority": "api.twitter.com",
     "accept-encoding": "gzip",
@@ -43,24 +61,24 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "DNT": "1"
   })
 
-template updateToken() =
+template updateAccount() =
   if resp.headers.hasKey(rlRemaining):
     let
       remaining = parseInt(resp.headers[rlRemaining])
       reset = parseInt(resp.headers[rlReset])
-    token.setRateLimit(api, remaining, reset)
+    account.setRateLimit(api, remaining, reset)
 
 template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
-  var token = await getToken(api)
-  if token.tok.len == 0:
+  var account = await getGuestAccount(api)
+  if account.oauthToken.len == 0:
     raise rateLimitError()
 
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(token)):
+    pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
@@ -79,19 +97,19 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
     fetchBody
 
-    release(token, used=true)
+    release(account, used=true)
 
     if resp.status == $Http400:
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
   except BadClientError as e:
-    release(token, used=true)
+    release(account, used=true)
     raise e
   except Exception as e:
-    echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
+    echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", account.id, ", url: ", url
     if "length" notin e.msg and "descriptor" notin e.msg:
-      release(token, invalid=true)
+      release(account, invalid=true)
     raise rateLimitError()
 
 proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
@@ -103,12 +121,12 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
       echo resp.status, ": ", body, " --- url: ", url
       result = newJNull()
 
-    updateToken()
+    updateAccount()
 
     let error = result.getError
     if error in {invalidToken, badToken}:
       echo "fetch error: ", result.getError
-      release(token, invalid=true)
+      release(account, invalid=true)
       raise rateLimitError()
 
 proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
@@ -117,11 +135,11 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
       echo resp.status, ": ", result, " --- url: ", url
       result.setLen(0)
 
-    updateToken()
+    updateAccount()
 
     if result.startsWith("{\"errors"):
       let errors = result.fromJson(Errors)
       if errors in {invalidToken, badToken}:
         echo "fetch error: ", errors
-        release(token, invalid=true)
+        release(account, invalid=true)
         raise rateLimitError()

src/consts.nim

@@ -2,17 +2,13 @@
 import uri, sequtils, strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
+  consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
+  consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  legacyUserTweets* = api / "1.1/timeline/user.json"
   photoRail* = api / "1.1/statuses/media_timeline.json"
-  userSearch* = api / "1.1/users/search.json"
-  tweetSearch* = api / "1.1/search/universal.json"
-
-  # oldUserTweets* = api / "2/timeline/profile"
 
   graphql = api / "graphql"
   graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"

src/nitter.nim

@@ -3,6 +3,7 @@ import asyncdispatch, strformat, logging
 from net import Port
 from htmlgen import a
 from os import getEnv
+from json import parseJson
 
 import jester
 
@@ -15,8 +16,14 @@ import routes/[
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
-let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
-let (cfg, fullCfg) = getConfig(configPath)
+let
+  configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+  (cfg, fullCfg) = getConfig(configPath)
+
+  accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
+  accounts = parseJson(readFile(accountsPath))
+
+initAccountPool(cfg, parseJson(readFile(accountsPath)))
 
 if not cfg.enableDebug:
   # Silence Jester's query warning
@@ -38,8 +45,6 @@ waitFor initRedisPool(cfg)
 stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
 stdout.flushFile
 
-asyncCheck initTokenPool(cfg)
-
 createUnsupportedRouter(cfg)
 createResolverRouter(cfg)
 createPrefRouter(cfg)