initial commit

.gitignore

@@ -0,0 +1 @@
+venv/

requirements.txt

@@ -0,0 +1,3 @@
+pytz==2024.2
+requests==2.32.3
+zstd==1.5.6.1

scraper.py

@@ -0,0 +1,115 @@
+import requests, re, json, pytz, zstd
+from datetime import datetime
+
+class TweetsScraper:
+    _GET_TWEETS_URL = 'https://api.x.com/graphql/MpOINUGH_YVb2BKjYZOPaQ/UserTweets'
+
+    # public non-logged-in access token (same for everyone, doesn't expire)
+    _AUTHORIZATION_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+
+    _HEADERS = {
+        "Host": "api.x.com",
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "content-type": "application/json",
+        "authorization": f"Bearer {_AUTHORIZATION_TOKEN}",
+        "x-twitter-client-language": "en",
+        "x-twitter-active-user": "yes",
+        "Origin": "https://x.com",
+        "Sec-GPC": "1",
+        "Connection": "keep-alive",
+        "Referer": "https://x.com/",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-site",
+        "TE": "trailers",
+    }
+    _FEATURES_USER_TWEETS = '{"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":false,"responsive_web_enhance_cards_enabled":false}'
+    _FIELD_TOGGLES_USER_TWEETS = '{"withArticlePlainText":false}'
+
+    def __init__(self):
+        self._session = requests.Session()
+
+    def _get_guest_token(self):
+        # Get guest token from x.com request
+        if "x-guest-token" in self._HEADERS.keys():
+            return
+
+        # Different headers necessary so we dont get a 400 response
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Host": "x.com",
+            "Pragma": "no-cache",
+            "Priority": "u=0, i",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0",
+            "x-client-transaction-id": "78k0T6XnCiJK2f5fZ5RwmyeKmiOHk8HFfWovTa6JQF4DRfIkyjpARHxQzi0pWxKPtzks0ezFLICv4xuxmIyokH1EBEWe7A",
+            "x-guest-token": "1881289785240932544"
+        }
+
+        res = self._session.get("https://x.com/?mx=2", headers=headers)
+
+        # find the guest token in the response
+        self._HEADERS["x-guest-token"] = res.text.split("gt=")[1].split(";")[0]
+
+    def get_tweets_anonymous(self, user):
+        self._get_guest_token()
+        
+        variables = {
+            "userId": user,
+            "count": 100,
+            "includePromotedContent": True,
+            "withQuickPromoteEligibilityTweetFields": True, 
+            "withVoice": True,
+            "withV2Timeline": True
+        }
+
+        res = self._session.get(self._GET_TWEETS_URL, params={"variables": json.dumps(variables, separators=(',', ':')), "features": self._FEATURES_USER_TWEETS, "fieldToggles": self._FIELD_TOGGLES_USER_TWEETS}, headers=self._HEADERS)
+
+        res_json = None
+        try:
+            res_json = json.loads(zstd.decompress(res.content))
+        except:
+            res_json = json.loads(res.text)
+
+        entries = [i for i in res_json['data']['user']['result']['timeline_v2']['timeline']['instructions'] if i['type'] == "TimelineAddEntries"][0]['entries']
+        return [Tweet(entry) for entry in entries if "tweet" in entry['entryId']]
+
+class Tweet():
+    def __init__(self, tweet_object):
+        tweet = tweet_object['content']['itemContent']['tweet_results']['result']
+        self.id = tweet['rest_id']
+        self.views = tweet['views']['count'] if "count" in tweet["views"].keys() else 0
+        self.text = tweet['legacy']['full_text']
+        self.likes = tweet['legacy']['favorite_count']
+        self.replies = tweet['legacy']['reply_count']
+        self.retweets = tweet['legacy']['retweet_count']
+        self.quotes = tweet['legacy']['quote_count']
+        self.date = datetime.strptime(tweet['legacy']['created_at'], "%a %b %d %H:%M:%S %z %Y").astimezone(pytz.utc)
+
+    def __repr__(self):
+        return f"L:{self.likes} \tRT:{self.replies} \tQ:{self.quotes} \tR:{self.replies} \tV:{self.views} \t{self.text}"
+
+    def __str__(self):
+        return f"L:{self.likes} \tRT:{self.replies} \tQ:{self.quotes} \tR:{self.replies} \tV:{self.views}\t {self.date.isoformat()}\t{self.text}"
+
+    def time_since_post(self):               
+        return datetime.now().astimezone(pytz.utc) - self.date
+
+
+if __name__ == "__main__":
+    tweets = TweetsScraper().get_tweets_anonymous("1279948441968246785") # pobnellion
+
+    for t in tweets:
+        print(t)