bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Bump GraphQL endpoints

Browse Source
This commit is contained in:
JustAnotherArchivist
2023-02-19 05:59:42 +00:00
parent 206907612d
commit c65e36a094
1 changed files with 24 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
snscrape/modules/twitter.py
View File

@@ -746,10 +746,10 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
elif apiType is _TwitterAPIType.GRAPHQL: elif apiType is _TwitterAPIType.GRAPHQL:
if 'user' in obj['data']: if 'user' in obj['data']:
# UserTweets, UserTweetsAndReplies # UserTweets, UserTweetsAndReplies
instructions = obj['data']['user']['result']['timeline']['timeline']['instructions'] instructions = obj['data']['user']['result']['timeline_v2']['timeline']['instructions']
else: else:
# TweetDetail # TweetDetail
instructions = obj['data'].get('threaded_conversation_with_injections', {}).get('instructions', []) instructions = obj['data'].get('threaded_conversation_with_injections_v2', {}).get('instructions', [])
tweetCount = 0 tweetCount = 0
for instruction in instructions: for instruction in instructions:
if 'addEntries' in instruction: if 'addEntries' in instruction:
@@ -1682,22 +1682,25 @@ class TwitterProfileScraper(TwitterUserScraper):
'withReactionsPerspective': False, 'withReactionsPerspective': False,
'withSuperFollowsTweetFields': True, 'withSuperFollowsTweetFields': True,
'withVoice': True, 'withVoice': True,
'withV2Timeline': False, 'withV2Timeline': True,
} }
variables = paginationVariables.copy() variables = paginationVariables.copy()
del variables['cursor'] del variables['cursor']
features = { features = {
'responsive_web_twitter_blue_verified_badge_is_enabled': True, 'responsive_web_twitter_blue_verified_badge_is_enabled': True,
'responsive_web_graphql_exclude_directive_enabled': False,
'verified_phone_label_enabled': False, 'verified_phone_label_enabled': False,
'responsive_web_graphql_timeline_navigation_enabled': True, 'responsive_web_graphql_timeline_navigation_enabled': True,
'view_counts_public_visibility_enabled': True, 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False,
'view_counts_everywhere_api_enabled': True, 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False,
'longform_notetweets_consumption_enabled': False,
'tweetypie_unmention_optimization_enabled': True, 'tweetypie_unmention_optimization_enabled': True,
'responsive_web_uc_gql_enabled': True,
'vibe_api_enabled': True, 'vibe_api_enabled': True,
'responsive_web_edit_tweet_api_enabled': True, 'responsive_web_edit_tweet_api_enabled': True,
'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True, 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True,
'view_counts_everywhere_api_enabled': True,
'longform_notetweets_consumption_enabled': True,
'tweet_awards_web_tipping_enabled': False,
'freedom_of_speech_not_reach_fetch_enabled': False,
'standardized_nudges_misinfo': True, 'standardized_nudges_misinfo': True,
'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False, 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False,
'interactive_text_enabled': True, 'interactive_text_enabled': True,
@@ -1709,8 +1712,8 @@ class TwitterProfileScraper(TwitterUserScraper):
paginationParams = {'variables': paginationVariables, 'features': features} paginationParams = {'variables': paginationVariables, 'features': features}
gotPinned = False gotPinned = False
for obj in self._iter_api_data('https://twitter.com/i/api/graphql/W3HCLclD2VauuL6RcQm9MA/UserTweetsAndReplies', _TwitterAPIType.GRAPHQL, params, paginationParams): for obj in self._iter_api_data('https://twitter.com/i/api/graphql/nrdle2catTyGnTyj1Qa7wA/UserTweetsAndReplies', _TwitterAPIType.GRAPHQL, params, paginationParams):
instructions = obj['data']['user']['result']['timeline']['timeline']['instructions'] instructions = obj['data']['user']['result']['timeline_v2']['timeline']['instructions']
if not gotPinned: if not gotPinned:
for instruction in instructions: for instruction in instructions:
if instruction['type'] == 'TimelinePinEntry': if instruction['type'] == 'TimelinePinEntry':
@@ -1766,29 +1769,31 @@ class TwitterTweetScraper(_TwitterAPIScraper):
'includePromotedContent': True, 'includePromotedContent': True,
'withCommunity': True, 'withCommunity': True,
'withQuickPromoteEligibilityTweetFields': True, 'withQuickPromoteEligibilityTweetFields': True,
'withBirdwatchNotes': True, 'withBirdwatchNotes': False,
'withSuperFollowsUserFields': True, 'withSuperFollowsUserFields': True,
'withDownvotePerspective': False, 'withDownvotePerspective': False,
'withReactionsMetadata': False, 'withReactionsMetadata': False,
'withReactionsPerspective': False, 'withReactionsPerspective': False,
'withSuperFollowsTweetFields': True, 'withSuperFollowsTweetFields': True,
'withVoice': True, 'withVoice': True,
'withV2Timeline': False, 'withV2Timeline': True,
} }
variables = paginationVariables.copy() variables = paginationVariables.copy()
del variables['cursor'], variables['referrer'] del variables['cursor'], variables['referrer']
features = { features = {
'responsive_web_twitter_blue_verified_badge_is_enabled': True, 'responsive_web_twitter_blue_verified_badge_is_enabled': True,
'responsive_web_graphql_exclude_directive_enabled': False,
'verified_phone_label_enabled': False, 'verified_phone_label_enabled': False,
'responsive_web_graphql_timeline_navigation_enabled': True, 'responsive_web_graphql_timeline_navigation_enabled': True,
'view_counts_public_visibility_enabled': True, 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False,
'view_counts_everywhere_api_enabled': True,
'longform_notetweets_consumption_enabled': False,
'tweetypie_unmention_optimization_enabled': True, 'tweetypie_unmention_optimization_enabled': True,
'responsive_web_uc_gql_enabled': True,
'vibe_api_enabled': True, 'vibe_api_enabled': True,
'responsive_web_edit_tweet_api_enabled': True, 'responsive_web_edit_tweet_api_enabled': True,
'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True, 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True,
'view_counts_everywhere_api_enabled': True,
'longform_notetweets_consumption_enabled': True,
'tweet_awards_web_tipping_enabled': False,
'freedom_of_speech_not_reach_fetch_enabled': False,
'standardized_nudges_misinfo': True, 'standardized_nudges_misinfo': True,
'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False, 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False,
'interactive_text_enabled': True, 'interactive_text_enabled': True,
@@ -1798,12 +1803,12 @@ class TwitterTweetScraper(_TwitterAPIScraper):
params = {'variables': variables, 'features': features} params = {'variables': variables, 'features': features}
paginationParams = {'variables': paginationVariables, 'features': features} paginationParams = {'variables': paginationVariables, 'features': features}
url = 'https://twitter.com/i/api/graphql/HQ_gjq7zDNvSiJOCSkwUEw/TweetDetail' url = 'https://twitter.com/i/api/graphql/NNiD2K-nEYUfXlMwGCocMQ/TweetDetail'
if self._mode is TwitterTweetScraperMode.SINGLE: if self._mode is TwitterTweetScraperMode.SINGLE:
obj = self._get_api_data(url, _TwitterAPIType.GRAPHQL, params = params) obj = self._get_api_data(url, _TwitterAPIType.GRAPHQL, params = params)
if not obj['data']: if not obj['data']:
return return
for instruction in obj['data']['threaded_conversation_with_injections']['instructions']: for instruction in obj['data']['threaded_conversation_with_injections_v2']['instructions']:
if instruction['type'] != 'TimelineAddEntries': if instruction['type'] != 'TimelineAddEntries':
continue continue
for entry in instruction['entries']: for entry in instruction['entries']:
@@ -1814,7 +1819,7 @@ class TwitterTweetScraper(_TwitterAPIScraper):
for obj in self._iter_api_data(url, _TwitterAPIType.GRAPHQL, params, paginationParams, direction = _ScrollDirection.BOTH): for obj in self._iter_api_data(url, _TwitterAPIType.GRAPHQL, params, paginationParams, direction = _ScrollDirection.BOTH):
if not obj['data']: if not obj['data']:
continue continue
yield from self._graphql_timeline_instructions_to_tweets(obj['data']['threaded_conversation_with_injections']['instructions'], includeConversationThreads = True) yield from self._graphql_timeline_instructions_to_tweets(obj['data']['threaded_conversation_with_injections_v2']['instructions'], includeConversationThreads = True)
elif self._mode is TwitterTweetScraperMode.RECURSE: elif self._mode is TwitterTweetScraperMode.RECURSE:
seenTweets = set() seenTweets = set()
queue = collections.deque() queue = collections.deque()
@@ -1828,7 +1833,7 @@ class TwitterTweetScraper(_TwitterAPIScraper):
for obj in self._iter_api_data(url, _TwitterAPIType.GRAPHQL, thisParams, thisPagParams, direction = _ScrollDirection.BOTH): for obj in self._iter_api_data(url, _TwitterAPIType.GRAPHQL, thisParams, thisPagParams, direction = _ScrollDirection.BOTH):
if not obj['data']: if not obj['data']:
continue continue
for tweet in self._graphql_timeline_instructions_to_tweets(obj['data']['threaded_conversation_with_injections']['instructions'], includeConversationThreads = True): for tweet in self._graphql_timeline_instructions_to_tweets(obj['data']['threaded_conversation_with_injections_v2']['instructions'], includeConversationThreads = True):
if tweet.id not in seenTweets: if tweet.id not in seenTweets:
yield tweet yield tweet
seenTweets.add(tweet.id) seenTweets.add(tweet.id)