From 4e70306f9976cf946e3366ed036d7f7347d378c6 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Tue, 21 Feb 2023 21:24:00 +0000 Subject: [PATCH] Deprecate Entity type There is no meaningful distinction from Items, and it complicates the integration of scrapers for user searches --- snscrape/base.py | 20 +++++++------------- snscrape/modules/facebook.py | 2 +- snscrape/modules/instagram.py | 2 +- snscrape/modules/mastodon.py | 2 +- snscrape/modules/telegram.py | 2 +- snscrape/modules/twitter.py | 4 ++-- snscrape/modules/vkontakte.py | 2 +- snscrape/modules/weibo.py | 2 +- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/snscrape/base.py b/snscrape/base.py index 6425dc7..f2e3486 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -1,3 +1,6 @@ +__all__ = ['DeprecatedFeatureWarning', 'IntWithGranularity', 'Item', 'Scraper', 'ScraperException'] + + import abc import copy import dataclasses @@ -97,7 +100,7 @@ class _JSONDataclass: @dataclasses.dataclass class Item(_JSONDataclass): - '''An abstract base class for an item returned by the scraper's get_items generator. + '''An abstract base class for an item returned by the scraper. An item can really be anything. The string representation should be useful for the CLI output (e.g. a direct URL for the item). ''' @@ -107,18 +110,6 @@ class Item(_JSONDataclass): pass -@dataclasses.dataclass -class Entity(_JSONDataclass): - '''An abstract base class for an entity returned by the scraper's entity property. - - An entity is typically the account of a person or organisation. The string representation should be the preferred direct URL to the entity's page on the network. - ''' - - @abc.abstractmethod - def __str__(self): - pass - - class IntWithGranularity(int): '''A number with an associated granularity @@ -296,3 +287,6 @@ def nonempty_string(name): raise ValueError('must not be an empty string') f.__name__ = name return f + + +__getattr__, __dir__ = _module_deprecation_helper(__all__, Entity = Item) diff --git a/snscrape/modules/facebook.py b/snscrape/modules/facebook.py index 6b6bbde..c2839f1 100644 --- a/snscrape/modules/facebook.py +++ b/snscrape/modules/facebook.py @@ -30,7 +30,7 @@ class FacebookPost(snscrape.base.Item): @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): username: str pageId: int name: str diff --git a/snscrape/modules/instagram.py b/snscrape/modules/instagram.py index 14483e0..1cd5db7 100644 --- a/snscrape/modules/instagram.py +++ b/snscrape/modules/instagram.py @@ -32,7 +32,7 @@ class InstagramPost(snscrape.base.Item): @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): username: str name: typing.Optional[str] followers: snscrape.base.IntWithGranularity diff --git a/snscrape/modules/mastodon.py b/snscrape/modules/mastodon.py index cfe69e4..653e83a 100644 --- a/snscrape/modules/mastodon.py +++ b/snscrape/modules/mastodon.py @@ -67,7 +67,7 @@ class PollOption: @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): account: str # @username@domain.invalid displayName: typing.Optional[str] = None displayNameWithCustomEmojis: typing.Optional[typing.List[typing.Union[str, 'CustomEmoji']]] = None diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index b49c186..d2140f8 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -39,7 +39,7 @@ class TelegramPost(snscrape.base.Item): @dataclasses.dataclass -class Channel(snscrape.base.Entity): +class Channel(snscrape.base.Item): username: str title: str verified: bool diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 5cfa4c5..ff240e7 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -457,7 +457,7 @@ class Tombstone(snscrape.base.Item): @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): # Most fields can be None if they're not known. username: str @@ -513,7 +513,7 @@ class UserRef: @dataclasses.dataclass -class Community(snscrape.base.Entity): +class Community(snscrape.base.Item): id: int name: str description: str diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index 3e431f3..1b577ae 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -75,7 +75,7 @@ class Video: @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): username: str name: str verified: bool diff --git a/snscrape/modules/weibo.py b/snscrape/modules/weibo.py index dff1fad..654fd62 100644 --- a/snscrape/modules/weibo.py +++ b/snscrape/modules/weibo.py @@ -34,7 +34,7 @@ class Post(snscrape.base.Item): @dataclasses.dataclass -class User(snscrape.base.Entity): +class User(snscrape.base.Item): screenname: str uid: int verified: bool