diff --git a/cisticola/scraper/bitchute.py b/cisticola/scraper/bitchute.py index 87cac2b..f5c79fb 100644 --- a/cisticola/scraper/bitchute.py +++ b/cisticola/scraper/bitchute.py @@ -24,8 +24,7 @@ class BitchuteScraper(Scraper): return username - # @logger.catch - @logger.catch(reraise = True) + @logger.catch def get_posts(self, channel: Channel, since: ScraperResult = None, archive_media: bool = True) -> Generator[ScraperResult, None, None]: session = requests.Session() diff --git a/docs/Makefile b/docs/Makefile index ab3e9be..8e2e9de 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,7 @@ SOURCEDIR = source BUILDDIR = build SPHINXAPIDOC = sphinx-apidoc -APIDOCFLAGS = --separate --private --module-first +APIDOCFLAGS = --separate --module-first MODULEPATH = ../cisticola SOURCEFILES = cisticola.* MODULEFILE = modules.rst diff --git a/docs/make.bat b/docs/make.bat index 3ab2ef7..8cd146a 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -11,7 +11,7 @@ set SOURCEDIR=source set BUILDDIR=build set SPHINXAPIDOC=sphinx-apidoc -set APIDOCFLAGS=--separate --private --module-first +set APIDOCFLAGS=--separate --module-first set MODULEPATH=../cisticola set SOURCEFILES=cisticola.* set MODULEFILE=modules.rst diff --git a/docs/source/about.rst b/docs/source/about.rst new file mode 100644 index 0000000..035ddaf --- /dev/null +++ b/docs/source/about.rst @@ -0,0 +1,23 @@ +About Cisticola +=============== + +The *cisticola* application enables users to easily collect, process, and analyze large-scale data from several social media platforms. + +Definitions +----------- +- *Platform*: a social media website, for example Telegram, YouTube, or Rumble. +- *Channel*: an account or group on a platform, for example Twitter users, Telegram private chat groups, YouTube channels, and Gab groups. +- *Post*: a single item created by a channel, for example a Telegram message, a Tweet, or a YouTube video. Posts can contain one or more media attachments. +- *Media*: a file uploaded to a platform by a channel as part of a post. + +Components +---------- +Cisticola has many components + +- :py:mod:`cisticola.base`: contains Object Relational Mapping (ORM) dataclasses that imperatively map to pre-defined SQL tables +- :py:mod:`cisticola.scraper`: contains platform-specific modules for scraping raw data from platforms. For example, the :py:mod:`cisticola.scraper.bitchute` module extracts raw data from Bitchute. +- :py:mod:`cisticola.transformer`: contains platform-specific modules for converting raw data into a standardized, cross-platform format. + +The data extracted by scrapers varies by platform, but typically includes media files attached to posts. + +Separating the "scraping" and "transforming" steps is useful because it ensures that no data is thrown away during the transormation. There may be some fields in the raw data that aren't included in the transformed format, but could be found to be useful in the future. \ No newline at end of file diff --git a/docs/source/cisticola.base.rst b/docs/source/cisticola.base.rst index db91e8b..8d7f53c 100644 --- a/docs/source/cisticola.base.rst +++ b/docs/source/cisticola.base.rst @@ -5,4 +5,3 @@ cisticola.base module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.rst b/docs/source/cisticola.rst index 22cdf67..8a475e9 100644 --- a/docs/source/cisticola.rst +++ b/docs/source/cisticola.rst @@ -5,7 +5,6 @@ cisticola package :members: :undoc-members: :show-inheritance: - :private-members: Subpackages ----------- diff --git a/docs/source/cisticola.scraper.base.rst b/docs/source/cisticola.scraper.base.rst index 1c6f6e2..2e5255a 100644 --- a/docs/source/cisticola.scraper.base.rst +++ b/docs/source/cisticola.scraper.base.rst @@ -5,4 +5,4 @@ cisticola.scraper.base module :members: :undoc-members: :show-inheritance: - :private-members: + :exclude-members: cookiestring diff --git a/docs/source/cisticola.scraper.bitchute.rst b/docs/source/cisticola.scraper.bitchute.rst index dc44b13..1f19a9f 100644 --- a/docs/source/cisticola.scraper.bitchute.rst +++ b/docs/source/cisticola.scraper.bitchute.rst @@ -5,4 +5,3 @@ cisticola.scraper.bitchute module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.gab.rst b/docs/source/cisticola.scraper.gab.rst index b0777c7..7bf4059 100644 --- a/docs/source/cisticola.scraper.gab.rst +++ b/docs/source/cisticola.scraper.gab.rst @@ -5,4 +5,3 @@ cisticola.scraper.gab module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.gettr.rst b/docs/source/cisticola.scraper.gettr.rst index 3275e9a..126bdab 100644 --- a/docs/source/cisticola.scraper.gettr.rst +++ b/docs/source/cisticola.scraper.gettr.rst @@ -5,4 +5,3 @@ cisticola.scraper.gettr module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.instagram.rst b/docs/source/cisticola.scraper.instagram.rst index 53ddc43..9288614 100644 --- a/docs/source/cisticola.scraper.instagram.rst +++ b/docs/source/cisticola.scraper.instagram.rst @@ -5,4 +5,3 @@ cisticola.scraper.instagram module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.odysee.rst b/docs/source/cisticola.scraper.odysee.rst index 491b0db..8ea022c 100644 --- a/docs/source/cisticola.scraper.odysee.rst +++ b/docs/source/cisticola.scraper.odysee.rst @@ -5,4 +5,3 @@ cisticola.scraper.odysee module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.rst b/docs/source/cisticola.scraper.rst index b93592c..a21acc1 100644 --- a/docs/source/cisticola.scraper.rst +++ b/docs/source/cisticola.scraper.rst @@ -5,13 +5,12 @@ cisticola.scraper package :members: :undoc-members: :show-inheritance: - :private-members: Submodules ---------- .. toctree:: - :maxdepth: 4 + :maxdepth: 1 cisticola.scraper.base cisticola.scraper.bitchute @@ -20,7 +19,6 @@ Submodules cisticola.scraper.instagram cisticola.scraper.odysee cisticola.scraper.rumble - cisticola.scraper.telegram_snscrape cisticola.scraper.telegram_telethon cisticola.scraper.twitter cisticola.scraper.vkontakte diff --git a/docs/source/cisticola.scraper.rumble.rst b/docs/source/cisticola.scraper.rumble.rst index 726c493..5b1cb32 100644 --- a/docs/source/cisticola.scraper.rumble.rst +++ b/docs/source/cisticola.scraper.rumble.rst @@ -5,4 +5,4 @@ cisticola.scraper.rumble module :members: :undoc-members: :show-inheritance: - :private-members: + :exclude-members: cookiestring diff --git a/docs/source/cisticola.scraper.telegram_snscrape.rst b/docs/source/cisticola.scraper.telegram_snscrape.rst deleted file mode 100644 index ffc9a7c..0000000 --- a/docs/source/cisticola.scraper.telegram_snscrape.rst +++ /dev/null @@ -1,8 +0,0 @@ -cisticola.scraper.telegram\_snscrape module -=========================================== - -.. automodule:: cisticola.scraper.telegram_snscrape - :members: - :undoc-members: - :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.telegram_telethon.rst b/docs/source/cisticola.scraper.telegram_telethon.rst index a41db65..9b19cea 100644 --- a/docs/source/cisticola.scraper.telegram_telethon.rst +++ b/docs/source/cisticola.scraper.telegram_telethon.rst @@ -5,4 +5,3 @@ cisticola.scraper.telegram\_telethon module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.twitter.rst b/docs/source/cisticola.scraper.twitter.rst index 9e557aa..be927c8 100644 --- a/docs/source/cisticola.scraper.twitter.rst +++ b/docs/source/cisticola.scraper.twitter.rst @@ -5,4 +5,3 @@ cisticola.scraper.twitter module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.vkontakte.rst b/docs/source/cisticola.scraper.vkontakte.rst index 405d70d..5df5c5a 100644 --- a/docs/source/cisticola.scraper.vkontakte.rst +++ b/docs/source/cisticola.scraper.vkontakte.rst @@ -5,4 +5,3 @@ cisticola.scraper.vkontakte module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.scraper.youtube.rst b/docs/source/cisticola.scraper.youtube.rst index e990195..7f8302e 100644 --- a/docs/source/cisticola.scraper.youtube.rst +++ b/docs/source/cisticola.scraper.youtube.rst @@ -5,4 +5,4 @@ cisticola.scraper.youtube module :members: :undoc-members: :show-inheritance: - :private-members: + :exclude-members: cookiestring diff --git a/docs/source/cisticola.transformer.base.rst b/docs/source/cisticola.transformer.base.rst index 0f57e13..647851e 100644 --- a/docs/source/cisticola.transformer.base.rst +++ b/docs/source/cisticola.transformer.base.rst @@ -5,4 +5,3 @@ cisticola.transformer.base module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.transformer.bitchute.rst b/docs/source/cisticola.transformer.bitchute.rst index 7427e9f..979048e 100644 --- a/docs/source/cisticola.transformer.bitchute.rst +++ b/docs/source/cisticola.transformer.bitchute.rst @@ -5,4 +5,3 @@ cisticola.transformer.bitchute module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.transformer.gettr.rst b/docs/source/cisticola.transformer.gettr.rst new file mode 100644 index 0000000..95e2f64 --- /dev/null +++ b/docs/source/cisticola.transformer.gettr.rst @@ -0,0 +1,7 @@ +cisticola.transformer.gettr module +================================== + +.. automodule:: cisticola.transformer.gettr + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cisticola.transformer.rst b/docs/source/cisticola.transformer.rst index 358d955..a6e0f39 100644 --- a/docs/source/cisticola.transformer.rst +++ b/docs/source/cisticola.transformer.rst @@ -5,14 +5,17 @@ cisticola.transformer package :members: :undoc-members: :show-inheritance: - :private-members: Submodules ---------- .. toctree:: - :maxdepth: 4 + :maxdepth: 1 cisticola.transformer.base cisticola.transformer.bitchute + cisticola.transformer.gettr + cisticola.transformer.rumble + cisticola.transformer.telegram_telethon cisticola.transformer.twitter + cisticola.transformer.vkontakte diff --git a/docs/source/cisticola.transformer.rumble.rst b/docs/source/cisticola.transformer.rumble.rst new file mode 100644 index 0000000..f9b0320 --- /dev/null +++ b/docs/source/cisticola.transformer.rumble.rst @@ -0,0 +1,7 @@ +cisticola.transformer.rumble module +=================================== + +.. automodule:: cisticola.transformer.rumble + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cisticola.transformer.telegram_telethon.rst b/docs/source/cisticola.transformer.telegram_telethon.rst new file mode 100644 index 0000000..6ba323c --- /dev/null +++ b/docs/source/cisticola.transformer.telegram_telethon.rst @@ -0,0 +1,7 @@ +cisticola.transformer.telegram\_telethon module +=============================================== + +.. automodule:: cisticola.transformer.telegram_telethon + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cisticola.transformer.twitter.rst b/docs/source/cisticola.transformer.twitter.rst index 05f29fa..41a2d99 100644 --- a/docs/source/cisticola.transformer.twitter.rst +++ b/docs/source/cisticola.transformer.twitter.rst @@ -5,4 +5,3 @@ cisticola.transformer.twitter module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/cisticola.transformer.vkontakte.rst b/docs/source/cisticola.transformer.vkontakte.rst new file mode 100644 index 0000000..3d6c98d --- /dev/null +++ b/docs/source/cisticola.transformer.vkontakte.rst @@ -0,0 +1,7 @@ +cisticola.transformer.vkontakte module +====================================== + +.. automodule:: cisticola.transformer.vkontakte + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cisticola.utils.rst b/docs/source/cisticola.utils.rst index 6e5872a..eeddcb0 100644 --- a/docs/source/cisticola.utils.rst +++ b/docs/source/cisticola.utils.rst @@ -5,4 +5,3 @@ cisticola.utils module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/index.rst b/docs/source/index.rst index 3c12d81..be9e5a3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -4,5 +4,6 @@ Welcome to Cisticola's documentation! .. toctree:: :maxdepth: 1 + about quickstart cisticola \ No newline at end of file diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index c86d956..f6ca747 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -81,12 +81,6 @@ The *cisticola* application uses pytest_ for unit testing. To run the full test pipenv run pytest -To run the test suite without archiving media (which can take a long time), run the following command from the package root directory: - -.. code-block:: - - pipenv run pytest -m "not media" - To see the logging output from a test run, add the ``--capture=no`` flag to the command. Examples