updated documentation, minor fixes like excluding very long cookiestring from docs

2026-06-08 03:18:34 +03:00 · 2023-08-03 01:59:30 -05:00
parent e2142966e7
commit b8ddc400f3
30 changed files with 64 additions and 40 deletions
--- a/cisticola/scraper/bitchute.py
+++ b/cisticola/scraper/bitchute.py
@@ -24,8 +24,7 @@ class BitchuteScraper(Scraper):

        return username

-    # @logger.catch
-    @logger.catch(reraise = True)
+    @logger.catch
    def get_posts(self, channel: Channel, since: ScraperResult = None, archive_media: bool = True) -> Generator[ScraperResult, None, None]:

        session = requests.Session()
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -9,7 +9,7 @@ SOURCEDIR     = source
 BUILDDIR      = build

 SPHINXAPIDOC  = sphinx-apidoc
-APIDOCFLAGS   = --separate --private --module-first
+APIDOCFLAGS   = --separate --module-first
 MODULEPATH    = ../cisticola
 SOURCEFILES   = cisticola.*
 MODULEFILE    = modules.rst
--- a/docs/make.bat
+++ b/docs/make.bat
@@ -11,7 +11,7 @@ set SOURCEDIR=source
 set BUILDDIR=build

 set SPHINXAPIDOC=sphinx-apidoc
-set APIDOCFLAGS=--separate --private --module-first
+set APIDOCFLAGS=--separate --module-first
 set MODULEPATH=../cisticola
 set SOURCEFILES=cisticola.*
 set MODULEFILE=modules.rst
--- a/docs/source/about.rst
+++ b/docs/source/about.rst
@@ -0,0 +1,23 @@
+About Cisticola
+===============
+
+The *cisticola* application enables users to easily collect, process, and analyze large-scale data from several social media platforms.
+
+Definitions
+-----------
+- *Platform*: a social media website, for example Telegram, YouTube, or Rumble.
+- *Channel*: an account or group on a platform, for example Twitter users, Telegram private chat groups, YouTube channels, and Gab groups.
+- *Post*: a single item created by a channel, for example a Telegram message, a Tweet, or a YouTube video. Posts can contain one or more media attachments.
+- *Media*: a file uploaded to a platform by a channel as part of a post.
+
+Components
+----------
+Cisticola has many components
+
+- :py:mod:`cisticola.base`: contains Object Relational Mapping (ORM) dataclasses that imperatively map to pre-defined SQL tables
+- :py:mod:`cisticola.scraper`: contains platform-specific modules for scraping raw data from platforms. For example, the :py:mod:`cisticola.scraper.bitchute` module extracts raw data from Bitchute.
+- :py:mod:`cisticola.transformer`: contains platform-specific modules for converting raw data into a standardized, cross-platform format.
+
+The data extracted by scrapers varies by platform, but typically includes media files attached to posts. 
+
+Separating the "scraping" and "transforming" steps is useful because it ensures that no data is thrown away during the transormation. There may be some fields in the raw data that aren't included in the transformed format, but could be found to be useful in the future.
--- a/docs/source/cisticola.base.rst
+++ b/docs/source/cisticola.base.rst
@@ -5,4 +5,3 @@ cisticola.base module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.rst
+++ b/docs/source/cisticola.rst
@@ -5,7 +5,6 @@ cisticola package
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:

 Subpackages
 -----------
--- a/docs/source/cisticola.scraper.base.rst
+++ b/docs/source/cisticola.scraper.base.rst
@@ -5,4 +5,4 @@ cisticola.scraper.base module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
+   :exclude-members: cookiestring
--- a/docs/source/cisticola.scraper.bitchute.rst
+++ b/docs/source/cisticola.scraper.bitchute.rst
@@ -5,4 +5,3 @@ cisticola.scraper.bitchute module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.gab.rst
+++ b/docs/source/cisticola.scraper.gab.rst
@@ -5,4 +5,3 @@ cisticola.scraper.gab module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.gettr.rst
+++ b/docs/source/cisticola.scraper.gettr.rst
@@ -5,4 +5,3 @@ cisticola.scraper.gettr module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.instagram.rst
+++ b/docs/source/cisticola.scraper.instagram.rst
@@ -5,4 +5,3 @@ cisticola.scraper.instagram module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.odysee.rst
+++ b/docs/source/cisticola.scraper.odysee.rst
@@ -5,4 +5,3 @@ cisticola.scraper.odysee module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.rst
+++ b/docs/source/cisticola.scraper.rst
@@ -5,13 +5,12 @@ cisticola.scraper package
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:

 Submodules
 ----------

 .. toctree::
-   :maxdepth: 4
+   :maxdepth: 1

   cisticola.scraper.base
   cisticola.scraper.bitchute
@@ -20,7 +19,6 @@ Submodules
   cisticola.scraper.instagram
   cisticola.scraper.odysee
   cisticola.scraper.rumble
-   cisticola.scraper.telegram_snscrape
   cisticola.scraper.telegram_telethon
   cisticola.scraper.twitter
   cisticola.scraper.vkontakte
--- a/docs/source/cisticola.scraper.rumble.rst
+++ b/docs/source/cisticola.scraper.rumble.rst
@@ -5,4 +5,4 @@ cisticola.scraper.rumble module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
+   :exclude-members: cookiestring
--- a/docs/source/cisticola.scraper.telegram_snscrape.rst
+++ b/docs/source/cisticola.scraper.telegram_snscrape.rst
@@ -1,8 +0,0 @@
-cisticola.scraper.telegram\_snscrape module
-===========================================
-
-.. automodule:: cisticola.scraper.telegram_snscrape
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.telegram_telethon.rst
+++ b/docs/source/cisticola.scraper.telegram_telethon.rst
@@ -5,4 +5,3 @@ cisticola.scraper.telegram\_telethon module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.twitter.rst
+++ b/docs/source/cisticola.scraper.twitter.rst
@@ -5,4 +5,3 @@ cisticola.scraper.twitter module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.vkontakte.rst
+++ b/docs/source/cisticola.scraper.vkontakte.rst
@@ -5,4 +5,3 @@ cisticola.scraper.vkontakte module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.scraper.youtube.rst
+++ b/docs/source/cisticola.scraper.youtube.rst
@@ -5,4 +5,4 @@ cisticola.scraper.youtube module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
+   :exclude-members: cookiestring
--- a/docs/source/cisticola.transformer.base.rst
+++ b/docs/source/cisticola.transformer.base.rst
@@ -5,4 +5,3 @@ cisticola.transformer.base module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.transformer.bitchute.rst
+++ b/docs/source/cisticola.transformer.bitchute.rst
@@ -5,4 +5,3 @@ cisticola.transformer.bitchute module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.transformer.gettr.rst
+++ b/docs/source/cisticola.transformer.gettr.rst
@@ -0,0 +1,7 @@
+cisticola.transformer.gettr module
+==================================
+
+.. automodule:: cisticola.transformer.gettr
+   :members:
+   :undoc-members:
+   :show-inheritance:
--- a/docs/source/cisticola.transformer.rst
+++ b/docs/source/cisticola.transformer.rst
@@ -5,14 +5,17 @@ cisticola.transformer package
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:

 Submodules
 ----------

 .. toctree::
-   :maxdepth: 4
+   :maxdepth: 1

   cisticola.transformer.base
   cisticola.transformer.bitchute
+   cisticola.transformer.gettr
+   cisticola.transformer.rumble
+   cisticola.transformer.telegram_telethon
   cisticola.transformer.twitter
+   cisticola.transformer.vkontakte
--- a/docs/source/cisticola.transformer.rumble.rst
+++ b/docs/source/cisticola.transformer.rumble.rst
@@ -0,0 +1,7 @@
+cisticola.transformer.rumble module
+===================================
+
+.. automodule:: cisticola.transformer.rumble
+   :members:
+   :undoc-members:
+   :show-inheritance:
--- a/docs/source/cisticola.transformer.telegram_telethon.rst
+++ b/docs/source/cisticola.transformer.telegram_telethon.rst
@@ -0,0 +1,7 @@
+cisticola.transformer.telegram\_telethon module
+===============================================
+
+.. automodule:: cisticola.transformer.telegram_telethon
+   :members:
+   :undoc-members:
+   :show-inheritance:
--- a/docs/source/cisticola.transformer.twitter.rst
+++ b/docs/source/cisticola.transformer.twitter.rst
@@ -5,4 +5,3 @@ cisticola.transformer.twitter module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/cisticola.transformer.vkontakte.rst
+++ b/docs/source/cisticola.transformer.vkontakte.rst
@@ -0,0 +1,7 @@
+cisticola.transformer.vkontakte module
+======================================
+
+.. automodule:: cisticola.transformer.vkontakte
+   :members:
+   :undoc-members:
+   :show-inheritance:
--- a/docs/source/cisticola.utils.rst
+++ b/docs/source/cisticola.utils.rst
@@ -5,4 +5,3 @@ cisticola.utils module
   :members:
   :undoc-members:
   :show-inheritance:
-   :private-members:
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -4,5 +4,6 @@ Welcome to Cisticola's documentation!
 .. toctree::
  :maxdepth: 1

+  about
  quickstart
  cisticola
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -81,12 +81,6 @@ The *cisticola* application uses pytest_ for unit testing. To run the full test

    pipenv run pytest

-To run the test suite without archiving media (which can take a long time), run the following command from the package root directory:
-
-.. code-block::
-
-    pipenv run pytest -m "not media"
-
 To see the logging output from a test run, add the ``--capture=no`` flag to the command. 

 Examples