added missing docstrings, created Makefile target for sphinx-apidoc, added quickstart page for installation and configuration instructions

docs/Makefile

@@ -8,12 +8,24 @@ SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
 
+SPHINXAPIDOC  = sphinx-apidoc
+APIDOCFLAGS   = --separate --private --module-first
+MODULEPATH    = ../cisticola
+SOURCEFILES   = cisticola.*
+MODULEFILE    = modules.rst
+
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 .PHONY: help Makefile
 
+# Custom process and flags for generating Sphinx sources
+apidoc:
+	rm $(SOURCEDIR)/$(SOURCEFILES)
+	$(SPHINXAPIDOC) $(APIDOCFLAGS) -o "$(SOURCEDIR)" "$(MODULEPATH)"
+	rm $(SOURCEDIR)/$(MODULEFILE)
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

docs/make.bat

@@ -10,6 +10,12 @@ if "%SPHINXBUILD%" == "" (
 set SOURCEDIR=source
 set BUILDDIR=build
 
+set SPHINXAPIDOC=sphinx-apidoc
+set APIDOCFLAGS=--separate --private --module-first
+set MODULEPATH=../cisticola
+set SOURCEFILES=cisticola.*
+set MODULEFILE=modules.rst
+
 if "%1" == "" goto help
 
 %SPHINXBUILD% >NUL 2>NUL
@@ -28,6 +34,11 @@ if errorlevel 9009 (
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 
+:apidoc
+	del %SOURCEDIR%\%SOURCEFILES%
+	%SPHINXAPIDOC% %APIDOCFLAGS% -o %SOURCEDIR% %MODULEPATH%
+	del %SOURCEDIR%\%MODULEFILE%
+
 :help
 %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 

docs/source/cisticola.rst

@@ -23,3 +23,4 @@ Submodules
    :maxdepth: 4
 
    cisticola.base
+   cisticola.utils

docs/source/cisticola.scraper.instagram.rst

@@ -0,0 +1,8 @@
+cisticola.scraper.instagram module
+==================================
+
+.. automodule:: cisticola.scraper.instagram
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:

docs/source/cisticola.scraper.rst

@@ -17,9 +17,11 @@ Submodules
    cisticola.scraper.bitchute
    cisticola.scraper.gab
    cisticola.scraper.gettr
+   cisticola.scraper.instagram
    cisticola.scraper.odysee
    cisticola.scraper.rumble
    cisticola.scraper.telegram_snscrape
    cisticola.scraper.telegram_telethon
    cisticola.scraper.twitter
-   cisticola.scraper.utils
+   cisticola.scraper.vkontakte
+   cisticola.scraper.youtube

docs/source/cisticola.scraper.utils.rst

@@ -1,8 +0,0 @@
-cisticola.scraper.utils module
-==============================
-
-.. automodule:: cisticola.scraper.utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :private-members:

docs/source/cisticola.scraper.vkontakte.rst

@@ -0,0 +1,8 @@
+cisticola.scraper.vkontakte module
+==================================
+
+.. automodule:: cisticola.scraper.vkontakte
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:

docs/source/cisticola.scraper.youtube.rst

@@ -0,0 +1,8 @@
+cisticola.scraper.youtube module
+================================
+
+.. automodule:: cisticola.scraper.youtube
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:

docs/source/cisticola.transformer.bitchute.rst

@@ -0,0 +1,8 @@
+cisticola.transformer.bitchute module
+=====================================
+
+.. automodule:: cisticola.transformer.bitchute
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:

docs/source/cisticola.transformer.rst

@@ -14,4 +14,5 @@ Submodules
    :maxdepth: 4
 
    cisticola.transformer.base
+   cisticola.transformer.bitchute
    cisticola.transformer.twitter

docs/source/cisticola.utils.rst

@@ -0,0 +1,8 @@
+cisticola.utils module
+======================
+
+.. automodule:: cisticola.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :private-members:

docs/source/index.rst

@@ -2,16 +2,7 @@ Welcome to Cisticola's documentation!
 =====================================
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Contents:
+  :maxdepth: 1
 
-   cisticola
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+  quickstart
+  cisticola

docs/source/quickstart.rst

@@ -0,0 +1,96 @@
+Quickstart
+==========
+
+Installation
+------------
+
+The *cisticola* application uses pipenv_ for dependency management. To install the dependencies of *cisticola*, first install pipenv using the following command:
+
+.. code-block::
+
+    pip install pipenv
+
+and then install the dependencies using the following command from the package root directory:
+
+.. code-block::
+
+    pipenv install
+
+To install the necessary dependencies for building the documentation and running unit tests, run the following command from the package root directory:
+
+.. code-block::
+
+    pipenv install --dev
+
+Environment Variables
+---------------------
+
+Three of the scrapers in *cisticola* (:py:mod:`~cisticola.scraper.gab.GabScraper`,  :py:mod:`~cisticola.scraper.instagram.InstagramScraper`, and :py:mod:`~cisticola.scraper.telegram_telethon.TelegramTelethonScraper`) require platform credentials to work correctly. 
+
+Gab
+"""
+
+The Gab credentials can be configured by running the following command from the root directory:
+
+.. code-block::
+
+    pipenv run garc configure 
+
+which will direct you to provide the username and password for your Gab account.
+
+Instagram
+"""""""""
+
+The Instagram credentials can be configured by setting the following environment variables, either in the project's ``.env`` file or in the system's environment:
+
+- ``INSTAGRAM_USERNAME``: username of your Instagram account
+- ``INSTAGRAM_PASSWORD``: password of your Instagram account
+
+Telegram Telethon
+"""""""""""""""""
+
+The Telegram credentials can be configured by setting the following environment variables, either in the project's ``.env`` file or in the system's environment:
+
+- ``TELEGRAM_API_ID``: API ID number for your Telegram application
+- ``TELEGRAM_API_HASH``: API hash for your Telegram application
+- ``TELEGRAM_PHONE``: phone number for the account corresponding to your your Telegram application
+
+If you do not already have a Telegram application, you can create one by following the instructions on `this page`_.
+
+Documentation
+-------------
+
+The *cisticola* application uses Sphinx_ to generate and display its documentation. To build the documentation in the HTML format, run the following command from the ``docs/`` directory:
+
+.. code-block::
+
+    pipenv run make html
+
+For developers, if changes are made to the package structure or additional modules are created, you can update the Sphinx source ``*.rst`` files by running the following command from the ``docs/`` directory:
+
+.. code-block::
+
+    pipenv run make apidoc
+
+Testing
+-------
+
+The *cisticola* application uses pytest_ for unit testing. To run the test suite, run the following command from the package root directory:
+
+.. code-block::
+
+    pipenv run pytest
+
+Examples
+--------
+
+An example of a *cisticola* ingest file ``russian_telegram_ingest.py`` is included in the package root directory, showing how the list of channels to scrape is defined, and how the :py:mod:`~cisticola.scraper.base.ScraperController` and :py:mod:`~cisticola.transformer.base.Transformer` classes are used. To run the ingest script, run the following command from the package root directory:
+
+.. code-block::
+
+    pipenv run python russian_telegram_ingest.py
+
+.. _pipenv: https://pipenv.pypa.io/en/latest/
+.. _Sphinx: https://www.sphinx-doc.org/en/master/
+.. _pytest: https://docs.pytest.org/en/7.1.x/
+.. _this page: https://core.telegram.org/api/obtaining_api_id