mirror of
https://github.com/bellingcat/cisticola.git
synced 2026-06-11 12:58:33 +03:00
354 lines
28 KiB
HTML
354 lines
28 KiB
HTML
|
|
<!DOCTYPE html>
|
|
|
|
<html>
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
|
|
|
<title>cisticola package — Cisticola documentation</title>
|
|
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
|
|
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
|
<script src="_static/jquery.js"></script>
|
|
<script src="_static/underscore.js"></script>
|
|
<script src="_static/doctools.js"></script>
|
|
<link rel="index" title="Index" href="genindex.html" />
|
|
<link rel="search" title="Search" href="search.html" />
|
|
<link rel="next" title="cisticola.scraper package" href="cisticola.scraper.html" />
|
|
<link rel="prev" title="cisticola" href="modules.html" />
|
|
|
|
<link rel="stylesheet" href="_static/custom.css" type="text/css" />
|
|
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />
|
|
|
|
</head><body>
|
|
|
|
|
|
<div class="document">
|
|
<div class="documentwrapper">
|
|
<div class="bodywrapper">
|
|
|
|
|
|
<div class="body" role="main">
|
|
|
|
<section id="cisticola-package">
|
|
<h1>cisticola package<a class="headerlink" href="#cisticola-package" title="Permalink to this headline">¶</a></h1>
|
|
<section id="subpackages">
|
|
<h2>Subpackages<a class="headerlink" href="#subpackages" title="Permalink to this headline">¶</a></h2>
|
|
<div class="toctree-wrapper compound">
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="cisticola.scraper.html">cisticola.scraper package</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.scraper.html#submodules">Submodules</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.scraper.html#module-cisticola.scraper.bitchute">cisticola.scraper.bitchute module</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.scraper.html#module-cisticola.scraper.gettr">cisticola.scraper.gettr module</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.scraper.html#module-cisticola.scraper.twitter">cisticola.scraper.twitter module</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.scraper.html#module-cisticola.scraper">Module contents</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="cisticola.transformer.html">cisticola.transformer package</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.transformer.html#submodules">Submodules</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.transformer.html#module-cisticola.transformer.twitter">cisticola.transformer.twitter module</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="cisticola.transformer.html#module-cisticola.transformer">Module contents</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</section>
|
|
<section id="submodules">
|
|
<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline">¶</a></h2>
|
|
</section>
|
|
<section id="module-cisticola.base">
|
|
<span id="cisticola-base-module"></span><h2>cisticola.base module<a class="headerlink" href="#module-cisticola.base" title="Permalink to this headline">¶</a></h2>
|
|
<dl class="py class">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel">
|
|
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">cisticola.base.</span></span><span class="sig-name descname"><span class="pre">Channel</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">id</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">platform_id</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">category</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">followers</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">platform</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">country</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">influencer</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">public</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">chat</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">notes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.base.Channel" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.category">
|
|
<span class="sig-name descname"><span class="pre">category</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.category" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.chat">
|
|
<span class="sig-name descname"><span class="pre">chat</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#cisticola.base.Channel.chat" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.country">
|
|
<span class="sig-name descname"><span class="pre">country</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.country" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.followers">
|
|
<span class="sig-name descname"><span class="pre">followers</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#cisticola.base.Channel.followers" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.id">
|
|
<span class="sig-name descname"><span class="pre">id</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#cisticola.base.Channel.id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.influencer">
|
|
<span class="sig-name descname"><span class="pre">influencer</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.influencer" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.name">
|
|
<span class="sig-name descname"><span class="pre">name</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.name" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.notes">
|
|
<span class="sig-name descname"><span class="pre">notes</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.notes" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.platform">
|
|
<span class="sig-name descname"><span class="pre">platform</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.platform" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.platform_id">
|
|
<span class="sig-name descname"><span class="pre">platform_id</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.platform_id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.public">
|
|
<span class="sig-name descname"><span class="pre">public</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#cisticola.base.Channel.public" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.Channel.url">
|
|
<span class="sig-name descname"><span class="pre">url</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.Channel.url" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="py class">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult">
|
|
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">cisticola.base.</span></span><span class="sig-name descname"><span class="pre">ScraperResult</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">scraper</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">platform</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">channel</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">platform_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">date</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">raw_data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">date_archived</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.base.ScraperResult" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
|
<p>A minimally processed result from a scraper</p>
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.channel">
|
|
<span class="sig-name descname"><span class="pre">channel</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.channel" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.date">
|
|
<span class="sig-name descname"><span class="pre">date</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">datetime.datetime</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.date" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.date_archived">
|
|
<span class="sig-name descname"><span class="pre">date_archived</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">datetime.datetime</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.date_archived" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.id">
|
|
<span class="sig-name descname"><span class="pre">id</span></span><a class="headerlink" href="#cisticola.base.ScraperResult.id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.platform">
|
|
<span class="sig-name descname"><span class="pre">platform</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.platform" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.platform_id">
|
|
<span class="sig-name descname"><span class="pre">platform_id</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.platform_id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.raw_data">
|
|
<span class="sig-name descname"><span class="pre">raw_data</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.raw_data" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.ScraperResult.scraper">
|
|
<span class="sig-name descname"><span class="pre">scraper</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.ScraperResult.scraper" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="py class">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult">
|
|
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">cisticola.base.</span></span><span class="sig-name descname"><span class="pre">TransformedResult</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">raw_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">scraper</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">transformer</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">platform</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">channel</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">date</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">date_archived</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">content</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">author_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">author_username</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.base.TransformedResult" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
|
<p>An object with fields for columns in the analysis table</p>
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.author_id">
|
|
<span class="sig-name descname"><span class="pre">author_id</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.author_id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.author_username">
|
|
<span class="sig-name descname"><span class="pre">author_username</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.author_username" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.channel">
|
|
<span class="sig-name descname"><span class="pre">channel</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.channel" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.content">
|
|
<span class="sig-name descname"><span class="pre">content</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.content" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.date">
|
|
<span class="sig-name descname"><span class="pre">date</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">datetime.datetime</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.date" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.date_archived">
|
|
<span class="sig-name descname"><span class="pre">date_archived</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">datetime.datetime</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.date_archived" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.id">
|
|
<span class="sig-name descname"><span class="pre">id</span></span><a class="headerlink" href="#cisticola.base.TransformedResult.id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.platform">
|
|
<span class="sig-name descname"><span class="pre">platform</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.platform" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.raw_id">
|
|
<span class="sig-name descname"><span class="pre">raw_id</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.raw_id" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.scraper">
|
|
<span class="sig-name descname"><span class="pre">scraper</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.scraper" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.transformer">
|
|
<span class="sig-name descname"><span class="pre">transformer</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.transformer" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py attribute">
|
|
<dt class="sig sig-object py" id="cisticola.base.TransformedResult.url">
|
|
<span class="sig-name descname"><span class="pre">url</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#cisticola.base.TransformedResult.url" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
</section>
|
|
<section id="module-cisticola">
|
|
<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-cisticola" title="Permalink to this headline">¶</a></h2>
|
|
<dl class="py class">
|
|
<dt class="sig sig-object py" id="cisticola.ETLController">
|
|
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">cisticola.</span></span><span class="sig-name descname"><span class="pre">ETLController</span></span><a class="headerlink" href="#cisticola.ETLController" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
|
<p>This class will transform the raw_data tables into a format more conducive to analysis.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="py class">
|
|
<dt class="sig sig-object py" id="cisticola.ScraperController">
|
|
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">cisticola.</span></span><span class="sig-name descname"><span class="pre">ScraperController</span></span><a class="headerlink" href="#cisticola.ScraperController" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
|
<p>Registers scrapers, uses them to generate ScraperResults. Synchronizes
|
|
everything with database via ORM.</p>
|
|
<dl class="py method">
|
|
<dt class="sig sig-object py" id="cisticola.ScraperController.connect_to_db">
|
|
<span class="sig-name descname"><span class="pre">connect_to_db</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">engine</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.ScraperController.connect_to_db" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py method">
|
|
<dt class="sig sig-object py" id="cisticola.ScraperController.register_scraper">
|
|
<span class="sig-name descname"><span class="pre">register_scraper</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">scraper</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="cisticola.scraper.html#cisticola.scraper.Scraper" title="cisticola.scraper.Scraper"><span class="pre">cisticola.scraper.Scraper</span></a></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.ScraperController.register_scraper" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py method">
|
|
<dt class="sig sig-object py" id="cisticola.ScraperController.scrape_channels">
|
|
<span class="sig-name descname"><span class="pre">scrape_channels</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">channels</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#cisticola.base.Channel" title="cisticola.base.Channel"><span class="pre">cisticola.base.Channel</span></a><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#cisticola.ScraperController.scrape_channels" title="Permalink to this definition">¶</a></dt>
|
|
<dd></dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
</section>
|
|
</section>
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
|
<div class="sphinxsidebarwrapper">
|
|
<h1 class="logo"><a href="index.html">Cisticola</a></h1>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<h3>Navigation</h3>
|
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
|
<ul class="current">
|
|
<li class="toctree-l1 current"><a class="reference internal" href="modules.html">cisticola</a><ul class="current">
|
|
<li class="toctree-l2 current"><a class="current reference internal" href="#">cisticola package</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
|
|
<div class="relations">
|
|
<h3>Related Topics</h3>
|
|
<ul>
|
|
<li><a href="index.html">Documentation overview</a><ul>
|
|
<li><a href="modules.html">cisticola</a><ul>
|
|
<li>Previous: <a href="modules.html" title="previous chapter">cisticola</a></li>
|
|
<li>Next: <a href="cisticola.scraper.html" title="next chapter">cisticola.scraper package</a></li>
|
|
</ul></li>
|
|
</ul></li>
|
|
</ul>
|
|
</div>
|
|
<div id="searchbox" style="display: none" role="search">
|
|
<h3 id="searchlabel">Quick search</h3>
|
|
<div class="searchformwrapper">
|
|
<form class="search" action="search.html" method="get">
|
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
|
<input type="submit" value="Go" />
|
|
</form>
|
|
</div>
|
|
</div>
|
|
<script>$('#searchbox').show(0);</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
<div class="clearer"></div>
|
|
</div>
|
|
<div class="footer">
|
|
©2022, Bellingcat.
|
|
|
|
|
|
|
Powered by <a href="http://sphinx-doc.org/">Sphinx 4.4.0</a>
|
|
& <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.12</a>
|
|
|
|
|
|
|
<a href="_sources/cisticola.rst.txt"
|
|
rel="nofollow">Page source</a>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
</body>
|
|
</html> |