Fix import order

.github/dependabot.yml

@@ -1,11 +0,0 @@
-version: 2
-updates:
-- package-ecosystem: "pip"
-  directory: "/"
-  schedule:
-    interval: "daily"
-  open-pull-requests-limit: 10
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: "daily"

Pipfile

@@ -4,8 +4,32 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-vk-api = "*"
+vk-api = ">=11.9.9"
-yt-dlp = "*"
+yt-dlp = ">=2023.2.17"
+flake8 = "*"
+mypy = ">=0.961"
+black = ">=22.3.0"
+isort = ">=5.10.1"
+pytest = "*"
+pytest-sphinx = "*"
+pytest-cov = "*"
+twine = ">=1.11.0"
+sphinx = "<5.1.0,>=4.3.0"
+furo = ">=2022.6.4.1"
+myst-parser = "<0.19.0,>=0.15.2"
+sphinx-copybutton = ">=0.5.0"
+sphinx-autobuild = ">=2021.3.14"
+sphinx-autodoc-typehints = "*"
+python-dotenv = ">=0.21.1"
+brotli = ">=1.0.9"
+certifi = ">=2022.12.7"
+charset-normalizer = ">=3.0.1"
+idna = ">=3.4"
+mutagen = ">=1.46.0"
+pycryptodomex = ">=3.17"
+requests = ">=2.28.2"
+urllib3 = ">=1.26.14"
+websockets = ">=10.4"
 
 [dev-packages]
 sphinx-copybutton = "==0.5.0"

dev-requirements.txt

@@ -2,11 +2,11 @@
 flake8
 
 # Static type checking
-mypy==0.961
+mypy>=0.961
 
 # Automatic code formatting
-black==22.3.0
+black>=22.3.0
-isort==5.10.1
+isort>=5.10.1
 
 # Running tests
 pytest
@@ -24,19 +24,20 @@ wheel
 Sphinx>=4.3.0,<5.1.0
 
 # Sphinx theme: https://sphinx-themes.org/sample-sites/furo/
-furo==2022.6.4.1
+furo>=2022.6.4.1
 
 # Lets Sphinx parse markdown files in addition to rst.
 myst-parser>=0.15.2,<0.19.0
 
 # Adds a copy button to code examples in the docs.
-sphinx-copybutton==0.5.0
+sphinx-copybutton>=0.5.0
 
 # Live rebuilding and reloading of docs for developing locally.
-sphinx-autobuild==2021.3.14
+sphinx-autobuild>=2021.3.14
 
 # Automatically adds types to docs
 sphinx-autodoc-typehints
 
 # For parsing and comparing version numbers.
 packaging
+python-dotenv>=0.21.1

requirements.txt

@@ -5,11 +5,15 @@
 #    pipenv lock --requirements
 #
 
-certifi==2022.6.15
+# -i https://pypi.org/simple
-charset-normalizer==2.0.12
+brotli>=1.0.9; platform_python_implementation >= 'CPython'
-idna==3.3
+certifi>=2022.12.7; python_version >= '3.6'
-requests==2.28.0
+charset-normalizer>=3.0.1; python_version >= '3.6'
-urllib3==1.26.9
+idna>=3.4; python_version >= '3.5'
-vk-api==11.9.8
+mutagen>=1.46.0; python_version >= '3.7'
-python-dotenv==0.20.0
+pycryptodomex>=3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-yt-dlp==2022.7.18
+requests>=2.28.2; python_version >= '3.7' and python_version < '4'
+urllib3>=1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+vk-api>=11.9.9
+websockets>=10.4; python_version >= '3.7'
+yt-dlp>=2023.2.17

setup.py

@@ -44,7 +44,10 @@ setup(
         "Programming Language :: Python :: 3",
     ],
     keywords=["scraper", "vk", "vkontakte", "vk-api", "media-downloader"],
-    url="https://github.com/bellingcat/vk-url-scraper",
+    project_urls={
+        "Code": "https://github.com/bellingcat/vk-url-scraper",
+        "Documentation": "https://vk-url-scraper.readthedocs.io/en/latest/",
+    },
     author="Bellingcat",
     author_email="tech@bellingcat.com",
     license="MIT",

tests/scraper_test.py

@@ -19,7 +19,6 @@ def test_login_custom_file():
     VkScraper(
         os.environ["VK_USERNAME"],
         os.environ["VK_PASSWORD"],
-        os.environ.get("VK_TOKEN"),
         session_file=session_filename,
     )
     assert os.path.isfile(session_filename)
@@ -81,7 +80,7 @@ def test_scrape_wall_url_with_photos():
         == "Хабаровск\nАллея героев\nПомолимся об укокоении воинов:\nАлександра, Игоря, Эдуарда, \nДионисия, Евгения, Александра, Артемия, Иннокентия, Андрея."
     )
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 6, 15, 10, 37, 24))
-    assert len(res[0]["payload"]) == 16
+    assert len(res[0]["payload"]) == 17
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["photo"]
     assert len(res[0]["attachments"]["photo"]) == 9
@@ -93,7 +92,7 @@ def test_scrape_wall_url_with_photos_inner_videos_and_links_with_inner_photos():
     assert res[0]["id"] == "wall-17315087_74182"
     assert res[0]["text"] == ""
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 11, 1, 9))
-    assert len(res[0]["payload"]) == 15
+    assert len(res[0]["payload"]) == 17
     assert len(res[0]["attachments"].keys()) == 3
     for k in ["photo", "link", "video"]:
         assert k in list(res[0]["attachments"].keys())
@@ -139,7 +138,7 @@ def test_scrape_video_only():
     assert len(res) == 1
     assert res[0]["id"] == "video38556806_456251917"
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 5, 42, 38))
-    assert len(res[0]["payload"]) == 31
+    assert len(res[0]["payload"]) == 34
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["video"]
 

vk_url_scraper/scraper.py

@@ -3,7 +3,7 @@ import re
 import shutil
 from collections import defaultdict
 from datetime import datetime
-from typing import List
+from typing import List, Optional
 from urllib.parse import urlparse
 
 import requests
@@ -43,7 +43,7 @@ class VkScraper:
         self,
         username: str,
         password: str,
-        token: str = None,
+        token: Optional[str] = None,
         session_file="vk_config.v2.json",
         captcha_handler=captcha_handler,
     ) -> None:
@@ -61,7 +61,7 @@ class VkScraper:
         token : str
             Access token received after authenticating, can be found in the vl_config.v2.json file
         session_file : str
-            File name where the VK session is saved so future logins are easier
+            File name where the VK session is saved so future logins are easier, this will not be created if token is passed
         captcha_handler : func
             Function that can receive a vk_api captcha instance and help the user solve it, default is a complete CLI handler
         """

vk_url_scraper/version.py

@@ -2,7 +2,7 @@ _MAJOR = "0"
 _MINOR = "3"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "6"
+_PATCH = "15"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""