fix vk_api dependency changes

https://blog.readthedocs.com/use-build-os-config/

https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python

.env.example

@@ -0,0 +1,3 @@
+VK_USERNAME="your username"
+VK_PASSWORD="your password"
+VK_TOKEN="optional token"

.github/actions/setup-venv/action.yml

@@ -16,6 +16,11 @@ runs:
       with:
         python-version: ${{ inputs.python-version }}
 
+    - shell: bash
+      run: |
+        # install ffmpeg
+        sudo apt install ffmpeg
+
     - shell: bash
       run: |
         # Install prerequisites.

.github/dependabot.yml

@@ -1,11 +0,0 @@
-version: 2
-updates:
-- package-ecosystem: "pip"
-  directory: "/"
-  schedule:
-    interval: "daily"
-  open-pull-requests-limit: 10
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: "daily"

.github/workflows/main.yml

@@ -20,6 +20,7 @@ env:
   PYTHONPATH: ./
   VK_USERNAME: ${{ secrets.VK_USERNAME }} 
   VK_PASSWORD: ${{ secrets.VK_PASSWORD }}
+  VK_TOKEN: ${{ secrets.VK_TOKEN }}
 
 jobs:
   checks:
@@ -30,10 +31,10 @@ jobs:
       fail-fast: false
       matrix:
         python: ['3.7', '3.10']
-        task: #  --show-capture=no on purpose
+        task: # --show-capture=no on purpose, -s for captchas
           - name: Test
             run: |
-              pytest  --show-capture=no --color=yes tests/
+              pytest -s --show-capture=no --color=yes tests/
 
         include:
           - python: '3.10'

.gitignore

@@ -1,6 +1,7 @@
 .env
 vk_config.v2.json
 output/
+tmp*/
 # build artifacts
 
 .eggs/

.readthedocs.yaml

@@ -4,8 +4,12 @@ sphinx:
   configuration: docs/source/conf.py
   fail_on_warning: false
 
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.8"
+    
 python:
-  version: "3.8"
   install:
     - requirements: requirements.txt
     - requirements: dev-requirements.txt

Pipfile

@@ -4,8 +4,32 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-vk-api = "*"
-yt-dlp = "*"
+yt-dlp = ">=2023.2.17"
+flake8 = "*"
+mypy = ">=0.961"
+black = ">=22.3.0"
+isort = ">=5.10.1"
+pytest = "*"
+pytest-sphinx = "*"
+pytest-cov = "*"
+twine = ">=1.11.0"
+sphinx = "<5.1.0,>=4.3.0"
+furo = ">=2022.6.4.1"
+myst-parser = "<0.19.0,>=0.15.2"
+sphinx-copybutton = ">=0.5.0"
+sphinx-autobuild = ">=2021.3.14"
+sphinx-autodoc-typehints = "*"
+python-dotenv = ">=0.21.1"
+brotli = ">=1.0.9"
+certifi = ">=2023.7.22"
+charset-normalizer = ">=3.0.1"
+idna = ">=3.4"
+mutagen = ">=1.46.0"
+pycryptodomex = ">=3.17"
+requests = ">=2.28.2"
+urllib3 = ">=1.26.14"
+websockets = ">=10.4"
+vk-api = {ref = "77b5a0d51a6bbf54d59554332f28a488615fbd6c", git = "git+https://github.com/python273/vk_api.git"}
 
 [dev-packages]
 sphinx-copybutton = "==0.5.0"
@@ -18,7 +42,7 @@ pytest-sphinx = "*"
 pytest-cov = "*"
 twine = ">=1.11.0"
 sphinx = ">=4.3.0,<5.1.0"
-furo = "==2022.6.4.1"
+furo = "==2022.6.21"
 myst-parser = ">=0.15.2,<0.19.0"
 sphinx-autobuild = "==2021.3.14"
 sphinx-autodoc-typehints = "*"
@@ -26,3 +50,6 @@ python-dotenv = "*"
 
 [requires]
 python_version = "3.9"
+
+[pipenv]
+allow_prereleases = true

README.md

@@ -1,7 +1,13 @@
 # vk-url-scraper
-Library to scrape data and especially media links (videos and photos) from vk.com URLs.
+Python library to scrape data, and especially media links like videos and photos, from vk.com URLs.
 
-You can use it via the [command line](#command-line-usage) or as a [python library](#python-library-usage).
+
+[![PyPI version](https://badge.fury.io/py/vk-url-scraper.svg)](https://badge.fury.io/py/vk-url-scraper)
+[![PyPI download month](https://img.shields.io/pypi/dm/vk-url-scraper.svg)](https://pypi.python.org/pypi/vk-url-scraper/)
+[![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest)
+
+
+You can use it via the [command line](#command-line-usage) or as a [python library](#python-library-usage), check the **[documentation](https://vk-url-scraper.readthedocs.io/en/latest/)**.
 
 ## Installation
 You can install the most recent release from [pypi](https://pypi.org/project/vk-url-scraper/) via `pip install vk-url-scraper`.
@@ -14,12 +20,16 @@ To use the library you will need a valid username/password combination for vk.co
 vk_url_scraper --help
 
 # scrape a URL and get the JSON result in the console
-vk_url_scraper -username "username here" --password "password here" --urls https://vk.com/wall12345_6789
+vk_url_scraper --username "username here" --password "password here" --urls https://vk.com/wall12345_6789
 # OR
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789
 # you can also have multiple urls
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789 https://vk.com/photo-12345_6789 https://vk.com/video12345_6789
 
+# you can pass a token as well to avoid always authenticating 
+# and possibly getting captcha prompts
+# you can fetch the token from the vk_config.v2.json file generated under by searching for "access_token"
+vk_url_scraper -u "username" -p "password" -t "vktoken goes here" --urls https://vk.com/wall12345_6789
 
 # save the JSON output into a file
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789 > output.json
@@ -45,7 +55,7 @@ res = vks.scrape("https://vk.com/wall-1_398461")
 
 # scrape any "video" URL
 res = vks.scrape("https://vk.com/video-6596301_145810025")
-print(res[0]["text]) # eg: -> to get the text from code
+print(res[0]["text"]) # eg: -> to get the text from code
 ```
 
 ```python
@@ -79,7 +89,7 @@ see [docs] for all available functions.
 2. To run all checks to `make run-checks` (fixes style) or individually
    1. To fix style: `black .` and `isort .` -> `flake8 .` to validate lint
    2. To do type checking: `mypy .`
-   3. To test: `pytest .` (`pytest -v --color=yes --doctest-modules tests/ vk_url_scraper/` to user verbose, colors, and test docstring examples)
+   3. To test: `pytest .` (`pytest -v --color=yes --doctest-modules tests/ vk_url_scraper/` to use verbose, colors, and test docstring examples)
 3. `make docs` to generate shpynx docs -> edit [config.py](docs/source/conf.py) if needed
 
 To test the command line interface available in [__main__.py](__vk_url_scraper/__main__.py) you need to pass the `-m` option to python like so: `python -m vk_url_scraper -u "" -p "" --urls ...`
@@ -90,6 +100,7 @@ To test the command line interface available in [__main__.py](__vk_url_scraper/_
 2. run `./scripts/release.sh` to create a tag and push, alternatively
    1. `git tag vx.y.z` to tag version
    2. `git push origin vx.y.z` -> this will trigger workflow and put project on [pypi](https://pypi.org/project/vk-url-scraper/)
+3. go to https://readthedocs.org/ to deploy new docs version (if webhook is not setup)
 
 ### Fixing a failed release
 
@@ -99,4 +110,4 @@ If for some reason the GitHub Actions release workflow failed with an error that
 git tag -l | xargs git tag -d && git fetch -t
 ```
 
-Then repeat the steps above.
+Then repeat the steps above.

dev-requirements.txt

@@ -2,11 +2,11 @@
 flake8
 
 # Static type checking
-mypy==0.961
+mypy>=0.961
 
 # Automatic code formatting
-black==22.3.0
-isort==5.10.1
+black>=22.3.0
+isort>=5.10.1
 
 # Running tests
 pytest
@@ -24,19 +24,20 @@ wheel
 Sphinx>=4.3.0,<5.1.0
 
 # Sphinx theme: https://sphinx-themes.org/sample-sites/furo/
-furo==2022.6.4.1
+furo>=2022.6.4.1
 
 # Lets Sphinx parse markdown files in addition to rst.
 myst-parser>=0.15.2,<0.19.0
 
 # Adds a copy button to code examples in the docs.
-sphinx-copybutton==0.5.0
+sphinx-copybutton>=0.5.0
 
 # Live rebuilding and reloading of docs for developing locally.
-sphinx-autobuild==2021.3.14
+sphinx-autobuild>=2021.3.14
 
 # Automatically adds types to docs
 sphinx-autodoc-typehints
 
 # For parsing and comparing version numbers.
 packaging
+python-dotenv>=0.21.1

requirements.txt

@@ -5,11 +5,15 @@
 #    pipenv lock --requirements
 #
 
-certifi==2022.6.15
-charset-normalizer==2.0.12
-idna==3.3
-requests==2.28.0
-urllib3==1.26.9
-vk-api==11.9.8
-python-dotenv==0.20.0
-yt-dlp==2022.5.18
+# -i https://pypi.org/simple
+brotli>=1.0.9; platform_python_implementation >= 'CPython'
+certifi>=2022.12.7; python_version >= '3.6'
+charset-normalizer>=3.0.1; python_version >= '3.6'
+idna>=3.4; python_version >= '3.5'
+mutagen>=1.46.0; python_version >= '3.7'
+pycryptodomex>=3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+requests>=2.28.2; python_version >= '3.7' and python_version < '4'
+urllib3>=1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+vk-api>=11.9.9
+websockets>=10.4; python_version >= '3.7'
+yt-dlp>=2023.2.17

setup.py

@@ -33,7 +33,7 @@ with open("vk_url_scraper/version.py", "r") as version_file:
 setup(
     name="vk-url-scraper",
     version=VERSION["VERSION"],
-    description="",
+    description="Scrape VK URLs to fetch info and media - python API or command line tool.",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     classifiers=[
@@ -43,8 +43,11 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3",
     ],
-    keywords="",
-    url="https://github.com/bellingcat/vk-url-scraper",
+    keywords=["scraper", "vk", "vkontakte", "vk-api", "media-downloader"],
+    project_urls={
+        "Code": "https://github.com/bellingcat/vk-url-scraper",
+        "Documentation": "https://vk-url-scraper.readthedocs.io/en/latest/",
+    },
     author="Bellingcat",
     author_email="tech@bellingcat.com",
     license="MIT",
@@ -55,4 +58,9 @@ setup(
     install_requires=read_requirements("requirements.txt"),
     extras_require={"dev": read_requirements("dev-requirements.txt")},
     python_requires=">=3.7",
+    entry_points={
+        "console_scripts": [
+            "vk_url_scraper=vk_url_scraper.__main__:main",
+        ],
+    },
 )

tests/scraper_test.py

@@ -14,9 +14,23 @@ def test_login_fail():
         VkScraper("invalid", "combination")
 
 
+# disabled due to CI
+# def test_login_custom_file():
+#     session_filename = "test-session.json"
+#     VkScraper(
+#         os.environ["VK_USERNAME"],
+#         os.environ["VK_PASSWORD"],
+#         session_file=session_filename,
+#     )
+#     assert os.path.isfile(session_filename)
+#     os.unlink(session_filename)
+
+
 def test_login_success():
     global vks
-    vks = VkScraper(os.environ["VK_USERNAME"], os.environ["VK_PASSWORD"])
+    vks = VkScraper(
+        os.environ["VK_USERNAME"], os.environ["VK_PASSWORD"], os.environ.get("VK_TOKEN")
+    )
 
 
 def test_scrape_empty_urll():
@@ -67,7 +81,7 @@ def test_scrape_wall_url_with_photos():
         == "Хабаровск\nАллея героев\nПомолимся об укокоении воинов:\nАлександра, Игоря, Эдуарда, \nДионисия, Евгения, Александра, Артемия, Иннокентия, Андрея."
     )
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 6, 15, 10, 37, 24))
-    assert len(res[0]["payload"]) == 16
+    assert len(res[0]["payload"]) == 18
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["photo"]
     assert len(res[0]["attachments"]["photo"]) == 9
@@ -79,7 +93,7 @@ def test_scrape_wall_url_with_photos_inner_videos_and_links_with_inner_photos():
     assert res[0]["id"] == "wall-17315087_74182"
     assert res[0]["text"] == ""
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 11, 1, 9))
-    assert len(res[0]["payload"]) == 15
+    assert len(res[0]["payload"]) == 18
     assert len(res[0]["attachments"].keys()) == 3
     for k in ["photo", "link", "video"]:
         assert k in list(res[0]["attachments"].keys())
@@ -99,7 +113,7 @@ def test_scrape_download_multiple_media():
             "wall-17315087_74182_2.jpg",
             "wall-17315087_74182_3.jpg",
             "wall-17315087_74182_4.jpg",
-            "wall-17315087_74182_0.mkv",
+            "wall-17315087_74182_0.mp4",
         }
         found_files = set(os.listdir(tempdir))
         assert len(expect_files) == len(expect_files & found_files)
@@ -114,7 +128,7 @@ def test_scrape_photo_only():
         == "Делимся расписанием конкурса [https://vk.com/wall-1_399468|«Код Петербурга»]. Все важные этапы — на одной схеме \n\nЕсли участвуете, обязательно сохраните себе. Так будет удобнее планировать работу над проектом, и вы точно не упустите лучший момент для отправки сервиса на модерацию."
     )
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 6, 7, 9, 43))
-    assert len(res[0]["payload"]) == 15
+    assert len(res[0]["payload"]) == 16
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["photo"]
     assert len(res[0]["attachments"]["photo"]) == 1
@@ -125,7 +139,7 @@ def test_scrape_video_only():
     assert len(res) == 1
     assert res[0]["id"] == "video38556806_456251917"
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 5, 42, 38))
-    assert len(res[0]["payload"]) == 31
+    assert len(res[0]["payload"]) == 34
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["video"]
 
@@ -135,4 +149,22 @@ def test_scrape_video_only2():
     with tempfile.TemporaryDirectory(dir="./") as tempdir:
         vks.download_media(res, tempdir)
         found_files = set(os.listdir(tempdir))
-        assert "video-17546758_456239898_0.mkv.webm" in found_files
+        assert "video-17546758_456239898_0.mp4" in found_files
+
+
+def test_scrape_private_video():
+    """
+    > Some videos are kept private and cannot be accessed without a passkey . In this case, send the ID in {owner_id}_{video_id}_{access_key}.
+    From https://dev.vk.com/ru/method/video.get
+    """
+    res = vks.scrape("https://vk.com/wall-127774884_178565")
+
+    with tempfile.TemporaryDirectory(dir="./") as tempdir:
+        vks.download_media(res, tempdir)
+        expect_files = {
+            "wall-127774884_178565_0.mp4",
+            "wall-127774884_178565_1.mp4",
+            "wall-127774884_178565_2.mp4",
+        }
+        found_files = set(os.listdir(tempdir))
+        assert len(expect_files) == len(expect_files & found_files)

vk_url_scraper/__init__.py

@@ -1,2 +1,2 @@
 from .scraper import VkScraper
-from .utils import DateTimeEncoder, mkdir_if_not_exists
+from .utils import DateTimeEncoder, suppress_stdout

vk_url_scraper/__main__.py

@@ -19,7 +19,7 @@ def get_argument_parser():
         action="store",
         dest="username",
         required=True,
-        help="username for a valid vk.com account",
+        help="username for a valid vk.com account (pass empty if using --token)",
     )
     parser.add_argument(
         "-p",
@@ -27,7 +27,15 @@ def get_argument_parser():
         action="store",
         dest="password",
         required=True,
-        help="password for the valid vk.com account",
+        help="password for the valid vk.com account (pass empty if using --token)",
+    )
+    parser.add_argument(
+        "-t",
+        "--token",
+        action="store",
+        dest="token",
+        required=False,
+        help="optional token, when passed username/password authentication will not be done - good to avoid captcha issues",
     )
     parser.add_argument(
         "-d",
@@ -50,7 +58,7 @@ def get_argument_parser():
 def main():
     parser = get_argument_parser()
     args = parser.parse_args()
-    vks = VkScraper(args.username, args.password)
+    vks = VkScraper(args.username, args.password, args.token)
     text = " ".join(args.urls)
     res = vks.scrape(text)
     res_json = json.dumps(res, ensure_ascii=False, indent=4, cls=DateTimeEncoder)

vk_url_scraper/scraper.py

@@ -1,15 +1,16 @@
 import os
 import re
+import shutil
 from collections import defaultdict
 from datetime import datetime
-from typing import List
+from typing import List, Optional
 from urllib.parse import urlparse
 
 import requests
 import vk_api  # used to get api_token after authentication
 import yt_dlp  # to download videos from url
 
-from .utils import mkdir_if_not_exists
+from .utils import captcha_handler, suppress_stdout
 
 
 class VkScraper:
@@ -36,12 +37,20 @@ class VkScraper:
 
     WALL_PATTERN = re.compile(r"(wall.{0,1}\d+_\d+)")
     PHOTO_PATTERN = re.compile(r"(photo.{0,1}\d+_\d+)")
-    VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+)")
+    VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+(?:_\w+)?)")
 
-    def __init__(self, username: str, password: str) -> None:
+    def __init__(
+        self,
+        username: str,
+        password: str,
+        token: Optional[str] = None,
+        session_file="vk_config.v2.json",
+        captcha_handler=captcha_handler,
+    ) -> None:
         """Initializes the scraper.
 
-        This function receives a username and password and performs authentication on vk.com to then call api endpoints
+        This function receives a username and password (or access token) and performs
+        authentication on vk.com to then call api endpoints. If token is passed, authentication will not be performed again.
 
         Parameters
         ----------
@@ -49,9 +58,22 @@ class VkScraper:
             Username on vk.com, can be a phone number or email
         password : str
             Matching password on vk.com
+        token : str
+            Access token received after authenticating, can be found in the vl_config.v2.json file
+        session_file : str
+            File name where the VK session is saved so future logins are easier, this will not be created if token is passed
+        captcha_handler : func
+            Function that can receive a vk_api captcha instance and help the user solve it, default is a complete CLI handler
         """
-        self.session = vk_api.VkApi(username, password)
-        self.session.auth(token_only=True)
+        self.session = vk_api.VkApi(
+            username,
+            password,
+            token=token,
+            config_filename=session_file,
+            captcha_handler=captcha_handler,
+        )
+        if token is None or len(token) == 0:
+            self.session.auth(token_only=True)
 
     def scrape(self, url: str) -> List:
         """Scrapes a URL for multiple possibilities of inner links such as wall, video, photo, ...
@@ -122,10 +144,11 @@ class VkScraper:
                     first_type = a["type"]
                     attachment = a[first_type]
                     if first_type == "video":
+                        video_path = f'video{attachment["owner_id"]}_{attachment["id"]}'
+                        if "access_key" in attachment:
+                            video_path += f"_{attachment['access_key']}"
                         attachments["video"].extend(
-                            self.scrape_videos(f'video{attachment["owner_id"]}_{attachment["id"]}')[
-                                0
-                            ]
+                            self.scrape_videos(video_path)[0]
                             .get("attachments", {})
                             .get("video", [""])
                         )
@@ -298,7 +321,7 @@ class VkScraper:
         headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
-        mkdir_if_not_exists(destination)
+        os.makedirs(destination, exist_ok=True)
         downloaded = []
         for r in results:
             for k, attachments in r["attachments"].items():
@@ -311,17 +334,29 @@ class VkScraper:
                             f.write(d.content)
                             downloaded.append(filename)
                 elif k == "video":
-                    for i, url in enumerate(attachments):
-                        filename = os.path.join(destination, f"{r['id']}_{i}.mkv")
-                        ydl = yt_dlp.YoutubeDL(
-                            {
-                                "outtmpl": filename,
-                                "quiet": True,
-                                "restrictfilenames": True,
-                                "forcefilename": True,
-                            }
-                        )
-                        info = ydl.extract_info(url, download=True)
-                        filename = ydl.prepare_filename(info)
-                        downloaded.append(filename)
+                    with suppress_stdout():  # ytdlp is not 100% quiet
+                        for i, url in enumerate(attachments):
+                            filename = os.path.join(destination, f"{r['id']}_{i}.%(ext)s")
+                            ydl = yt_dlp.YoutubeDL(
+                                {
+                                    "format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+                                    "merge_output_format": "mp4",
+                                    "retries": 5,
+                                    "noplaylist": True,
+                                    "outtmpl": filename,
+                                    "quiet": True,
+                                    "restrictfilenames": True,
+                                    "forcefilename": True,
+                                    "simulate": False,
+                                }
+                            )
+                            info = ydl.extract_info(url, download=True)
+                            filename = ydl.prepare_filename(info)
+                            if "unknown_video" in filename:
+                                old_filename = filename
+                                filename = shutil.copy(
+                                    filename, filename.replace("unknown_video", "mp4")
+                                )
+                                os.remove(old_filename)
+                            downloaded.append(filename)
         return downloaded

vk_url_scraper/utils.py

@@ -1,5 +1,7 @@
 import json
 import os
+import sys
+from contextlib import contextmanager
 from datetime import datetime
 
 
@@ -11,6 +13,21 @@ class DateTimeEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, o)
 
 
-def mkdir_if_not_exists(folder):
-    if not os.path.exists(folder):
-        os.makedirs(folder)
+def captcha_handler(captcha):
+    key = input(
+        f"CAPTCHA DETECTED, please solve it and input the solution. url= {captcha.get_url()} :"
+    ).strip()
+    return captcha.try_again(key)
+
+
+@contextmanager
+def suppress_stdout():
+    # https://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/
+    # this is used to silence ytdlp which does not fully respects quite=True and outputs filenames to the console
+    with open(os.devnull, "w") as devnull:
+        old_stdout = sys.stdout
+        sys.stdout = devnull
+        try:
+            yield
+        finally:
+            sys.stdout = old_stdout

vk_url_scraper/version.py

@@ -1,8 +1,8 @@
 _MAJOR = "0"
-_MINOR = "2"
+_MINOR = "3"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "2"
+_PATCH = "27"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""