Bump version to v0.3.2 for release

.env.example

@@ -0,0 +1,3 @@
+VK_USERNAME="your username"
+VK_PASSWORD="your password"
+VK_TOKEN="optional token"

.github/workflows/main.yml

@@ -20,6 +20,7 @@ env:
   PYTHONPATH: ./
   VK_USERNAME: ${{ secrets.VK_USERNAME }} 
   VK_PASSWORD: ${{ secrets.VK_PASSWORD }}
+  VK_TOKEN: ${{ secrets.VK_TOKEN }}
 
 jobs:
   checks:
@@ -29,11 +30,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ['3.7', '3.10']
+        # python: ['3.7', '3.10']
-        task: #  --show-capture=no on purpose
+        python: ['3.10']
+        task: # --show-capture=no on purpose
           - name: Test
             run: |
-              pytest  --show-capture=no --color=yes tests/
+              pytest --show-capture=no --color=yes tests/
 
         include:
           - python: '3.10'

README.md

@@ -20,6 +20,9 @@ vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall1
 # you can also have multiple urls
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789 https://vk.com/photo-12345_6789 https://vk.com/video12345_6789
 
+# you can pass a token as well to avoid always authenticating 
+# and possibly getting captch prompts
+vk_url_scraper -u "username" -p "password" -t "vktoken goes here" --urls https://vk.com/wall12345_6789
 
 # save the JSON output into a file
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789 > output.json

setup.py

@@ -33,7 +33,7 @@ with open("vk_url_scraper/version.py", "r") as version_file:
 setup(
     name="vk-url-scraper",
     version=VERSION["VERSION"],
-    description="",
+    description="Scrape VK URLs to fetch info and media - python API or command line tool.",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     classifiers=[
@@ -43,7 +43,7 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3",
     ],
-    keywords="",
+    keywords=["scraper", "vk", "vkontakte", "vk-api", "media-downloader"],
     url="https://github.com/bellingcat/vk-url-scraper",
     author="Bellingcat",
     author_email="tech@bellingcat.com",
@@ -55,4 +55,9 @@ setup(
     install_requires=read_requirements("requirements.txt"),
     extras_require={"dev": read_requirements("dev-requirements.txt")},
     python_requires=">=3.7",
+    entry_points={
+        "console_scripts": [
+            "vk_url_scraper=vk_url_scraper.__main__:main",
+        ],
+    },
 )

tests/scraper_test.py

@@ -2,21 +2,24 @@ import datetime
 import os
 import tempfile
 
-import pytest
-
 from vk_url_scraper import VkScraper
 
+# import pytest
+
+
 vks = None
 
 
-def test_login_fail():
+# def test_login_fail():
-    with pytest.raises(Exception):
+#     with pytest.raises(Exception):
-        VkScraper("invalid", "combination")
+#         VkScraper("invalid", "combination")
 
 
 def test_login_success():
     global vks
-    vks = VkScraper(os.environ["VK_USERNAME"], os.environ["VK_PASSWORD"])
+    vks = VkScraper(
+        os.environ["VK_USERNAME"], os.environ["VK_PASSWORD"], os.environ.get("VK_TOKEN")
+    )
 
 
 def test_scrape_empty_urll():
@@ -131,5 +134,12 @@ def test_scrape_video_only():
 
 
 def test_scrape_video_only2():
-    res = vks.scrape("https://vk.com/video-1_456239018")
+    res = vks.scrape("https://vk.com/video-17546758_456239898")
-    print(res[0]["attachments"]["video"][0])
+    with tempfile.TemporaryDirectory(dir="./") as tempdir:
+        vks.download_media(res, tempdir)
+        found_files = set(os.listdir(tempdir))
+        # different systems might attribute different extension
+        assert (
+            "video-17546758_456239898_0.webm" in found_files
+            or "video-17546758_456239898_0.mp4" in found_files
+        )

vk_url_scraper/__main__.py

@@ -29,6 +29,14 @@ def get_argument_parser():
         required=True,
         help="password for the valid vk.com account",
     )
+    parser.add_argument(
+        "-t",
+        "--token",
+        action="store",
+        dest="token",
+        required=False,
+        help="optional token, when passed authentication will not be performed - good to avoid captcha issues",
+    )
     parser.add_argument(
         "-d",
         "--download",
@@ -50,7 +58,7 @@ def get_argument_parser():
 def main():
     parser = get_argument_parser()
     args = parser.parse_args()
-    vks = VkScraper(args.username, args.password)
+    vks = VkScraper(args.username, args.password, args.token)
     text = " ".join(args.urls)
     res = vks.scrape(text)
     res_json = json.dumps(res, ensure_ascii=False, indent=4, cls=DateTimeEncoder)

vk_url_scraper/scraper.py

@@ -9,7 +9,7 @@ import requests
 import vk_api  # used to get api_token after authentication
 import yt_dlp  # to download videos from url
 
-from .utils import mkdir_if_not_exists
+from .utils import captcha_handler, mkdir_if_not_exists
 
 
 class VkScraper:
@@ -38,10 +38,13 @@ class VkScraper:
     PHOTO_PATTERN = re.compile(r"(photo.{0,1}\d+_\d+)")
     VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+)")
 
-    def __init__(self, username: str, password: str) -> None:
+    def __init__(
+        self, username: str, password: str, token: str = None, captcha_handler=captcha_handler
+    ) -> None:
         """Initializes the scraper.
 
-        This function receives a username and password and performs authentication on vk.com to then call api endpoints
+        This function receives a username and password (or access token) and performs
+        authentication on vk.com to then call api endpoints. If token is passed, authentication will not be performed again.
 
         Parameters
         ----------
@@ -49,9 +52,14 @@ class VkScraper:
             Username on vk.com, can be a phone number or email
         password : str
             Matching password on vk.com
+        token : str
+            Access token received after authenticating, can be found in the vl_config.v2.json file
         """
-        self.session = vk_api.VkApi(username, password)
+        self.session = vk_api.VkApi(
-        self.session.auth(token_only=True)
+            username, password, token=token, captcha_handler=captcha_handler
+        )
+        if token is None or len(token) == 0:
+            self.session.auth(token_only=True)
 
     def scrape(self, url: str) -> List:
         """Scrapes a URL for multiple possibilities of inner links such as wall, video, photo, ...
@@ -312,8 +320,22 @@ class VkScraper:
                             downloaded.append(filename)
                 elif k == "video":
                     for i, url in enumerate(attachments):
-                        filename = os.path.join(destination, f"{r['id']}_{i}.mkv")
+                        filename = os.path.join(destination, f"{r['id']}_{i}.%(ext)s")
-                        ydl = yt_dlp.YoutubeDL({"outtmpl": filename, "quiet": True})
+                        ydl = yt_dlp.YoutubeDL(
-                        ydl.extract_info(url, download=True)
+                            {
+                                "outtmpl": filename,
+                                "quiet": True,
+                                "restrictfilenames": True,
+                                "forcefilename": True,
+                            }
+                        )
+                        info = ydl.extract_info(url, download=True)
+                        filename = ydl.prepare_filename(info)
+                        if "unknown_video" in filename:
+                            new_filename = filename.replace("unknown_video", "mkv")
+                            with open(filename, "rb") as vin, open(new_filename, "wb") as vout:
+                                vout.write(vin.read())
+                            os.remove(filename)
+                            filename = new_filename
                         downloaded.append(filename)
         return downloaded

vk_url_scraper/utils.py

@@ -14,3 +14,12 @@ class DateTimeEncoder(json.JSONEncoder):
 def mkdir_if_not_exists(folder):
     if not os.path.exists(folder):
         os.makedirs(folder)
+
+
+def captcha_handler(captcha):
+    print(
+        f"CAPTCHA DETECTED, please solve it and input the solution. {captcha.sid=} {captcha.get_url()=}",
+        flush=True,
+    )
+    key = input(f"Enter captcha code for {captcha.get_url()}:").strip()
+    return captcha.try_again(key)

vk_url_scraper/version.py

@@ -1,8 +1,8 @@
 _MAJOR = "0"
-_MINOR = "2"
+_MINOR = "3"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "1"
+_PATCH = "2"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""