Bump version to v0.3.11 for release

Bump yt-dlp from 2022.5.18 to 2022.7.18

.gitignore

@@ -1,6 +1,7 @@
 .env
 vk_config.v2.json
 output/
+tmp*/
 # build artifacts
 
 .eggs/

Pipfile

@@ -18,7 +18,7 @@ pytest-sphinx = "*"
 pytest-cov = "*"
 twine = ">=1.11.0"
 sphinx = ">=4.3.0,<5.1.0"
-furo = "==2022.6.4.1"
+furo = "==2022.6.21"
 myst-parser = ">=0.15.2,<0.19.0"
 sphinx-autobuild = "==2021.3.14"
 sphinx-autodoc-typehints = "*"
@@ -26,3 +26,6 @@ python-dotenv = "*"
 
 [requires]
 python_version = "3.9"
+
+[pipenv]
+allow_prereleases = true

README.md

@@ -1,7 +1,13 @@
 # vk-url-scraper
-Library to scrape data and especially media links (videos and photos) from vk.com URLs.
+Python library to scrape data, and especially media links like videos and photos, from vk.com URLs.
 
-You can use it via the [command line](#command-line-usage) or as a [python library](#python-library-usage).
+
+[![PyPI version](https://badge.fury.io/py/vk-url-scraper.svg)](https://badge.fury.io/py/vk-url-scraper)
+[![PyPI download month](https://img.shields.io/pypi/dm/vk-url-scraper.svg)](https://pypi.python.org/pypi/vk-url-scraper/)
+[![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest)
+
+
+You can use it via the [command line](#command-line-usage) or as a [python library](#python-library-usage), check the **[documentation](https://vk-url-scraper.readthedocs.io/en/latest/)**.
 
 ## Installation
 You can install the most recent release from [pypi](https://pypi.org/project/vk-url-scraper/) via `pip install vk-url-scraper`.
@@ -21,7 +27,8 @@ vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall1
 vk_url_scraper -u "username here" -p "password here" --urls https://vk.com/wall12345_6789 https://vk.com/photo-12345_6789 https://vk.com/video12345_6789
 
 # you can pass a token as well to avoid always authenticating 
-# and possibly getting captch prompts
+# and possibly getting captcha prompts
+# you can fetch the token from the bk_config.v2.json file generated under by searching for "access_token"
 vk_url_scraper -u "username" -p "password" -t "vktoken goes here" --urls https://vk.com/wall12345_6789
 
 # save the JSON output into a file
@@ -48,7 +55,7 @@ res = vks.scrape("https://vk.com/wall-1_398461")
 
 # scrape any "video" URL
 res = vks.scrape("https://vk.com/video-6596301_145810025")
-print(res[0]["text]) # eg: -> to get the text from code
+print(res[0]["text"]) # eg: -> to get the text from code
 ```
 
 ```python
@@ -93,6 +100,7 @@ To test the command line interface available in [__main__.py](__vk_url_scraper/_
 2. run `./scripts/release.sh` to create a tag and push, alternatively
    1. `git tag vx.y.z` to tag version
    2. `git push origin vx.y.z` -> this will trigger workflow and put project on [pypi](https://pypi.org/project/vk-url-scraper/)
+3. go to https://readthedocs.org/ to deploy new docs version (if webhook is not setup)
 
 ### Fixing a failed release
 
@@ -102,4 +110,4 @@ If for some reason the GitHub Actions release workflow failed with an error that
 git tag -l | xargs git tag -d && git fetch -t
 ```
 
-Then repeat the steps above.
+Then repeat the steps above.

requirements.txt

@@ -12,4 +12,4 @@ requests==2.28.0
 urllib3==1.26.9
 vk-api==11.9.8
 python-dotenv==0.20.0
-yt-dlp==2022.5.18
+yt-dlp==2022.7.18

setup.py

@@ -44,7 +44,10 @@ setup(
         "Programming Language :: Python :: 3",
     ],
     keywords=["scraper", "vk", "vkontakte", "vk-api", "media-downloader"],
-    url="https://github.com/bellingcat/vk-url-scraper",
+    project_urls={
+        "Code": "https://github.com/bellingcat/vk-url-scraper",
+        "Documentation": "https://vk-url-scraper.readthedocs.io/en/latest/",
+    },
     author="Bellingcat",
     author_email="tech@bellingcat.com",
     license="MIT",

tests/scraper_test.py

@@ -14,6 +14,17 @@ def test_login_fail():
         VkScraper("invalid", "combination")
 
 
+def test_login_custom_file():
+    session_filename = "test-session.json"
+    VkScraper(
+        os.environ["VK_USERNAME"],
+        os.environ["VK_PASSWORD"],
+        session_file=session_filename,
+    )
+    assert os.path.isfile(session_filename)
+    os.unlink(session_filename)
+
+
 def test_login_success():
     global vks
     vks = VkScraper(
@@ -69,7 +80,7 @@ def test_scrape_wall_url_with_photos():
         == "Хабаровск\nАллея героев\nПомолимся об укокоении воинов:\nАлександра, Игоря, Эдуарда, \nДионисия, Евгения, Александра, Артемия, Иннокентия, Андрея."
     )
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 6, 15, 10, 37, 24))
-    assert len(res[0]["payload"]) == 16
+    assert len(res[0]["payload"]) == 17
     assert len(res[0]["attachments"].keys()) == 1
     assert list(res[0]["attachments"].keys()) == ["photo"]
     assert len(res[0]["attachments"]["photo"]) == 9
@@ -81,7 +92,7 @@ def test_scrape_wall_url_with_photos_inner_videos_and_links_with_inner_photos():
     assert res[0]["id"] == "wall-17315087_74182"
     assert res[0]["text"] == ""
     assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 11, 1, 9))
-    assert len(res[0]["payload"]) == 15
+    assert len(res[0]["payload"]) == 17
     assert len(res[0]["attachments"].keys()) == 3
     for k in ["photo", "link", "video"]:
         assert k in list(res[0]["attachments"].keys())

vk_url_scraper/scraper.py

@@ -40,7 +40,12 @@ class VkScraper:
     VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+)")
 
     def __init__(
-        self, username: str, password: str, token: str = None, captcha_handler=captcha_handler
+        self,
+        username: str,
+        password: str,
+        token: str = None,
+        session_file="vk_config.v2.json",
+        captcha_handler=captcha_handler,
     ) -> None:
         """Initializes the scraper.
 
@@ -55,9 +60,17 @@ class VkScraper:
             Matching password on vk.com
         token : str
             Access token received after authenticating, can be found in the vl_config.v2.json file
+        session_file : str
+            File name where the VK session is saved so future logins are easier, this will not be created if token is passed
+        captcha_handler : func
+            Function that can receive a vk_api captcha instance and help the user solve it, default is a complete CLI handler
         """
         self.session = vk_api.VkApi(
-            username, password, token=token, captcha_handler=captcha_handler
+            username,
+            password,
+            token=token,
+            config_filename=session_file,
+            captcha_handler=captcha_handler,
         )
         if token is None or len(token) == 0:
             self.session.auth(token_only=True)

vk_url_scraper/utils.py

@@ -15,7 +15,7 @@ class DateTimeEncoder(json.JSONEncoder):
 
 def captcha_handler(captcha):
     key = input(
-        f"CAPTCHA DETECTED, please solve it and input the solution. {captcha.sid=} {captcha.get_url()=}:"
+        f"CAPTCHA DETECTED, please solve it and input the solution. url={captcha.get_url()}:"
     ).strip()
     return captcha.try_again(key)
 

vk_url_scraper/version.py

@@ -2,7 +2,7 @@ _MAJOR = "0"
 _MINOR = "3"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "4"
+_PATCH = "11"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""