bellingcat/reddit-post-scraping-tool
1
0
Fork 0
mirror of https://github.com/bellingcat/reddit-post-scraping-tool.git synced 2026-06-12 13:38:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

dev 1.9.0.0

Browse Source
This commit is contained in:
Richard Mwewa
2023-08-31 20:43:00 +02:00
committed by GitHub
parent b5d52e4bb5
commit 0023a974fc
21 changed files with 8849 additions and 2518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
RPST GUI/RPST/ApiHandler.vb
View File

@@ -12,27 +12,32 @@ Public Class ApiHandler
Public Property UpdatesEndpoint As String = "https://api.github.com/repos/bellingcat/reddit-post-scraping-tool/releases/latest"
''' <summary>
''' Scrape Reddit data.
''' Asyncrosnously scrape Reddit data.
''' </summary>
''' <returns>Json object containing scraped data.</returns>
Public Function ScrapeReddit(subreddit As String, listing As String, limit As Integer, timeframe As String) As JObject
Public Async Function ScrapeRedditAsync(subreddit As String, listing As String, limit As Integer, timeframe As String) As Task(Of JObject)
Dim ApiEndpoint As String = $"https://reddit.com/r/{subreddit}/{listing}.json?limit={limit}&t={timeframe}"
Return GetJObjectFromEndpoint(ApiEndpoint)
Return Await GetJObjectFromEndpointAsync(endpoint:=ApiEndpoint)
End Function
''' <summary>
''' Gets remote version information from the repository release page.
''' Asyncrosnously gets remote version information from the repository release page.
''' </summary>
''' <returns>Json object containing update data.</returns>
Public Function CheckUpdates() As JObject
Return GetJObjectFromEndpoint(UpdatesEndpoint)
Public Async Function CheckUpdatesAsync() As Task(Of JObject)
Return Await GetJObjectFromEndpointAsync(endpoint:=UpdatesEndpoint)
End Function
Private Function GetJObjectFromEndpoint(endpoint As String) As JObject
''' <summary>
''' Asyncronously retrieves a JObject from the specified endpoint.
''' </summary>
''' <param name="endpoint">The URL endpoint to retrieve data from.</param>
''' <returns>A JObject containing the retrieved data.</returns>
Private Async Function GetJObjectFromEndpointAsync(endpoint As String) As Task(Of JObject)
Try
Using httpClient As New HttpClient()
httpClient.DefaultRequestHeaders.Add("User-Agent", headers)
Dim response As HttpResponseMessage = httpClient.GetAsync(endpoint).Result
httpClient.DefaultRequestHeaders.Add("User-Agent", Headers)
Dim response As HttpResponseMessage = Await httpClient.GetAsync(endpoint)
If response.IsSuccessStatusCode Then
Dim json As String = response.Content.ReadAsStringAsync().Result
Dim data As JObject = JsonConvert.DeserializeObject(Of JObject)(json)