WIP

threadpool

#nim c -d:ssl --threads:on a13dlme_threadpool.nim

import httpclient,re,strutils,uri,nimquery
from xmltree import `$`
from htmlparser import parseHtml
from streams import newStringStream
import std/locks,threadpool

let ta_url = "https://13dl.me/list/popular/"
var lock: Lock
var counter {.guard: lock.} = 0

proc scrape(url: string) =

    let data = re"""(?<=title\=\").+?(?=\")"""
    let pa_tf = re"""^Page"""
    let p_tf = re"""^Prev"""
    let ne = re"""(?<=href\=\").+?(?=\")"""

    let client0  = newHttpClient()
    let response = client0.get(url)
    let html = parseHtml(newStringStream(response.body))
    client0.close()

    let a_tag = html.querySelectorAll("a")

    var next_url = ""

    for v in a_tag:
        let vv = findall($v,data)
        if len(vv) > 0:
            if (vv[0] == "" or vv[0] == "Home" or vv[0] == "Popular Manga" or vv[0] == "Newest"):
                discard
            else:
                if (match(vv[0],pa_tf)) or (match(vv[0],p_tf)):
                    discard
                else:
                    if vv[0] == "Next":
                        next_url = findall($v,ne)[0]
                        break
                    else:
                        withLock lock:
                            counter += 1
                            let num = counter 
                            echo num.intToStr & " | " & vv[0]
                        echo "------------------------------------------------------"

lock.initLock

for i in 0..1000:
    let url = ta_url & "home/?p=" & i.intToStr
    spawn scrape(url)

sync()
Edit
Pub: 16 May 2022 06:24 UTC
Edit: 18 May 2022 13:57 UTC
Views: 241