WIP

gorutine &sync.WaitGroup{}

//go get github.com/PuerkitoBio/goquery
package main

import (
    "fmt"
    "github.com/PuerkitoBio/goquery"
    "regexp"
    "sync"
    "strconv"
    "log"
)

func main() {
    url := "https://13dl.me/list/popular/"

    var count int = 0
    var num int = 0

    wg := &sync.WaitGroup{}

    var mu sync.Mutex
    job := make(chan string)
    var flag0 bool = true

//    for flag0 == true {
    for i := 0; i < 1000; i++{
        if flag0 == false {
            break
        }

        wg.Add(1)

        var ta_url string
        ta_url = url + "home/?p=" + strconv.Itoa(num)

        go func(){
            defer wg.Done()

            mj := <- job
            fmt.Println(mj)

            var flag2 bool = true

            doc, err := goquery.NewDocument(mj)
            if err != nil {
                log.Println("err",err)
                flag2 = false
            }
            if flag2 != true {
              mu.Lock()
                flag0 = false
              mu.Unlock()
            } 

            atag := doc.Find("a")
            re1 := regexp.MustCompile(`^Home`)
            re2 := regexp.MustCompile(`^Popular Manga`)
            re3 := regexp.MustCompile(`^Page\s\d*`)
            re4 := regexp.MustCompile(`^Prev`)
            re5 := regexp.MustCompile(`^Newest`)
            re0 := regexp.MustCompile(`^Next`)

            var title string

            atag.Each(func(i int, s *goquery.Selection){
                str1, _ := s.Attr("title")
//              str2, _ := s.Attr("href")
                title = str1
                if re0.MatchString(title) == true {
//
                } else if ( title == "" || re1.MatchString(title) == true || re2.MatchString(title) == true || re3.MatchString(title) == true || re4.MatchString(title) == true || re4.MatchString(title) == true || re5.MatchString(title) == true ) {
                } else {
                  mu.Lock()
                    count ++
                    fmt.Println(count, title)
                  mu.Unlock()
                }
            })
            atag = nil
            doc = nil
        }()

        job <- ta_url
        num++
    }
    wg.Wait()
}

SQLite3

//go get github.com/mattn/go-sqlite3
//go get github.com/PuerkitoBio/goquery

package main

import (
    "fmt"
    "github.com/PuerkitoBio/goquery"
    "regexp"
    "sync"
    "strconv"
    "log"
    "database/sql"
    _ "github.com/mattn/go-sqlite3"
    "os"
)

func createTable(db *sql.DB) {
    createTableSQL := `CREATE TABLE manga(
    "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,        
    "title" TEXT
    );` // SQL Statement for Create Table

    log.Println("Create table...")

    statement, err := db.Prepare(createTableSQL) // Prepare SQL Statement
    if err != nil {
        log.Fatal(err.Error())
    }
    statement.Exec() // Execute SQL Statements
    log.Println("table created.")
}

func main() {
    url := "https://13dl.me/list/popular/"
    /////////////// DB
    file, err := os.Create("a13dl.db") // Create SQLite file
    if err != nil {
        log.Fatal(err.Error())
    }
    file.Close()
    log.Println("db created.")
    file = nil

    sqliteDatabase, _ := sql.Open("sqlite3", "./a13dl.db") // Open the SQLite File
    defer sqliteDatabase.Close() // Defer Closing the database
    createTable(sqliteDatabase) // Create Database Tables
    /////////////// 

    var count int = 0
    var num int = 0

    wg := &sync.WaitGroup{}
//  var wg sync.WaitGroup
    var mu sync.Mutex
    job := make(chan string)
//    message := make(chan string)
    var flag0 bool = true

//    for flag0 == true {
    for i := 0; i < 1000; i++{
        if flag0 == false {
            break
        }

        wg.Add(1)

        var ta_url string
        ta_url = url + "home/?p=" + strconv.Itoa(num)
        //ta_url = url

        go func(){
            defer wg.Done()

            mj := <- job
//            fmt.Println(mj)

//          var ta_url string
            var flag2 bool = true

//          ta_url = url + "home/?p=" + strconv.Itoa(mj)
//          doc, err := goquery.NewDocument(ta_url)
            doc, err := goquery.NewDocument(mj)
            if err != nil {
                log.Println("err",err)
                flag2 = false
            }
            if flag2 != true {
              mu.Lock()
                flag0 = false
              mu.Unlock()
            } 

            atag := doc.Find("a")
            re1 := regexp.MustCompile(`^Home`)
            re2 := regexp.MustCompile(`^Popular Manga`)
            re3 := regexp.MustCompile(`^Page\s\d*`)
            re4 := regexp.MustCompile(`^Prev`)
            re5 := regexp.MustCompile(`^Newest`)
            re0 := regexp.MustCompile(`^Next`)

            var title string
//            var flag bool

            atag.Each(func(i int, s *goquery.Selection){
                str1, _ := s.Attr("title")
//              str2, _ := s.Attr("href")
                title = str1
                if re0.MatchString(title) == true {
//                    message <- str2
//                    flag = true
                } else if ( title == "" || re1.MatchString(title) == true || re2.MatchString(title) == true || re3.MatchString(title) == true || re4.MatchString(title) == true || re4.MatchString(title) == true || re5.MatchString(title) == true ) {
                } else {
                  mu.Lock()
                    count ++
                    fmt.Println(count, title)
                    /////////////// DB
                    statement, _ := sqliteDatabase.Prepare("INSERT INTO manga(id,title) VALUES (?,?) ;")
                    statement.Exec(count,title)
                    log.Println(count,"...inserted!!")
                    ///////////////
                  mu.Unlock()
                }
            })
            atag = nil
            doc = nil
//            if flag != true {
////                message <- ""
//              mu.Lock()
//                flag0 = false
//              mu.Unlock()
//            }
        }()

//        job <- num
        job <- ta_url
        num++
//        msg := <-message
//        if msg != "" {
//            //url = msg
//            fmt.Println(msg)
//        } else {
//        }
    }
    wg.Wait()
}
Edit
Pub: 15 May 2022 10:33 UTC
Edit: 23 May 2022 07:20 UTC
Views: 269