WIP
gorutine &sync.WaitGroup{}
SQLite3
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | //go get github.com/mattn/go-sqlite3
//go get github.com/PuerkitoBio/goquery
package main
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"regexp"
"sync"
"strconv"
"log"
"database/sql"
_ "github.com/mattn/go-sqlite3"
"os"
)
func createTable(db *sql.DB) {
createTableSQL := `CREATE TABLE manga(
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"title" TEXT
);` // SQL Statement for Create Table
log.Println("Create table...")
statement, err := db.Prepare(createTableSQL) // Prepare SQL Statement
if err != nil {
log.Fatal(err.Error())
}
statement.Exec() // Execute SQL Statements
log.Println("table created.")
}
func main() {
url := "https://13dl.me/list/popular/"
/////////////// DB
file, err := os.Create("a13dl.db") // Create SQLite file
if err != nil {
log.Fatal(err.Error())
}
file.Close()
log.Println("db created.")
file = nil
sqliteDatabase, _ := sql.Open("sqlite3", "./a13dl.db") // Open the SQLite File
defer sqliteDatabase.Close() // Defer Closing the database
createTable(sqliteDatabase) // Create Database Tables
///////////////
var count int = 0
var num int = 0
wg := &sync.WaitGroup{}
// var wg sync.WaitGroup
var mu sync.Mutex
job := make(chan string)
// message := make(chan string)
var flag0 bool = true
// for flag0 == true {
for i := 0; i < 1000; i++{
if flag0 == false {
break
}
wg.Add(1)
var ta_url string
ta_url = url + "home/?p=" + strconv.Itoa(num)
//ta_url = url
go func(){
defer wg.Done()
mj := <- job
// fmt.Println(mj)
// var ta_url string
var flag2 bool = true
// ta_url = url + "home/?p=" + strconv.Itoa(mj)
// doc, err := goquery.NewDocument(ta_url)
doc, err := goquery.NewDocument(mj)
if err != nil {
log.Println("err",err)
flag2 = false
}
if flag2 != true {
mu.Lock()
flag0 = false
mu.Unlock()
}
atag := doc.Find("a")
re1 := regexp.MustCompile(`^Home`)
re2 := regexp.MustCompile(`^Popular Manga`)
re3 := regexp.MustCompile(`^Page\s\d*`)
re4 := regexp.MustCompile(`^Prev`)
re5 := regexp.MustCompile(`^Newest`)
re0 := regexp.MustCompile(`^Next`)
var title string
// var flag bool
atag.Each(func(i int, s *goquery.Selection){
str1, _ := s.Attr("title")
// str2, _ := s.Attr("href")
title = str1
if re0.MatchString(title) == true {
// message <- str2
// flag = true
} else if ( title == "" || re1.MatchString(title) == true || re2.MatchString(title) == true || re3.MatchString(title) == true || re4.MatchString(title) == true || re4.MatchString(title) == true || re5.MatchString(title) == true ) {
} else {
mu.Lock()
count ++
fmt.Println(count, title)
/////////////// DB
statement, _ := sqliteDatabase.Prepare("INSERT INTO manga(id,title) VALUES (?,?) ;")
statement.Exec(count,title)
log.Println(count,"...inserted!!")
///////////////
mu.Unlock()
}
})
atag = nil
doc = nil
// if flag != true {
//// message <- ""
// mu.Lock()
// flag0 = false
// mu.Unlock()
// }
}()
// job <- num
job <- ta_url
num++
// msg := <-message
// if msg != "" {
// //url = msg
// fmt.Println(msg)
// } else {
// }
}
wg.Wait()
}
|