Goroutine
SQLite3
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | //go get github.com/mattn/go-sqlite3
//go get github.com/PuerkitoBio/goquery
package main
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"regexp"
"sync"
"strconv"
"log"
"database/sql"
_ "github.com/mattn/go-sqlite3"
"os"
)
func createTable(db *sql.DB) {
createTableSQL := `CREATE TABLE manga(
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"title" TEXT
);` // SQL Statement for Create Table
log.Println("Create table...")
statement, err := db.Prepare(createTableSQL) // Prepare SQL Statement
if err != nil {
log.Fatal(err.Error())
}
statement.Exec() // Execute SQL Statements
log.Println("table created.")
}
func main() {
url := "https://13dl.me/list/popular/"
/////////////// DB
file, err := os.Create("a13dl.db") // Create SQLite file
if err != nil {
log.Fatal(err.Error())
}
file.Close()
log.Println("db created.")
file = nil
sqliteDatabase, _ := sql.Open("sqlite3", "./a13dl.db") // Open the SQLite File
defer sqliteDatabase.Close() // Defer Closing the database
createTable(sqliteDatabase) // Create Database Tables
///////////////
var count int = 0
var num int = 0
wg := &sync.WaitGroup{}
// var wg sync.WaitGroup
var mu sync.Mutex
job := make(chan string)
var flag0 bool = true
// for flag0 == true {
for i := 0; i < 1000; i++{
if flag0 == false {
break
}
wg.Add(1)
var ta_url string
ta_url = url + "home/?p=" + strconv.Itoa(num)
//ta_url = url
go func(){
defer wg.Done()
mj := <- job
// fmt.Println(mj)
var flag2 bool = true
doc, err := goquery.NewDocument(mj)
if err != nil {
log.Println("err",err)
flag2 = false
}
if flag2 != true {
mu.Lock()
flag0 = false
mu.Unlock()
}
atag := doc.Find("a")
re1 := regexp.MustCompile(`^Home`)
re2 := regexp.MustCompile(`^Popular Manga`)
re3 := regexp.MustCompile(`^Page\s\d*`)
re4 := regexp.MustCompile(`^Prev`)
re5 := regexp.MustCompile(`^Newest`)
re0 := regexp.MustCompile(`^Next`)
var title string
atag.Each(func(i int, s *goquery.Selection){
str1, _ := s.Attr("title")
// str2, _ := s.Attr("href")
title = str1
if re0.MatchString(title) == true {
// //unless
} else if ( title == "" || re1.MatchString(title) == true || re2.MatchString(title) == true || re3.MatchString(title) == true || re4.MatchString(title) == true || re4.MatchString(title) == true || re5.MatchString(title) == true ) {
} else {
mu.Lock()
count ++
fmt.Println(count, title)
/////////////// DB
statement, _ := sqliteDatabase.Prepare("INSERT INTO manga(id,title) VALUES (?,?) ;")
statement.Exec(count,title)
log.Println(count,"...inserted!!")
///////////////
mu.Unlock()
}
})
atag = nil
doc = nil
}()
job <- ta_url
num++
}
wg.Wait()
}
|