Step 2
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | # encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require 'nokogiri'
# ./youtube__/youtube.db
#
#SQL =<<EOS
#create table youtube (
# id INTEGER PRIMARY KEY AUTOINCREMENT,
# videoid text,
# chan_id text,
# publ_id text,
# title text
# );
#EOS
db = SQLite3::Database.open("./youtube__/youtube.db")
def working(target,id_pack)
puts target
begin
resp_0 = Net::HTTP.get_response(URI.parse(target))
rescue =>e
puts e.message
sleep 1
return nil
end
doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
#title_tag = doc.css('title')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
if i == 40 then
json_str = element.to_s[58..-11]
#<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
end
}
doc = nil
begin
script40 = JSON.parse(json_str)
rescue => e
puts e
return nil
end
ids = id_pack.new("videoid","publisheddate","channelid","title")
script40.each {|y,x|
if y.to_s == "contents"
puts"-"*30
puts "URL: #{target}"
ids.vi = target.sub("https://www.youtube.com/watch?v=","")
match_date = x.to_s.match(/\"dateText\"=\>\{\"simpleText\"=\>\"(.{10})/)
if match_date != nil
ids.da = match_date[1]
ch_id = $'.match(/\"browseId\"=\>\"(.*?)\",/)
if ch_id != nil
#puts "channelId: #{$1}"
ids.ch = $1
end
end
match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
if match1 != nil
# puts""
# puts"-"*30
#puts "title: #{$1}"
ids.ti = $1
# puts $~
end
end
}
#struct data
return ids
end
threads = []
iiii = 0
mute = Mutex.new
id_pack = Struct.new("Id_pack",:vi,:da,:ch,:ti)
lastid = db.execute("SELECT id FROM youtube order by id DESC limit 1")
db.execute("SELECT id,videoid FROM youtube").each do |videoid|
row = videoid[1]
num = videoid[0]
puts "#{row} #{num}"
str1 = row.gsub("\"","")
if iiii < 10 && num < lastid[0][0]
iiii += 1
threads << Thread.new do
target = 'https://www.youtube.com/watch?v=' << str1
ids = working(target,id_pack)
if ids != nil
mute.synchronize do
db.transaction do
v_id = str1
date = ids.da
chid = ids.ch
title = ids.ti
sth = db.prepare("update youtube set chan_id=?, publ_id=? where id=?")
sth.execute(chid,date,num)
end
end
end
end
else
iiii = 0
target = 'https://www.youtube.com/watch?v=' << str1
ids = working(target,id_pack)
if ids != nil
mute.synchronize do
db.transaction do
v_id = str1
date = ids.da
chid = ids.ch
title = ids.ti
sth = db.prepare("update youtube set chan_id=?, publ_id=? where id=?")
sth.execute(chid,date,num)
end
end
threads.each(&:join)
end
end
end
exit
|