# encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require "csv"
require 'nokogiri'
SQL =<<EOS
create table youtube (
id INTEGER PRIMARY KEY,
videoid text,
chan_id text,
publ_id text,
title text
);
EOS
words = ARGV[0]
#system("mkdir" ,"youtube__3")
db = SQLite3::Database.open("./youtube__3/#{words}.db")
db.execute(SQL)
keywords = URI.encode_www_form(search_query: words)
target = 'https://www.youtube.com/results?' << keywords
resp_0 = Net::HTTP.get_response(URI.parse(target))
#print resp_0.body
doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
if i == 33 then
json_str = element.to_s[58..-11]
#<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
end
}
doc = nil
script33 = JSON.parse(json_str)
videoid_list = []
videotitle_list = []
script33.each do |y,x|
if y == 'contents'
x.each do |yy,xx|
match = xx.to_s.match(/(\"videoId\"\=\>.{13})/)
if match != nil
temp = $&[11..-1]
videoid_list.push(temp)
while $'.match(/(\"videoId\"\=\>[^\[].{12})/) != nil do
videoid_list.push($&[11..-1])
end
end
match2 = xx.to_s.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/)
if match2 != nil
temp = $1
videotitle_list.push(temp)
while $'.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/) != nil do
videotitle_list.push($1)
end
end
end
end
end
videoid_list.uniq!
videotitle_list.pop # trash scan
videotitle_list.uniq!
id_list = []
title_list = []
videoid_list.each_with_index {|content,ind|
if videotitle_list[ind] != nil
# puts "-"*20
# puts "#{ind} https://www.youtube.com./watch?v=#{content[1..-2]}"
# puts videotitle_list[ind]
mmmm = videotitle_list[ind].match(/#{words}/i)
if mmmm != nil
id_list.push(content)
title_list.push(videotitle_list[ind])
end
else
# puts "-"*20
# puts "no title found"
# puts "#{ind} https://www.youtube.com./watch?v=#{content[1..-2]}"
end
}
videoid_list.clear
videotitle_list.clear
id_list = id_list.zip(title_list)
def work(target,words)
begin
resp_1 = Net::HTTP.get_response(URI.parse(target))
rescue => e
puts e
sleep 1
return nil
end
doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
if i == 40 then
json_str = element.to_s[58..-11]
end
}
script_tag = nil
title_tag = doc.css('title')
doc = nil
mmmm = title_tag[0].to_s.match(/#{words}/i)
if mmmm == nil
#next
return nil
end
title_tag = nil
begin
script40 = JSON.parse(json_str)
rescue => e
puts e
return nil
end
videoid_list2 = []
videotitle_list2 = []
# videodate_list2 = []
script40.each {|y,x|
if y.to_s == "contents"
# match_date = x.to_s.match(/\"dateText\"\=\>\{\"simpleText\"\=\>(\".{10}\")\},/)
# if match_date != nil
# puts match_date[1]
# end
match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
if match1 != nil
# videotitle_list2.push($1)
# puts""
# puts"-"*30
# puts $1
# puts $~
# puts"-"*30
tempstrings = $'
while tempstrings.match(/\"title\"=\>\{\"accessibility\"=\>\{\"accessibilityData\"=\>\{\"label\"=\>"(.*?)\"\}\},/) do
if $0 == nil
break
end
# puts" _"*20
# puts""
# puts $1
tempstrings = $'
videotitle_list2.push($1)
match_videoid = /\"commandMetadata\"=\>\{\"webCommandMetadata\"=\>\{\"url\"=\>\"\/watch\?v=(.{11}\"),/ =~ $'
if match_videoid != nil
# puts ("\"" + $1)
videoid_list2.push("\"" + $1)
end
end
else
# puts "-"*30
# puts y x
# puts "can't find the title"
next
end
## match2 = x.to_s.match(/\"videoId\"\=\>(.{13})/)
## if match2 != nil
## videoid_list2.push($1)
## while $'.match(/\"videoId\"\=\>(.{13})/) do
## videoid_list2.push($1)
## end
## end
end
}
videoid_list2.uniq!
videotitle_list2.uniq!
# videoid_list2.shift
# videotitle_list2.shift
# videodate_list2.shift
videoid_list3 = []
videotitle_list3 = []
videoid_list2.each_with_index {|content,ind|
if videotitle_list2[ind] == nil
# puts "-"*30
# puts "error"
# puts ind,content
# puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
# puts "-"*30
next
end
mmmm = videotitle_list2[ind].match(/#{words}/i)
if mmmm == nil
# puts "-"*30
# puts ind,content
# puts videotitle_list2[ind]
# puts "skip"
next
end
#puts "-"*30
#puts ind,content
videoid_list3.push(content[0..-1])
#puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
#puts videotitle_list2[ind]
videotitle_list3.push(videotitle_list2[ind])
}
videoid_list2.clear
videotitle_list2.clear
ziped_list = videoid_list3.zip(videotitle_list3)
# ziped_list.each_with_index do |list,ind|
# puts "#{ind}: #{list[0]}"
# puts "#{ind}: #{list[1]}"
# puts "#{ind}: #{list[2]}"
# end
# return videoid_list3
return ziped_list
end
def working2(target,words,id_pack)
begin
resp_1 = Net::HTTP.get_response(URI.parse(target))
rescue =>e
puts e.message
sleep 1
return nil
end
doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
title_tag = doc.css('title')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
if i == 40 then
json_str = element.to_s[58..-11]
#<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
end
}
doc = nil
begin
script40 = JSON.parse(json_str)
rescue => e
puts e
return nil
end
# id_list = []
puts title_tag.text
ids = id_pack.new("videoid","publisheddate","channelid","title")
script40.each {|y,x|
if y.to_s == "contents"
puts"-"*30
puts "URL: #{target}"
ids.vi = target.sub("https://www.youtube.com/watch?v=","")
match_date = x.to_s.match(/\"dateText\"=\>\{\"simpleText\"=\>\"(.{10})/)
if match_date != nil
# id_list.push(match_date[1])
ids.da = match_date[1]
ch_id = $'.match(/\"browseId\"=\>\"(.*?)\",/)
if ch_id != nil
#puts "channelId: #{$1}"
# id_list.push($1)
ids.ch = $1
end
end
match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
if match1 != nil
# videotitle_list2.push($1)
# puts""
# puts"-"*30
#puts "title: #{$1}"
# id_list.push($1)
ids.ti = $1
# puts $~
end
end
}
#struct data
return ids
end
mute = Mutex.new
counter = 0
db_counter = 0
pick = db.execute("SELECT id from youtube order by id DESC limit 1")
if pick[0] != nil # last db index
db_counter = pick[0][0]
db_counter += 1 # next db inde = last + 1
end
id_pack = Struct.new("Id_pack",:vi,:da,:ch,:ti)
while id_list.size > 0 && counter < 20000 do
counter += 1
threads = []
30.times do |k|
threads << Thread.new do
#mute.synchronize do
tempwork = id_list.shift
if tempwork == nil
next
end
target = "https://www.youtube.com/watch?v=" << tempwork[0][1..-2]
# youtube_data.videoid = tempwork[0][1..-2]
id_title_list = work(target,words)
if id_title_list != nil
id_list.concat(id_title_list)
end
#end
end
end
threads.each(&:join)
id_list.uniq!
threads2 = []
del_list = []
id_list.each_with_index {|data,num|
# if num == db_counter
v_id = data[0].gsub("\"","")
title = data[1].delete("\t\r\n")
flag1 = db.execute "SELECT id FROM youtube WHERE videoid=?",v_id
if flag1[0] != nil #db has same videoid
del_list << num
next
end
db.transaction do
sth = db.prepare("insert into youtube (id,videoid,title) values(?,?,?)")
sth.execute(db_counter,v_id,title)
end
threads2 << Thread.new do
flag2 = db.execute "SELECT id,chan_id FROM youtube WHERE videoid=?",v_id
id_number = 0
if flag2[0] != nil
id_number = flag2[0][0]
target2 = 'https://www.youtube.com/watch?v=' << v_id
ids = working2(target2,words,id_pack)
if ids != nil
mute.synchronize do
db.transaction do
#v_id = ids.vi
#date = id_list[0]
date = ids.da
#chid = id_list[1]
chid = ids.ch
#title = id_list[2]
title2 = ids.ti
sth = db.prepare("update youtube set chan_id=?, publ_id=? where id=?")
#sth = db.prepare("insert into youtube (videoid,chan_id,publ_id,title) values(?,?,?,?)")
sth.execute(chid,date,id_number)
#sth.execute(v_id,chid,date,title2)
end
end
end
end
end
db_counter += 1
# end
}
threads2.each(&:join)
del_list.each do |v|
puts id_list[v]
end
puts id_list.size
end
db.close
exit