# encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require 'nokogiri'
SQL =<<EOS
create table youtube (
id INTEGER PRIMARY KEY,
videoid text,
chan_id text,
publ_id text,
dura_id text,
title text
);
EOS
words = ARGV[0]
system("mkdir" ,"youtube__2")
db = SQLite3::Database.open("./youtube__2/#{words}2.db")
db.execute(SQL)
keywords = URI.encode_www_form(search_query: words)
target = 'https://www.youtube.com/results?' << keywords
resp_0 = Net::HTTP.get_response(URI.parse(target))
doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
#if i == 33 then
if i == 43
json_str = element.to_s[58..-11]
#<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
end
}
doc = nil
script33 = JSON.parse(json_str)
videoid_list = []
videotitle_list = []
script33.each do |k,v| #json hash
if k == 'contents'
v.each do |kk,vv|
match = vv.to_s.match(/(\"videoId\"\=\>.{13})/)
if match != nil
temp = $&[11..-1] #videoid without "
videoid_list.push(temp)
while $'.match(/(\"videoId\"\=\>[^\[].{12})/) != nil do
videoid_list.push($&[11..-1]) #videoid without "
end
end
match2 = vv.to_s.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/)
if match2 != nil
temp = $1
videotitle_list.push(temp)
while $'.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/) != nil do
videotitle_list.push($1)
end
end
end
end
end
videoid_list.uniq! # delete same videoids values
videotitle_list.pop # trash last regexp match strings
videotitle_list.uniq! # delete same titles
id_list = [] # array for function retuern value zip_list
title_list = [] # array for function retuern value zip_list
videoid_list.each_with_index {|content,ind|
if videotitle_list[ind] != nil # check 'keywords' is in video titles
# puts "-"*20
# puts "#{ind} https://www.youtube.com./watch?v=#{content[1..-2]}"
# puts videotitle_list[ind]
mmmm = videotitle_list[ind].match(/#{words}/i)
if mmmm != nil
id_list.push(content) # videoid
title_list.push(videotitle_list[ind]) # video title
end
else # no title strings case
# puts "-"*20
# puts "no title found"
# puts "#{ind} https://www.youtube.com./watch?v=#{content[1..-2]}"
end
}
videoid_list.clear
videotitle_list.clear
id_list = id_list.zip(title_list)
def work(target,words)
begin
resp_1 = Net::HTTP.get_response(URI.parse(target))
rescue => e
puts e
sleep 1
return nil
end
doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
#if i == 40 then
if i == 43
json_str = element.to_s[58..-11]
end
}
script_tag = nil
title_tag = doc.css('title')
doc = nil
mmmm = title_tag[0].to_s.match(/#{words}/i) # check 'keywords' in the videotitle
if mmmm == nil
#next
return nil
end
title_tag = nil
begin
script40 = JSON.parse(json_str)
rescue => e
puts e
return nil
end
videoid_list2 = []
videotitle_list2 = []
script40.each {|k,v|
if k.to_s == "contents"
match1 = v.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
if match1 != nil
# videotitle_list2.push($1)
# puts""
# puts"-"*30
# puts $1
# puts $~
# puts"-"*30
tempstrings = $' # after matched point strings
while tempstrings.match(/\"title\"=\>\{\"accessibility\"=\>\{\"accessibilityData\"=\>\{\"label\"=\>"(.*?)\"\}\},/) do
if $0 == nil
break
end
# puts" _"*20
# puts""
# puts $1
tempstrings = $'
videotitle_list2.push($1)
match_videoid = /\"commandMetadata\"=\>\{\"webCommandMetadata\"=\>\{\"url\"=\>\"\/watch\?v=(.{11}\"),/ =~ $'
if match_videoid != nil
# puts ("\"" + $1)
videoid_list2.push("\"" + $1)
end
end
else # no match title
# puts "-"*30
# puts y x
# puts "can't find the title"
next
end
end
}
videoid_list2.uniq!
videotitle_list2.uniq!
videoid_list3 = []
videotitle_list3 = []
videoid_list2.each_with_index {|content,ind|
if videotitle_list2[ind] == nil
# puts "-"*30
# puts "error"
# puts ind,content
# puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
# puts "-"*30
next
end
mmmm = videotitle_list2[ind].match(/#{words}/i)
if mmmm == nil
# puts "-"*30
# puts ind,content
# puts videotitle_list2[ind]
# puts "skip"
next
end
#puts "-"*30
#puts ind,content
videoid_list3.push(content[0..-1])
#puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
#puts videotitle_list2[ind]
videotitle_list3.push(videotitle_list2[ind])
}
videoid_list2.clear
videotitle_list2.clear
ziped_list = videoid_list3.zip(videotitle_list3)
# ziped_list.each_with_index do |list,ind|
# puts "#{ind}: #{list[0]}"
# puts "#{ind}: #{list[1]}"
# puts "#{ind}: #{list[2]}"
# end
return ziped_list
end
def working2(target,words,id_pack)
begin
resp_1 = Net::HTTP.get_response(URI.parse(target))
rescue =>e
puts e.message
sleep 1
return nil
end
doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
title_tag = doc.css('title')
script_tag = doc.css('script')
json_str = ""
script_tag.each_with_index {|element,i|
if i == 40 then
json_str = element.to_s[58..-11]
#<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
end
}
begin
script40 = JSON.parse(json_str)
rescue => e
puts e
return nil
end
ids = id_pack.new("videoid","publisheddate","channelid","title","duration")
meta_tag = doc.css('meta')
dura_id = ""
meta_tag.each do |element|
duration = element.to_s.match(/\<meta itemprop=\"duration\" content=\"(PT.*?S)\"\>/)
if duration != nil
ids.du = $1
dura_id = $1
end
end
meta_tag = nil
doc = nil
puts title_tag.text
script40.each {|y,x|
if y.to_s == "contents"
puts"-"*30
puts "URL: #{target}"
ids.vi = target.sub("https://www.youtube.com/watch?v=","")
match_date = x.to_s.match(/\"dateText\"=\>\{\"simpleText\"=\>\"(.{10})/)
if match_date != nil
ids.da = match_date[1]
ch_id = $'.match(/\"browseId\"=\>\"(.*?)\",/)
if ch_id != nil
#puts "channelId: #{$1}"
ids.ch = $1
end
end
match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
if match1 != nil
# puts""
# puts"-"*30
#puts "title: #{$1}"
ids.ti = $1
# puts $~
end
end
}
#struct data
return ids
end
mute = Mutex.new
counter = 0
db_counter = 0
pick = db.execute("SELECT id from youtube order by id DESC limit 1")
if pick[0] != nil # last db index
db_counter = pick[0][0]
db_counter += 1 # next db inde = last + 1
end
id_pack = Struct.new("Id_pack",:vi,:da,:ch,:ti,:du)
while id_list.size > 0 && counter < 20000 do
counter += 1
threads = []
30.times do |k|
threads << Thread.new do
#mute.synchronize do
tempwork = id_list.shift
if tempwork == nil
next
end
target = "https://www.youtube.com/watch?v=" << tempwork[0][1..-2]
id_title_list = work(target,words)
if id_title_list != nil
id_list.concat(id_title_list)
end
#end
end
end
threads.each(&:join)
id_list.uniq!
threads2 = []
del_list = []
id_list.each_with_index {|data,num|
v_id = data[0].gsub("\"","")
title = data[1].delete("\t\r\n")
flag1 = db.execute "SELECT id FROM youtube WHERE videoid=?",v_id
if flag1[0] != nil #db has same videoid
del_list << data
next
end
db.transaction do
sth = db.prepare("insert into youtube (id,videoid,title) values(?,?,?)")
sth.execute(db_counter,v_id,title)
end
threads2 << Thread.new do
flag2 = db.execute "SELECT id,chan_id FROM youtube WHERE videoid=?",v_id
id_number = 0
if flag2[0] != nil
id_number = flag2[0][0]
end
target2 = 'https://www.youtube.com/watch?v=' << v_id
ids = working2(target2,words,id_pack)
if ids != nil
mute.synchronize do
db.transaction do
#v_id = ids.vi
date = ids.da
chid = ids.ch
title2 = ids.ti
duration = ids.du
sth = db.prepare("update youtube set chan_id=?, publ_id=?,dura_id=? where id=?")
sth.execute(chid,date,duration,id_number)
end
end
end
end
db_counter += 1
}
threads2.each(&:join)
del_list.each do |v|
puts v
#id_list.delete(v)
end
puts id_list.size
end
exit