# encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require 'nokogiri'

SQL =<<EOS                                                                     
create table youtube (
    id INTEGER PRIMARY KEY,
    videoid text,
    chan_id text,
    publ_id text,
    dura_id text,
    title text
    );
EOS

words = ARGV[0]
system("mkdir" ,"youtube__2")
db = SQLite3::Database.open("./youtube__2/#{words}2.db")
db.execute(SQL)

keywords = URI.encode_www_form(search_query: words)
target = 'https://www.youtube.com/results?' << keywords
resp_0 = Net::HTTP.get_response(URI.parse(target))

doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
script_tag = doc.css('script')

json_str = ""
script_tag.each_with_index {|element,i|
  #if i == 33 then
  if i == 43
    json_str = element.to_s[58..-11]
    #<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
  end
}
doc = nil

script33 = JSON.parse(json_str)

videoid_list = []
videotitle_list = []

script33.each do |k,v| #json hash
  if k == 'contents'
    v.each do |kk,vv|
      match =  vv.to_s.match(/(\"videoId\"\=\>.{13})/)
      if match != nil
        temp = $&[11..-1] #videoid without "
        videoid_list.push(temp)
        while $'.match(/(\"videoId\"\=\>[^\[].{12})/) != nil do
          videoid_list.push($&[11..-1]) #videoid without "
        end
      end
      match2 =  vv.to_s.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/)
      if match2 != nil
        temp = $1
        videotitle_list.push(temp)
          while $'.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/) != nil do
            videotitle_list.push($1)
        end
      end
    end
  end
end

videoid_list.uniq! # delete same videoids values
videotitle_list.pop # trash last regexp match strings
videotitle_list.uniq! # delete same titles

id_list = [] # array for function retuern value zip_list 
title_list = [] # array for function retuern value zip_list

videoid_list.each_with_index {|content,ind|
  if videotitle_list[ind] != nil # check 'keywords' is in video titles
#    puts "-"*20
#    puts "#{ind}  https://www.youtube.com./watch?v=#{content[1..-2]}"
#    puts videotitle_list[ind]
    mmmm = videotitle_list[ind].match(/#{words}/i)
    if mmmm != nil
      id_list.push(content) # videoid
      title_list.push(videotitle_list[ind]) # video title
    end
  else # no title strings case
#    puts "-"*20
#    puts "no title found"
#    puts "#{ind}  https://www.youtube.com./watch?v=#{content[1..-2]}"
  end
}

videoid_list.clear
videotitle_list.clear

id_list = id_list.zip(title_list)

def work(target,words)
  begin
    resp_1 = Net::HTTP.get_response(URI.parse(target))
  rescue => e
    puts e
    sleep 1
    return nil
  end

  doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
  script_tag = doc.css('script')

  json_str = ""
  script_tag.each_with_index {|element,i|
    #if i == 40 then
    if i == 43
      json_str = element.to_s[58..-11]
    end
  }
  script_tag = nil

  title_tag = doc.css('title')
  doc = nil

  mmmm = title_tag[0].to_s.match(/#{words}/i) # check 'keywords' in the videotitle
  if mmmm == nil
    #next
    return nil
  end

  title_tag = nil
  begin
    script40 = JSON.parse(json_str)
  rescue => e
    puts e
    return nil
  end

  videoid_list2 = []
  videotitle_list2 = []

  script40.each {|k,v|
    if k.to_s == "contents"
      match1 = v.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
      if match1 != nil
#        videotitle_list2.push($1)
#        puts""
#        puts"-"*30 
#        puts $1
#        puts $~
#        puts"-"*30 

        tempstrings = $' # after matched point strings
        while tempstrings.match(/\"title\"=\>\{\"accessibility\"=\>\{\"accessibilityData\"=\>\{\"label\"=\>"(.*?)\"\}\},/) do
          if $0 == nil
            break
          end
#          puts" _"*20 
#          puts""
#          puts $1
          tempstrings = $'
          videotitle_list2.push($1)
          match_videoid = /\"commandMetadata\"=\>\{\"webCommandMetadata\"=\>\{\"url\"=\>\"\/watch\?v=(.{11}\"),/ =~ $'
          if match_videoid != nil
#            puts ("\"" + $1)
            videoid_list2.push("\"" + $1)
          end
        end
      else # no match title
#        puts "-"*30
#        puts y x
#        puts "can't find the title"
        next
      end
    end

  }

  videoid_list2.uniq! 
  videotitle_list2.uniq!

  videoid_list3 = []
  videotitle_list3 = []

  videoid_list2.each_with_index {|content,ind|
    if videotitle_list2[ind] == nil
#      puts "-"*30
#      puts "error"
#      puts ind,content 
#      puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
#      puts "-"*30
      next
    end
    mmmm = videotitle_list2[ind].match(/#{words}/i)
    if mmmm == nil
#      puts "-"*30
#      puts ind,content 
#      puts videotitle_list2[ind]
#      puts "skip"
      next
    end
    #puts "-"*30
    #puts ind,content 
    videoid_list3.push(content[0..-1])
    #puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
    #puts videotitle_list2[ind]
    videotitle_list3.push(videotitle_list2[ind])
  }

  videoid_list2.clear
  videotitle_list2.clear

  ziped_list = videoid_list3.zip(videotitle_list3)
#  ziped_list.each_with_index do |list,ind|
#    puts "#{ind}: #{list[0]}" 
#    puts "#{ind}: #{list[1]}" 
#    puts "#{ind}: #{list[2]}" 
#  end

  return ziped_list
end

def working2(target,words,id_pack)
  begin
    resp_1 = Net::HTTP.get_response(URI.parse(target))
  rescue =>e
    puts e.message
    sleep 1
    return nil
  end
  doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
  title_tag = doc.css('title')
  script_tag = doc.css('script')
  json_str = ""
  script_tag.each_with_index {|element,i|
    if i == 40 then
      json_str = element.to_s[58..-11]
      #<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
    end
  }

  begin
    script40 = JSON.parse(json_str)
  rescue => e
    puts e
    return nil
  end
  ids = id_pack.new("videoid","publisheddate","channelid","title","duration")

  meta_tag = doc.css('meta')
  dura_id = ""
  meta_tag.each do |element|
    duration = element.to_s.match(/\<meta itemprop=\"duration\" content=\"(PT.*?S)\"\>/)
    if duration != nil
      ids.du = $1
      dura_id = $1
    end
  end
  meta_tag = nil

  doc = nil
  puts title_tag.text

  script40.each {|y,x|
    if y.to_s == "contents"
      puts"-"*30 
      puts "URL: #{target}"
      ids.vi = target.sub("https://www.youtube.com/watch?v=","")
      match_date = x.to_s.match(/\"dateText\"=\>\{\"simpleText\"=\>\"(.{10})/)
      if match_date != nil
        ids.da = match_date[1]

        ch_id = $'.match(/\"browseId\"=\>\"(.*?)\",/) 
        if ch_id != nil
          #puts "channelId: #{$1}"
          ids.ch = $1
        end
      end
      match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
      if match1 != nil
#        puts""
#        puts"-"*30 
        #puts "title: #{$1}"
        ids.ti = $1
#        puts $~
      end
    end
  }

  #struct data
  return ids
end

mute = Mutex.new
counter = 0
db_counter = 0
pick = db.execute("SELECT id from youtube order by id DESC limit 1")
if pick[0] != nil # last db index
  db_counter = pick[0][0]
  db_counter += 1 # next db inde = last + 1 
end

id_pack = Struct.new("Id_pack",:vi,:da,:ch,:ti,:du)

while id_list.size > 0 && counter < 20000 do
  counter += 1
  threads = []

  30.times do |k|
      threads << Thread.new do
        #mute.synchronize do
            tempwork = id_list.shift
            if tempwork == nil
              next
            end
            target = "https://www.youtube.com/watch?v=" << tempwork[0][1..-2]
            id_title_list = work(target,words)
            if id_title_list != nil
              id_list.concat(id_title_list)
            end
        #end
      end
  end
  threads.each(&:join)
  id_list.uniq!

  threads2 = []
  del_list = []

  id_list.each_with_index {|data,num|
    v_id = data[0].gsub("\"","")
    title = data[1].delete("\t\r\n")
    flag1 = db.execute "SELECT id FROM youtube WHERE videoid=?",v_id
    if flag1[0] != nil #db has same videoid
      del_list << data
      next
    end

    db.transaction do
      sth = db.prepare("insert into youtube (id,videoid,title) values(?,?,?)")
      sth.execute(db_counter,v_id,title)
    end

    threads2 << Thread.new do

      flag2 = db.execute "SELECT id,chan_id FROM youtube WHERE videoid=?",v_id
      id_number = 0
      if flag2[0] != nil
        id_number = flag2[0][0]
      end

        target2 = 'https://www.youtube.com/watch?v=' << v_id
        ids = working2(target2,words,id_pack)
        if ids != nil
            mute.synchronize do

            db.transaction do
              #v_id = ids.vi
              date = ids.da
              chid = ids.ch
              title2 = ids.ti
              duration = ids.du
              sth = db.prepare("update youtube set chan_id=?, publ_id=?,dura_id=? where id=?")
              sth.execute(chid,date,duration,id_number)
            end

          end
        end
      end
    db_counter += 1
  }
  threads2.each(&:join)

  del_list.each do |v|
    puts v
    #id_list.delete(v)
  end

  puts id_list.size
end

exit
Edit
Pub: 12 Nov 2022 14:42 UTC
Edit: 24 Nov 2022 06:12 UTC
Views: 179