Step 1

# encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require 'nokogiri'

SQL =<<EOS                                                                     
create table youtube (
    id INTEGER PRIMARY KEY,
    videoid text,
    chan_id text,
    publ_id text,
    title text
    );
EOS

system("mkdir" ,"youtube__")
db = SQLite3::Database.open("./youtube__/youtube.db")
db.execute(SQL)

words = "ransomware"
keywords = URI.encode_www_form(search_query: words)
target = 'https://www.youtube.com/results?' << keywords
resp_0 = Net::HTTP.get_response(URI.parse(target))

doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
script_tag = doc.css('script')

json_str = ""
script_tag.each_with_index {|element,i|
  if i == 33 then
    json_str = element.to_s[58..-11]
   #<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
  end
}
doc = nil

script33 = JSON.parse(json_str)

videoid_list = []
videotitle_list = []
script33.each do |y,x|
  if y == 'contents'
    x.each do |yy,xx|
      match =  xx.to_s.match(/(\"videoId\"=\>.{13})/)
      if match != nil
        temp = $&[11..-1]
        videoid_list.push(temp)
        while $'.match(/(\"videoId\"=\>[^\[].{12})/) != nil do
          videoid_list.push($&[11..-1])
        end
      end
      match2 =  xx.to_s.match(/\"title\"=\>\{\"runs.+?\[\{\"text\"=\>(.*?)\}\],/)
      if match2 != nil
        temp = $1
        videotitle_list.push(temp)
          while $'.match(/\"title\"\=\>\{\"runs.+?\[\{\"text\"\=\>(.*?)\}\],/) != nil do
            videotitle_list.push($1)
        end
      end
    end
  end
end

videoid_list.uniq!
videotitle_list.pop # trash scan
videotitle_list.uniq!

id_list = []
title_list = []

videoid_list.each_with_index {|content,ind|
  if videotitle_list[ind] != nil
#    puts "-"*20
#    puts "#{ind}  https://www.youtube.com./watch?v=#{content[1..-2]}"
#    puts videotitle_list[ind]
    mmmm = videotitle_list[ind].match(/#{words}/i)
    if mmmm != nil
      id_list.push(content)
      title_list.push(videotitle_list[ind])
    end
  else
#    puts "-"*20
#    puts "no title found"
#    puts "#{ind}  https://www.youtube.com./watch?v=#{content[1..-2]}"

  end
}

videoid_list.clear
videotitle_list.clear

id_list = id_list.zip(title_list)

def work(target,words)
  begin
    resp_1 = Net::HTTP.get_response(URI.parse(target))
  rescue => e
    puts e
    sleep 1
    return nil
  end
  doc = Nokogiri::HTML.parse(resp_1.body, nil,'utf-8')
  script_tag = doc.css('script')

  json_str = ""
  script_tag.each_with_index {|element,i|
    if i == 40 then
      json_str = element.to_s[58..-11]
    end
  }
  script_tag = nil

  title_tag = doc.css('title')
  doc = nil
  mmmm = title_tag[0].to_s.match(/#{words}/i)
  if mmmm == nil
    return nil
  end

  title_tag = nil

  begin
    script40 = JSON.parse(json_str)
  rescue => e
    puts e
    return nil
  end

  videoid_list2 = []
  videotitle_list2 = []

  script40.each {|y,x|
    if y.to_s == "contents"
      match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
      if match1 != nil
#        puts""
#        puts"-"*30 
#        puts $1
#        puts $~
#        puts"-"*30 

        tempstrings = $'
        while tempstrings.match(/\"title\"=\>\{\"accessibility\"=\>\{\"accessibilityData\"=\>\{\"label\"=\>"(.*?)\"\}\},/) do
          if $0 == nil
            break
          end
#          puts" _"*20 
#          puts""
#          puts $1
          tempstrings = $'
          videotitle_list2.push($1)
          match_videoid = /\"commandMetadata\"=\>\{\"webCommandMetadata\"=\>\{\"url\"=\>\"\/watch\?v=(.{11}\"),/ =~ $'
          if match_videoid != nil
#            puts ("\"" + $1)
            videoid_list2.push("\"" + $1)
          end
        end
      else
#        puts "-"*30
#        puts y x
#        puts "can't find the title"
        next
      end
    end
  }

  videoid_list2.uniq!
  videotitle_list2.uniq!

  videoid_list3 = []
  videotitle_list3 = []

  videoid_list2.each_with_index {|content,ind|
    if videotitle_list2[ind] == nil
#      puts "-"*30
#      puts "error"
#      puts ind,content 
#      puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
#      puts "-"*30
      next
    end
    mmmm = videotitle_list2[ind].match(/#{words}/i)
    if mmmm == nil
#      puts "-"*30
#      puts ind,content 
#      puts videotitle_list2[ind]
#      puts "skip"
      next
    end
    #puts "-"*30
    #puts ind,content 
    videoid_list3.push(content[0..-1])
    #puts "https://www.youtube.com./watch?v=#{content[1..-2]}"
    #puts videotitle_list2[ind]
    videotitle_list3.push(videotitle_list2[ind])
  }
  videoid_list2.clear
  videotitle_list2.clear
  ziped_list = videoid_list3.zip(videotitle_list3)
  videoid_list3.clear

#  ziped_list.each_with_index do |list,ind|
#    puts "#{ind}: #{list[0]}" 
#    puts "#{ind}: #{list[1]}" 
#  end

  return ziped_list
end

#youtube = Struct.new("Youtube", :videoid, :title, :date)
#youtube_data = youtube.new("video_id","title","date")

#mute = Mutex.new
counter = 0
db_counter = 0
while id_list.size > 0 && counter < 20000 do
  counter += 1
  threads = []

  10.times do |k|
    if id_list.size > 0 
      threads << Thread.new do
        #mute.synchronize do

            tempwork = id_list.shift
            if tempwork == nil
              next
            end
            target = "https://www.youtube.com/watch?v=" << tempwork[0][1..-2]
            id_title_list = work(target,words)
            if id_title_list != nil
              id_list.concat(id_title_list)
            end

        #end
      end
    end
  end
  threads.each(&:join)
  id_list.uniq!

  db.transaction do
    id_list.each_with_index {|data,num|
      if num == db_counter
        v_id = data[0]
        title = data[1].delete("\t\r\n")
        sth = db.prepare("insert into youtube (id,videoid,title) values(?,?,?)")
        sth.execute(db_counter,v_id,title)
        db_counter += 1
      end
    }
  end
  puts id_list.size
end
Edit
Pub: 11 Nov 2022 09:42 UTC
Edit: 12 Nov 2022 11:38 UTC
Views: 141