Step 2

# encoding: UTF-8
require 'net/http'
require 'uri'
require 'sqlite3'
require 'time'
require 'json'
require 'nokogiri'

# ./youtube__/youtube.db
#
#SQL =<<EOS
#create table youtube (
#    id INTEGER PRIMARY KEY AUTOINCREMENT,
#    videoid text,
#    chan_id text,
#    publ_id text,
#    title text
#    );
#EOS

db = SQLite3::Database.open("./youtube__/youtube.db")

def working(target,id_pack)
  puts target
  begin
    resp_0 = Net::HTTP.get_response(URI.parse(target))
  rescue =>e
    puts e.message
    sleep 1
    return nil
  end
  doc = Nokogiri::HTML.parse(resp_0.body, nil,'utf-8')
  #title_tag = doc.css('title')
  script_tag = doc.css('script')
  json_str = ""
  script_tag.each_with_index {|element,i|
    if i == 40 then
      json_str = element.to_s[58..-11]
      #<script nonce="fX_rKtuwcvo7T-wFeZz4CQ">var ytInitialData =
    end
  }
  doc = nil

  begin
    script40 = JSON.parse(json_str)
  rescue => e
    puts e
    return nil
  end
  ids = id_pack.new("videoid","publisheddate","channelid","title")

  script40.each {|y,x|
    if y.to_s == "contents"
      puts"-"*30 
      puts "URL: #{target}"
      ids.vi = target.sub("https://www.youtube.com/watch?v=","")
      match_date = x.to_s.match(/\"dateText\"=\>\{\"simpleText\"=\>\"(.{10})/)
      if match_date != nil
        ids.da = match_date[1]

        ch_id = $'.match(/\"browseId\"=\>\"(.*?)\",/) 
        if ch_id != nil
          #puts "channelId: #{$1}"
          ids.ch = $1
        end
      end
      match1 = x.to_s.match(/\{\"title\"=\>\{\"runs\"=\>\[\{\"text\"=\>\"(.*?)\"/)
      if match1 != nil
#        puts""
#        puts"-"*30 
        #puts "title: #{$1}"
        ids.ti = $1
#        puts $~
      end
    end
  }

  #struct data
  return ids
end

threads = []
iiii = 0
mute = Mutex.new

id_pack = Struct.new("Id_pack",:vi,:da,:ch,:ti) 

lastid = db.execute("SELECT id FROM youtube order by id DESC limit 1")

db.execute("SELECT id,videoid FROM youtube").each do |videoid|
  row = videoid[1]
  num = videoid[0]
  puts "#{row} #{num}"

  str1 = row.gsub("\"","")
  if iiii < 10 && num < lastid[0][0]
    iiii += 1
    threads << Thread.new do
      target = 'https://www.youtube.com/watch?v=' << str1
      ids = working(target,id_pack)
      if ids != nil
        mute.synchronize do

          db.transaction do
            v_id = str1
            date = ids.da
            chid = ids.ch
            title = ids.ti
            sth = db.prepare("update youtube set chan_id=?, publ_id=? where id=?")
            sth.execute(chid,date,num)
          end

        end
      end
    end
  else 
    iiii = 0
    target = 'https://www.youtube.com/watch?v=' << str1
    ids = working(target,id_pack)
    if ids != nil
      mute.synchronize do

        db.transaction do
          v_id = str1
          date = ids.da
          chid = ids.ch
          title = ids.ti
          sth = db.prepare("update youtube set chan_id=?, publ_id=? where id=?")
          sth.execute(chid,date,num)
        end

      end

      threads.each(&:join)
    end
  end
end

exit
Edit
Pub: 12 Nov 2022 11:09 UTC
Edit: 12 Nov 2022 11:38 UTC
Views: 106