Fix notifications
This commit is contained in:
Omar Roth
parent
5e572c4610
commit
208f32661b
1 changed files with 145 additions and 35 deletions
180
src/helpers.cr
180
src/helpers.cr
|
|
@ -542,12 +542,12 @@ def login_req(login_form, f_req)
|
|||
return HTTP::Params.encode(data)
|
||||
end
|
||||
|
||||
def get_channel(id, client, db)
|
||||
def get_channel(id, client, db, refresh = true, pull_videos = true)
|
||||
if db.query_one?("SELECT EXISTS (SELECT true FROM channels WHERE id = $1)", id, as: Bool)
|
||||
channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
|
||||
|
||||
if Time.now - channel.updated > 1.minute
|
||||
channel = fetch_channel(id, client, db)
|
||||
if refresh && Time.now - channel.updated > 10.minutes
|
||||
channel = fetch_channel(id, client, db, pull_videos)
|
||||
channel_array = channel.to_a
|
||||
args = arg_array(channel_array)
|
||||
|
||||
|
|
@ -555,7 +555,7 @@ def get_channel(id, client, db)
|
|||
ON CONFLICT (id) DO UPDATE SET updated = $3", channel_array)
|
||||
end
|
||||
else
|
||||
channel = fetch_channel(id, client, db)
|
||||
channel = fetch_channel(id, client, db, pull_videos)
|
||||
args = arg_array(channel.to_a)
|
||||
db.exec("INSERT INTO channels VALUES (#{args})", channel.to_a)
|
||||
end
|
||||
|
|
@ -563,35 +563,112 @@ def get_channel(id, client, db)
|
|||
return channel
|
||||
end
|
||||
|
||||
def fetch_channel(ucid, client, db)
|
||||
def fetch_channel(ucid, client, db, pull_videos = true)
|
||||
rss = client.get("/feeds/videos.xml?channel_id=#{ucid}").body
|
||||
rss = XML.parse_html(rss)
|
||||
|
||||
db.exec("DELETE FROM channel_videos * WHERE ucid = $1", ucid)
|
||||
|
||||
rss.xpath_nodes("//feed/entry").each do |entry|
|
||||
video_id = entry.xpath_node("videoid").not_nil!.content
|
||||
title = entry.xpath_node("title").not_nil!.content
|
||||
published = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
|
||||
updated = Time.parse(entry.xpath_node("updated").not_nil!.content, "%FT%X%z")
|
||||
author = entry.xpath_node("author/name").not_nil!.content
|
||||
ucid = entry.xpath_node("channelid").not_nil!.content
|
||||
|
||||
video = ChannelVideo.new(video_id, title, published, updated, ucid, author)
|
||||
|
||||
video_array = video.to_a
|
||||
args = arg_array(video_array)
|
||||
|
||||
db.exec("UPDATE users SET notifications = notifications || $1 \
|
||||
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video_id, published, ucid)
|
||||
|
||||
# TODO: Update record on conflict
|
||||
db.exec("INSERT INTO channel_videos VALUES (#{args})\
|
||||
ON CONFLICT (id) DO NOTHING", video_array)
|
||||
author = rss.xpath_node(%q(//feed/title))
|
||||
if !author
|
||||
raise "Deleted or invalid channel"
|
||||
end
|
||||
author = author.content
|
||||
|
||||
author = rss.xpath_node("//feed/author/name").try &.content
|
||||
author ||= ""
|
||||
if !pull_videos
|
||||
rss.xpath_nodes("//feed/entry").each do |entry|
|
||||
video_id = entry.xpath_node("videoid").not_nil!.content
|
||||
title = entry.xpath_node("title").not_nil!.content
|
||||
published = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
|
||||
updated = Time.parse(entry.xpath_node("updated").not_nil!.content, "%FT%X%z")
|
||||
author = entry.xpath_node("author/name").not_nil!.content
|
||||
ucid = entry.xpath_node("channelid").not_nil!.content
|
||||
|
||||
video = ChannelVideo.new(video_id, title, published, Time.now, ucid, author)
|
||||
|
||||
db.exec("UPDATE users SET notifications = notifications || $1 \
|
||||
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, Time.now, ucid)
|
||||
|
||||
video_array = video.to_a
|
||||
args = arg_array(video_array)
|
||||
db.exec("INSERT INTO channel_videos VALUES (#{args})\
|
||||
ON CONFLICT (id) DO NOTHING", video_array)
|
||||
end
|
||||
else
|
||||
videos = [] of ChannelVideo
|
||||
page = 1
|
||||
|
||||
loop do
|
||||
url = produce_videos_url(ucid, page)
|
||||
response = client.get(url)
|
||||
|
||||
json = JSON.parse(response.body)
|
||||
content_html = json["content_html"].as_s
|
||||
if content_html.empty?
|
||||
# If we don't get anything, move on
|
||||
break
|
||||
end
|
||||
document = XML.parse_html(content_html)
|
||||
|
||||
document.xpath_nodes(%q(//li[contains(@class, "channels-content-item")])).each do |item|
|
||||
root = item.xpath_node(%q(div/div/div[@class="yt-lockup-content"]))
|
||||
if !root
|
||||
raise "could not find root"
|
||||
end
|
||||
|
||||
anchor = root.xpath_node(%q(h3[contains(@class,"yt-lockup-title")]/a))
|
||||
if !anchor
|
||||
raise "could not find anchor"
|
||||
end
|
||||
title = anchor.content.strip
|
||||
video_id = anchor["href"].lchop("/watch?v=")
|
||||
|
||||
published = root.xpath_node(%q(div[@class="yt-lockup-meta"]/ul/li[2]))
|
||||
if !published
|
||||
# This happens on Youtube red videos, here we just skip them
|
||||
next
|
||||
end
|
||||
published = published.content.split(" ")
|
||||
span = published[0].to_i
|
||||
case published[1]
|
||||
when .includes? "second"
|
||||
span = span.seconds
|
||||
when .includes? "minute"
|
||||
span = span.minutes
|
||||
when .includes? "hour"
|
||||
span = span.hours
|
||||
when .includes? "day"
|
||||
span = span.days
|
||||
when .includes? "week"
|
||||
span = span.weeks
|
||||
when .includes? "month"
|
||||
span = span.months
|
||||
when .includes? "year"
|
||||
span = span.years
|
||||
else
|
||||
raise "Unrecognized time: #{published[1]}"
|
||||
end
|
||||
|
||||
published = Time.now - span
|
||||
|
||||
videos << ChannelVideo.new(video_id, title, published, Time.now, ucid, author)
|
||||
end
|
||||
|
||||
if document.xpath_nodes(%q(//li[contains(@class, "channels-content-item")])).size < 30
|
||||
break
|
||||
end
|
||||
|
||||
page += 1
|
||||
end
|
||||
|
||||
db.exec("DELETE FROM channel_videos * WHERE ucid = $1", ucid)
|
||||
videos.each do |video|
|
||||
db.exec("UPDATE users SET notifications = notifications || $1 \
|
||||
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
|
||||
|
||||
video_array = video.to_a
|
||||
args = arg_array(video_array)
|
||||
db.exec("INSERT INTO channel_videos VALUES (#{args})", video_array)
|
||||
end
|
||||
end
|
||||
|
||||
channel = InvidiousChannel.new(ucid, author, Time.now)
|
||||
|
||||
|
|
@ -602,8 +679,8 @@ def get_user(sid, client, headers, db)
|
|||
if db.query_one?("SELECT EXISTS (SELECT true FROM users WHERE id = $1)", sid, as: Bool)
|
||||
user = db.query_one("SELECT * FROM users WHERE id = $1", sid, as: User)
|
||||
|
||||
if Time.now - user.updated > 1.minutes
|
||||
user = fetch_user(sid, client, headers)
|
||||
if Time.now - user.updated > 1.minute
|
||||
user = fetch_user(sid, client, headers, db)
|
||||
user_array = user.to_a
|
||||
args = arg_array(user_array)
|
||||
|
||||
|
|
@ -611,7 +688,7 @@ def get_user(sid, client, headers, db)
|
|||
ON CONFLICT (email) DO UPDATE SET id = $1, updated = $2, subscriptions = $4", user_array)
|
||||
end
|
||||
else
|
||||
user = fetch_user(sid, client, headers)
|
||||
user = fetch_user(sid, client, headers, db)
|
||||
user_array = user.to_a
|
||||
args = arg_array(user.to_a)
|
||||
|
||||
|
|
@ -622,7 +699,7 @@ def get_user(sid, client, headers, db)
|
|||
return user
|
||||
end
|
||||
|
||||
def fetch_user(sid, client, headers)
|
||||
def fetch_user(sid, client, headers, db)
|
||||
feed = client.get("/subscription_manager?disable_polymer=1", headers)
|
||||
feed = XML.parse_html(feed.body)
|
||||
|
||||
|
|
@ -630,9 +707,13 @@ def fetch_user(sid, client, headers)
|
|||
feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).each do |channel|
|
||||
if !["Popular on YouTube", "Music", "Sports", "Gaming"].includes? channel["title"]
|
||||
channel_id = channel["href"].lstrip("/channel/")
|
||||
get_channel(channel_id, client, PG_DB)
|
||||
|
||||
channels << channel_id
|
||||
begin
|
||||
channel = get_channel(channel_id, client, db, false, false)
|
||||
channels << channel.id
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -670,7 +751,7 @@ def decode_time(string)
|
|||
end
|
||||
|
||||
def produce_playlist_url(ucid, index)
|
||||
ucid = ucid.lstrip("UC")
|
||||
ucid = ucid.lchop("UC")
|
||||
ucid = "VLUU" + ucid
|
||||
|
||||
continuation = write_var_int(index)
|
||||
|
|
@ -702,6 +783,35 @@ def produce_playlist_url(ucid, index)
|
|||
return url
|
||||
end
|
||||
|
||||
def produce_videos_url(ucid, page)
|
||||
page = "#{page}"
|
||||
|
||||
meta = "\x12\x06videos \x00\x30\x01\x38\x01\x60\x01\x6a\x00\x7a"
|
||||
meta += page.size.to_u8.unsafe_chr
|
||||
meta += page
|
||||
meta += "\xb8\x01\x00"
|
||||
|
||||
meta = Base64.urlsafe_encode(meta)
|
||||
meta = URI.escape(meta)
|
||||
|
||||
continuation = "\x12"
|
||||
continuation += ucid.size.to_u8.unsafe_chr
|
||||
continuation += ucid
|
||||
continuation += "\x1a"
|
||||
continuation += meta.size.to_u8.unsafe_chr
|
||||
continuation += meta
|
||||
|
||||
continuation = continuation.size.to_u8.unsafe_chr + continuation
|
||||
continuation = "\xe2\xa9\x85\xb2\x02" + continuation
|
||||
|
||||
continuation = Base64.urlsafe_encode(continuation)
|
||||
continuation = URI.escape(continuation)
|
||||
|
||||
url = "/browse_ajax?continuation=#{continuation}"
|
||||
|
||||
return url
|
||||
end
|
||||
|
||||
def read_var_int(bytes)
|
||||
numRead = 0
|
||||
result = 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue