PrivateCoffee/invidious
4
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Merge pull request #2685 from SamantazFox/database-improvments

Browse source 
Database improvements
This commit is contained in:
TheFrenchGhosty 2022-01-05 09:40:39 +00:00 committed by GitHub
commit 326a362eb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 1237 additions and 447 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
src/invidious.cr
View file

@ -20,12 +20,13 @@ require "kemal"
require "athena-negotiation"
require "openssl/hmac"
require "option_parser"
require "pg"
require "sqlite3"
require "xml"
require "yaml"
require "compress/zip"
require "protodec/utils"
require "./invidious/database/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
require "./invidious/*"
@ -112,19 +113,19 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
# Check table integrity
if CONFIG.check_tables
check_enum(PG_DB, "privacy", PlaylistPrivacy)
Invidious::Database.check_enum(PG_DB, "privacy", PlaylistPrivacy)
check_table(PG_DB, "channels", InvidiousChannel)
check_table(PG_DB, "channel_videos", ChannelVideo)
check_table(PG_DB, "playlists", InvidiousPlaylist)
check_table(PG_DB, "playlist_videos", PlaylistVideo)
check_table(PG_DB, "nonces", Nonce)
check_table(PG_DB, "session_ids", SessionId)
check_table(PG_DB, "users", User)
check_table(PG_DB, "videos", Video)
Invidious::Database.check_table(PG_DB, "channels", InvidiousChannel)
Invidious::Database.check_table(PG_DB, "channel_videos", ChannelVideo)
Invidious::Database.check_table(PG_DB, "playlists", InvidiousPlaylist)
Invidious::Database.check_table(PG_DB, "playlist_videos", PlaylistVideo)
Invidious::Database.check_table(PG_DB, "nonces", Nonce)
Invidious::Database.check_table(PG_DB, "session_ids", SessionId)
Invidious::Database.check_table(PG_DB, "users", User)
Invidious::Database.check_table(PG_DB, "videos", Video)
if CONFIG.cache_annotations
check_table(PG_DB, "annotations", Annotation)
Invidious::Database.check_table(PG_DB, "annotations", Annotation)
end
end
@ -246,8 +247,8 @@ before_all do |env|
# Invidious users only have SID
if !env.request.cookies.has_key? "SSID"
if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String)
user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
if email = Invidious::Database::SessionIDs.select_email(sid)
user = Invidious::Database::Users.select!(email: email)
csrf_token = generate_response(sid, {
":authorize_token",
":playlist_ajax",
@ -255,7 +256,7 @@ before_all do |env|
":subscription_ajax",
":token_ajax",
":watch_ajax",
}, HMAC_KEY, PG_DB, 1.week)
}, HMAC_KEY, 1.week)
preferences = user.preferences
env.set "preferences", preferences
@ -269,7 +270,7 @@ before_all do |env|
headers["Cookie"] = env.request.headers["Cookie"]
begin
user, sid = get_user(sid, headers, PG_DB, false)
user, sid = get_user(sid, headers, false)
csrf_token = generate_response(sid, {
":authorize_token",
":playlist_ajax",
@ -277,7 +278,7 @@ before_all do |env|
":subscription_ajax",
":token_ajax",
":watch_ajax",
}, HMAC_KEY, PG_DB, 1.week)
}, HMAC_KEY, 1.week)
preferences = user.preferences
env.set "preferences", preferences
@ -437,7 +438,7 @@ post "/watch_ajax" do |env|
end
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
next error_template(400, ex)
@ -457,10 +458,10 @@ post "/watch_ajax" do |env|
case action
when "action_mark_watched"
if !user.watched.includes? id
PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.email)
Invidious::Database::Users.mark_watched(user, id)
end
when "action_mark_unwatched"
PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email)
Invidious::Database::Users.mark_unwatched(user, id)
else
next error_json(400, "Unsupported action #{action}")
end
@ -574,7 +575,7 @@ post "/subscription_ajax" do |env|
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
next error_template(400, ex)
@ -598,16 +599,15 @@ post "/subscription_ajax" do |env|
# Sync subscriptions with YouTube
subscribe_ajax(channel_id, action, env.request.headers)
end
email = user.email
case action
when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id, PG_DB, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email)
get_channel(channel_id, false, false)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
when "action_remove_subscriptions"
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email)
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
next error_json(400, "Unsupported action #{action}")
end
@ -632,13 +632,14 @@ get "/subscription_manager" do |env|
end
user = user.as(User)
sid = sid.as(String)
if !user.password
# Refresh account
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
user, sid = get_user(sid, headers, PG_DB)
user, sid = get_user(sid, headers)
end
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
@ -648,20 +649,14 @@ get "/subscription_manager" do |env|
format = env.params.query["format"]?
format ||= "rss"
if user.subscriptions.empty?
values = "'{}'"
else
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel)
subscriptions = Invidious::Database::Channels.select(user.subscriptions)
subscriptions.sort_by!(&.author.downcase)
if action_takeout
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
playlists = Invidious::Database::Playlists.select_like_iv(user.email)
next JSON.build do |json|
json.object do
@ -677,7 +672,7 @@ get "/subscription_manager" do |env|
json.field "privacy", playlist.privacy.to_s
json.field "videos" do
json.array do
PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id|
Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
json.string video_id
end
end
@ -762,20 +757,20 @@ post "/data_control" do |env|
user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
Invidious::Database::Users.update_subscriptions(user)
end
if body["watch_history"]?
user.watched += body["watch_history"].as_a.map(&.as_s)
user.watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
Invidious::Database::Users.update_watch_history(user)
end
if body["preferences"]?
user.preferences = Preferences.from_json(body["preferences"].to_json)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
Invidious::Database::Users.update_preferences(user)
end
if playlists = body["playlists"]?.try &.as_a?
@ -788,8 +783,8 @@ post "/data_control" do |env|
next if !description
next if !privacy
playlist = create_playlist(PG_DB, title, privacy, user)
PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id)
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
@ -798,7 +793,7 @@ post "/data_control" do |env|
next if !video_id
begin
video = get_video(video_id, PG_DB)
video = get_video(video_id)
rescue ex
next
end
@ -815,11 +810,8 @@ post "/data_control" do |env|
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id)
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
end
end
end
@ -837,18 +829,18 @@ post "/data_control" do |env|
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
Invidious::Database::Users.update_subscriptions(user)
when "import_freetube"
user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
md["channel_id"]
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe_subscriptions"
body = JSON.parse(body)
user.subscriptions += body["subscriptions"].as_a.compact_map do |channel|
@ -865,9 +857,9 @@ post "/data_control" do |env|
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe"
Compress::Zip::Reader.open(IO::Memory.new(body)) do |file|
file.each_entry do |entry|
@ -879,14 +871,14 @@ post "/data_control" do |env|
user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v="))
user.watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
Invidious::Database::Users.update_watch_history(user)
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
Invidious::Database::Users.update_subscriptions(user)
db.close
tempfile.delete
@ -914,7 +906,7 @@ get "/change_password" do |env|
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY)
templated "change_password"
end
@ -940,7 +932,7 @@ post "/change_password" do |env|
end
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
next error_template(400, ex)
end
@ -970,7 +962,7 @@ post "/change_password" do |env|
end
new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10)
PG_DB.exec("UPDATE users SET password = $1 WHERE email = $2", new_password.to_s, user.email)
Invidious::Database::Users.update_password(user, new_password.to_s)
env.redirect referer
end
@ -988,7 +980,7 @@ get "/delete_account" do |env|
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY)
templated "delete_account"
end
@ -1009,14 +1001,14 @@ post "/delete_account" do |env|
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
next error_template(400, ex)
end
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email)
PG_DB.exec("DELETE FROM session_ids * WHERE email = $1", user.email)
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
@ -1040,7 +1032,7 @@ get "/clear_watch_history" do |env|
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
templated "clear_watch_history"
end
@ -1061,12 +1053,12 @@ post "/clear_watch_history" do |env|
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
next error_template(400, ex)
end
PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email)
Invidious::Database::Users.clear_watch_history(user)
env.redirect referer
end
@ -1083,7 +1075,7 @@ get "/authorize_token" do |env|
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY)
scopes = env.params.query["scopes"]?.try &.split(",")
scopes ||= [] of String
@ -1114,7 +1106,7 @@ post "/authorize_token" do |env|
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
next error_template(400, ex)
end
@ -1123,7 +1115,7 @@ post "/authorize_token" do |env|
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB)
access_token = generate_token(user.email, scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
@ -1158,8 +1150,7 @@ get "/token_manager" do |env|
end
user = user.as(User)
tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1 ORDER BY issued DESC", user.email, as: {session: String, issued: Time})
tokens = Invidious::Database::SessionIDs.select_all(user.email)
templated "token_manager"
end
@ -1188,7 +1179,7 @@ post "/token_ajax" do |env|
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
next error_template(400, ex)
@ -1208,7 +1199,7 @@ post "/token_ajax" do |env|
case action
when .starts_with? "action_revoke_token"
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email)
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
next error_json(400, "Unsupported action #{action}")
end

42
src/invidious/channels/channels.cr
View file

@ -114,7 +114,7 @@ class ChannelRedirect < Exception
end
end
def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10)
finished_channel = Channel(String | Nil).new
spawn do
@ -130,7 +130,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
active_threads += 1
spawn do
begin
get_channel(ucid, db, refresh, pull_all_videos)
get_channel(ucid, refresh, pull_all_videos)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
@ -151,28 +151,21 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
return final
end
def get_channel(id, db, refresh = true, pull_all_videos = true)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
def get_channel(id, refresh = true, pull_all_videos = true)
if channel = Invidious::Database::Channels.select(id)
if refresh && Time.utc - channel.updated > 10.minutes
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
db.exec("INSERT INTO channels VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array)
channel = fetch_channel(id, pull_all_videos: pull_all_videos)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
else
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array)
channel = fetch_channel(id, pull_all_videos: pull_all_videos)
Invidious::Database::Channels.insert(channel)
end
return channel
end
def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
def fetch_channel(ucid, pull_all_videos = true, locale = nil)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}")
@ -241,15 +234,11 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid)
Invidious::Database::Users.add_notification(video)
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
@ -284,13 +273,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
was_insert = Invidious::Database::ChannelVideos.insert(video)
Invidious::Database::Users.add_notification(video) if was_insert
end
end

24
src/invidious/database/annotations.cr Normal file
View file

@ -0,0 +1,24 @@
require "./base.cr"
module Invidious::Database::Annotations
extend self
def insert(id : String, annotations : String)
request = <<-SQL
INSERT INTO annotations
VALUES ($1, $2)
ON CONFLICT DO NOTHING
SQL
PG_DB.exec(request, id, annotations)
end
def select(id : String) : Annotation?
request = <<-SQL
SELECT * FROM annotations
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: Annotation)
end
end

110
src/invidious/database/base.cr Normal file
View file

@ -0,0 +1,110 @@
require "pg"
module Invidious::Database
extend self
def check_enum(db, enum_name, struct_type = nil)
return # TODO
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
LOGGER.info("check_enum: CREATE TYPE #{enum_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql"))
end
end
end
def check_table(db, table_name, struct_type = nil)
# Create table if it doesn't exist
begin
db.exec("SELECT * FROM #{table_name} LIMIT 0")
rescue ex
LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql"))
end
end
return if !struct_type
struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT")
return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
if !column_array[i]?
new_column = column_types.select(&.starts_with?(name))[0]
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
next
end
# Column doesn't exist
if !column_array.includes? name
new_column = column_types.select(&.starts_with?(name))[0]
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
end
# Column exists but in the wrong position, rotate
if struct_array.includes? column_array[i]
until name == column_array[i]
new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new")
# There's a column we didn't expect
if !new_column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
column_array = get_column_array(db, table_name)
next
end
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
column_array = get_column_array(db, table_name)
end
else
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
end
end
end
return if column_array.size <= struct_array.size
column_array.each do |column|
if !struct_array.includes? column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
def get_column_array(db, table_name)
column_array = [] of String
db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
rs.column_count.times do |i|
column = rs.as(PG::ResultSet).field(i)
column_array << column.name
end
end
return column_array
end
end

149
src/invidious/database/channels.cr Normal file
View file

@ -0,0 +1,149 @@
require "./base.cr"
#
# This module contains functions related to the "channels" table.
#
module Invidious::Database::Channels
extend self
# -------------------
# Insert / delete
# -------------------
def insert(channel : InvidiousChannel, update_on_conflict : Bool = false)
channel_array = channel.to_a
request = <<-SQL
INSERT INTO channels
VALUES (#{arg_array(channel_array)})
SQL
if update_on_conflict
request += <<-SQL
ON CONFLICT (id) DO UPDATE
SET author = $2, updated = $3
SQL
end
PG_DB.exec(request, args: channel_array)
end
# -------------------
# Update
# -------------------
def update_author(id : String, author : String)
request = <<-SQL
UPDATE channels
SET updated = $1, author = $2, deleted = false
WHERE id = $3
SQL
PG_DB.exec(request, Time.utc, author, id)
end
def update_mark_deleted(id : String)
request = <<-SQL
UPDATE channels
SET updated = $1, deleted = true
WHERE id = $2
SQL
PG_DB.exec(request, Time.utc, id)
end
# -------------------
# Select
# -------------------
def select(id : String) : InvidiousChannel?
request = <<-SQL
SELECT * FROM channels
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: InvidiousChannel)
end
def select(ids : Array(String)) : Array(InvidiousChannel)?
return [] of InvidiousChannel if ids.empty?
values = ids.map { |id| %(('#{id}')) }.join(",")
request = <<-SQL
SELECT * FROM channels
WHERE id = ANY(VALUES #{values})
SQL
return PG_DB.query_all(request, as: InvidiousChannel)
end
end
#
# This module contains functions related to the "channel_videos" table.
#
module Invidious::Database::ChannelVideos
extend self
# -------------------
# Insert
# -------------------
# This function returns the status of the query (i.e: success?)
def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
if with_premiere_timestamp
last_items = "premiere_timestamp = $9, views = $10"
else
last_items = "views = $10"
end
request = <<-SQL
INSERT INTO channel_videos
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (id) DO UPDATE
SET title = $2, published = $3, updated = $4, ucid = $5,
author = $6, length_seconds = $7, live_now = $8, #{last_items}
RETURNING (xmax=0) AS was_insert
SQL
return PG_DB.query_one(request, *video.to_tuple, as: Bool)
end
# -------------------
# Select
# -------------------
def select(ids : Array(String)) : Array(ChannelVideo)
return [] of ChannelVideo if ids.empty?
request = <<-SQL
SELECT * FROM channel_videos
WHERE id IN (#{arg_array(ids)})
ORDER BY published DESC
SQL
return PG_DB.query_all(request, args: ids, as: ChannelVideo)
end
def select_notfications(ucid : String, since : Time) : Array(ChannelVideo)
request = <<-SQL
SELECT * FROM channel_videos
WHERE ucid = $1 AND published > $2
ORDER BY published DESC
LIMIT 15
SQL
return PG_DB.query_all(request, ucid, since, as: ChannelVideo)
end
def select_popular_videos : Array(ChannelVideo)
request = <<-SQL
SELECT DISTINCT ON (ucid) *
FROM channel_videos
WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
PG_DB.query_all(request, as: ChannelVideo)
end
end

46
src/invidious/database/nonces.cr Normal file
View file

@ -0,0 +1,46 @@
require "./base.cr"
module Invidious::Database::Nonces
extend self
# -------------------
# Insert
# -------------------
def insert(nonce : String, expire : Time)
request = <<-SQL
INSERT INTO nonces
VALUES ($1, $2)
ON CONFLICT DO NOTHING
SQL
PG_DB.exec(request, nonce, expire)
end
# -------------------
# Update
# -------------------
def update_set_expired(nonce : String)
request = <<-SQL
UPDATE nonces
SET expire = $1
WHERE nonce = $2
SQL
PG_DB.exec(request, Time.utc(1990, 1, 1), nonce)
end
# -------------------
# Select
# -------------------
def select(nonce : String) : Tuple(String, Time)?
request = <<-SQL
SELECT * FROM nonces
WHERE nonce = $1
SQL
return PG_DB.query_one?(request, nonce, as: {String, Time})
end
end

257
src/invidious/database/playlists.cr Normal file
View file

@ -0,0 +1,257 @@
require "./base.cr"
#
# This module contains functions related to the "playlists" table.
#
module Invidious::Database::Playlists
extend self
# -------------------
# Insert / delete
# -------------------
def insert(playlist : InvidiousPlaylist)
playlist_array = playlist.to_a
request = <<-SQL
INSERT INTO playlists
VALUES (#{arg_array(playlist_array)})
SQL
PG_DB.exec(request, args: playlist_array)
end
# this function is a bit special: it will also remove all videos
# related to the given playlist ID in the "playlist_videos" table,
# in addition to deleting said ID from "playlists".
def delete(id : String)
request = <<-SQL
DELETE FROM playlist_videos * WHERE plid = $1;
DELETE FROM playlists * WHERE id = $1
SQL
PG_DB.exec(request, id)
end
# -------------------
# Update
# -------------------
def update(id : String, title : String, privacy, description, updated)
request = <<-SQL
UPDATE playlists
SET title = $1, privacy = $2, description = $3, updated = $4
WHERE id = $5
SQL
PG_DB.exec(request, title, privacy, description, updated, id)
end
def update_description(id : String, description)
request = <<-SQL
UPDATE playlists
SET description = $1
WHERE id = $2
SQL
PG_DB.exec(request, description, id)
end
def update_subscription_time(id : String)
request = <<-SQL
UPDATE playlists
SET subscribed = $1
WHERE id = $2
SQL
PG_DB.exec(request, Time.utc, id)
end
def update_video_added(id : String, index : String | Int64)
request = <<-SQL
UPDATE playlists
SET index = array_append(index, $1),
video_count = cardinality(index) + 1,
updated = $2
WHERE id = $3
SQL
PG_DB.exec(request, index, Time.utc, id)
end
def update_video_removed(id : String, index : String | Int64)
request = <<-SQL
UPDATE playlists
SET index = array_remove(index, $1),
video_count = cardinality(index) - 1,
updated = $2
WHERE id = $3
SQL
PG_DB.exec(request, index, Time.utc, id)
end
# -------------------
# Salect
# -------------------
def select(*, id : String, raise_on_fail : Bool = false) : InvidiousPlaylist?
request = <<-SQL
SELECT * FROM playlists
WHERE id = $1
SQL
if raise_on_fail
return PG_DB.query_one(request, id, as: InvidiousPlaylist)
else
return PG_DB.query_one?(request, id, as: InvidiousPlaylist)
end
end
def select_all(*, author : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1
SQL
return PG_DB.query_all(request, author, as: InvidiousPlaylist)
end
# -------------------
# Salect (filtered)
# -------------------
def select_like_iv(email : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
ORDER BY created
SQL
PG_DB.query_all(request, email, as: InvidiousPlaylist)
end
def select_not_like_iv(email : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1 AND id NOT LIKE 'IV%'
ORDER BY created
SQL
PG_DB.query_all(request, email, as: InvidiousPlaylist)
end
def select_user_created_playlists(email : String) : Array({String, String})
request = <<-SQL
SELECT id,title FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
SQL
PG_DB.query_all(request, email, as: {String, String})
end
# -------------------
# Misc checks
# -------------------
# Check if given playlist ID exists
def exists?(id : String) : Bool
request = <<-SQL
SELECT id FROM playlists
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: String).nil?
end
# Count how many playlist a user has created.
def count_owned_by(author : String) : Int64
request = <<-SQL
SELECT count(*) FROM playlists
WHERE author = $1
SQL
return PG_DB.query_one?(request, author, as: Int64) || 0_i64
end
end
#
# This module contains functions related to the "playlist_videos" table.
#
module Invidious::Database::PlaylistVideos
extend self
private alias VideoIndex = Int64 | Array(Int64)
# -------------------
# Insert / Delete
# -------------------
def insert(video : PlaylistVideo)
video_array = video.to_a
request = <<-SQL
INSERT INTO playlist_videos
VALUES (#{arg_array(video_array)})
SQL
PG_DB.exec(request, args: video_array)
end
def delete(index)
request = <<-SQL
DELETE FROM playlist_videos *
WHERE index = $1
SQL
PG_DB.exec(request, index)
end
# -------------------
# Salect
# -------------------
def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo)
request = <<-SQL
SELECT * FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT $3
OFFSET $4
SQL
return PG_DB.query_all(request, plid, index, limit, offset, as: PlaylistVideo)
end
def select_index(plid : String, vid : String) : Int64?
request = <<-SQL
SELECT index FROM playlist_videos
WHERE plid = $1 AND id = $2
LIMIT 1
SQL
return PG_DB.query_one?(request, plid, vid, as: Int64)
end
def select_one_id(plid : String, index : VideoIndex) : String?
request = <<-SQL
SELECT id FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT 1
SQL
return PG_DB.query_one?(request, plid, index, as: String)
end
def select_ids(plid : String, index : VideoIndex, limit = 500) : Array(String)
request = <<-SQL
SELECT id FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT $3
SQL
return PG_DB.query_all(request, plid, index, limit, as: String)
end
end

74
src/invidious/database/sessions.cr Normal file
View file

@ -0,0 +1,74 @@
require "./base.cr"
module Invidious::Database::SessionIDs
extend self
# -------------------
# Insert
# -------------------
def insert(sid : String, email : String, handle_conflicts : Bool = false)
request = <<-SQL
INSERT INTO session_ids
VALUES ($1, $2, $3)
SQL
request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts
PG_DB.exec(request, sid, email, Time.utc)
end
# -------------------
# Delete
# -------------------
def delete(*, sid : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE id = $1
SQL
PG_DB.exec(request, sid)
end
def delete(*, email : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE email = $1
SQL
PG_DB.exec(request, email)
end
def delete(*, sid : String, email : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE id = $1 AND email = $2
SQL
PG_DB.exec(request, sid, email)
end
# -------------------
# Select
# -------------------
def select_email(sid : String) : String?
request = <<-SQL
SELECT email FROM session_ids
WHERE id = $1
SQL
PG_DB.query_one?(request, sid, as: String)
end
def select_all(email : String) : Array({session: String, issued: Time})
request = <<-SQL
SELECT id, issued FROM session_ids
WHERE email = $1
ORDER BY issued DESC
SQL
PG_DB.query_all(request, email, as: {session: String, issued: Time})
end
end

49
src/invidious/database/statistics.cr Normal file
View file

@ -0,0 +1,49 @@
require "./base.cr"
module Invidious::Database::Statistics
extend self
# -------------------
# User stats
# -------------------
def count_users_total : Int64
request = <<-SQL
SELECT count(*) FROM users
SQL
PG_DB.query_one(request, as: Int64)
end
def count_users_active_1m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '6 months'
SQL
PG_DB.query_one(request, as: Int64)
end
def count_users_active_6m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '1 month'
SQL
PG_DB.query_one(request, as: Int64)
end
# -------------------
# Channel stats
# -------------------
def channel_last_update : Time?
request = <<-SQL
SELECT updated FROM channels
ORDER BY updated DESC
LIMIT 1
SQL
PG_DB.query_one?(request, as: Time)
end
end

218
src/invidious/database/users.cr Normal file
View file

@ -0,0 +1,218 @@
require "./base.cr"
module Invidious::Database::Users
extend self
# -------------------
# Insert / delete
# -------------------
def insert(user : User, update_on_conflict : Bool = false)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
request = <<-SQL
INSERT INTO users
VALUES (#{arg_array(user_array)})
SQL
if update_on_conflict
request += <<-SQL
ON CONFLICT (email) DO UPDATE
SET updated = $1, subscriptions = $3
SQL
end
PG_DB.exec(request, args: user_array)
end
def delete(user : User)
request = <<-SQL
DELETE FROM users *
WHERE email = $1
SQL
PG_DB.exec(request, user.email)
end
# -------------------
# Update (history)
# -------------------
def update_watch_history(user : User)
request = <<-SQL
UPDATE users
SET watched = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.watched, user.email)
end
def mark_watched(user : User, vid : String)
request = <<-SQL
UPDATE users
SET watched = array_append(watched, $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def mark_unwatched(user : User, vid : String)
request = <<-SQL
UPDATE users
SET watched = array_remove(watched, $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def clear_watch_history(user : User)
request = <<-SQL
UPDATE users
SET watched = '{}'
WHERE email = $1
SQL
PG_DB.exec(request, user.email)
end
# -------------------
# Update (channels)
# -------------------
def update_subscriptions(user : User)
request = <<-SQL
UPDATE users
SET feed_needs_update = true, subscriptions = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.subscriptions, user.email)
end
def subscribe_channel(user : User, ucid : String)
request = <<-SQL
UPDATE users
SET feed_needs_update = true,
subscriptions = array_append(subscriptions,$1)
WHERE email = $2
SQL
PG_DB.exec(request, ucid, user.email)
end
def unsubscribe_channel(user : User, ucid : String)
request = <<-SQL
UPDATE users
SET feed_needs_update = true,
subscriptions = array_remove(subscriptions, $1)
WHERE email = $2
SQL
PG_DB.exec(request, ucid, user.email)
end
# -------------------
# Update (notifs)
# -------------------
def add_notification(video : ChannelVideo)
request = <<-SQL
UPDATE users
SET notifications = array_append(notifications, $1),
feed_needs_update = true
WHERE $2 = ANY(subscriptions)
SQL
PG_DB.exec(request, video.id, video.ucid)
end
def remove_notification(user : User, vid : String)
request = <<-SQL
UPDATE users
SET notifications = array_remove(notifications, $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def clear_notifications(user : User)
request = <<-SQL
UPDATE users
SET notifications = $1, updated = $2
WHERE email = $3
SQL
PG_DB.exec(request, [] of String, Time.utc, user)
end
# -------------------
# Update (misc)
# -------------------
def update_preferences(user : User)
request = <<-SQL
UPDATE users
SET preferences = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.preferences.to_json, user.email)
end
def update_password(user : User, pass : String)
request = <<-SQL
UPDATE users
SET password = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.email, pass)
end
# -------------------
# Select
# -------------------
def select(*, email : String) : User?
request = <<-SQL
SELECT * FROM users
WHERE email = $1
SQL
return PG_DB.query_one?(request, email, as: User)
end
# Same as select, but can raise an exception
def select!(*, email : String) : User
request = <<-SQL
SELECT * FROM users
WHERE email = $1
SQL
return PG_DB.query_one(request, email, as: User)
end
def select(*, token : String) : User?
request = <<-SQL
SELECT * FROM users
WHERE token = $1
SQL
return PG_DB.query_one?(request, token, as: User)
end
def select_notifications(user : User) : Array(String)
request = <<-SQL
SELECT notifications
FROM users
WHERE email = $1
SQL
return PG_DB.query_one(request, user.email, as: Array(String))
end
end

43
src/invidious/database/videos.cr Normal file
View file

@ -0,0 +1,43 @@
require "./base.cr"
module Invidious::Database::Videos
extend self
def insert(video : Video)
request = <<-SQL
INSERT INTO videos
VALUES ($1, $2, $3)
ON CONFLICT (id) DO NOTHING
SQL
PG_DB.exec(request, video.id, video.info.to_json, video.updated)
end
def delete(id)
request = <<-SQL
DELETE FROM videos *
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
def update(video : Video)
request = <<-SQL
UPDATE videos
SET (id, info, updated) = ($1, $2, $3)
WHERE id = $1
SQL
PG_DB.exec(request, video.id, video.info.to_json, video.updated)
end
def select(id : String) : Video?
request = <<-SQL
SELECT * FROM videos
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: Video)
end
end

10
src/invidious/helpers/handlers.cr
View file

@ -97,18 +97,18 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s)
scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil)
scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil)
if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String)
user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email)
end
elsif sid = env.request.cookies["SID"]?.try &.value
if sid.starts_with? "v1:"
raise "Cannot use token as SID"
end
if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String)
user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
if email = Invidious::Database::SessionIDs.select_email(sid)
user = Invidious::Database::Users.select!(email: email)
end
scopes = [":*"]

118
src/invidious/helpers/helpers.cr
View file

@ -60,112 +60,7 @@ def html_to_content(description_html : String)
return description
end
def check_enum(db, enum_name, struct_type = nil)
return # TODO
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
LOGGER.info("check_enum: CREATE TYPE #{enum_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql"))
end
end
end
def check_table(db, table_name, struct_type = nil)
# Create table if it doesn't exist
begin
db.exec("SELECT * FROM #{table_name} LIMIT 0")
rescue ex
LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql"))
end
end
return if !struct_type
struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT")
return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
if !column_array[i]?
new_column = column_types.select(&.starts_with?(name))[0]
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
next
end
# Column doesn't exist
if !column_array.includes? name
new_column = column_types.select(&.starts_with?(name))[0]
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
end
# Column exists but in the wrong position, rotate
if struct_array.includes? column_array[i]
until name == column_array[i]
new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new")
# There's a column we didn't expect
if !new_column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
column_array = get_column_array(db, table_name)
next
end
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
column_array = get_column_array(db, table_name)
end
else
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
end
end
end
return if column_array.size <= struct_array.size
column_array.each do |column|
if !struct_array.includes? column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
def get_column_array(db, table_name)
column_array = [] of String
db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
rs.column_count.times do |i|
column = rs.as(PG::ResultSet).field(i)
column_array << column.name
end
end
return column_array
end
def cache_annotation(db, id, annotations)
def cache_annotation(id, annotations)
if !CONFIG.cache_annotations
return
end
@ -183,7 +78,7 @@ def cache_annotation(db, id, annotations)
end
end
db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations
Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations
end
def create_notification_stream(env, topics, connection_channel)
@ -204,7 +99,7 @@ def create_notification_stream(env, topics, connection_channel)
published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id, PG_DB)
video = get_video(video_id)
video.published = published
response = JSON.parse(video.to_json(locale, nil))
@ -235,11 +130,12 @@ def create_notification_stream(env, topics, connection_channel)
spawn do
begin
if since
since_unix = Time.unix(since.not_nil!)
topics.try &.each do |topic|
case topic
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
@ -280,7 +176,7 @@ def create_notification_stream(env, topics, connection_channel)
next
end
video = get_video(video_id, PG_DB)
video = get_video(video_id)
video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale, nil))

14
src/invidious/helpers/tokens.cr
View file

@ -1,8 +1,8 @@
require "crypto/subtle"
def generate_token(email, scopes, expire, key, db)
def generate_token(email, scopes, expire, key)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc)
Invidious::Database::SessionIDs.insert(session, email)
token = {
"session" => session,
@ -19,7 +19,7 @@ def generate_token(email, scopes, expire, key, db)
return token.to_json
end
def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false)
def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
expire = Time.utc + expire
token = {
@ -30,7 +30,7 @@ def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = fa
if use_nonce
nonce = Random::Secure.hex(16)
db.exec("INSERT INTO nonces VALUES ($1, $2) ON CONFLICT DO NOTHING", nonce, expire)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end
@ -63,7 +63,7 @@ def sign_token(key, hash)
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip
end
def validate_request(token, session, request, key, db, locale = nil)
def validate_request(token, session, request, key, locale = nil)
case token
when String
token = JSON.parse(URI.decode_www_form(token)).as_h
@ -92,9 +92,9 @@ def validate_request(token, session, request, key, db, locale = nil)
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
raise InfoException.new("Erroneous token")
end

2
src/invidious/helpers/utils.cr
View file

@ -1,5 +1,3 @@
require "db"
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
def ci_lower_bound(pos, n)
if n == 0

9
src/invidious/jobs/pull_popular_videos_job.cr
View file

@ -1,11 +1,4 @@
class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
QUERY = <<-SQL
SELECT DISTINCT ON (ucid) *
FROM channel_videos
WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
private getter db : DB::Database
@ -14,7 +7,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
def begin
loop do
videos = db.query_all(QUERY, as: ChannelVideo)
videos = Invidious::Database::ChannelVideos.select_popular_videos
.sort_by!(&.published)
.reverse!

4
src/invidious/jobs/refresh_channels_job.cr
View file

@ -35,11 +35,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
lim_fibers = max_fibers
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
Invidious::Database::Channels.update_author(id, channel.author)
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
Invidious::Database::Channels.update_mark_deleted(id)
else
lim_fibers = 1
LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")

2
src/invidious/jobs/refresh_feeds_job.cr
View file

@ -25,7 +25,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
spawn do
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
column_array = Invidious::Database.get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")

10
src/invidious/jobs/statistics_refresh_job.cr
View file

@ -47,12 +47,14 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
private def refresh_stats
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64)
users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
users["total"] = Invidious::Database::Statistics.count_users_total
users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
end
end

34
src/invidious/playlists.cr
View file

@ -125,7 +125,7 @@ struct Playlist
json.field "videos" do
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id)
videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
@ -200,12 +200,12 @@ struct InvidiousPlaylist
json.field "videos" do
json.array do
if !offset || offset == 0
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64)
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id)
videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id)
videos.each_with_index do |video, index|
video.to_json(json, offset + index)
end
@ -225,7 +225,8 @@ struct InvidiousPlaylist
end
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
@ -246,7 +247,7 @@ struct InvidiousPlaylist
end
end
def create_playlist(db, title, privacy, user)
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
@ -261,15 +262,12 @@ def create_playlist(db, title, privacy, user)
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(db, user, playlist)
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
@ -282,10 +280,7 @@ def subscribe_playlist(db, user, playlist)
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
Invidious::Database::Playlists.insert(playlist)
return playlist
end
@ -327,9 +322,9 @@ def produce_playlist_continuation(id, index)
return continuation
end
def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
def get_playlist(plid, locale, refresh = true, force_refresh = false)
if plid.starts_with? "IV"
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise InfoException.new("Playlist does not exist.")
@ -409,7 +404,7 @@ def fetch_playlist(plid, locale)
})
end
def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil)
def get_playlist_videos(playlist, offset, locale = nil, video_id = nil)
# Show empy playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
@ -417,8 +412,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil)
end
if playlist.is_a? InvidiousPlaylist
db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3",
playlist.id, playlist.index, offset, as: PlaylistVideo)
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if video_id
initial_data = YoutubeAPI.next({

2
src/invidious/routes/api/manifest.cr
View file

@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, PG_DB, region: region)
video = get_video(id, region: region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex

69
src/invidious/routes/api/v1/authenticated.cr
View file

@ -22,12 +22,11 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User)
begin
preferences = Preferences.from_json(env.request.body || "{}")
user.preferences = Preferences.from_json(env.request.body || "{}")
rescue
preferences = user.preferences
end
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email)
Invidious::Database::Users.update_preferences(user)
env.response.status_code = 204
end
@ -45,7 +44,7 @@ module Invidious::Routes::API::V1::Authenticated
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(PG_DB, user, max_results, page)
videos, notifications = get_subscription_feed(user, max_results, page)
JSON.build do |json|
json.object do
@ -72,13 +71,7 @@ module Invidious::Routes::API::V1::Authenticated
env.response.content_type = "application/json"
user = env.get("user").as(User)
if user.subscriptions.empty?
values = "'{}'"
else
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel)
subscriptions = Invidious::Database::Channels.select(user.subscriptions)
JSON.build do |json|
json.array do
@ -99,8 +92,8 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid
get_channel(ucid, PG_DB, false, false)
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
get_channel(ucid, false, false)
Invidious::Database::Users.subscribe_channel(user, ucid)
end
# For Google accounts, access tokens don't have enough information to
@ -116,7 +109,7 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"]
PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email)
Invidious::Database::Users.unsubscribe_channel(user, ucid)
env.response.status_code = 204
end
@ -127,7 +120,7 @@ module Invidious::Routes::API::V1::Authenticated
env.response.content_type = "application/json"
user = env.get("user").as(User)
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist)
playlists = Invidious::Database::Playlists.select_all(author: user.email)
JSON.build do |json|
json.array do
@ -153,11 +146,11 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(400, "Invalid privacy setting.")
end
if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_json(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(PG_DB, title, privacy, user)
playlist = create_playlist(title, privacy, user)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
env.response.status_code = 201
{
@ -172,9 +165,12 @@ module Invidious::Routes::API::V1::Authenticated
env.response.content_type = "application/json"
user = env.get("user").as(User)
plid = env.params.url["plid"]
plid = env.params.url["plid"]?
if !plid || plid.empty?
return error_json(400, "A playlist ID is required")
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -195,7 +191,8 @@ module Invidious::Routes::API::V1::Authenticated
updated = playlist.updated
end
PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid)
Invidious::Database::Playlists.update(plid, title, privacy, description, updated)
env.response.status_code = 204
end
@ -207,7 +204,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -216,8 +213,7 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(403, "Invalid user")
end
PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid)
PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid)
Invidious::Database::Playlists.delete(plid)
env.response.status_code = 204
end
@ -230,7 +226,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -249,7 +245,7 @@ module Invidious::Routes::API::V1::Authenticated
end
begin
video = get_video(video_id, PG_DB)
video = get_video(video_id)
rescue ex
return error_json(500, ex)
end
@ -266,11 +262,8 @@ module Invidious::Routes::API::V1::Authenticated
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid)
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(plid, playlist_video.index)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}"
env.response.status_code = 201
@ -289,7 +282,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
index = env.params.url["index"].to_i64(16)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -302,8 +295,8 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(404, "Playlist does not contain index")
end
PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index)
PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid)
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(plid, index)
env.response.status_code = 204
end
@ -318,7 +311,7 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User)
scopes = env.get("scopes").as(Array(String))
tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time})
tokens = Invidious::Database::SessionIDs.select_all(user.email)
JSON.build do |json|
json.array do
@ -360,7 +353,7 @@ module Invidious::Routes::API::V1::Authenticated
if sid = env.get?("sid").try &.as(String)
env.response.content_type = "text/html"
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
return templated "authorize_token"
else
env.response.content_type = "application/json"
@ -374,7 +367,7 @@ module Invidious::Routes::API::V1::Authenticated
end
end
access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB)
access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
@ -406,9 +399,9 @@ module Invidious::Routes::API::V1::Authenticated
# Allow tokens to revoke other tokens with correct scope
if session == env.get("session").as(String)
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session)
Invidious::Database::SessionIDs.delete(sid: session)
elsif scopes_include_scope(scopes, "GET:tokens")
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session)
Invidious::Database::SessionIDs.delete(sid: session)
else
return error_json(400, "Cannot revoke session #{session}")
end

2
src/invidious/routes/api/v1/misc.cr
View file

@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Misc
end
begin
playlist = get_playlist(PG_DB, plid, locale)
playlist = get_playlist(plid, locale)
rescue ex : InfoException
return error_json(404, ex)
rescue ex

10
src/invidious/routes/api/v1/videos.cr
View file

@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Videos
region = env.params.query["region"]?
begin
video = get_video(id, PG_DB, region: region)
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
@ -36,7 +36,7 @@ module Invidious::Routes::API::V1::Videos
# getting video info.
begin
video = get_video(id, PG_DB, region: region)
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
@ -157,7 +157,7 @@ module Invidious::Routes::API::V1::Videos
region = env.params.query["region"]?
begin
video = get_video(id, PG_DB, region: region)
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos
case source
when "archive"
if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation))
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')
@ -271,7 +271,7 @@ module Invidious::Routes::API::V1::Videos
annotations = response.body
cache_annotation(PG_DB, id, annotations)
cache_annotation(id, annotations)
end
else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")

14
src/invidious/routes/embed.cr
View file

@ -6,9 +6,9 @@ module Invidious::Routes::Embed
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
playlist = get_playlist(plid, locale: locale)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset, locale: locale)
rescue ex
return error_template(500, ex)
end
@ -30,7 +30,7 @@ module Invidious::Routes::Embed
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(PG_DB, env.params.query, plid, id)
continuation = process_continuation(env.params.query, plid, id)
if md = env.params.query["playlist"]?
.try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
@ -60,9 +60,9 @@ module Invidious::Routes::Embed
if plid
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
playlist = get_playlist(plid, locale: locale)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset, locale: locale)
rescue ex
return error_template(500, ex)
end
@ -119,7 +119,7 @@ module Invidious::Routes::Embed
subscriptions ||= [] of String
begin
video = get_video(id, PG_DB, region: params.region)
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
@ -137,7 +137,7 @@ module Invidious::Routes::Embed
# end
if notifications && notifications.includes? id
PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
end

33
src/invidious/routes/feeds.cr
View file

@ -15,13 +15,14 @@ module Invidious::Routes::Feeds
user = user.as(User)
items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
# TODO: make a single DB call and separate the items here?
items_created = Invidious::Database::Playlists.select_like_iv(user.email)
items_created.map! do |item|
item.author = ""
item
end
items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
items_saved = Invidious::Database::Playlists.select_not_like_iv(user.email)
items_saved.map! do |item|
item.author = ""
item
@ -83,7 +84,7 @@ module Invidious::Routes::Feeds
headers["Cookie"] = env.request.headers["Cookie"]
if !user.password
user, sid = get_user(sid, headers, PG_DB)
user, sid = get_user(sid, headers)
end
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
@ -93,14 +94,13 @@ module Invidious::Routes::Feeds
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(PG_DB, user, max_results, page)
videos, notifications = get_subscription_feed(user, max_results, page)
# "updated" here is used for delivering new notifications, so if
# we know a user has looked at their feed e.g. in the past 10 minutes,
# they've already seen a video posted 20 minutes ago, and don't need
# to be notified.
PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc,
user.email)
Invidious::Database::Users.clear_notifications(user)
user.notifications = [] of String
env.set "user", user
@ -220,7 +220,7 @@ module Invidious::Routes::Feeds
haltf env, status_code: 403
end
user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User)
user = Invidious::Database::Users.select(token: token.strip)
if !user
haltf env, status_code: 403
end
@ -234,7 +234,7 @@ module Invidious::Routes::Feeds
params = HTTP::Params.parse(env.params.query["params"]? || "")
videos, notifications = get_subscription_feed(PG_DB, user, max_results, page)
videos, notifications = get_subscription_feed(user, max_results, page)
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
@ -264,8 +264,8 @@ module Invidious::Routes::Feeds
path = env.request.path
if plid.starts_with? "IV"
if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale)
if playlist = Invidious::Database::Playlists.select(id: plid)
videos = get_playlist_videos(playlist, offset: 0, locale: locale)
return XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
@ -364,7 +364,7 @@ module Invidious::Routes::Feeds
if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]?
PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid)
elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]?
PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid)
Invidious::Database::Playlists.update_subscription_time(plid)
else
haltf env, status_code: 400
end
@ -393,7 +393,7 @@ module Invidious::Routes::Feeds
published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
video = get_video(id, PG_DB, force_refresh: true)
video = get_video(id, force_refresh: true)
# Deliver notifications to `/api/v1/auth/notifications`
payload = {
@ -416,13 +416,8 @@ module Invidious::Routes::Feeds
views: video.views,
})
was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3,
updated = $4, ucid = $5, author = $6, length_seconds = $7,
live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1),
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
Invidious::Database::Users.add_notification(video) if was_insert
end
end

35
src/invidious/routes/login.cr
View file

@ -275,7 +275,7 @@ module Invidious::Routes::Login
raise "Couldn't get SID."
end
user, sid = get_user(sid, headers, PG_DB)
user, sid = get_user(sid, headers)
# We are now logged in
traceback << "done.<br/>"
@ -303,8 +303,8 @@ module Invidious::Routes::Login
end
if env.request.cookies["PREFS"]?
preferences = env.get("preferences").as(Preferences)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email)
user.preferences = env.get("preferences").as(Preferences)
Invidious::Database::Users.update_preferences(user)
cookie = env.request.cookies["PREFS"]
cookie.expires = Time.utc(1990, 1, 1)
@ -327,7 +327,7 @@ module Invidious::Routes::Login
return error_template(401, "Password is a required field")
end
user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User)
user = Invidious::Database::Users.select(email: email)
if user
if !user.password
@ -336,7 +336,7 @@ module Invidious::Routes::Login
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc)
Invidious::Database::SessionIDs.insert(sid, email)
if Kemal.config.ssl || CONFIG.https_only
secure = true
@ -393,9 +393,9 @@ module Invidious::Routes::Login
prompt = ""
if captcha_type == "image"
captcha = generate_captcha(HMAC_KEY, PG_DB)
captcha = generate_captcha(HMAC_KEY)
else
captcha = generate_text_captcha(HMAC_KEY, PG_DB)
captcha = generate_text_captcha(HMAC_KEY)
end
return templated "login"
@ -412,7 +412,7 @@ module Invidious::Routes::Login
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
begin
validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale)
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
@ -427,7 +427,7 @@ module Invidious::Routes::Login
error_exception = Exception.new
tokens.each do |token|
begin
validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, answer, env.request, HMAC_KEY, locale)
found_valid_captcha = true
rescue ex
error_exception = ex
@ -449,13 +449,8 @@ module Invidious::Routes::Login
end
end
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array)
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc)
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
@ -475,8 +470,8 @@ module Invidious::Routes::Login
end
if env.request.cookies["PREFS"]?
preferences = env.get("preferences").as(Preferences)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email)
user.preferences = env.get("preferences").as(Preferences)
Invidious::Database::Users.update_preferences(user)
cookie = env.request.cookies["PREFS"]
cookie.expires = Time.utc(1990, 1, 1)
@ -506,12 +501,12 @@ module Invidious::Routes::Login
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid)
Invidious::Database::SessionIDs.delete(sid: sid)
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)

62
src/invidious/routes/playlists.cr
View file

@ -12,7 +12,7 @@ module Invidious::Routes::Playlists
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY)
templated "create_playlist"
end
@ -31,7 +31,7 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
@ -46,11 +46,11 @@ module Invidious::Routes::Playlists
return error_template(400, "Invalid privacy setting.")
end
if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_template(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(PG_DB, title, privacy, user)
playlist = create_playlist(title, privacy, user)
env.redirect "/playlist?list=#{playlist.id}"
end
@ -66,8 +66,8 @@ module Invidious::Routes::Playlists
user = user.as(User)
playlist_id = env.params.query["list"]
playlist = get_playlist(PG_DB, playlist_id, locale)
subscribe_playlist(PG_DB, user, playlist)
playlist = get_playlist(playlist_id, locale)
subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
end
@ -85,12 +85,16 @@ module Invidious::Routes::Playlists
sid = sid.as(String)
plid = env.params.query["list"]?
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if !plid || plid.empty?
return error_template(400, "A playlist ID is required")
end
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY)
templated "delete_playlist"
end
@ -112,18 +116,17 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid)
PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid)
Invidious::Database::Playlists.delete(plid)
env.redirect "/feed/playlists"
end
@ -149,7 +152,7 @@ module Invidious::Routes::Playlists
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -158,12 +161,12 @@ module Invidious::Routes::Playlists
end
begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale)
rescue ex
videos = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
templated "edit_playlist"
end
@ -185,12 +188,12 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -207,7 +210,7 @@ module Invidious::Routes::Playlists
updated = playlist.updated
end
PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid)
Invidious::Database::Playlists.update(plid, title, privacy, description, updated)
env.redirect "/playlist?list=#{plid}"
end
@ -233,7 +236,7 @@ module Invidious::Routes::Playlists
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -283,7 +286,7 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
@ -311,7 +314,7 @@ module Invidious::Routes::Playlists
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist)
playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
rescue ex
if redirect
@ -342,7 +345,7 @@ module Invidious::Routes::Playlists
video_id = env.params.query["video_id"]
begin
video = get_video(video_id, PG_DB)
video = get_video(video_id)
rescue ex
if redirect
return error_template(500, ex)
@ -363,15 +366,12 @@ module Invidious::Routes::Playlists
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id)
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
when "action_remove_video"
index = env.params.query["set_video_id"]
PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index)
PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id)
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
when "action_move_video_before"
# TODO: Playlist stub
else
@ -405,7 +405,7 @@ module Invidious::Routes::Playlists
end
begin
playlist = get_playlist(PG_DB, plid, locale)
playlist = get_playlist(plid, locale)
rescue ex
return error_template(500, ex)
end
@ -422,7 +422,7 @@ module Invidious::Routes::Playlists
end
begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale)
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
end

20
src/invidious/routes/preferences.cr
View file

@ -170,11 +170,12 @@ module Invidious::Routes::PreferencesRoute
vr_mode: vr_mode,
show_nick: show_nick,
save_player_pos: save_player_pos,
}.to_json).to_json
}.to_json)
if user = env.get? "user"
user = user.as(User)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email)
user.preferences = preferences
Invidious::Database::Users.update_preferences(user)
if CONFIG.admins.includes? user.email
CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home
@ -220,10 +221,10 @@ module Invidious::Routes::PreferencesRoute
end
if CONFIG.domain
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years,
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years,
secure: secure, http_only: true)
else
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years,
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years,
secure: secure, http_only: true)
end
end
@ -241,18 +242,15 @@ module Invidious::Routes::PreferencesRoute
if user = env.get? "user"
user = user.as(User)
preferences = user.preferences
case preferences.dark_mode
case user.preferences.dark_mode
when "dark"
preferences.dark_mode = "light"
user.preferences.dark_mode = "light"
else
preferences.dark_mode = "dark"
user.preferences.dark_mode = "dark"
end
preferences = preferences.to_json
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email)
Invidious::Database::Users.update_preferences(user)
else
preferences = env.get("preferences").as(Preferences)

2
src/invidious/routes/video_playback.cr
View file

@ -263,7 +263,7 @@ module Invidious::Routes::VideoPlayback
haltf env, status_code: 400, response: "TESTING"
end
video = get_video(id, PG_DB, region: region)
video = get_video(id, region: region)
fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
url = fmt.try &.["url"]?.try &.as_s

8
src/invidious/routes/watch.cr
View file

@ -39,7 +39,7 @@ module Invidious::Routes::Watch
end
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(PG_DB, env.params.query, plid, id)
continuation = process_continuation(env.params.query, plid, id)
nojs = env.params.query["nojs"]?
@ -60,7 +60,7 @@ module Invidious::Routes::Watch
env.params.query.delete_all("listen")
begin
video = get_video(id, PG_DB, region: params.region)
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
@ -76,11 +76,11 @@ module Invidious::Routes::Watch
env.params.query.delete_all("iv_load_policy")
if watched && !watched.includes? id
PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
Invidious::Database::Users.mark_watched(user.as(User), id)
end
if notifications && notifications.includes? id
PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
end

59
src/invidious/users.cr
View file

@ -29,43 +29,31 @@ struct User
end
end
def get_user(sid, headers, db, refresh = true)
if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String)
user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
def get_user(sid, headers, refresh = true)
if email = Invidious::Database::SessionIDs.select_email(sid)
user = Invidious::Database::Users.select!(email: email)
if refresh && Time.utc - user.updated > 1.minute
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
user, sid = fetch_user(sid, headers)
db.exec("INSERT INTO users VALUES (#{args}) \
ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array)
db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \
ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc)
Invidious::Database::Users.insert(user, update_on_conflict: true)
Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
begin
view_name = "subscriptions_#{sha256(user.email)}"
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
rescue ex
end
end
else
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user.to_a)
user, sid = fetch_user(sid, headers)
db.exec("INSERT INTO users VALUES (#{args}) \
ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array)
db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \
ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc)
Invidious::Database::Users.insert(user, update_on_conflict: true)
Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
begin
view_name = "subscriptions_#{sha256(user.email)}"
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
rescue ex
end
end
@ -73,7 +61,7 @@ def get_user(sid, headers, db, refresh = true)
return user, sid
end
def fetch_user(sid, headers, db)
def fetch_user(sid, headers)
feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
feed = XML.parse_html(feed.body)
@ -86,7 +74,7 @@ def fetch_user(sid, headers, db)
end
end
channels = get_batch_channels(channels, db, false, false)
channels = get_batch_channels(channels, false, false)
email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
if email
@ -130,7 +118,7 @@ def create_user(sid, email, password)
return user, sid
end
def generate_captcha(key, db)
def generate_captcha(key)
second = Random::Secure.rand(12)
second_angle = second * 30
second = second * 5
@ -182,16 +170,16 @@ def generate_captcha(key, db)
return {
question: image,
tokens: {generate_response(answer, {":login"}, key, db, use_nonce: true)},
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
def generate_text_captcha(key, db)
def generate_text_captcha(key)
response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
response = JSON.parse(response)
tokens = response["a"].as_a.map do |answer|
generate_response(answer.as_s, {":login"}, key, db, use_nonce: true)
generate_response(answer.as_s, {":login"}, key, use_nonce: true)
end
return {
@ -232,20 +220,16 @@ def subscribe_ajax(channel_id, action, env_headers)
end
end
def get_subscription_feed(db, user, max_results = 40, page = 1)
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email,
as: Array(String))
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
args = arg_array(notifications)
notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo)
notifications = Invidious::Database::ChannelVideos.select(notifications)
videos = [] of ChannelVideo
notifications.sort_by!(&.published).reverse!
@ -311,8 +295,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
else nil # Ignore
end
notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String))
notifications = Invidious::Database::Users.select_notifications(user)
notifications = videos.select { |v| notifications.includes? v.id }
videos = videos - notifications
end

14
src/invidious/videos.cr
View file

@ -993,8 +993,8 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
return params
end
def get_video(id, db, refresh = true, region = nil, force_refresh = false)
if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region
def get_video(id, refresh = true, region = nil, force_refresh = false)
if (video = Invidious::Database::Videos.select(id)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
if (refresh &&
@ -1003,17 +1003,15 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false)
force_refresh
begin
video = fetch_video(id, region)
db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated)
Invidious::Database::Videos.update(video)
rescue ex
db.exec("DELETE FROM videos * WHERE id = $1", id)
Invidious::Database::Videos.delete(id)
raise ex
end
end
else
video = fetch_video(id, region)
if !region
db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated)
end
Invidious::Database::Videos.insert(video) if !region
end
return video
@ -1058,7 +1056,7 @@ def itag_to_metadata?(itag : JSON::Any)
return VIDEO_FORMATS[itag.to_s]?
end
def process_continuation(db, query, plid, id)
def process_continuation(query, plid, id)
continuation = nil
if plid
if index = query["index"]?.try &.to_i?

2
src/invidious/views/playlist.ecr
View file

@ -61,7 +61,7 @@
<div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
<% else %>
<% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %>
<% if Invidious::Database::Playlists.exists?(playlist.id) %>
<div class="pure-u-1-3"><a href="/subscribe_playlist?list=<%= plid %>"><i class="icon ion-md-add"></i></a></div>
<% else %>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>

2
src/invidious/views/watch.ecr
View file

@ -138,7 +138,7 @@ we're going to need to do it here in order to allow for translations.
</p>
<% if user %>
<% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %>
<% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %>
<% if !playlists.empty? %>
<form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post">
<div class="pure-control-group">