Clean up data import/export
This commit is contained in:
Omar Roth
parent
6066615553
commit
b77c73df0d
2 changed files with 82 additions and 68 deletions
|
|
@ -13,7 +13,6 @@ dependencies:
|
||||||
github: detectlanguage/detectlanguage-crystal
|
github: detectlanguage/detectlanguage-crystal
|
||||||
kemal:
|
kemal:
|
||||||
github: kemalcr/kemal
|
github: kemalcr/kemal
|
||||||
commit: b389022
|
|
||||||
pg:
|
pg:
|
||||||
github: will/crystal-pg
|
github: will/crystal-pg
|
||||||
|
|
||||||
|
|
|
||||||
149
src/invidious.cr
149
src/invidious.cr
|
|
@ -1230,83 +1230,85 @@ post "/data_control" do |env|
|
||||||
case part.name
|
case part.name
|
||||||
when "import_invidious"
|
when "import_invidious"
|
||||||
body = JSON.parse(body)
|
body = JSON.parse(body)
|
||||||
body["subscriptions"]?.try &.as_a.each do |ucid|
|
|
||||||
ucid = ucid.as_s
|
|
||||||
|
|
||||||
if !user.subscriptions.includes? ucid
|
if body["subscriptions"]?
|
||||||
|
user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s }
|
||||||
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
|
user.subscriptions.each do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
|
|
||||||
user.subscriptions << ucid
|
|
||||||
rescue ex
|
rescue ex
|
||||||
next
|
next
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
end
|
end
|
||||||
|
|
||||||
if body["watch_history"]?
|
if body["watch_history"]?
|
||||||
watched = user.watched + body["watch_history"].as_a.map { |a| a.as_s }
|
user.watched += body["watch_history"].as_a.map { |a| a.as_s }
|
||||||
watched.uniq!
|
user.watched.uniq!
|
||||||
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", watched, user.email)
|
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
|
||||||
end
|
end
|
||||||
|
|
||||||
if body["preferences"]?
|
if body["preferences"]?
|
||||||
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", body["preferences"].to_json, user.email)
|
user.preferences = Preferences.from_json(body["preferences"].to_json)
|
||||||
|
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
|
||||||
end
|
end
|
||||||
when "import_youtube"
|
when "import_youtube"
|
||||||
subscriptions = XML.parse(body)
|
subscriptions = XML.parse(body)
|
||||||
subscriptions.xpath_nodes(%q(//outline[@type="rss"])).each do |channel|
|
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
|
||||||
ucid = channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||||
|
end
|
||||||
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
if !user.subscriptions.includes? ucid
|
user.subscriptions.each do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
|
next
|
||||||
user.subscriptions << ucid
|
|
||||||
rescue ex
|
|
||||||
next
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
when "import_freetube"
|
when "import_freetube"
|
||||||
body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).each do |md|
|
user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
|
||||||
ucid = md["channel_id"]
|
md["channel_id"]
|
||||||
|
end
|
||||||
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
if !user.subscriptions.includes? ucid
|
user.subscriptions.each do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
|
next
|
||||||
user.subscriptions << ucid
|
|
||||||
rescue ex
|
|
||||||
next
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
when "import_newpipe_subscriptions"
|
when "import_newpipe_subscriptions"
|
||||||
body = JSON.parse(body)
|
body = JSON.parse(body)
|
||||||
body["subscriptions"].as_a.each do |channel|
|
user.subscriptions += body["subscriptions"].as_a.map do |channel|
|
||||||
ucid = channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||||
|
end
|
||||||
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
if !user.subscriptions.includes? ucid
|
user.subscriptions.each do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
|
next
|
||||||
user.subscriptions << ucid
|
|
||||||
rescue ex
|
|
||||||
next
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
when "import_newpipe"
|
when "import_newpipe"
|
||||||
Zip::Reader.open(body) do |file|
|
Zip::Reader.open(IO::Memory.new(body)) do |file|
|
||||||
file.each_entry do |entry|
|
file.each_entry do |entry|
|
||||||
if entry.filename == "newpipe.db"
|
if entry.filename == "newpipe.db"
|
||||||
# We do this because the SQLite driver cannot parse a database from an IO
|
# We do this because the SQLite driver cannot parse a database from an IO
|
||||||
|
|
@ -1314,30 +1316,29 @@ post "/data_control" do |env|
|
||||||
# video URLs can **only** be watch history, so this works okay for now.
|
# video URLs can **only** be watch history, so this works okay for now.
|
||||||
|
|
||||||
db = entry.io.gets_to_end
|
db = entry.io.gets_to_end
|
||||||
db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/) do |md|
|
|
||||||
id = md["id"]
|
|
||||||
|
|
||||||
if !user.watched.includes? id
|
user.watched += db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/).map do |md|
|
||||||
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE email = $2", id, user.email)
|
md["id"]
|
||||||
user.watched << id
|
end
|
||||||
|
user.watched.uniq!
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
|
||||||
|
|
||||||
|
user.subscriptions += db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/).map do |md|
|
||||||
|
md["ucid"]
|
||||||
|
end
|
||||||
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
|
user.subscriptions.each do |ucid|
|
||||||
|
begin
|
||||||
|
client = make_client(YT_URL)
|
||||||
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
|
next
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/) do |md|
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
ucid = md["ucid"]
|
|
||||||
|
|
||||||
if !user.subscriptions.includes? ucid
|
|
||||||
begin
|
|
||||||
client = make_client(YT_URL)
|
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
|
||||||
|
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
|
|
||||||
user.subscriptions << ucid
|
|
||||||
rescue ex
|
|
||||||
next
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -1629,6 +1630,20 @@ get "/feed/subscriptions" do |env|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
get "/feed/history" do |env|
|
||||||
|
user = env.get? "user"
|
||||||
|
referer = get_referer(env)
|
||||||
|
|
||||||
|
if user
|
||||||
|
user = user.as(User)
|
||||||
|
watched = user.watched.reverse
|
||||||
|
|
||||||
|
templated "history"
|
||||||
|
else
|
||||||
|
env.redirect referer
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
get "/feed/channel/:ucid" do |env|
|
get "/feed/channel/:ucid" do |env|
|
||||||
env.response.content_type = "text/xml"
|
env.response.content_type = "text/xml"
|
||||||
ucid = env.params.url["ucid"]
|
ucid = env.params.url["ucid"]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue