structables/main.py
Kumi 455218f7b6
feat: stream proxy content dynamically
Refactored the proxy route to stream data dynamically from URLs starting
with specific domains, rather than loading it all at once. This change
introduces a subfunction within the proxy route handler to manage
streaming, which iterates through data chunks and yields them until
completion. The new approach aims to enhance performance and resource
utilization by avoiding the download of the entire content before
sending it to the client. Additionally, error handling has been
tightened, specifically for HTTP errors and missing 'content-type'
headers, improving the robustness and reliability of the proxy feature.
2024-03-29 08:17:56 +01:00

1266 lines
39 KiB
Python

#!/usr/bin/env python
from flask import (
Flask,
render_template,
request,
redirect,
Response,
stream_with_context,
)
from urllib.parse import quote, unquote, urlencode
from urllib.request import Request, urlopen
from urllib.error import HTTPError
from traceback import print_exc
from urllib.parse import urljoin, urlparse
from argparse import ArgumentParser
from configparser import ConfigParser
from werkzeug.exceptions import BadRequest, abort, InternalServerError, NotFound
from bs4 import BeautifulSoup
import os
import json
import re
import logging
import pathlib
logging.basicConfig(level=logging.DEBUG)
global_ibles = {}
def proxy(url):
logging.debug(f"Generating proxy URL for {url}")
return f"/proxy/?url={url}"
def get_typesense_api_key():
logging.debug("Getting Typesense API key...")
data = urlopen("https://www.instructables.com/")
soup = BeautifulSoup(data.read().decode(), "html.parser")
scripts = soup.select("script")
for script in scripts:
if "typesense" in script.text and (
matches := re.search(r'"typesenseApiKey":\s?"(.*?)"', script.text)
):
api_key = matches.group(1)
logging.debug(f"Identified Typesense API key as {api_key}")
return api_key
logging.error("Failed to get Typesense API key")
TYPESENSE_API_KEY = get_typesense_api_key()
def projects_search(
query="*",
category="",
channel="",
filter_by="",
page=1,
per_page=50,
query_by="title,stepBody,screenName",
sort_by="publishDate:desc",
timeout=5,
):
if category:
if filter_by:
filter_by += " && "
filter_by += f"category:={category}"
if channel:
if filter_by:
filter_by += " && "
filter_by += f"channel:={channel}"
query = quote(query)
filter_by = quote(filter_by)
logging.debug(
f"Searching projects with query {query} and filter {filter_by}, page {page}"
)
projects_headers = {"x-typesense-api-key": TYPESENSE_API_KEY}
request_args = {
"q": query,
"query_by": query_by,
"page": page,
"sort_by": sort_by,
"include_fields": "title,urlString,coverImageUrl,screenName,favorites,views,primaryClassification,featureFlag,prizeLevel,IMadeItCount",
"filter_by": filter_by,
"per_page": per_page,
}
args_str = "&".join([f"{key}={value}" for key, value in request_args.items()])
projects_request = Request(
f"https://www.instructables.com/api_proxy/search/collections/projects/documents/search?{args_str}",
headers=projects_headers,
)
projects_data = urlopen(projects_request, timeout=timeout)
project_obj = json.loads(projects_data.read().decode())
project_ibles = project_obj["hits"]
logging.debug(f"Got {len(project_ibles)} projects")
return project_ibles, project_obj["out_of"]
def update_data():
logging.debug("Updating data...")
channels = []
sitemap_data = urlopen("https://www.instructables.com/sitemap/")
sitemap_soup = BeautifulSoup(sitemap_data.read().decode(), "html.parser")
main = sitemap_soup.select("div.sitemap-content")[0]
groups = []
for group in main.select("div.group-section"):
channels.append(group.select("h2 a")[0].text.lower())
global_ibles["/projects"] = []
project_ibles, total = projects_search(filter_by="featureFlag:=true")
while len(global_ibles["/projects"]) <= 0:
for ible in project_ibles:
link = f"/{ible['document']['urlString']}"
img = proxy(ible["document"]["coverImageUrl"])
title = ible["document"]["title"]
author = ible["document"]["screenName"]
author_link = f"/member/{author}"
channel = ible["document"]["primaryClassification"]
channel_link = f"/channel/{channel}"
views = ible["document"]["views"]
favorites = ible["document"]["favorites"]
global_ibles["/projects"].append(
{
"link": link,
"img": img,
"title": title,
"author": author,
"author_link": author_link,
"channel": channel,
"channel_link": channel_link,
"views": views,
"favorites": favorites,
}
)
debugmode = os.environ.get("FLASK_DEBUG", os.environ.get("STRUCTABLES_DEBUG", False))
invidious = os.environ.get("STRUCTABLES_INVIDIOUS")
unsafe = os.environ.get("STRUCTABLES_UNSAFE", False)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
"-p",
"--port",
default=8002,
type=int,
help="Port to listen on",
)
parser.add_argument(
"-d",
"--debug",
action="store_true",
help="Enable debug mode",
)
parser.add_argument(
"-l",
"--listen-host",
default="127.0.0.1",
help="Host to listen on",
)
parser.add_argument(
"-I",
"--invidious",
help="URL to Invidious instance, e.g. https://invidious.private.coffee/",
)
parser.add_argument(
"-u",
"--unsafe",
action="store_true",
help="Display iframes regardless of origin",
)
args = parser.parse_args()
if args.debug:
debugmode = True
if args.invidious:
invidious = args.invidious
if args.unsafe:
unsafe = True
print("Loading initial data...")
update_data()
print("Started!")
app = Flask(__name__, template_folder="templates", static_folder="static")
if debugmode:
app.logger.setLevel(logging.DEBUG)
@app.route("/cron/")
def cron():
update_data()
return "OK"
def explore_lists(soup):
list_ = []
for ible in soup.select(".home-content-explore-ible"):
link = ible.a["href"]
img = proxy(ible.select("a img")[0].get("data-src"))
alt = ible.select("a img")[0].get("alt")
title = ible.select("div strong a")[0].text
author = ible.select("div span.ible-author a")[0].text
author_link = ible.select("div span.ible-author a")[0].get("href")
channel = ible.select("div span.ible-channel a")[0].text
channel_link = ible.select("div span.ible-channel a")[0].get("href")
views = 0
if ible.select("span.ible-views") != []:
views = ible.select("span.ible-views")[0].text
favorites = 0
if ible.select("span.ible-favorites") != []:
favorites = ible.select("span.ible-favorites")[0].text
list_.append(
{
"link": link,
"img": img,
"alt": alt,
"title": title,
"author": author,
"author_link": author_link,
"channel": channel,
"channel_link": channel_link,
"favorites": favorites,
"views": views,
}
)
return list_
def member_header(header):
avatar = proxy(
header.select("div.profile-avatar-container img.profile-avatar")[0].get("src")
)
title = header.select("div.profile-top div.profile-headline h1.profile-title")[
0
].text
profile_top = header.select("div.profile-top")[0]
# stats_text = profile_top.select("div.profile-header-stats")[0]
# stats_num = header.select("div.profile-top div.profile-header-stats")[1]
location = header.select("span.member-location")
if location != []:
location = location[0].text
else:
location = 0
signup = header.select("span.member-signup-date")
if signup != []:
signup = signup[0].text
else:
signup = 0
instructables = header.select("span.ible-count")
if instructables != []:
instructables = instructables[0].text
else:
instructables = 0
views = header.select("span.total-views")
if views != []:
views = views[0].text
else:
views = 0
comments = header.select("span.total-comments")
if comments != []:
comments = comments[0].text
else:
comments = 0
followers = header.select("span.follower-count")
if followers != []:
followers = followers[0].text
else:
followers = 0
bio = header.select("span.member-bio")
if bio != []:
bio = bio[0].text
else:
bio = ""
return {
"avatar": avatar,
"title": title,
"location": location,
"signup": signup,
"instructables": instructables,
"views": views,
"comments": comments,
"followers": followers,
"bio": bio,
}
def category_page(name, teachers=False):
path = urlparse(request.path).path
page = request.args.get("page", 1, type=int)
ibles = []
channels = []
contests = []
for channel in global_ibles["/projects"]:
if (
channel["channel"].startswith(name.lower())
and channel["channel"] not in channels
):
channels.append(channel["channel"])
category_ibles, total = projects_search(
category=name, page=page, filter_by="featureFlag:=true"
)
for ible in category_ibles:
link = f"/{ible['document']['urlString']}"
img = proxy(ible["document"]["coverImageUrl"])
title = ible["document"]["title"]
author = ible["document"]["screenName"]
author_link = f"/member/{author}"
channel = ible["document"]["primaryClassification"]
channel_link = f"/channel/{channel}"
views = ible["document"]["views"]
favorites = ible["document"]["favorites"]
ibles.append(
{
"link": link,
"img": img,
"title": title,
"author": author,
"author_link": author_link,
"channel": channel,
"channel_link": channel_link,
"views": views,
"favorites": favorites,
}
)
return render_template(
"category.html",
title=name,
channels=channels,
ibles=ibles,
contests=contests,
path=path,
)
def get_pagination(request, total, per_page=1):
pagination = []
args = request.args.copy()
current = int(args.pop("page", 1))
query_string = urlencode(args)
total_pages = int(total / per_page)
if query_string:
query_string = "&" + query_string
if current > 1:
pagination.append(
{
"link": f"?page={current - 1}{query_string}",
"text": "Previous",
"disabled": False,
"active": False,
}
)
for page in range(max(current - 5, 1), min(current + 5, total_pages)):
if page == current:
pagination.append(
{
"link": f"?page={page}{query_string}",
"text": page,
"disabled": False,
"active": True,
}
)
else:
pagination.append(
{
"link": f"?page={page}{query_string}",
"text": page,
"disabled": False,
"active": False,
}
)
if current < total_pages:
pagination.append(
{
"link": f"?page={current + 1}{query_string}",
"text": "Next",
"disabled": False,
"active": False,
}
)
return pagination
def project_list(head, sort="", per_page=20):
head = f"{head + ' ' if head != '' else ''}Projects" + sort
path = urlparse(request.path).path
page = request.args.get("page", 1, type=int)
if path in ("/projects/", "/projects"):
ibles = global_ibles["/projects"]
total = len(ibles)
else:
if "projects" in path.split("/"):
ibles = []
parts = path.split("/")
category = parts[1]
channel = "" if parts[2] == "projects" else parts[2]
project_ibles, total = projects_search(
category=category, channel=channel, per_page=per_page, page=page
)
elif "search" in path.split("/"):
ibles = []
query = (
request.args.get("q") if request.method == "GET" else request.form["q"]
)
project_ibles, total = projects_search(
query=query,
filter_by="",
per_page=per_page,
page=page,
query_by="title,screenName",
)
else:
abort(404)
for ible in project_ibles:
link = f"/{ible['document']['urlString']}"
img = proxy(ible["document"]["coverImageUrl"])
title = ible["document"]["title"]
author = ible["document"]["screenName"]
author_link = f"/member/{author}"
channel = ible["document"]["primaryClassification"]
channel_link = f"/channel/{channel}"
views = ible["document"]["views"]
favorites = ible["document"]["favorites"]
ibles.append(
{
"link": link,
"img": img,
"title": title,
"author": author,
"author_link": author_link,
"channel": channel,
"channel_link": channel_link,
"views": views,
"favorites": favorites,
}
)
return render_template(
"projects.html",
title=head,
ibles=ibles,
path=path,
pagination=get_pagination(request, total, per_page),
)
@app.route("/sitemap/")
@app.route("/sitemap/<path:path>")
def route_sitemap(path=""):
try:
data = urlopen("https://www.instructables.com/sitemap/" + path)
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
main = soup.select("div.sitemap-content")[0]
group_section = main.select("div.group-section")
if group_section:
groups = []
for group in group_section:
category = group.select("h2 a")[0].text
category_link = group.select("h2 a")[0].get("href")
channels = []
for li in group.select("ul.sitemap-listing li"):
channel = li.a.text
channel_link = li.a["href"]
channels.append([channel, channel_link])
groups.append([category, category_link, channels])
else:
groups = []
channels = []
for li in main.select("ul.sitemap-listing li"):
channel = li.a.text
channel_link = li.a["href"]
channels.append([channel, channel_link])
groups.append(["", "", channels])
return render_template("sitemap.html", title="Sitemap", groups=groups)
@app.route("/contest/archive/")
def route_contest_archive():
page = 1
if request.args.get("page") != None:
page = request.args.get("page")
try:
data = urlopen(f"https://www.instructables.com/contest/archive/?page={page}")
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
main = soup.select("div#contest-archive-wrapper")[0]
contest_count = main.select("p.contest-count")[0].text
contest_list = []
for index, year in enumerate(main.select("div.contest-archive-list h2")):
year_list = main.select(
"div.contest-archive-list div.contest-archive-list-year"
)[index]
year_name = year.text
month_list = []
for month in year_list.select("div.contest-archive-list-month"):
month_name = month.select("h3")[0].text
month_contest_list = []
for p in month.select("p"):
date = p.select("span")[0].text
link = p.select("a")[0].get("href")
title = p.select("a")[0].text
month_contest_list.append([date, link, title])
month_list.append([month_name, month_contest_list])
contest_list.append([year_name, month_list])
pagination = main.select("nav.pagination ul.pagination")[0]
return render_template(
"archives.html",
title=f"Contest Archives (Page {page})",
page=page,
contest_count=contest_count,
pagination=pagination,
contest_list=contest_list,
)
@app.route("/contest/<contest>/")
def route_contest(contest):
try:
data = urlopen(f"https://www.instructables.com/contest/{contest}/")
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
title = soup.select('meta[property="og:title"]')[0].get("content")
body = soup.select("div#contest-wrapper")[0]
img = proxy(body.select("div#contest-masthead img")[0].get("src"))
entry_count = body.select("li.entries-nav-btn")[0].text.split(" ")[0]
prizes = body.select("li.prizes-nav-btn")[0].text.split(" ")[0]
info = body.select("div.contest-body-column-left")[0]
info.select("div#site-announcements-page")[0].decompose()
info.select("h3")[0].decompose()
info.select("div#contest-body-nav")[0].decompose()
info = str(info).replace("https://www.instructables.com", "/")
entries = body.select("span.contest-entity-count")[0].text
entry_list = []
for entry in body.select("div.contest-entries-list div.contest-entries-list-ible"):
link = entry.a["href"]
entry_img = proxy(entry.select("a noscript img")[0].get("src"))
entry_title = entry.select("a.ible-title")[0].text
author = entry.select("div span.ible-author a")[0].text
author_link = entry.select("div span.ible-author a")[0].get("href")
channel = entry.select("div span.ible-channel a")[0].text
channel_link = entry.select("div span.ible-channel a")[0].get("href")
views = entry.select(".ible-views")[0].text
entry_list.append(
{
"link": link,
"entry_img": entry_img,
"entry_title": entry_title,
"author": author,
"author_link": author_link,
"channel": channel,
"channel_link": channel_link,
"views": views,
}
)
return render_template(
"contest.html",
title=title,
img=img,
entry_count=entry_count,
prizes=prizes,
info=info,
entry_list=entry_list,
)
@app.route("/contest/")
def route_contests():
try:
data = urlopen("https://www.instructables.com/contest/")
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
contest_count = str(soup.select("p.contest-count")[0])
contests = []
for contest in soup.select("div#cur-contests div.row-fluid div.contest-banner"):
link = contest.select("div.contest-banner-inner a")[0].get("href")
img = proxy(contest.select("div.contest-banner-inner a img")[0].get("src"))
alt = contest.select("div.contest-banner-inner a img")[0].get("alt")
deadline = contest.select("span.contest-meta-deadline")[0].get("data-deadline")
prizes = contest.select("span.contest-meta-count")[0].text
entries = contest.select("span.contest-meta-count")[1].text
contests.append(
{
"link": link,
"img": img,
"alt": alt,
"deadline": deadline,
"prizes": prizes,
"entries": entries,
}
)
closed = []
for display in soup.select("div.contest-winner-display"):
link = display.select("div.contest-banner-inner a")[0].get("href")
img = proxy(display.select("div.contest-banner-inner a img")[0].get("src"))
alt = display.select("div.contest-banner-inner a img")[0].get("alt")
featured_items = []
for featured_item in display.select("ul.featured-items li"):
item_link = featured_item.select("div.ible-thumb a")[0].get("href")
item_img = proxy(featured_item.select("div.ible-thumb a img")[0].get("src"))
item_title = featured_item.select("a.title")[0].text
item_author = featured_item.select("a.author")[0].text
item_author_link = featured_item.select("a.author")[0].get("href")
featured_items.append(
{
"link": item_link,
"img": item_img,
"title": item_title,
"author": item_author,
"author_link": item_author_link,
}
)
closed.append(
{"link": link, "img": img, "alt": alt, "featured_items": featured_items}
)
return render_template(
"contests.html",
title="Contests",
contest_count=contest_count,
contests=contests,
closed=closed,
)
@app.route("/<category>/<channel>/projects/")
def route_channel_projects(category, channel):
return project_list(channel.title())
@app.route("/<category>/<channel>/projects/<sort>/")
def route_channel_projects_sort(category, channel, sort):
return project_list(
channel.title(),
" Sorted by " + sort.title(),
)
@app.route("/<category>/projects/")
def route_category_projects(category):
return project_list(category.title())
@app.route("/<category>/projects/<sort>/")
def route_category_projects_sort(category, sort):
return project_list(category.title(), " Sorted by " + sort.title())
@app.route("/projects/")
def route_projects():
return project_list("")
@app.route("/search", methods=["POST", "GET"])
def route_search():
return project_list("Search")
@app.route("/projects/<sort>/")
def route_projects_sort(sort):
return project_list("", " Sorted by " + sort.title())
@app.route("/circuits/")
def route_circuits():
return category_page("Circuits")
@app.route("/workshop/")
def route_workshop():
return category_page("Workshop")
@app.route("/craft/")
def route_craft():
return category_page("Craft")
@app.route("/cooking/")
def route_cooking():
return category_page("Cooking")
@app.route("/living/")
def route_living():
return category_page("Living")
@app.route("/outside/")
def route_outside():
return category_page("Outside")
@app.route("/teachers/")
def route_teachers():
return category_page("Teachers", True)
@app.route("/member/<member>/instructables/")
def route_member_instructables(member):
try:
data = urlopen(f"https://www.instructables.com/member/{member}/instructables/")
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
header = soup.select(".profile-header.profile-header-social")[0]
header_content = member_header(header)
ibles = soup.select("ul.ible-list-items")[0]
ible_list = []
for ible in ibles.select("li"):
link = ible.select("div.thumbnail-image")[0].a.get("href")
img = proxy(ible.select("div.thumbnail-image a noscript img")[0].get("src"))
title = ible.select("div.caption-inner a.title")[0].text
stats = ible.select("div.ible-stats-right-col")[0]
views = 0
if stats.select("span.ible-views") != []:
views = stats.select("span.ible-views")[0].text
favorites = 0
if stats.select("span.ible-favorites") != []:
favorites = stats.select("span.ible-favorites")[0].text
ible_list.append(
{
"link": link,
"img": img,
"title": title,
"views": views,
"favorites": favorites,
}
)
return render_template(
"member-instructables.html",
title=f"{header_content['title']}'s Instructables",
header_content=header_content,
ibles=ible_list,
)
@app.route("/member/<member>/")
def route_member(member):
headers = {
"User-Agent": "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0"
}
request = Request(
f"https://www.instructables.com/member/{member}/", headers=headers
)
try:
data = urlopen(request)
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
header_content = member_header(soup)
body = soup.select("div.member-profile-body")[0]
ible_list = body.select(".boxed-content.promoted-content")
ible_list_title = ""
ibles = []
if ible_list != []:
ible_list = ible_list[0]
ible_list_title = ible_list.select("h2.module-title")[0].text
for ible in ible_list.select("ul.promoted-items li"):
ible_title = ible.get("data-title")
ible_link = ible.select("div.image-wrapper")[0].a.get("href")
ible_img = proxy(ible.select("div.image-wrapper a img")[0].get("src"))
ibles.append({"title": ible_title, "link": ible_link, "img": ible_img})
ach_list = body.select(
"div.two-col-section div.right-col-section.centered-sidebar div.boxed-content.about-me"
)
ach_list_title = ""
achs = []
if len(ach_list) > 1:
ach_list = ach_list[1]
ach_list_title = ach_list.select("h2.module-title")[0].text
for ach in ach_list.select(
"div.achievements-section.main-achievements.contest-achievements div.achievement-item:not(.two-column-filler)"
):
ach_title = ach.select("div.achievement-info span.achievement-title")[
0
].text
ach_desc = ach.select("div.achievement-info span.achievement-description")[
0
].text
achs.append([ach_title, ach_desc])
return render_template(
"member.html",
title=header_content["title"] + "'s Profile",
header_content=header_content,
ible_list_title=ible_list_title,
ibles=ibles,
ach_list_title=ach_list_title,
achs=achs,
)
@app.route("/<article>/")
def route_article(article):
try:
data = urlopen(
f"https://www.instructables.com/json-api/showInstructableModel?urlString={article}"
)
data = json.loads(data.read().decode())
except HTTPError as e:
abort(e.code)
try:
title = data["title"]
author = data["author"]["screenName"]
author_link = f"/member/{author}"
category = data["classifications"][0]["title"]
category_slug = data["classifications"][0]["name"]
category_link = f"/{category_slug}/"
channel = data["classifications"][0]["channels"][0]["title"]
channel_slug = data["classifications"][0]["channels"][0]["name"]
channel_link = f"/{category_slug}/{channel_slug}/"
views = data["views"]
favorites = data["favorites"]
if "steps" in data:
steps = []
if "supplies" in data:
supplies = data["supplies"]
supplies_files = []
if "suppliesFiles" in data:
supplies_files = data["suppliesFiles"]
data["steps"].insert(
1, {"title": "Supplies", "body": supplies, "files": supplies_files}
)
for step in data["steps"]:
step_title = step["title"]
print(step_title)
step_imgs = []
step_videos = [] # TODO: Check if this is still required
step_iframes = []
step_downloads = []
for file in step["files"]:
print(file)
if file["image"] and not "embedType" in "file":
step_imgs.append(
{"src": proxy(file["downloadUrl"]), "alt": file["name"]}
)
elif not file["image"]:
step_downloads.append(
{"src": proxy(file["downloadUrl"]), "name": file["name"]}
)
else: # Leaves us with embeds
embed_code = file["embedHtmlCode"]
soup = BeautifulSoup(embed_code, "html.parser")
iframe = soup.select("iframe")[0]
src = iframe.get("src")
if src.startswith("https://content.instructables.com"):
src = src.replace(
"https://content.instructables.com",
f"/proxy/?url={src}",
)
elif invidious and src.startswith("https://www.youtube.com"):
src = src.replace("https://www.youtube.com", invidious)
elif not unsafe:
src = "/iframe/?url=" + quote(src)
step_iframes.append(
{
"src": src,
"width": file.get("width"),
"height": file.get("height"),
}
)
step_text = step["body"]
step_text = step_text.replace(
"https://content.instructables.com",
"/proxy/?url=https://content.instructables.com",
)
steps.append(
{
"title": step_title,
"imgs": step_imgs,
"text": step_text,
"videos": step_videos,
"iframes": step_iframes,
"downloads": step_downloads,
}
)
comments_list = []
comment_count = 0
# TODO: Fix comments
# comments = body.select("section.discussion")[0]
# comment_count = comments.select("h2")[0].text
# comment_list = comments.select("div.posts")
# if comment_list != []:
# comment_list = comment_list[0]
# comments_list = []
# replies_used = 0
# for comment in comment_list.select(".post.js-comment:not(.reply)"):
# comment_votes = comment.select(".votes")[0].text
# comment_author_img_src = proxy(comment.select(".avatar a noscript img")[0].get("src"))
# comment_author_img_alt = comment.select(".avatar a noscript img")[0].get("alt")
# comment_author = comment.select(".posted-by a")[0].text
# comment_author_link = comment.select(".posted-by a")[0].get("href")
# comment_date = comment.select(".posted-by p.posted-date")[0].text
# comment_text = comment.select("div.text p")[0]
# comment_reply_count = comment.select("button.js-show-replies")
# if comment_reply_count != []:
# comment_reply_count = comment_reply_count[0].get("data-num-hidden")
# else:
# comment_reply_count = 0
# reply_list = []
# for index, reply in enumerate(comment_list.select(".post.js-comment:not(.reply) ~ .post.js-comment.reply.hide:has(~.post.js-comment:not(.reply))")[replies_used:int(comment_reply_count) + replies_used]):
# reply_votes = reply.select(".votes")[0].text
# reply_author_img_src = proxy(reply.select(".avatar a noscript img")[0].get("src"))
# reply_author_img_alt = reply.select(".avatar a noscript img")[0].get("alt")
# reply_author = reply.select(".posted-by a")[0].text
# reply_author_link = reply.select(".posted-by a")[0].get("href")
# reply_date = reply.select(".posted-by p.posted-date")[0].text
# reply_text = reply.select("div.text p")[0]
# reply_list.append([reply_votes, reply_author_img_src, reply_author_img_alt, reply_author, reply_author_link, reply_date, reply_text])
# replies_used += 1
# comments_list.append([comment_votes, comment_author_img_src, comment_author_img_alt, comment_author, comment_author_link, comment_date, comment_text, comment_reply_count, reply_list])
return render_template(
"article.html",
title=title,
author=author,
author_link=author_link,
category=category,
category_link=category_link,
channel=channel,
channel_link=channel_link,
views=views,
favorites=favorites,
steps=steps,
comment_count=comment_count,
comments_list=comments_list,
enumerate=enumerate,
)
else:
## Collections
thumbnails = []
for thumbnail in data["instructables"]:
text = thumbnail["title"]
link = thumbnail["showUrl"]
img = proxy(thumbnail["downloadUrl"])
thumbnail_title = thumbnail["title"]
thumbnail_author = thumbnail["author"]["screenName"]
thumbnail_author_link = f"/member/{thumbnail_author}"
thumbnail_channel = thumbnail["classifications"][0]["channels"][0][
"title"
]
thumbnail_category = thumbnail["classifications"][0]["title"]
thumbnail_channel_link = f"/{thumbnail_category}/{thumbnail_channel}"
thumbnails.append(
{
"text": text,
"link": link,
"img": img,
"title": thumbnail_title,
"author": thumbnail_author,
"author_link": thumbnail_author_link,
"channel": thumbnail_channel,
"channel_link": thumbnail_channel_link,
}
)
print(thumbnails[-1])
return render_template(
"collection.html",
title=title,
author=author,
author_link=author_link,
category=category,
category_link=category_link,
channel=channel,
channel_link=channel_link,
views=views,
favorites=favorites,
thumbnails=thumbnails,
)
except Exception:
print_exc()
raise InternalServerError()
@app.route("/<category>/<channel>/")
def route_channel_redirect(category, channel):
# TODO: Just check if the channel exists
if (
category == "circuits"
or category == "workshop"
or category == "craft"
or category == "cooking"
or category == "living"
or category == "outside"
or category == "teachers"
):
return redirect(f"/{category}/{channel}/projects/", 307)
else:
raise NotFound()
@app.route("/")
def route_explore():
try:
data = urlopen("https://www.instructables.com/")
except HTTPError as e:
abort(e.code)
soup = BeautifulSoup(data.read().decode(), "html.parser")
explore = soup.select(".home-content-explore-wrap")[0]
title = explore.select("h2")[0].text
circuits = explore_lists(
explore.select(".home-content-explore-category-circuits")[0]
)
workshop = explore_lists(
explore.select(".home-content-explore-category-workshop")[0]
)
craft = explore_lists(explore.select(".home-content-explore-category-craft")[0])
cooking = explore_lists(explore.select(".home-content-explore-category-cooking")[0])
living = explore_lists(explore.select(".home-content-explore-category-living")[0])
outside = explore_lists(explore.select(".home-content-explore-category-outside")[0])
teachers = explore_lists(
explore.select(".home-content-explore-category-teachers")[0]
)
return render_template(
"index.html",
title=title,
sections=[
("Circuits", "/circuits", circuits),
("Workshop", "/workshop", workshop),
("Craft", "/craft", craft),
("Cooking", "/cooking", cooking),
("Living", "/living", living),
("Outside", "/outside", outside),
("Teachers", "/teachers", teachers),
],
)
@app.route("/proxy/")
def route_proxy():
url = request.args.get("url")
if url is not None:
if url.startswith("https://cdn.instructables.com/") or url.startswith(
"https://content.instructables.com/"
):
def generate():
# Subfunction to allow streaming the data instead of
# downloading all of it at once
try:
with urlopen(unquote(url)) as data:
while True:
chunk = data.read(1024 * 1024)
if not chunk:
break
yield chunk
except HTTPError as e:
abort(e.code)
try:
with urlopen(unquote(url)) as data:
content_type = data.headers["content-type"]
except HTTPError as e:
abort(e.code)
except KeyError:
raise InternalServerError()
return Response(generate(), content_type=content_type)
else:
raise BadRequest()
else:
raise BadRequest()
@app.route("/iframe/")
def route_iframe():
url = request.args.get("url")
url = unquote(url)
if url != None:
return render_template("iframe.html", url=url)
else:
raise BadRequest()
@app.route("/privacypolicy/")
def privacypolicy():
content = "No privacy policy found."
try:
with (pathlib.Path(__file__).parent / "privacy.txt").open() as f:
content = f.read()
except:
pass
return render_template(
"privacypolicy.html", title="Privacy Policy", content=content
)
@app.errorhandler(404)
def not_found(e):
return render_template("404.html")
@app.errorhandler(400)
def bad_request(e):
return render_template("400.html")
@app.errorhandler(429)
def too_many_requests(e):
return render_template("429.html")
@app.errorhandler(500)
def internal_server_error(e):
return render_template("500.html")
if __name__ == "__main__":
app.run(port=args.port, host=args.listen_host, debug=debugmode)