Replaced selenium with playwright
Fixed requirements Added source links to footer
This commit is contained in:
parent
586b916ed5
commit
823a44f664
8 changed files with 483 additions and 2941 deletions
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
venv/
|
||||||
|
*.pyc
|
||||||
|
__pycache__/
|
||||||
|
.vscode
|
16
README.md
16
README.md
|
@ -1,24 +1,36 @@
|
||||||
<div align="center">
|
<div align="center">
|
||||||
<img src="static/img/logo.png">
|
<img src="static/img/logo.png">
|
||||||
<h1>Indestructables</h1>
|
<h1>Indestructables</h1>
|
||||||
An open source alternative front-end to Instructables
|
An open source alternative front-end to Instructables. This is a fork of <a href="https://codeberg.org/snowcatridge10/indestructables">snowcatridge10's Indestructables</a> to use Playwright instead of Selenium.
|
||||||
|
|
||||||
|
<a href="https://matrix.to/#/#indestructables:fedora.im">snowcatridge10's Matrix Room</a>
|
||||||
|
|
||||||
<a href="https://matrix.to/#/#indestructables:fedora.im">Matrix</a>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
# Instances
|
# Instances
|
||||||
|
|
||||||
None, yet!
|
None, yet!
|
||||||
|
|
||||||
# Run your own instance
|
# Run your own instance
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
|
First, create a virtual environment with `python3 -m venv venv` and activate it with `source venv/bin/activate`. Then, install the dependencies with:
|
||||||
|
|
||||||
`pip3 install -r requirements.txt`.
|
`pip3 install -r requirements.txt`.
|
||||||
|
|
||||||
For the production environment, you also need the uWSGI Python3 plugin. On Debian, it can be installed via `apt install uwsgi-plugin-python3`
|
For the production environment, you also need the uWSGI Python3 plugin. On Debian, it can be installed via `apt install uwsgi-plugin-python3`
|
||||||
|
|
||||||
|
Furthermore, you need to install the Chromium binary used by Playwright. You can do this by running `playwright install chromium`.
|
||||||
|
|
||||||
## Production
|
## Production
|
||||||
|
|
||||||
1. Clone the repository
|
1. Clone the repository
|
||||||
2. Run `uwsgi --plugin python3 --http-socket 0.0.0.0:8002 --wsgi-file main.py --callable app --processes 4 --threads 2`
|
2. Run `uwsgi --plugin python3 --http-socket 0.0.0.0:8002 --wsgi-file main.py --callable app --processes 4 --threads 2`
|
||||||
3. Point your reverse proxy to http://localhost:8002
|
3. Point your reverse proxy to http://localhost:8002
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
1. Clone the repository
|
1. Clone the repository
|
||||||
2. Run `python3 main.py`
|
2. Run `python3 main.py`
|
||||||
3. Connect to http://localhost:8002
|
3. Connect to http://localhost:8002
|
||||||
|
|
BIN
chromedriver
BIN
chromedriver
Binary file not shown.
BIN
chromedriver.exe
BIN
chromedriver.exe
Binary file not shown.
2777
geckodriver.log
2777
geckodriver.log
File diff suppressed because it is too large
Load diff
621
main.py
621
main.py
|
@ -1,47 +1,64 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
from flask import Flask, render_template, request, redirect, Response, stream_with_context
|
from flask import (
|
||||||
|
Flask,
|
||||||
|
render_template,
|
||||||
|
request,
|
||||||
|
redirect,
|
||||||
|
Response,
|
||||||
|
stream_with_context,
|
||||||
|
)
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from urllib.parse import quote, unquote
|
from urllib.parse import quote, unquote
|
||||||
from traceback import print_exc
|
from traceback import print_exc
|
||||||
from requests_html import HTMLSession
|
from requests_html import HTMLSession
|
||||||
|
from playwright.sync_api import sync_playwright
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
|
||||||
from selenium import webdriver
|
|
||||||
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
|
||||||
from selenium.webdriver.common.by import By
|
|
||||||
from selenium.webdriver.support.ui import WebDriverWait
|
|
||||||
from selenium.webdriver.support import expected_conditions as EC
|
|
||||||
from selenium.webdriver.common.keys import Keys
|
|
||||||
from selenium.webdriver.firefox.options import Options
|
|
||||||
from selenium.webdriver.chrome.options import Options
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
debugmode = False
|
debugmode = False
|
||||||
|
|
||||||
try:
|
if __name__ == "__main__":
|
||||||
if sys.argv[1] == "debug":
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"-p",
|
||||||
|
"--port",
|
||||||
|
default=8002,
|
||||||
|
type=int,
|
||||||
|
help="Port to listen on",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-d",
|
||||||
|
"--debug",
|
||||||
|
action="store_true",
|
||||||
|
help="Enable debug mode",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-l",
|
||||||
|
"--listen-host",
|
||||||
|
default="127.0.0.1",
|
||||||
|
help="Host to listen on",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.debug:
|
||||||
debugmode = True
|
debugmode = True
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
print("Loading...")
|
print("Loading...")
|
||||||
|
|
||||||
def proxy(src):
|
def proxy(src):
|
||||||
return "/proxy/?url=" + quote(str(src))
|
return "/proxy/?url=" + quote(str(src))
|
||||||
|
|
||||||
instance_root_url = "http://127.0.0.1:8002"
|
def get_instance_root_url(request):
|
||||||
|
return request.url_root
|
||||||
|
|
||||||
chrome_options = Options()
|
playwright = sync_playwright().start()
|
||||||
chrome_options.add_argument("--headless")
|
browser = playwright.chromium.launch(headless=True)
|
||||||
|
page = browser.new_page()
|
||||||
if os.name == 'nt':
|
|
||||||
# Windows
|
|
||||||
driver = webdriver.Chrome('./chromedriver.exe', options=chrome_options)
|
|
||||||
else:
|
|
||||||
# Linux
|
|
||||||
driver = webdriver.Chrome('./chromedriver', options=chrome_options)
|
|
||||||
|
|
||||||
channels = []
|
channels = []
|
||||||
|
|
||||||
|
@ -59,49 +76,75 @@ global_ibles = {}
|
||||||
|
|
||||||
global_ibles["/projects"] = []
|
global_ibles["/projects"] = []
|
||||||
|
|
||||||
driver.get("https://www.instructables.com/projects")
|
page.goto("https://www.instructables.com/projects")
|
||||||
|
|
||||||
while len(global_ibles["/projects"]) <= 0:
|
while len(global_ibles["/projects"]) <= 0:
|
||||||
for ible in driver.find_elements(By.CLASS_NAME, "ibleCard__QPJVm"):
|
for ible in page.query_selector_all(".ibleCard__QPJVm"):
|
||||||
link = ible.find_elements(By.CSS_SELECTOR, "a")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
link = (
|
||||||
img = proxy(ible.find_elements(By.CSS_SELECTOR, "img")[0].get_attribute("src"))
|
ible.query_selector("a")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
|
img = proxy(ible.query_selector("img").get_attribute("src"))
|
||||||
|
|
||||||
title = ible.find_elements(By.CLASS_NAME, "title__t0fGQ")[0].text
|
title = ible.query_selector(".title__t0fGQ").inner_text()
|
||||||
author = ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0].text
|
author = ible.query_selector("a[href^='/member/']").inner_text()
|
||||||
author_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
author_link = (
|
||||||
|
ible.query_selector("a[href^='/member/']")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
|
|
||||||
channel = "TEST"
|
channel = "TEST"
|
||||||
channel_link = "TEST"
|
channel_link = "TEST"
|
||||||
|
|
||||||
for c in channels:
|
for c in channels:
|
||||||
try:
|
try:
|
||||||
channel = ible.find_elements(By.CSS_SELECTOR, "a[href^='/" + c + "']")[0].text
|
channel = ible.query_selector("a[href^='/" + c + "']").inner_text()
|
||||||
channel_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/" + c + "']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
channel_link = (
|
||||||
|
ible.query_selector("a[href^='/" + c + "']")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
except:
|
except:
|
||||||
try:
|
try:
|
||||||
channel = ible.find_elements(By.CSS_SELECTOR, "a[href^='/projects/']")[0].text
|
channel = ible.query_selector("a[href^='/projects/']").inner_text()
|
||||||
channel_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/projects/']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
channel_link = (
|
||||||
|
ible.query_selector("a[href^='/projects/']")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
stats = ible.find_elements(By.CLASS_NAME, "stats__GFKyl")[0]
|
stats = ible.query_selector(".stats__GFKyl")
|
||||||
views = 0
|
views = 0
|
||||||
if stats.find_elements(By.CSS_SELECTOR, "div[title$=' views']") != []:
|
if stats.query_selector("div[title$=' views']"):
|
||||||
views = stats.find_elements(By.CSS_SELECTOR, "div[title$=' views']")[0].text
|
views = stats.query_selector("div[title$=' views']").inner_text()
|
||||||
favorites = 0
|
favorites = 0
|
||||||
if stats.find_elements(By.CSS_SELECTOR, "div[title$=' favorites']") != []:
|
if stats.query_selector("div[title$=' favorites']"):
|
||||||
favorites = stats.find_elements(By.CSS_SELECTOR, "div[title$=' favorites']")[0].text
|
favorites = stats.query_selector("div[title$=' favorites']").inner_text()
|
||||||
|
|
||||||
global_ibles["/projects"].append([link, img, title, author, author_link, channel, channel_link, views, favorites])
|
global_ibles["/projects"].append(
|
||||||
|
[
|
||||||
|
link,
|
||||||
|
img,
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
favorites,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
firefox_capabilities = DesiredCapabilities.FIREFOX
|
browser.close()
|
||||||
firefox_capabilities['marionette'] = True
|
playwright.stop()
|
||||||
firefox_capabilities['binary'] = "C:/Program Files/Mozilla Firefox/firefox.exe"
|
|
||||||
options = Options()
|
|
||||||
options.add_argument("--headless")
|
|
||||||
|
|
||||||
print("Started!")
|
print("Started!")
|
||||||
|
|
||||||
|
|
||||||
def explore_lists(soup):
|
def explore_lists(soup):
|
||||||
list_ = []
|
list_ = []
|
||||||
for ible in soup.select(".home-content-explore-ible"):
|
for ible in soup.select(".home-content-explore-ible"):
|
||||||
|
@ -119,12 +162,30 @@ def explore_lists(soup):
|
||||||
favorites = 0
|
favorites = 0
|
||||||
if ible.select("span.ible-favorites") != []:
|
if ible.select("span.ible-favorites") != []:
|
||||||
favorites = ible.select("span.ible-favorites")[0].text
|
favorites = ible.select("span.ible-favorites")[0].text
|
||||||
list_.append([link, img, alt, title, author, author_link, channel, channel_link, favorites, views])
|
list_.append(
|
||||||
|
[
|
||||||
|
link,
|
||||||
|
img,
|
||||||
|
alt,
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
favorites,
|
||||||
|
views,
|
||||||
|
]
|
||||||
|
)
|
||||||
return list_
|
return list_
|
||||||
|
|
||||||
|
|
||||||
def member_header(header):
|
def member_header(header):
|
||||||
avatar = proxy(header.select("div.profile-avatar-container img.profile-avatar")[0].get("src"))
|
avatar = proxy(
|
||||||
title = header.select("div.profile-top div.profile-headline h1.profile-title")[0].text
|
header.select("div.profile-avatar-container img.profile-avatar")[0].get("src")
|
||||||
|
)
|
||||||
|
title = header.select("div.profile-top div.profile-headline h1.profile-title")[
|
||||||
|
0
|
||||||
|
].text
|
||||||
|
|
||||||
profile_top = header.select("div.profile-top")[0]
|
profile_top = header.select("div.profile-top")[0]
|
||||||
|
|
||||||
|
@ -133,7 +194,6 @@ def member_header(header):
|
||||||
# stats_text = profile_top.select("div.profile-header-stats")[0]
|
# stats_text = profile_top.select("div.profile-header-stats")[0]
|
||||||
# stats_num = header.select("div.profile-top div.profile-header-stats")[1]
|
# stats_num = header.select("div.profile-top div.profile-header-stats")[1]
|
||||||
|
|
||||||
|
|
||||||
location = header.select("span.member-location")
|
location = header.select("span.member-location")
|
||||||
if location != []:
|
if location != []:
|
||||||
location = location[0].text
|
location = location[0].text
|
||||||
|
@ -146,7 +206,6 @@ def member_header(header):
|
||||||
else:
|
else:
|
||||||
signup = 0
|
signup = 0
|
||||||
|
|
||||||
|
|
||||||
instructables = header.select("span.ible-count")
|
instructables = header.select("span.ible-count")
|
||||||
if instructables != []:
|
if instructables != []:
|
||||||
instructables = instructables[0].text
|
instructables = instructables[0].text
|
||||||
|
@ -177,25 +236,42 @@ def member_header(header):
|
||||||
else:
|
else:
|
||||||
bio = ""
|
bio = ""
|
||||||
|
|
||||||
return [avatar, title, location, signup, instructables, views, comments, followers, bio]
|
return [
|
||||||
|
avatar,
|
||||||
|
title,
|
||||||
|
location,
|
||||||
|
signup,
|
||||||
|
instructables,
|
||||||
|
views,
|
||||||
|
comments,
|
||||||
|
followers,
|
||||||
|
bio,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def category_page(path, name, teachers=False):
|
def category_page(path, name, teachers=False):
|
||||||
data = requests.get("https://www.instructables.com" + path)
|
data = requests.get("https://www.instructables.com" + path)
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
channels = []
|
channels = []
|
||||||
for card in soup.select("div.scrollable-cards-inner div.scrollable-card"):
|
for card in soup.select("div.scrollable-cards-inner div.scrollable-card"):
|
||||||
link = card.a["href"]
|
link = card.a["href"]
|
||||||
img = proxy(card.select(f"a{' noscript' if teachers else ''} img")[0].get("src"))
|
img = proxy(
|
||||||
|
card.select(f"a{' noscript' if teachers else ''} img")[0].get("src")
|
||||||
|
)
|
||||||
title = card.select("a img")[0].get("alt")
|
title = card.select("a img")[0].get("alt")
|
||||||
|
|
||||||
channels.append([link, title, img])
|
channels.append([link, title, img])
|
||||||
|
|
||||||
ibles = []
|
ibles = []
|
||||||
for ible in soup.select("div.category-landing-projects-list div.category-landing-projects-ible"):
|
for ible in soup.select(
|
||||||
|
"div.category-landing-projects-list div.category-landing-projects-ible"
|
||||||
|
):
|
||||||
link = ible.a["href"]
|
link = ible.a["href"]
|
||||||
img = proxy(ible.select("a noscript img")[0].get("src"))
|
img = proxy(ible.select("a noscript img")[0].get("src"))
|
||||||
|
|
||||||
|
@ -214,75 +290,134 @@ def category_page(path, name, teachers=False):
|
||||||
if stats.select("span.ible-favorites") != []:
|
if stats.select("span.ible-favorites") != []:
|
||||||
favorites = stats.select("span.ible-favorites")[0].text
|
favorites = stats.select("span.ible-favorites")[0].text
|
||||||
|
|
||||||
ibles.append([link, img, title, author, author_link, channel, channel_link, views, favorites])
|
ibles.append(
|
||||||
|
[
|
||||||
|
link,
|
||||||
|
img,
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
favorites,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
contests = []
|
contests = []
|
||||||
for contest in soup.select("div.category-landing-contests-list div.category-landing-contests-item"):
|
for contest in soup.select(
|
||||||
|
"div.category-landing-contests-list div.category-landing-contests-item"
|
||||||
|
):
|
||||||
link = contest.a["href"]
|
link = contest.a["href"]
|
||||||
img = proxy(contest.select("a noscript img")[0].get("src"))
|
img = proxy(contest.select("a noscript img")[0].get("src"))
|
||||||
title = contest.select("a img")[0].get("alt")
|
title = contest.select("a img")[0].get("alt")
|
||||||
|
|
||||||
contests.append([link, img, title])
|
contests.append([link, img, title])
|
||||||
|
|
||||||
return render_template("category.html", data=[name, channels, ibles, contests, path])
|
return render_template(
|
||||||
|
"category.html", data=[name, channels, ibles, contests, path]
|
||||||
|
)
|
||||||
|
|
||||||
def project_list(path, head, sort=''):
|
|
||||||
#driver = webdriver.Firefox(options=options, capabilities=firefox_capabilities)
|
|
||||||
|
|
||||||
driver.get("https://www.instructables.com" + path)
|
def project_list(path, head, sort=""):
|
||||||
|
playwright = sync_playwright().start()
|
||||||
|
browser = playwright.chromium.launch(headless=True)
|
||||||
|
page = browser.new_page()
|
||||||
|
|
||||||
|
page.goto(urljoin("https://www.instructables.com", path))
|
||||||
|
|
||||||
head = f"{head + ' ' if head != '' else ''}Projects" + sort
|
head = f"{head + ' ' if head != '' else ''}Projects" + sort
|
||||||
path_ = path.rsplit('/', 1)[0]
|
path_ = path.rsplit("/", 1)[0]
|
||||||
|
|
||||||
if path == "/projects/" or path == "/projects":
|
if path == "/projects/" or path == "/projects":
|
||||||
ibles = global_ibles["/projects"]
|
ibles = global_ibles["/projects"]
|
||||||
else:
|
else:
|
||||||
ibles = []
|
ibles = []
|
||||||
|
|
||||||
for ible in driver.find_elements(By.CLASS_NAME, "ibleCard__QPJVm"):
|
for ible in page.query_selector_all(".ibleCard__QPJVm"):
|
||||||
link = ible.find_elements(By.CSS_SELECTOR, "a")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
link = (
|
||||||
img = proxy(ible.find_elements(By.CSS_SELECTOR, "img")[0].get_attribute("src"))
|
ible.query_selector("a")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
|
img = proxy(
|
||||||
|
ible.find_elements(By.CSS_SELECTOR, "img")[0].get_attribute("src")
|
||||||
|
)
|
||||||
|
|
||||||
title = ible.find_elements(By.CLASS_NAME, "title__t0fGQ")[0].text
|
title = ible.find_elements(By.CLASS_NAME, "title__t0fGQ")[0].text
|
||||||
author = ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0].text
|
author = ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0].text
|
||||||
author_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
author_link = (
|
||||||
|
ible.find_elements(By.CSS_SELECTOR, "a[href^='/member/']")[0]
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
|
|
||||||
channel = "TEST"
|
channel = "TEST"
|
||||||
channel_link = "TEST"
|
channel_link = "TEST"
|
||||||
|
|
||||||
for c in channels:
|
for c in channels:
|
||||||
try:
|
try:
|
||||||
channel = ible.find_elements(By.CSS_SELECTOR, "a[href^='/" + c + "']")[0].text
|
channel = ible.query_selector("a[href^='/" + c + "']").inner_text()
|
||||||
channel_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/" + c + "']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
channel_link = (
|
||||||
|
ible.query_selector("a[href^='/" + c + "']")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
except:
|
except:
|
||||||
try:
|
try:
|
||||||
channel = ible.find_elements(By.CSS_SELECTOR, "a[href^='/projects/']")[0].text
|
channel = ible.query_selector("a[href^='/projects/'] span").inner_text()
|
||||||
channel_link = ible.find_elements(By.CSS_SELECTOR, "a[href^='/projects/']")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
channel_link = (
|
||||||
|
ible.query_selector("a[href^='/projects/']")
|
||||||
|
.get_attribute("href")
|
||||||
|
.replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
stats = ible.find_elements(By.CLASS_NAME, "stats__GFKyl")[0]
|
stats = ible.query_selector(".stats__GFKyl")
|
||||||
views = 0
|
views = 0
|
||||||
if stats.find_elements(By.CSS_SELECTOR, "div[title$=' views']") != []:
|
|
||||||
views = stats.find_elements(By.CSS_SELECTOR, "div[title$=' views']")[0].text
|
|
||||||
favorites = 0
|
|
||||||
if stats.find_elements(By.CSS_SELECTOR, "div[title$=' favorites']") != []:
|
|
||||||
favorites = stats.find_elements(By.CSS_SELECTOR, "div[title$=' favorites']")[0].text
|
|
||||||
|
|
||||||
ibles.append([link, img, title, author, author_link, channel, channel_link, views, favorites])
|
if stats.query_selector("div[title$=' views']"):
|
||||||
|
views = stats.query_selector("div[title$=' views']").inner_text()
|
||||||
|
|
||||||
|
favorites = 0
|
||||||
|
|
||||||
|
if stats.query_selector("div[title$=' favorites']"):
|
||||||
|
favorites = stats.query_selector("div[title$=' favorites']").inner_text()
|
||||||
|
|
||||||
|
ibles.append(
|
||||||
|
[
|
||||||
|
link,
|
||||||
|
img,
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
favorites,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
if len(ibles) >= 8:
|
if len(ibles) >= 8:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
browser.close()
|
||||||
|
playwright.stop()
|
||||||
|
|
||||||
return render_template("projects.html", data=[head, ibles, path_])
|
return render_template("projects.html", data=[head, ibles, path_])
|
||||||
|
|
||||||
|
|
||||||
app = Flask(__name__, template_folder="templates", static_folder="static")
|
app = Flask(__name__, template_folder="templates", static_folder="static")
|
||||||
|
|
||||||
@app.route('/sitemap/')
|
|
||||||
|
@app.route("/sitemap/")
|
||||||
def route_sitemap():
|
def route_sitemap():
|
||||||
data = requests.get(f"https://www.instructables.com/sitemap/")
|
data = requests.get(f"https://www.instructables.com/sitemap/")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -301,14 +436,17 @@ def route_sitemap():
|
||||||
|
|
||||||
return render_template("sitemap.html", data=groups)
|
return render_template("sitemap.html", data=groups)
|
||||||
|
|
||||||
@app.route('/contest/archive/')
|
|
||||||
|
@app.route("/contest/archive/")
|
||||||
def route_contest_archive():
|
def route_contest_archive():
|
||||||
page = 1
|
page = 1
|
||||||
if request.args.get("page") != None:
|
if request.args.get("page") != None:
|
||||||
page = request.args.get("page")
|
page = request.args.get("page")
|
||||||
data = requests.get(f"https://www.instructables.com/contest/archive/?page={page}")
|
data = requests.get(f"https://www.instructables.com/contest/archive/?page={page}")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -318,7 +456,9 @@ def route_contest_archive():
|
||||||
|
|
||||||
contest_list = []
|
contest_list = []
|
||||||
for index, year in enumerate(main.select("div.contest-archive-list h2")):
|
for index, year in enumerate(main.select("div.contest-archive-list h2")):
|
||||||
year_list = main.select("div.contest-archive-list div.contest-archive-list-year")[index]
|
year_list = main.select(
|
||||||
|
"div.contest-archive-list div.contest-archive-list-year"
|
||||||
|
)[index]
|
||||||
year_name = year.text
|
year_name = year.text
|
||||||
month_list = []
|
month_list = []
|
||||||
for month in year_list.select("div.contest-archive-list-month"):
|
for month in year_list.select("div.contest-archive-list-month"):
|
||||||
|
@ -334,31 +474,35 @@ def route_contest_archive():
|
||||||
|
|
||||||
pagination = main.select("nav.pagination ul.pagination")[0]
|
pagination = main.select("nav.pagination ul.pagination")[0]
|
||||||
|
|
||||||
return render_template("archives.html", data=[page, contest_count, pagination, contest_list])
|
return render_template(
|
||||||
|
"archives.html", data=[page, contest_count, pagination, contest_list]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/contest/<contest>/')
|
@app.route("/contest/<contest>/")
|
||||||
def route_contest(contest):
|
def route_contest(contest):
|
||||||
data = requests.get(f"https://www.instructables.com/contest/{contest}/")
|
data = requests.get(f"https://www.instructables.com/contest/{contest}/")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
title = soup.select("meta[property=\"og:title\"]")[0].get("content")
|
title = soup.select('meta[property="og:title"]')[0].get("content")
|
||||||
|
|
||||||
body = soup.select("div#contest-wrapper")[0]
|
body = soup.select("div#contest-wrapper")[0]
|
||||||
|
|
||||||
img = proxy(body.select("div#contest-masthead img")[0].get("src"))
|
img = proxy(body.select("div#contest-masthead img")[0].get("src"))
|
||||||
|
|
||||||
entry_count = body.select("li.entries-nav-btn")[0].text.split(' ')[0]
|
entry_count = body.select("li.entries-nav-btn")[0].text.split(" ")[0]
|
||||||
prizes = body.select("li.prizes-nav-btn")[0].text.split(' ')[0]
|
prizes = body.select("li.prizes-nav-btn")[0].text.split(" ")[0]
|
||||||
|
|
||||||
info = body.select("div.contest-body-column-left")[0]
|
info = body.select("div.contest-body-column-left")[0]
|
||||||
info.select("div#site-announcements-page")[0].decompose()
|
info.select("div#site-announcements-page")[0].decompose()
|
||||||
info.select("h3")[0].decompose()
|
info.select("h3")[0].decompose()
|
||||||
info.select("div#contest-body-nav")[0].decompose()
|
info.select("div#contest-body-nav")[0].decompose()
|
||||||
info = str(info).replace("https://www.instructables.com", '')
|
info = str(info).replace("https://www.instructables.com", "{instance_root_url}")
|
||||||
|
|
||||||
entries = body.select("span.contest-entity-count")[0].text
|
entries = body.select("span.contest-entity-count")[0].text
|
||||||
|
|
||||||
|
@ -373,16 +517,31 @@ def route_contest(contest):
|
||||||
channel_link = entry.select("div span.ible-channel a")[0].get("href")
|
channel_link = entry.select("div span.ible-channel a")[0].get("href")
|
||||||
views = entry.select(".ible-views")[0].text
|
views = entry.select(".ible-views")[0].text
|
||||||
|
|
||||||
entry_list.append([link, entry_img, entry_title, author, author_link, channel, channel_link, views])
|
entry_list.append(
|
||||||
|
[
|
||||||
|
link,
|
||||||
|
entry_img,
|
||||||
|
entry_title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
return render_template("contest.html", data=[title, img, entry_count, prizes, info, entry_list])
|
return render_template(
|
||||||
|
"contest.html", data=[title, img, entry_count, prizes, info, entry_list]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/contest/')
|
@app.route("/contest/")
|
||||||
def route_contests():
|
def route_contests():
|
||||||
data = requests.get("https://www.instructables.com/contest/")
|
data = requests.get("https://www.instructables.com/contest/")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -391,8 +550,8 @@ def route_contests():
|
||||||
contests = []
|
contests = []
|
||||||
for contest in soup.select("div#cur-contests div.row-fluid div.contest-banner"):
|
for contest in soup.select("div#cur-contests div.row-fluid div.contest-banner"):
|
||||||
link = contest.select("div.contest-banner-inner a")[0].get("href")
|
link = contest.select("div.contest-banner-inner a")[0].get("href")
|
||||||
img = proxy(contest.select("div.contest-banner-inner a img")[0].get("src"))
|
img = proxy(contest.select("div.contest-banner-inner a img")[0].get("src"))
|
||||||
alt = contest.select("div.contest-banner-inner a img")[0].get("alt")
|
alt = contest.select("div.contest-banner-inner a img")[0].get("alt")
|
||||||
deadline = contest.select("span.contest-meta-deadline")[0].get("data-deadline")
|
deadline = contest.select("span.contest-meta-deadline")[0].get("data-deadline")
|
||||||
prizes = contest.select("span.contest-meta-count")[0].text
|
prizes = contest.select("span.contest-meta-count")[0].text
|
||||||
entries = contest.select("span.contest-meta-count")[1].text
|
entries = contest.select("span.contest-meta-count")[1].text
|
||||||
|
@ -412,76 +571,104 @@ def route_contests():
|
||||||
item_author = featured_item.select("a.author")[0].text
|
item_author = featured_item.select("a.author")[0].text
|
||||||
item_author_link = featured_item.select("a.author")[0].get("href")
|
item_author_link = featured_item.select("a.author")[0].get("href")
|
||||||
|
|
||||||
featured_items.append([item_link, item_img, item_title, item_author, item_author_link])
|
featured_items.append(
|
||||||
|
[item_link, item_img, item_title, item_author, item_author_link]
|
||||||
|
)
|
||||||
closed.append([link, img, alt, featured_items])
|
closed.append([link, img, alt, featured_items])
|
||||||
|
|
||||||
return render_template("contests.html", data=[contest_count, contests, closed])
|
return render_template("contests.html", data=[contest_count, contests, closed])
|
||||||
|
|
||||||
@app.route('/<category>/<channel>/projects/')
|
|
||||||
|
@app.route("/<category>/<channel>/projects/")
|
||||||
def route_channel_projects(category, channel):
|
def route_channel_projects(category, channel):
|
||||||
return project_list(f"/{category}/{channel}/projects/", channel.title())
|
return project_list(f"/{category}/{channel}/projects/", channel.title())
|
||||||
|
|
||||||
@app.route('/<category>/<channel>/projects/<sort>/')
|
|
||||||
def route_channel_projects_sort(category, channel, sort):
|
|
||||||
return project_list(f"/{category}/{channel}/projects/{sort}", channel.title(), " Sorted by " + sort.title())
|
|
||||||
|
|
||||||
@app.route('/<category>/projects/')
|
@app.route("/<category>/<channel>/projects/<sort>/")
|
||||||
|
def route_channel_projects_sort(category, channel, sort):
|
||||||
|
return project_list(
|
||||||
|
f"/{category}/{channel}/projects/{sort}",
|
||||||
|
channel.title(),
|
||||||
|
" Sorted by " + sort.title(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/<category>/projects/")
|
||||||
def route_category_projects(category):
|
def route_category_projects(category):
|
||||||
return project_list(f"/{category}/projects/", category.title())
|
return project_list(f"/{category}/projects/", category.title())
|
||||||
|
|
||||||
@app.route('/<category>/projects/<sort>/')
|
|
||||||
|
@app.route("/<category>/projects/<sort>/")
|
||||||
def route_category_projects_sort(category, sort):
|
def route_category_projects_sort(category, sort):
|
||||||
return project_list(f"/{category}/projects/{sort}", category.title(), " Sorted by " + sort.title())
|
return project_list(
|
||||||
|
f"/{category}/projects/{sort}", category.title(), " Sorted by " + sort.title()
|
||||||
|
)
|
||||||
|
|
||||||
@app.route('/projects/')
|
|
||||||
|
@app.route("/projects/")
|
||||||
def route_projects():
|
def route_projects():
|
||||||
return project_list("/projects/", '')
|
return project_list("/projects/", "")
|
||||||
|
|
||||||
@app.route('/search')
|
|
||||||
|
@app.route("/search")
|
||||||
def route_search():
|
def route_search():
|
||||||
return project_list("/search/?q=" + request.args['q'] + "&projects=all", 'Search')
|
return project_list("/search/?q=" + request.args["q"] + "&projects=all", "Search")
|
||||||
|
|
||||||
@app.route('/projects/<sort>/')
|
|
||||||
|
@app.route("/projects/<sort>/")
|
||||||
def route_projects_sort(sort):
|
def route_projects_sort(sort):
|
||||||
return project_list(f"/projects/{sort}", '', " Sorted by " + sort.title())
|
return project_list(f"/projects/{sort}", "", " Sorted by " + sort.title())
|
||||||
|
|
||||||
@app.route('/circuits/')
|
|
||||||
|
@app.route("/circuits/")
|
||||||
def route_circuits():
|
def route_circuits():
|
||||||
return category_page("/circuits/", "Circuits")
|
return category_page("/circuits/", "Circuits")
|
||||||
|
|
||||||
@app.route('/workshop/')
|
|
||||||
|
@app.route("/workshop/")
|
||||||
def route_workshop():
|
def route_workshop():
|
||||||
return category_page("/workshop/", "Workshop")
|
return category_page("/workshop/", "Workshop")
|
||||||
|
|
||||||
@app.route('/craft/')
|
|
||||||
|
@app.route("/craft/")
|
||||||
def route_craft():
|
def route_craft():
|
||||||
return category_page("/craft/", "Craft")
|
return category_page("/craft/", "Craft")
|
||||||
|
|
||||||
@app.route('/cooking/')
|
|
||||||
|
@app.route("/cooking/")
|
||||||
def route_cooking():
|
def route_cooking():
|
||||||
return category_page("/cooking/", "Cooking")
|
return category_page("/cooking/", "Cooking")
|
||||||
|
|
||||||
@app.route('/living/')
|
|
||||||
|
@app.route("/living/")
|
||||||
def route_living():
|
def route_living():
|
||||||
return category_page("/living/", "Living")
|
return category_page("/living/", "Living")
|
||||||
|
|
||||||
@app.route('/outside/')
|
|
||||||
|
@app.route("/outside/")
|
||||||
def route_outside():
|
def route_outside():
|
||||||
return category_page("/outside/", "Outside")
|
return category_page("/outside/", "Outside")
|
||||||
|
|
||||||
@app.route('/teachers/')
|
|
||||||
|
@app.route("/teachers/")
|
||||||
def route_teachers():
|
def route_teachers():
|
||||||
return category_page("/teachers/", "Teachers", True)
|
return category_page("/teachers/", "Teachers", True)
|
||||||
|
|
||||||
@app.route('/sitemap/projects/<category>/<subcategory>')
|
|
||||||
def route_sitemap_circuits(category, subcategory):
|
|
||||||
return category_page("/" + category + "/" + subcategory, subcategory + " - " + category)
|
|
||||||
|
|
||||||
@app.route('/member/<member>/instructables/')
|
@app.route("/sitemap/projects/<category>/<subcategory>")
|
||||||
|
def route_sitemap_circuits(category, subcategory):
|
||||||
|
return category_page(
|
||||||
|
"/" + category + "/" + subcategory, subcategory + " - " + category
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/member/<member>/instructables/")
|
||||||
def route_member_instructables(member):
|
def route_member_instructables(member):
|
||||||
data = requests.get(f"https://www.instructables.com/member/{member}/instructables")
|
data = requests.get(f"https://www.instructables.com/member/{member}/instructables")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -505,18 +692,24 @@ def route_member_instructables(member):
|
||||||
|
|
||||||
ible_list.append([link, img, title, views, favorites])
|
ible_list.append([link, img, title, views, favorites])
|
||||||
|
|
||||||
return render_template("member-instructables.html", data=header_content + [ible_list])
|
return render_template(
|
||||||
|
"member-instructables.html", data=header_content + [ible_list]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/member/<member>/')
|
@app.route("/member/<member>/")
|
||||||
def route_member(member):
|
def route_member(member):
|
||||||
headers = {
|
headers = {
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0'
|
"User-Agent": "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0"
|
||||||
}
|
}
|
||||||
|
|
||||||
data = requests.get(f"https://www.instructables.com/member/{member}/", headers=headers)
|
data = requests.get(
|
||||||
|
f"https://www.instructables.com/member/{member}/", headers=headers
|
||||||
|
)
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -539,8 +732,9 @@ def route_member(member):
|
||||||
|
|
||||||
ibles.append([ible_title, ible_link, ible_img])
|
ibles.append([ible_title, ible_link, ible_img])
|
||||||
|
|
||||||
|
ach_list = body.select(
|
||||||
ach_list = body.select("div.two-col-section div.right-col-section.centered-sidebar div.boxed-content.about-me")
|
"div.two-col-section div.right-col-section.centered-sidebar div.boxed-content.about-me"
|
||||||
|
)
|
||||||
|
|
||||||
ach_list_title = ""
|
ach_list_title = ""
|
||||||
achs = []
|
achs = []
|
||||||
|
@ -548,19 +742,30 @@ def route_member(member):
|
||||||
if len(ach_list) > 1:
|
if len(ach_list) > 1:
|
||||||
ach_list = ach_list[1]
|
ach_list = ach_list[1]
|
||||||
ach_list_title = ach_list.select("h2.module-title")[0].text
|
ach_list_title = ach_list.select("h2.module-title")[0].text
|
||||||
for ach in ach_list.select("div.achievements-section.main-achievements.contest-achievements div.achievement-item:not(.two-column-filler)"):
|
for ach in ach_list.select(
|
||||||
ach_title = ach.select("div.achievement-info span.achievement-title")[0].text
|
"div.achievements-section.main-achievements.contest-achievements div.achievement-item:not(.two-column-filler)"
|
||||||
ach_desc = ach.select("div.achievement-info span.achievement-description")[0].text
|
):
|
||||||
|
ach_title = ach.select("div.achievement-info span.achievement-title")[
|
||||||
|
0
|
||||||
|
].text
|
||||||
|
ach_desc = ach.select("div.achievement-info span.achievement-description")[
|
||||||
|
0
|
||||||
|
].text
|
||||||
achs.append([ach_title, ach_desc])
|
achs.append([ach_title, ach_desc])
|
||||||
|
|
||||||
return render_template("member.html", data=header_content + [ible_list_title, ibles, ach_list_title, achs])
|
return render_template(
|
||||||
|
"member.html",
|
||||||
|
data=header_content + [ible_list_title, ibles, ach_list_title, achs],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/<article>/')
|
@app.route("/<article>/")
|
||||||
def route_article(article):
|
def route_article(article):
|
||||||
data = requests.get(f"https://www.instructables.com/{article}/")
|
data = requests.get(f"https://www.instructables.com/{article}/")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -597,13 +802,18 @@ def route_article(article):
|
||||||
step_imgs = []
|
step_imgs = []
|
||||||
for img in step.select("div.no-js-photoset img"):
|
for img in step.select("div.no-js-photoset img"):
|
||||||
step_imgs.append([proxy(img.get("src")), img.get("alt")])
|
step_imgs.append([proxy(img.get("src")), img.get("alt")])
|
||||||
|
|
||||||
step_videos = []
|
step_videos = []
|
||||||
for img in step.select("video"):
|
for img in step.select("video"):
|
||||||
step_videos.append([proxy(img.get("src"))])
|
step_videos.append([proxy(img.get("src"))])
|
||||||
|
|
||||||
step_text = str(step.select("div.step-body")[0])
|
step_text = str(step.select("div.step-body")[0])
|
||||||
step_text = step_text.replace("https://content.instructables.com", "/proxy/?url=https://content.instructables.com")
|
step_text = step_text.replace(
|
||||||
|
"https://content.instructables.com",
|
||||||
|
"{instance_root_url}/proxy/?url=https://content.instructables.com".format(
|
||||||
|
instance_root_url=get_instance_root_url(request)
|
||||||
|
),
|
||||||
|
)
|
||||||
steps.append([step_title, step_imgs, step_text, step_videos])
|
steps.append([step_title, step_imgs, step_text, step_videos])
|
||||||
|
|
||||||
comments_list = []
|
comments_list = []
|
||||||
|
@ -645,44 +855,123 @@ def route_article(article):
|
||||||
# replies_used += 1
|
# replies_used += 1
|
||||||
|
|
||||||
# comments_list.append([comment_votes, comment_author_img_src, comment_author_img_alt, comment_author, comment_author_link, comment_date, comment_text, comment_reply_count, reply_list])
|
# comments_list.append([comment_votes, comment_author_img_src, comment_author_img_alt, comment_author, comment_author_link, comment_date, comment_text, comment_reply_count, reply_list])
|
||||||
return render_template("article.html", data=[title, author, author_link, category, category_link, channel, channel_link, views, favorites, steps, comment_count, comments_list], enumerate=enumerate)
|
return render_template(
|
||||||
|
"article.html",
|
||||||
|
data=[
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
category,
|
||||||
|
category_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
favorites,
|
||||||
|
steps,
|
||||||
|
comment_count,
|
||||||
|
comments_list,
|
||||||
|
],
|
||||||
|
enumerate=enumerate,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
## Collections
|
## Collections
|
||||||
thumbnails = []
|
thumbnails = []
|
||||||
for thumbnail in soup.select("ul#thumbnails-list li"):
|
for thumbnail in soup.select("ul#thumbnails-list li"):
|
||||||
text = link = img = thumbnail_title = thumbnail_author = thumbnail_author_link = thumbnail_channel = thumbnail_channel_link = ''
|
text = (
|
||||||
|
link
|
||||||
|
) = (
|
||||||
|
img
|
||||||
|
) = (
|
||||||
|
thumbnail_title
|
||||||
|
) = (
|
||||||
|
thumbnail_author
|
||||||
|
) = (
|
||||||
|
thumbnail_author_link
|
||||||
|
) = thumbnail_channel = thumbnail_channel_link = ""
|
||||||
|
|
||||||
if thumbnail.select("div.thumbnail > p") != []:
|
if thumbnail.select("div.thumbnail > p") != []:
|
||||||
text = thumbnail.select("div.thumbnail > p")[0]
|
text = thumbnail.select("div.thumbnail > p")[0]
|
||||||
if thumbnail.select("div.thumbnail div.thumbnail-image"):
|
if thumbnail.select("div.thumbnail div.thumbnail-image"):
|
||||||
link = thumbnail.select("div.thumbnail div.thumbnail-image a")[0].get("href")
|
link = thumbnail.select("div.thumbnail div.thumbnail-image a")[
|
||||||
img = proxy(thumbnail.select("div.thumbnail div.thumbnail-image a img")[0].get("src"))
|
0
|
||||||
thumbnail_title = thumbnail.select("div.thumbnail div.thumbnail-info h3.title a")[0].text
|
].get("href")
|
||||||
thumbnail_author = thumbnail.select("div.thumbnail div.thumbnail-info span.author a")[0].text
|
img = proxy(
|
||||||
thumbnail_author_link = thumbnail.select("div.thumbnail div.thumbnail-info span.author a")[0].get("href")
|
thumbnail.select("div.thumbnail div.thumbnail-image a img")[
|
||||||
thumbnail_channel = thumbnail.select("div.thumbnail div.thumbnail-info span.origin a")[0].text
|
0
|
||||||
thumbnail_channel_link = thumbnail.select("div.thumbnail div.thumbnail-info span.origin a")[0].get("href")
|
].get("src")
|
||||||
thumbnails.append([text, link, img, thumbnail_title, thumbnail_author, thumbnail_author_link, thumbnail_channel, thumbnail_channel_link])
|
)
|
||||||
|
thumbnail_title = thumbnail.select(
|
||||||
|
"div.thumbnail div.thumbnail-info h3.title a"
|
||||||
|
)[0].text
|
||||||
|
thumbnail_author = thumbnail.select(
|
||||||
|
"div.thumbnail div.thumbnail-info span.author a"
|
||||||
|
)[0].text
|
||||||
|
thumbnail_author_link = thumbnail.select(
|
||||||
|
"div.thumbnail div.thumbnail-info span.author a"
|
||||||
|
)[0].get("href")
|
||||||
|
thumbnail_channel = thumbnail.select(
|
||||||
|
"div.thumbnail div.thumbnail-info span.origin a"
|
||||||
|
)[0].text
|
||||||
|
thumbnail_channel_link = thumbnail.select(
|
||||||
|
"div.thumbnail div.thumbnail-info span.origin a"
|
||||||
|
)[0].get("href")
|
||||||
|
thumbnails.append(
|
||||||
|
[
|
||||||
|
text,
|
||||||
|
link,
|
||||||
|
img,
|
||||||
|
thumbnail_title,
|
||||||
|
thumbnail_author,
|
||||||
|
thumbnail_author_link,
|
||||||
|
thumbnail_channel,
|
||||||
|
thumbnail_channel_link,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
return render_template(
|
||||||
return render_template("collection.html", data=[title, author, author_link, category, category_link, channel, channel_link, views, favorites, thumbnails])
|
"collection.html",
|
||||||
|
data=[
|
||||||
|
title,
|
||||||
|
author,
|
||||||
|
author_link,
|
||||||
|
category,
|
||||||
|
category_link,
|
||||||
|
channel,
|
||||||
|
channel_link,
|
||||||
|
views,
|
||||||
|
favorites,
|
||||||
|
thumbnails,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
print_exc()
|
print_exc()
|
||||||
return Response(render_template("404.html"), status=404)
|
return Response(render_template("404.html"), status=404)
|
||||||
|
|
||||||
@app.route('/<category>/<channel>/')
|
|
||||||
|
@app.route("/<category>/<channel>/")
|
||||||
def route_channel_redirect(category, channel):
|
def route_channel_redirect(category, channel):
|
||||||
if category == "circuits" or category == "workshop" or category == "craft" or category == "cooking" or category == "living" or category == "outside" or category == "teachers":
|
if (
|
||||||
|
category == "circuits"
|
||||||
|
or category == "workshop"
|
||||||
|
or category == "craft"
|
||||||
|
or category == "cooking"
|
||||||
|
or category == "living"
|
||||||
|
or category == "outside"
|
||||||
|
or category == "teachers"
|
||||||
|
):
|
||||||
return redirect(f"/{category}/{channel}/projects/", 307)
|
return redirect(f"/{category}/{channel}/projects/", 307)
|
||||||
else:
|
else:
|
||||||
return Response(render_template("404.html"), status=404)
|
return Response(render_template("404.html"), status=404)
|
||||||
|
|
||||||
@app.route('/')
|
|
||||||
|
@app.route("/")
|
||||||
def route_explore():
|
def route_explore():
|
||||||
data = requests.get("https://www.instructables.com/")
|
data = requests.get("https://www.instructables.com/")
|
||||||
if data.status_code != 200:
|
if data.status_code != 200:
|
||||||
return Response(render_template(str(data.status_code) + ".html"), status=data.status_code)
|
return Response(
|
||||||
|
render_template(str(data.status_code) + ".html"), status=data.status_code
|
||||||
|
)
|
||||||
|
|
||||||
soup = BeautifulSoup(data.text, "html.parser")
|
soup = BeautifulSoup(data.text, "html.parser")
|
||||||
|
|
||||||
|
@ -690,22 +979,33 @@ def route_explore():
|
||||||
|
|
||||||
title = explore.select("h2")[0].text
|
title = explore.select("h2")[0].text
|
||||||
|
|
||||||
circuits = explore_lists(explore.select(".home-content-explore-category-circuits")[0])
|
circuits = explore_lists(
|
||||||
workshop = explore_lists(explore.select(".home-content-explore-category-workshop")[0])
|
explore.select(".home-content-explore-category-circuits")[0]
|
||||||
|
)
|
||||||
|
workshop = explore_lists(
|
||||||
|
explore.select(".home-content-explore-category-workshop")[0]
|
||||||
|
)
|
||||||
craft = explore_lists(explore.select(".home-content-explore-category-craft")[0])
|
craft = explore_lists(explore.select(".home-content-explore-category-craft")[0])
|
||||||
cooking = explore_lists(explore.select(".home-content-explore-category-cooking")[0])
|
cooking = explore_lists(explore.select(".home-content-explore-category-cooking")[0])
|
||||||
living = explore_lists(explore.select(".home-content-explore-category-living")[0])
|
living = explore_lists(explore.select(".home-content-explore-category-living")[0])
|
||||||
outside = explore_lists(explore.select(".home-content-explore-category-outside")[0])
|
outside = explore_lists(explore.select(".home-content-explore-category-outside")[0])
|
||||||
teachers = explore_lists(explore.select(".home-content-explore-category-teachers")[0])
|
teachers = explore_lists(
|
||||||
|
explore.select(".home-content-explore-category-teachers")[0]
|
||||||
|
)
|
||||||
|
|
||||||
return render_template("index.html", data=[title, circuits, workshop, craft, cooking, living, outside, teachers])
|
return render_template(
|
||||||
|
"index.html",
|
||||||
|
data=[title, circuits, workshop, craft, cooking, living, outside, teachers],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/proxy/')
|
@app.route("/proxy/")
|
||||||
def route_proxy():
|
def route_proxy():
|
||||||
url = request.args.get("url")
|
url = request.args.get("url")
|
||||||
if url != None:
|
if url != None:
|
||||||
if url.startswith("https://cdn.instructables.com/") or url.startswith("https://content.instructables.com/"):
|
if url.startswith("https://cdn.instructables.com/") or url.startswith(
|
||||||
|
"https://content.instructables.com/"
|
||||||
|
):
|
||||||
data = requests.get(unquote(url))
|
data = requests.get(unquote(url))
|
||||||
return Response(data.content, content_type=data.headers["content-type"])
|
return Response(data.content, content_type=data.headers["content-type"])
|
||||||
else:
|
else:
|
||||||
|
@ -717,5 +1017,6 @@ def route_proxy():
|
||||||
def not_found(e):
|
def not_found(e):
|
||||||
return render_template("404.html")
|
return render_template("404.html")
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
app.run(port=8002, debug=debugmode)
|
if __name__ == "__main__":
|
||||||
|
app.run(port=args.port, host=args.listen_host, debug=debugmode)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
bs4
|
bs4
|
||||||
requests
|
requests
|
||||||
flask
|
flask
|
||||||
selenium
|
requests-html
|
||||||
|
playwright
|
|
@ -1,7 +1,8 @@
|
||||||
<footer>
|
<footer>
|
||||||
<hr>
|
<hr>
|
||||||
<center>
|
<center>
|
||||||
<p><a href="">Source code (AGPLv3)</a></p>
|
<p><a href="https://git.kumi/PrivateCoffee/indestructables">Modified Source Code (AGPLv3)</a></p>
|
||||||
|
<p><a href="https://codeberg.org/snowcatridge10/indestructables">Original Source Code (AGPLv3)</a></p>
|
||||||
<p><a href="">View privacy policy.</a></p>
|
<p><a href="">View privacy policy.</a></p>
|
||||||
</center>
|
</center>
|
||||||
</footer>
|
</footer>
|
Loading…
Reference in a new issue