PrivateCoffee/structables
4
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add cron endpoint to update global_ibles

Browse source
This commit is contained in:
Kumi 2023-06-03 09:46:28 +02:00
parent fbb0fea2d5
commit 75ec8b961f
Signed by: kumi
GPG key ID: ECBCC9082395383F
1 changed files with 99 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
main.py
View file

@ -20,42 +20,12 @@ from argparse import ArgumentParser
import os
debugmode = False
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
"-p",
"--port",
default=8002,
type=int,
help="Port to listen on",
)
parser.add_argument(
"-d",
"--debug",
action="store_true",
help="Enable debug mode",
)
parser.add_argument(
"-l",
"--listen-host",
default="127.0.0.1",
help="Host to listen on",
)
args = parser.parse_args()
if args.debug:
debugmode = True
print("Loading...")
global_ibles = {}
def proxy(src):
return "/proxy/?url=" + quote(str(src))
def get_instance_root_url(request):
return request.url_root
def update_data():
playwright = sync_playwright().start()
browser = playwright.chromium.launch(headless=True)
page = browser.new_page()
@ -72,8 +42,6 @@ groups = []
for group in main.select("div.group-section"):
channels.append(group.select("h2 a")[0].text.lower())
global_ibles = {}
global_ibles["/projects"] = []
page.goto("https://www.instructables.com/projects")
@ -142,8 +110,49 @@ while len(global_ibles["/projects"]) <= 0:
browser.close()
playwright.stop()
debugmode = False
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
"-p",
"--port",
default=8002,
type=int,
help="Port to listen on",
)
parser.add_argument(
"-d",
"--debug",
action="store_true",
help="Enable debug mode",
)
parser.add_argument(
"-l",
"--listen-host",
default="127.0.0.1",
help="Host to listen on",
)
args = parser.parse_args()
if args.debug:
debugmode = True
print("Loading...")
update_data()
print("Started!")
app = Flask(__name__, template_folder="templates", static_folder="static")
def get_instance_root_url(request):
return request.url_root
@app.route("/cron/")
def cron():
update_data()
return "OK"
def explore_lists(soup):
list_ = []
@ -407,10 +416,6 @@ def project_list(path, head, sort=""):
return render_template("projects.html", data=[head, ibles, path_])
app = Flask(__name__, template_folder="templates", static_folder="static")
@app.route("/sitemap/")
def route_sitemap():
data = requests.get(f"https://www.instructables.com/sitemap/")