Add cron endpoint to update global_ibles
This commit is contained in:
parent
fbb0fea2d5
commit
75ec8b961f
1 changed files with 99 additions and 94 deletions
81
main.py
81
main.py
|
@ -20,42 +20,12 @@ from argparse import ArgumentParser
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
debugmode = False
|
global_ibles = {}
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
parser = ArgumentParser()
|
|
||||||
parser.add_argument(
|
|
||||||
"-p",
|
|
||||||
"--port",
|
|
||||||
default=8002,
|
|
||||||
type=int,
|
|
||||||
help="Port to listen on",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-d",
|
|
||||||
"--debug",
|
|
||||||
action="store_true",
|
|
||||||
help="Enable debug mode",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-l",
|
|
||||||
"--listen-host",
|
|
||||||
default="127.0.0.1",
|
|
||||||
help="Host to listen on",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.debug:
|
|
||||||
debugmode = True
|
|
||||||
|
|
||||||
print("Loading...")
|
|
||||||
|
|
||||||
def proxy(src):
|
def proxy(src):
|
||||||
return "/proxy/?url=" + quote(str(src))
|
return "/proxy/?url=" + quote(str(src))
|
||||||
|
|
||||||
def get_instance_root_url(request):
|
def update_data():
|
||||||
return request.url_root
|
|
||||||
|
|
||||||
playwright = sync_playwright().start()
|
playwright = sync_playwright().start()
|
||||||
browser = playwright.chromium.launch(headless=True)
|
browser = playwright.chromium.launch(headless=True)
|
||||||
page = browser.new_page()
|
page = browser.new_page()
|
||||||
|
@ -72,8 +42,6 @@ groups = []
|
||||||
for group in main.select("div.group-section"):
|
for group in main.select("div.group-section"):
|
||||||
channels.append(group.select("h2 a")[0].text.lower())
|
channels.append(group.select("h2 a")[0].text.lower())
|
||||||
|
|
||||||
global_ibles = {}
|
|
||||||
|
|
||||||
global_ibles["/projects"] = []
|
global_ibles["/projects"] = []
|
||||||
|
|
||||||
page.goto("https://www.instructables.com/projects")
|
page.goto("https://www.instructables.com/projects")
|
||||||
|
@ -142,8 +110,49 @@ while len(global_ibles["/projects"]) <= 0:
|
||||||
browser.close()
|
browser.close()
|
||||||
playwright.stop()
|
playwright.stop()
|
||||||
|
|
||||||
|
debugmode = False
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"-p",
|
||||||
|
"--port",
|
||||||
|
default=8002,
|
||||||
|
type=int,
|
||||||
|
help="Port to listen on",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-d",
|
||||||
|
"--debug",
|
||||||
|
action="store_true",
|
||||||
|
help="Enable debug mode",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-l",
|
||||||
|
"--listen-host",
|
||||||
|
default="127.0.0.1",
|
||||||
|
help="Host to listen on",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.debug:
|
||||||
|
debugmode = True
|
||||||
|
|
||||||
|
print("Loading...")
|
||||||
|
|
||||||
|
update_data()
|
||||||
|
|
||||||
print("Started!")
|
print("Started!")
|
||||||
|
|
||||||
|
app = Flask(__name__, template_folder="templates", static_folder="static")
|
||||||
|
|
||||||
|
def get_instance_root_url(request):
|
||||||
|
return request.url_root
|
||||||
|
|
||||||
|
@app.route("/cron/")
|
||||||
|
def cron():
|
||||||
|
update_data()
|
||||||
|
return "OK"
|
||||||
|
|
||||||
def explore_lists(soup):
|
def explore_lists(soup):
|
||||||
list_ = []
|
list_ = []
|
||||||
|
@ -407,10 +416,6 @@ def project_list(path, head, sort=""):
|
||||||
|
|
||||||
return render_template("projects.html", data=[head, ibles, path_])
|
return render_template("projects.html", data=[head, ibles, path_])
|
||||||
|
|
||||||
|
|
||||||
app = Flask(__name__, template_folder="templates", static_folder="static")
|
|
||||||
|
|
||||||
|
|
||||||
@app.route("/sitemap/")
|
@app.route("/sitemap/")
|
||||||
def route_sitemap():
|
def route_sitemap():
|
||||||
data = requests.get(f"https://www.instructables.com/sitemap/")
|
data = requests.get(f"https://www.instructables.com/sitemap/")
|
||||||
|
|
Loading…
Reference in a new issue