commit
This commit is contained in:
parent
01622a6e9e
commit
3d1a1f6971
2 changed files with 72 additions and 36 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -1 +1,5 @@
|
||||||
geckodriver.log
|
geckodriver.log
|
||||||
|
venv/
|
||||||
|
*.pyc
|
||||||
|
__pycache__/
|
||||||
|
.vscode
|
54
main.py
54
main.py
|
@ -1,13 +1,5 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
from flask import Flask, render_template, request, redirect, Response, stream_with_context
|
|
||||||
import requests
|
|
||||||
import re
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from urllib.parse import quote, unquote
|
|
||||||
from traceback import print_exc
|
|
||||||
from requests_html import HTMLSession
|
|
||||||
|
|
||||||
from selenium import webdriver
|
from selenium import webdriver
|
||||||
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
|
@ -16,6 +8,15 @@ from selenium.webdriver.support import expected_conditions as EC
|
||||||
from selenium.webdriver.common.keys import Keys
|
from selenium.webdriver.common.keys import Keys
|
||||||
from selenium.webdriver.firefox.options import Options
|
from selenium.webdriver.firefox.options import Options
|
||||||
from selenium.webdriver.chrome.options import Options
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
|
||||||
|
from flask import Flask, render_template, request, redirect, Response, stream_with_context
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from urllib.parse import quote, unquote
|
||||||
|
from traceback import print_exc
|
||||||
|
from requests_html import HTMLSession
|
||||||
|
from argparse import ArgumentParser
|
||||||
import os
|
import os
|
||||||
|
|
||||||
debugmode = False
|
debugmode = False
|
||||||
|
@ -26,6 +27,31 @@ try:
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"-p",
|
||||||
|
"--port",
|
||||||
|
default=8002,
|
||||||
|
type=int,
|
||||||
|
help="Port to listen on",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-d",
|
||||||
|
"--debug",
|
||||||
|
action="store_true",
|
||||||
|
help="Enable debug mode",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-l",
|
||||||
|
"--listen-host",
|
||||||
|
default="127.0.0.1",
|
||||||
|
help="Host to listen on",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.debug:
|
||||||
|
debugmode = True
|
||||||
|
|
||||||
print("Loading...")
|
print("Loading...")
|
||||||
|
|
||||||
def proxy(src):
|
def proxy(src):
|
||||||
|
@ -59,9 +85,10 @@ global_ibles = {}
|
||||||
|
|
||||||
global_ibles["/projects"] = []
|
global_ibles["/projects"] = []
|
||||||
|
|
||||||
driver.get("https://www.instructables.com/projects")
|
def update_data():
|
||||||
|
driver.get("https://www.instructables.com/projects")
|
||||||
|
|
||||||
while len(global_ibles["/projects"]) <= 0:
|
while len(global_ibles["/projects"]) <= 0:
|
||||||
for ible in driver.find_elements(By.CLASS_NAME, "ibleCard__QPJVm"):
|
for ible in driver.find_elements(By.CLASS_NAME, "ibleCard__QPJVm"):
|
||||||
link = ible.find_elements(By.CSS_SELECTOR, "a")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
link = ible.find_elements(By.CSS_SELECTOR, "a")[0].get_attribute("href").replace("https://www.instructables.com", instance_root_url)
|
||||||
img = proxy(ible.find_elements(By.CSS_SELECTOR, "img")[0].get_attribute("src"))
|
img = proxy(ible.find_elements(By.CSS_SELECTOR, "img")[0].get_attribute("src"))
|
||||||
|
@ -713,9 +740,14 @@ def route_proxy():
|
||||||
else:
|
else:
|
||||||
return Response(render_template("400.html"), status=400)
|
return Response(render_template("400.html"), status=400)
|
||||||
|
|
||||||
|
@app.route("/cron/")
|
||||||
|
def cron():
|
||||||
|
update_data()
|
||||||
|
return "OK"
|
||||||
|
|
||||||
@app.errorhandler(404)
|
@app.errorhandler(404)
|
||||||
def not_found(e):
|
def not_found(e):
|
||||||
return render_template("404.html")
|
return render_template("404.html")
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
app.run(port=8002, debug=debugmode)
|
app.run(port=args.port, host=args.listen_host, debug=debugmode)
|
||||||
|
|
Loading…
Reference in a new issue