feat: switch to JSON API for fetching contest data
Updated the contest data fetching logic to use the JSON API instead of scraping the HTML directly. This change improves the reliability and performance of data retrieval by utilizing a structured API endpoint. - Replaced HTML parsing with JSON decoding - Simplified the contest details extraction - Updated template to display new data structure Addresses issues with parsing inconsistencies and changes in HTML structure.
This commit is contained in:
parent
380041ec1b
commit
c633a3ec47
GPG key ID:
ECBCC9082395383F
2 changed files with 61 additions and 95 deletions
|
|
@ -49,7 +49,7 @@ def init_contest_routes(app):
|
||||||
"total_pages": total_pages,
|
"total_pages": total_pages,
|
||||||
"has_prev": page > 1,
|
"has_prev": page > 1,
|
||||||
"has_next": page < total_pages,
|
"has_next": page < total_pages,
|
||||||
"limit": limit
|
"limit": limit,
|
||||||
}
|
}
|
||||||
|
|
||||||
return render_template(
|
return render_template(
|
||||||
|
|
@ -125,68 +125,33 @@ def init_contest_routes(app):
|
||||||
@app.route("/contest/")
|
@app.route("/contest/")
|
||||||
def route_contests():
|
def route_contests():
|
||||||
try:
|
try:
|
||||||
data = urlopen("https://www.instructables.com/contest/")
|
# Fetch current contests from the JSON API
|
||||||
|
response = urlopen(
|
||||||
|
"https://www.instructables.com/json-api/getCurrentContests?limit=50&offset=0"
|
||||||
|
)
|
||||||
|
data = json.loads(response.read().decode())
|
||||||
except HTTPError as e:
|
except HTTPError as e:
|
||||||
abort(e.code)
|
abort(e.code)
|
||||||
|
except Exception as e:
|
||||||
|
abort(500) # Handle other exceptions such as JSON decode errors
|
||||||
|
|
||||||
soup = BeautifulSoup(data.read().decode(), "html.parser")
|
contests = data.get("contests", [])
|
||||||
|
contest_list = []
|
||||||
contest_count = "0"
|
for contest in contests:
|
||||||
|
contest_details = {
|
||||||
contests = []
|
"link": f"https://www.instructables.com/{contest['urlString']}",
|
||||||
for contest in soup.select("div#cur-contests div.row-fluid div.contest-banner"):
|
"img": contest["bannerUrlMedium"],
|
||||||
link = contest.select("div.contest-banner-inner a")[0].get("href")
|
"alt": contest["title"],
|
||||||
img = proxy(contest.select("div.contest-banner-inner a img")[0].get("src"))
|
"title": contest["title"],
|
||||||
alt = contest.select("div.contest-banner-inner a img")[0].get("alt")
|
"deadline": contest["deadline"],
|
||||||
deadline = contest.select("span.contest-meta-deadline")[0].get(
|
"prizes": contest["prizeCount"],
|
||||||
"data-deadline"
|
"entries": contest["numEntries"],
|
||||||
)
|
}
|
||||||
prizes = contest.select("span.contest-meta-count")[0].text
|
contest_list.append(contest_details)
|
||||||
entries = contest.select("span.contest-meta-count")[1].text
|
|
||||||
|
|
||||||
contests.append(
|
|
||||||
{
|
|
||||||
"link": link,
|
|
||||||
"img": img,
|
|
||||||
"alt": alt,
|
|
||||||
"deadline": deadline,
|
|
||||||
"prizes": prizes,
|
|
||||||
"entries": entries,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
closed = []
|
|
||||||
for display in soup.select("div.contest-winner-display"):
|
|
||||||
link = display.select("div.contest-banner-inner a")[0].get("href")
|
|
||||||
img = proxy(display.select("div.contest-banner-inner a img")[0].get("src"))
|
|
||||||
alt = display.select("div.contest-banner-inner a img")[0].get("alt")
|
|
||||||
featured_items = []
|
|
||||||
for featured_item in display.select("ul.featured-items li"):
|
|
||||||
item_link = featured_item.select("div.ible-thumb a")[0].get("href")
|
|
||||||
item_img = proxy(
|
|
||||||
featured_item.select("div.ible-thumb a img")[0].get("src")
|
|
||||||
)
|
|
||||||
item_title = featured_item.select("a.title")[0].text
|
|
||||||
item_author = featured_item.select("a.author")[0].text
|
|
||||||
item_author_link = featured_item.select("a.author")[0].get("href")
|
|
||||||
|
|
||||||
featured_items.append(
|
|
||||||
{
|
|
||||||
"link": item_link,
|
|
||||||
"img": item_img,
|
|
||||||
"title": item_title,
|
|
||||||
"author": item_author,
|
|
||||||
"author_link": item_author_link,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
closed.append(
|
|
||||||
{"link": link, "img": img, "alt": alt, "featured_items": featured_items}
|
|
||||||
)
|
|
||||||
|
|
||||||
return render_template(
|
return render_template(
|
||||||
"contests.html",
|
"contests.html",
|
||||||
title="Contests",
|
title="Contests",
|
||||||
contest_count=contest_count,
|
contest_count=len(contest_list),
|
||||||
contests=contests,
|
contests=contest_list,
|
||||||
closed=closed,
|
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,40 +1,41 @@
|
||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
|
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<center>
|
<center>
|
||||||
<h1>{{ title }}</h1>
|
<h1>{{ title }}</h1>
|
||||||
{{ contest_count|safe }}
|
<p>Total Running Contests: {{ contest_count }}</p>
|
||||||
<br>
|
<br>
|
||||||
<div class="contest-list">
|
<div class="contest-list">
|
||||||
{% for contest in contests %}
|
{% for contest in contests %}
|
||||||
<div class="contest-list-item">
|
<div class="contest-list-item">
|
||||||
<a href="{{ contest.link }}">
|
<a href="{{ contest.link }}">
|
||||||
<img src="{{ contest.img }}" alt="{{ contest.alt }}" style="max-width:500px;">
|
<img src="{{ contest.img }}" alt="{{ contest.alt }}" style="max-width:500px;">
|
||||||
</a>
|
</a>
|
||||||
<p>Closes {{ contest.deadline }}</p>
|
<h2>{{ contest.title }}</h2>
|
||||||
<p class="ible-small">{{ contest.prizes }} Prizes, {{ contest.entries }} Entries</p>
|
<p>Closes: {{ contest.deadline }}</p>
|
||||||
</div>
|
<p class="ible-small">{{ contest.prizes }} Prizes, {{ contest.entries }} Entries</p>
|
||||||
{% endfor %}
|
</div>
|
||||||
</div>
|
{% endfor %}
|
||||||
<div class="closed-contests" id="contest-winners">
|
</div>
|
||||||
<h2>Winner's Circle</h2>
|
<div class="closed-contests" id="contest-winners">
|
||||||
{% for closed in closed %}
|
<h2>Winner's Circle</h2>
|
||||||
<div class="closed-contest">
|
{% for closed in closed %}
|
||||||
<a href="{{ closed.link }}"><img class="closed-contest-contest" src="{{ closed.img }}"
|
<div class="closed-contest">
|
||||||
alt="{{ closed.alt }}"></a>
|
<a href="{{ closed.link }}"><img class="closed-contest-contest" src="{{ closed.img }}"
|
||||||
{% for featured_items in closed.featured_items %}
|
alt="{{ closed.alt }}"></a>
|
||||||
<div class="closed-contest-winner">
|
{% for featured_items in closed.featured_items %}
|
||||||
<a href="{{ featured_items.link }}">
|
<div class="closed-contest-winner">
|
||||||
<img class="closed-contest-winner-img" src="{{ featured_items.img }}"
|
<a href="{{ featured_items.link }}">
|
||||||
alt="{{ featured_items.title}}">
|
<img class="closed-contest-winner-img" src="{{ featured_items.img }}"
|
||||||
<br>
|
alt="{{ featured_items.title}}">
|
||||||
<b>{{ featured_items.title }}</b>
|
<br>
|
||||||
</a>
|
<b>{{ featured_items.title }}</b>
|
||||||
<p>by <a href="{{ featured_items.author_link }}">{{ featured_items.author }}</a></p>
|
</a>
|
||||||
</div>
|
<p>by <a href="{{ featured_items.author_link }}">{{ featured_items.author }}</a></p>
|
||||||
{% endfor %}
|
</div>
|
||||||
</div>
|
{% endfor %}
|
||||||
{% endfor %}
|
</div>
|
||||||
</div>
|
{% endfor %}
|
||||||
</center>
|
</div>
|
||||||
{% endblock %}
|
</center>
|
||||||
|
{% endblock %}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue