refactor: apply consistent string styling and error handling
Some checks are pending
Docker / build (push) Waiting to run

Applied consistent use of double quotes for strings across the codebase to improve readability and maintain consistency. Updated exception handling to explicitly catch specific exceptions for better error management. Refactored context and view functions for cleaner and more structured output. Removed unused imports and unnecessary comments to streamline the code.
This commit is contained in:
Kumi 2024-11-12 14:09:33 +01:00
parent 7cc291d758
commit 8e48bce180
Signed by: kumi
GPG key ID: ECBCC9082395383F
7 changed files with 3337 additions and 2262 deletions

View file

@ -2,4 +2,4 @@ from django.apps import AppConfig
class chartsConfig(AppConfig):
name = 'charts'
name = "charts"

View file

@ -1,43 +1,46 @@
import aiohttp
import asyncio
import json
from .synchronous import *
from .synchronous import data_prep_posts, data_prep_comments, update_rank, update_dominance
import datetime
from datetime import date, timedelta
from .models import Coin, Social, P2Pool
import requests
import pygsheets
####################################################################################
# Asynchronous get block data from xmrchain
####################################################################################
async def get_block_data(session, block: str):
url = 'https://localmonero.co/blocks/api/get_block_data/' + block
url = "https://localmonero.co/blocks/api/get_block_data/" + block
async with session.get(url) as res:
data = await res.read()
data = json.loads(data)
data['provider'] = 'localmonero'
data["provider"] = "localmonero"
if res.status < 299:
data['success'] = True
data["success"] = True
else:
data['success'] = False
data["success"] = False
return data
####################################################################################
# Asynchronous get network data from xmrchain
####################################################################################
async def get_network_data(session, block: str):
url = 'https://xmrchain.net/api/networkinfo'
url = "https://xmrchain.net/api/networkinfo"
async with session.get(url) as res:
data = await res.read()
data = json.loads(data)
data['provider'] = 'xmrchain'
data["provider"] = "xmrchain"
if res.status < 299:
data['success'] = True
data["success"] = True
else:
data['success'] = False
data["success"] = False
return data
####################################################################################
# Asynchronous get coinmarketcap data for price USD and BTC
####################################################################################
@ -47,25 +50,33 @@ async def get_coinmarketcap_data(session, symbol: str, convert: str):
file.close()
url = data["metrics_provider"][0]["price_url_old"] + symbol
parameters = {'convert': convert,}
headers = {'Accepts': 'application/json', data["metrics_provider"][0]["api_key_name"]: data["metrics_provider"][0]["api_key_value"],}
parameters = {
"convert": convert,
}
headers = {
"Accepts": "application/json",
data["metrics_provider"][0]["api_key_name"]: data["metrics_provider"][0][
"api_key_value"
],
}
async with session.get(url, headers=headers, params=parameters) as res:
data = await res.read()
data = json.loads(data)
data['provider'] = 'coinmarketcap'
data["provider"] = "coinmarketcap"
if res.status < 299:
try:
if data['data'][symbol.upper()]['cmc_rank']:
data['success'] = True
if data["data"][symbol.upper()]["cmc_rank"]:
data["success"] = True
else:
data['success'] = False
except:
data['success'] = False
data["success"] = False
except Exception:
data["success"] = False
else:
data['success'] = False
data["success"] = False
return data
####################################################################################
# Asynchronous get coinmetrics data for any coin inside URL
####################################################################################
@ -77,141 +88,147 @@ async def get_coin_data(session, symbol, url):
async with session.get(url) as res:
data = await res.read()
data = json.loads(data)
data_aux = data['data']
data_aux = data["data"]
for item in data_aux:
day, hour = str(item['time']).split('T')
day = datetime.datetime.strptime(day, '%Y-%m-%d')
day = datetime.datetime.strftime(day, '%Y-%m-%d')
day, hour = str(item["time"]).split("T")
day = datetime.datetime.strptime(day, "%Y-%m-%d")
day = datetime.datetime.strftime(day, "%Y-%m-%d")
try:
coin = Coin.objects.filter(name=symbol).get(date=day)
except:
except Exception:
coin = Coin()
try:
coin.name = symbol
coin.date = day
try:
coin.priceusd = float(item['PriceUSD'])
except:
coin.priceusd = float(item["PriceUSD"])
except Exception:
coin.priceusd = 0
try:
coin.pricebtc = float(item['PriceBTC'])
except:
coin.pricebtc = float(item["PriceBTC"])
except Exception:
coin.pricebtc = 0
try:
coin.inflation = float(item['IssContPctAnn'])
coin.stocktoflow = (100/coin.inflation)**1.65
except:
coin.inflation = float(item["IssContPctAnn"])
coin.stocktoflow = (100 / coin.inflation) ** 1.65
except Exception:
coin.inflation = 0
coin.stocktoflow = 0
try:
coin.supply = float(item['SplyCur'])
except:
coin.supply = float(item["SplyCur"])
except Exception:
coin.supply = 0
try:
coin.fee = float(item['FeeTotNtv'])
except:
coin.fee = float(item["FeeTotNtv"])
except Exception:
coin.fee = 0
try:
coin.revenue = float(item['RevNtv'])
except:
coin.revenue = float(item["RevNtv"])
except Exception:
coin.revenue = 0
try:
coin.hashrate = float(item['HashRate'])
except:
coin.hashrate = float(item["HashRate"])
except Exception:
coin.hashrate = 0
try:
coin.transactions = float(item['TxCnt'])
except:
coin.transactions = float(item["TxCnt"])
except Exception:
coin.transactions = 0
coin.save()
count += 1
except:
except Exception:
pass
try:
url = data['next_page_url']
url = data["next_page_url"]
update = True
except:
except Exception:
update = False
break
return count
####################################################################################
# Asynchronous get social metrics from reddit
####################################################################################
async def get_social_data(session, symbol):
yesterday = datetime.datetime.strftime(date.today()-timedelta(1), '%Y-%m-%d')
yesterday = datetime.datetime.strftime(date.today() - timedelta(1), "%Y-%m-%d")
try:
social = Social.objects.filter(name=symbol).get(date=yesterday)
except:
url = 'https://www.reddit.com/r/'+ symbol +'/about.json'
except Exception:
url = "https://www.reddit.com/r/" + symbol + "/about.json"
async with session.get(url, headers={'User-agent': 'Checking new social data'}) as res:
async with session.get(
url, headers={"User-agent": "Checking new social data"}
) as res:
data = await res.read()
data = json.loads(data)
data = data['data']
data = data["data"]
social = Social()
social.name = symbol
social.date = yesterday
social.subscriberCount = data['subscribers']
social.subscriberCount = data["subscribers"]
timestamp1 = int(datetime.datetime.timestamp(datetime.datetime.strptime(yesterday, '%Y-%m-%d')))
timestamp1 = int(
datetime.datetime.timestamp(
datetime.datetime.strptime(yesterday, "%Y-%m-%d")
)
)
timestamp2 = int(timestamp1 - 7200)
limit = 1000
filters = []
data = data_prep_posts(symbol, timestamp2, timestamp1, filters, limit)
social.postsPerHour = len(data)/2
social.postsPerHour = len(data) / 2
data = data_prep_comments(symbol, timestamp2, timestamp1, filters, limit)
social.commentsPerHour = len(data)/2
social.commentsPerHour = len(data) / 2
social.save()
return True
####################################################################################
# Asynchronous get whole xmr data calling coinmarketcap and xmrchain
####################################################################################
async def update_xmr_data(yesterday, coin):
name = coin.name
#Coin.objects.filter(name=coin.name).filter(date=yesterday).delete()
# TODO: Why is this all commented out?
name = coin.name # noqa: F841
# Coin.objects.filter(name=coin.name).filter(date=yesterday).delete()
url = 'https://localmonero.co/blocks/api/get_stats'
url = "https://localmonero.co/blocks/api/get_stats"
response = requests.get(url)
data = json.loads(response.text)
height = int(data['height'])
difficulty = int(data['difficulty'])
hashrate = int(data['hashrate'])
supply = int(data['total_emission'])
blocksize = 0
height = int(data["height"]) # noqa: F841
difficulty = int(data["difficulty"]) # noqa: F841
hashrate = int(data["hashrate"]) # noqa: F841
supply = int(data["total_emission"]) # noqa: F841
blocksize = 0 # noqa: F841
actions = []
my_timeout = aiohttp.ClientTimeout(
total=10,
sock_connect=10,
sock_read=10
)
client_args = dict(
trust_env=True,
timeout=my_timeout
)
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
client_args = dict(trust_env=True, timeout=my_timeout)
async with aiohttp.ClientSession(**client_args) as session:
#for count in range(1, 1400):
# for count in range(1, 1400):
# block = str(height - count)
# actions.append(asyncio.ensure_future(get_block_data(session, block)))
actions.append(asyncio.ensure_future(get_coinmarketcap_data(session, 'xmr', 'USD')))
actions.append(asyncio.ensure_future(get_coinmarketcap_data(session, 'xmr', 'BTC')))
actions.append(
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "USD"))
)
actions.append(
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "BTC"))
)
try:
responses = await asyncio.gather(*actions, return_exceptions=True)
except asyncio.exceptions.TimeoutError:
print('Timeout!')
print("Timeout!")
errors = 0
success = 0
txs = 0
revenue = 0
fees = 0
priceusd = 0
success = 0 # noqa: F841
txs = 0 # noqa: F841
revenue = 0 # noqa: F841
fees = 0 # noqa: F841
priceusd = 0 # noqa: F841
pricebtc = 0
for response in responses:
if response:
@ -232,17 +249,19 @@ async def update_xmr_data(yesterday, coin):
# except:
# errors += 1
if response['provider'] == 'coinmarketcap':
if response["provider"] == "coinmarketcap":
try:
#priceusd = float(response['data']['XMR']['quote']['USD']['price'])
# priceusd = float(response['data']['XMR']['quote']['USD']['price'])
update_rank(response)
update_dominance(response)
except:
except Exception:
try:
pricebtc = float(response['data']['XMR']['quote']['BTC']['price'])
except:
pricebtc = float( # noqa: F841
response["data"]["XMR"]["quote"]["BTC"]["price"]
)
except Exception:
errors += 1
except:
except Exception:
errors += 1
else:
errors += 1
@ -290,6 +309,7 @@ async def update_xmr_data(yesterday, coin):
# return False
return True
####################################################################################
# Asynchronous get social and coins data
####################################################################################
@ -298,71 +318,61 @@ async def update_others_data(date):
data = json.load(file)
file.close()
url_btc = data["metrics_provider"][0]["metrics_url_new"] + 'btc/' + date
url_dash = data["metrics_provider"][0]["metrics_url_new"] + 'dash/' + date
url_grin = data["metrics_provider"][0]["metrics_url_new"] + 'grin/' + date
url_zec = data["metrics_provider"][0]["metrics_url_new"] + 'zec/' + date
url_btc = data["metrics_provider"][0]["metrics_url_new"] + "btc/" + date
url_dash = data["metrics_provider"][0]["metrics_url_new"] + "dash/" + date
url_grin = data["metrics_provider"][0]["metrics_url_new"] + "grin/" + date
url_zec = data["metrics_provider"][0]["metrics_url_new"] + "zec/" + date
actions = []
my_timeout = aiohttp.ClientTimeout(
total=10,
sock_connect=10,
sock_read=10
)
client_args = dict(
trust_env=True,
timeout=my_timeout
)
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
client_args = dict(trust_env=True, timeout=my_timeout)
async with aiohttp.ClientSession(**client_args) as session:
# reddit data
#actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
#actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
#actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
# actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
# actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
# actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
# coinmetrics data
actions.append(asyncio.ensure_future(get_coin_data(session, 'btc', url_btc)))
actions.append(asyncio.ensure_future(get_coin_data(session, 'dash', url_dash)))
actions.append(asyncio.ensure_future(get_coin_data(session, 'grin', url_grin)))
actions.append(asyncio.ensure_future(get_coin_data(session, 'zec', url_zec)))
actions.append(asyncio.ensure_future(get_coin_data(session, "btc", url_btc)))
actions.append(asyncio.ensure_future(get_coin_data(session, "dash", url_dash)))
actions.append(asyncio.ensure_future(get_coin_data(session, "grin", url_grin)))
actions.append(asyncio.ensure_future(get_coin_data(session, "zec", url_zec)))
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=False)))
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=True)))
try:
await asyncio.gather(*actions, return_exceptions=True)
except asyncio.exceptions.TimeoutError:
print('Timeout!')
print("Timeout!")
return True
####################################################################################
# Asynchronous get social and coins data
####################################################################################
async def update_social_data(symbol):
actions = []
my_timeout = aiohttp.ClientTimeout(
total=10,
sock_connect=10,
sock_read=10
)
client_args = dict(
trust_env=True,
timeout=my_timeout
)
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
client_args = dict(trust_env=True, timeout=my_timeout)
async with aiohttp.ClientSession(**client_args) as session:
# reddit data
actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
actions.append(asyncio.ensure_future(get_social_data(session, "Monero")))
actions.append(asyncio.ensure_future(get_social_data(session, "Bitcoin")))
actions.append(
asyncio.ensure_future(get_social_data(session, "Cryptocurrency"))
)
try:
await asyncio.gather(*actions, return_exceptions=True)
except asyncio.exceptions.TimeoutError:
print('Timeout!')
print("Timeout!")
return True
####################################################################################
# Asynchronous get p2pool and p2poolmini data and then save to google sheets
####################################################################################
@ -376,100 +386,120 @@ async def get_p2pool_data(session, mini):
else:
p2pool_stat.delete()
try:
coin = Coin.objects.filter(name='xmr').get(date=yesterday)
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
if coin.hashrate > 0:
update = True
else:
update = False
except:
except Exception:
update = False
except:
except Exception:
try:
coin = Coin.objects.filter(name='xmr').get(date=yesterday)
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
if coin.hashrate > 0:
update = True
else:
update = False
except:
except Exception:
update = False
if update:
p2pool_stat = P2Pool()
p2pool_stat.date = today
if not(mini):
async with session.get('https://p2pool.io/api/pool/stats') as res:
if not (mini):
async with session.get("https://p2pool.io/api/pool/stats") as res:
data = await res.read()
data = json.loads(data)
p2pool_stat.hashrate = data['pool_statistics']['hashRate']
p2pool_stat.percentage = 100*data['pool_statistics']['hashRate']/coin.hashrate
p2pool_stat.miners = data['pool_statistics']['miners']
p2pool_stat.totalhashes = data['pool_statistics']['totalHashes']
p2pool_stat.totalblocksfound = data['pool_statistics']['totalBlocksFound']
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
p2pool_stat.percentage = (
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
)
p2pool_stat.miners = data["pool_statistics"]["miners"]
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
p2pool_stat.totalblocksfound = data["pool_statistics"][
"totalBlocksFound"
]
p2pool_stat.mini = False
p2pool_stat.save()
print('p2pool saved!')
gc = pygsheets.authorize(service_file='service_account_credentials.json')
sh = gc.open('zcash_bitcoin')
wks = sh.worksheet_by_title('p2pool')
values_mat = wks.get_values(start=(3,1), end=(9999,3), returnas='matrix')
print("p2pool saved!")
gc = pygsheets.authorize(
service_file="service_account_credentials.json"
)
sh = gc.open("zcash_bitcoin")
wks = sh.worksheet_by_title("p2pool")
values_mat = wks.get_values(
start=(3, 1), end=(9999, 3), returnas="matrix"
)
k = len(values_mat)
date_aux = datetime.datetime.strptime(values_mat[k-1][0], '%Y-%m-%d')
date_aux2 = datetime.datetime.strftime(date.today(), '%Y-%m-%d')
date_aux2 = datetime.datetime.strptime(date_aux2, '%Y-%m-%d')
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
if date_aux < date_aux2:
cell = 'F' + str(k + 3)
cell = "F" + str(k + 3)
wks.update_value(cell, p2pool_stat.totalblocksfound)
cell = 'E' + str(k + 3)
cell = "E" + str(k + 3)
wks.update_value(cell, p2pool_stat.totalhashes)
cell = 'D' + str(k + 3)
cell = "D" + str(k + 3)
wks.update_value(cell, p2pool_stat.percentage)
cell = 'C' + str(k + 3)
cell = "C" + str(k + 3)
wks.update_value(cell, p2pool_stat.hashrate)
cell = 'B' + str(k + 3)
cell = "B" + str(k + 3)
wks.update_value(cell, p2pool_stat.miners)
cell = 'A' + str(k + 3)
wks.update_value(cell, datetime.datetime.strftime(p2pool_stat.date, '%Y-%m-%d'))
print('spreadsheet updated')
cell = "A" + str(k + 3)
wks.update_value(
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
)
print("spreadsheet updated")
else:
print('spreadsheet already with the latest data')
print("spreadsheet already with the latest data")
return data
else:
async with session.get('https://p2pool.io/mini/api/pool/stats') as res:
async with session.get("https://p2pool.io/mini/api/pool/stats") as res:
data = await res.read()
data = json.loads(data)
p2pool_stat.hashrate = data['pool_statistics']['hashRate']
p2pool_stat.percentage = 100*data['pool_statistics']['hashRate']/coin.hashrate
p2pool_stat.miners = data['pool_statistics']['miners']
p2pool_stat.totalhashes = data['pool_statistics']['totalHashes']
p2pool_stat.totalblocksfound = data['pool_statistics']['totalBlocksFound']
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
p2pool_stat.percentage = (
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
)
p2pool_stat.miners = data["pool_statistics"]["miners"]
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
p2pool_stat.totalblocksfound = data["pool_statistics"][
"totalBlocksFound"
]
p2pool_stat.mini = True
p2pool_stat.save()
print('p2pool_mini saved!')
gc = pygsheets.authorize(service_file='service_account_credentials.json')
sh = gc.open('zcash_bitcoin')
wks = sh.worksheet_by_title('p2poolmini')
values_mat = wks.get_values(start=(3,1), end=(9999,3), returnas='matrix')
print("p2pool_mini saved!")
gc = pygsheets.authorize(
service_file="service_account_credentials.json"
)
sh = gc.open("zcash_bitcoin")
wks = sh.worksheet_by_title("p2poolmini")
values_mat = wks.get_values(
start=(3, 1), end=(9999, 3), returnas="matrix"
)
k = len(values_mat)
date_aux = datetime.datetime.strptime(values_mat[k-1][0], '%Y-%m-%d')
date_aux2 = datetime.datetime.strftime(date.today(), '%Y-%m-%d')
date_aux2 = datetime.datetime.strptime(date_aux2, '%Y-%m-%d')
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
if date_aux < date_aux2:
cell = 'F' + str(k + 3)
cell = "F" + str(k + 3)
wks.update_value(cell, p2pool_stat.totalblocksfound)
cell = 'E' + str(k + 3)
cell = "E" + str(k + 3)
wks.update_value(cell, p2pool_stat.totalhashes)
cell = 'D' + str(k + 3)
cell = "D" + str(k + 3)
wks.update_value(cell, p2pool_stat.percentage)
cell = 'C' + str(k + 3)
cell = "C" + str(k + 3)
wks.update_value(cell, p2pool_stat.hashrate)
cell = 'B' + str(k + 3)
cell = "B" + str(k + 3)
wks.update_value(cell, p2pool_stat.miners)
cell = 'A' + str(k + 3)
wks.update_value(cell, datetime.datetime.strftime(p2pool_stat.date, '%Y-%m-%d'))
print('spreadsheet updated')
cell = "A" + str(k + 3)
wks.update_value(
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
)
print("spreadsheet updated")
else:
print('spreadsheet already with the latest data')
print("spreadsheet already with the latest data")
return data
else:
return False

View file

@ -1,7 +1,21 @@
from .models import *
from .models import Coin
from django import forms
class CoinForm(forms.ModelForm):
class Meta:
model = Coin
fields = ['name', 'date', 'priceusd', 'pricebtc', 'inflation', 'transactions', 'hashrate', 'supply', 'fee', 'revenue', 'blocksize', 'difficulty']
fields = [
"name",
"date",
"priceusd",
"pricebtc",
"inflation",
"transactions",
"hashrate",
"supply",
"fee",
"revenue",
"blocksize",
"difficulty",
]

View file

@ -1,8 +1,6 @@
from __future__ import unicode_literals
from django.contrib.postgres.fields import ArrayField
from django.contrib.auth.models import User
from django.db import models
from django.forms import BooleanField
# Create your models here.
class Coin(models.Model):
@ -20,6 +18,7 @@ class Coin(models.Model):
blocksize = models.FloatField(default="0")
difficulty = models.FloatField(default="0")
class Social(models.Model):
name = models.CharField(max_length=4)
date = models.DateField()
@ -27,6 +26,7 @@ class Social(models.Model):
commentsPerHour = models.FloatField()
postsPerHour = models.FloatField()
class Dominance(models.Model):
name = models.CharField(max_length=4)
date = models.DateField()
@ -35,6 +35,7 @@ class Dominance(models.Model):
def __str__(self):
return self.dominance
class Rank(models.Model):
name = models.CharField(max_length=4)
date = models.DateField()
@ -43,6 +44,7 @@ class Rank(models.Model):
def __str__(self):
return self.rank
class Sfmodel(models.Model):
date = models.DateField()
priceusd = models.FloatField()
@ -51,6 +53,7 @@ class Sfmodel(models.Model):
stocktoflow = models.FloatField()
greyline = models.FloatField()
class DailyData(models.Model):
# Date field
date = models.DateField()
@ -127,6 +130,7 @@ class DailyData(models.Model):
crypto_commentsPerHour = models.FloatField()
crypto_postsPerHour = models.FloatField()
class P2Pool(models.Model):
date = models.DateField()
mini = models.BooleanField()
@ -136,6 +140,7 @@ class P2Pool(models.Model):
totalhashes = models.IntegerField()
totalblocksfound = models.IntegerField()
class Withdrawal(models.Model):
date = models.DateTimeField(auto_now_add=True)
state = models.BooleanField()

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff