Kumi
8e48bce180
Some checks are pending
Docker / build (push) Waiting to run
Applied consistent use of double quotes for strings across the codebase to improve readability and maintain consistency. Updated exception handling to explicitly catch specific exceptions for better error management. Refactored context and view functions for cleaner and more structured output. Removed unused imports and unnecessary comments to streamline the code.
506 lines
21 KiB
Python
506 lines
21 KiB
Python
import aiohttp
|
|
import asyncio
|
|
import json
|
|
from .synchronous import data_prep_posts, data_prep_comments, update_rank, update_dominance
|
|
import datetime
|
|
from datetime import date, timedelta
|
|
from .models import Coin, Social, P2Pool
|
|
import requests
|
|
import pygsheets
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get block data from xmrchain
|
|
####################################################################################
|
|
async def get_block_data(session, block: str):
|
|
url = "https://localmonero.co/blocks/api/get_block_data/" + block
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "localmonero"
|
|
if res.status < 299:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get network data from xmrchain
|
|
####################################################################################
|
|
async def get_network_data(session, block: str):
|
|
url = "https://xmrchain.net/api/networkinfo"
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "xmrchain"
|
|
if res.status < 299:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get coinmarketcap data for price USD and BTC
|
|
####################################################################################
|
|
async def get_coinmarketcap_data(session, symbol: str, convert: str):
|
|
with open("settings.json") as file:
|
|
data = json.load(file)
|
|
file.close()
|
|
|
|
url = data["metrics_provider"][0]["price_url_old"] + symbol
|
|
parameters = {
|
|
"convert": convert,
|
|
}
|
|
headers = {
|
|
"Accepts": "application/json",
|
|
data["metrics_provider"][0]["api_key_name"]: data["metrics_provider"][0][
|
|
"api_key_value"
|
|
],
|
|
}
|
|
|
|
async with session.get(url, headers=headers, params=parameters) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "coinmarketcap"
|
|
if res.status < 299:
|
|
try:
|
|
if data["data"][symbol.upper()]["cmc_rank"]:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
except Exception:
|
|
data["success"] = False
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get coinmetrics data for any coin inside URL
|
|
####################################################################################
|
|
async def get_coin_data(session, symbol, url):
|
|
update = True
|
|
count = 0
|
|
|
|
while update:
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data_aux = data["data"]
|
|
for item in data_aux:
|
|
day, hour = str(item["time"]).split("T")
|
|
day = datetime.datetime.strptime(day, "%Y-%m-%d")
|
|
day = datetime.datetime.strftime(day, "%Y-%m-%d")
|
|
try:
|
|
coin = Coin.objects.filter(name=symbol).get(date=day)
|
|
except Exception:
|
|
coin = Coin()
|
|
|
|
try:
|
|
coin.name = symbol
|
|
coin.date = day
|
|
try:
|
|
coin.priceusd = float(item["PriceUSD"])
|
|
except Exception:
|
|
coin.priceusd = 0
|
|
try:
|
|
coin.pricebtc = float(item["PriceBTC"])
|
|
except Exception:
|
|
coin.pricebtc = 0
|
|
try:
|
|
coin.inflation = float(item["IssContPctAnn"])
|
|
coin.stocktoflow = (100 / coin.inflation) ** 1.65
|
|
except Exception:
|
|
coin.inflation = 0
|
|
coin.stocktoflow = 0
|
|
try:
|
|
coin.supply = float(item["SplyCur"])
|
|
except Exception:
|
|
coin.supply = 0
|
|
try:
|
|
coin.fee = float(item["FeeTotNtv"])
|
|
except Exception:
|
|
coin.fee = 0
|
|
try:
|
|
coin.revenue = float(item["RevNtv"])
|
|
except Exception:
|
|
coin.revenue = 0
|
|
try:
|
|
coin.hashrate = float(item["HashRate"])
|
|
except Exception:
|
|
coin.hashrate = 0
|
|
try:
|
|
coin.transactions = float(item["TxCnt"])
|
|
except Exception:
|
|
coin.transactions = 0
|
|
coin.save()
|
|
count += 1
|
|
except Exception:
|
|
pass
|
|
try:
|
|
url = data["next_page_url"]
|
|
update = True
|
|
except Exception:
|
|
update = False
|
|
break
|
|
return count
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social metrics from reddit
|
|
####################################################################################
|
|
async def get_social_data(session, symbol):
|
|
yesterday = datetime.datetime.strftime(date.today() - timedelta(1), "%Y-%m-%d")
|
|
try:
|
|
social = Social.objects.filter(name=symbol).get(date=yesterday)
|
|
except Exception:
|
|
url = "https://www.reddit.com/r/" + symbol + "/about.json"
|
|
|
|
async with session.get(
|
|
url, headers={"User-agent": "Checking new social data"}
|
|
) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data = data["data"]
|
|
|
|
social = Social()
|
|
social.name = symbol
|
|
social.date = yesterday
|
|
social.subscriberCount = data["subscribers"]
|
|
|
|
timestamp1 = int(
|
|
datetime.datetime.timestamp(
|
|
datetime.datetime.strptime(yesterday, "%Y-%m-%d")
|
|
)
|
|
)
|
|
timestamp2 = int(timestamp1 - 7200)
|
|
limit = 1000
|
|
filters = []
|
|
data = data_prep_posts(symbol, timestamp2, timestamp1, filters, limit)
|
|
social.postsPerHour = len(data) / 2
|
|
data = data_prep_comments(symbol, timestamp2, timestamp1, filters, limit)
|
|
social.commentsPerHour = len(data) / 2
|
|
social.save()
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get whole xmr data calling coinmarketcap and xmrchain
|
|
####################################################################################
|
|
async def update_xmr_data(yesterday, coin):
|
|
# TODO: Why is this all commented out?
|
|
name = coin.name # noqa: F841
|
|
# Coin.objects.filter(name=coin.name).filter(date=yesterday).delete()
|
|
|
|
url = "https://localmonero.co/blocks/api/get_stats"
|
|
response = requests.get(url)
|
|
data = json.loads(response.text)
|
|
height = int(data["height"]) # noqa: F841
|
|
difficulty = int(data["difficulty"]) # noqa: F841
|
|
hashrate = int(data["hashrate"]) # noqa: F841
|
|
supply = int(data["total_emission"]) # noqa: F841
|
|
blocksize = 0 # noqa: F841
|
|
actions = []
|
|
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# for count in range(1, 1400):
|
|
# block = str(height - count)
|
|
# actions.append(asyncio.ensure_future(get_block_data(session, block)))
|
|
actions.append(
|
|
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "USD"))
|
|
)
|
|
actions.append(
|
|
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "BTC"))
|
|
)
|
|
|
|
try:
|
|
responses = await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
errors = 0
|
|
success = 0 # noqa: F841
|
|
txs = 0 # noqa: F841
|
|
revenue = 0 # noqa: F841
|
|
fees = 0 # noqa: F841
|
|
priceusd = 0 # noqa: F841
|
|
pricebtc = 0
|
|
for response in responses:
|
|
if response:
|
|
try:
|
|
# if response['provider'] == 'localmonero':
|
|
# date_aux = response['block_header']['timestamp']
|
|
# if date_aux == yesterday:
|
|
# try:
|
|
# blocksize += int(response['block_header']['size'])
|
|
# for tx in response['block_header']['txs']:
|
|
# if tx['coinbase']:
|
|
# revenue += int(tx['xmr_outputs'])
|
|
# else:
|
|
# txs += 1
|
|
# fees += int(tx['tx_fee'])
|
|
# revenue += int(tx['tx_fee'])
|
|
# success += 1
|
|
# except:
|
|
# errors += 1
|
|
|
|
if response["provider"] == "coinmarketcap":
|
|
try:
|
|
# priceusd = float(response['data']['XMR']['quote']['USD']['price'])
|
|
update_rank(response)
|
|
update_dominance(response)
|
|
except Exception:
|
|
try:
|
|
pricebtc = float( # noqa: F841
|
|
response["data"]["XMR"]["quote"]["BTC"]["price"]
|
|
)
|
|
except Exception:
|
|
errors += 1
|
|
except Exception:
|
|
errors += 1
|
|
else:
|
|
errors += 1
|
|
|
|
# blocksize = blocksize/success
|
|
# revenue = float(revenue)/10**12
|
|
# fees = float(fees)/10**12
|
|
# inflation = 100*365*(revenue)/float(coin.supply)
|
|
# stocktoflow = (100/inflation)**1.65
|
|
# supply = coin.supply + revenue
|
|
|
|
# try:
|
|
# coin = Coin.objects.filter(name='xmr').get(date=yesterday)
|
|
# coin.name = name
|
|
# coin.date = datetime.datetime.strptime(yesterday, '%Y-%m-%d')
|
|
# coin.date = datetime.datetime.strftime(coin.date, '%Y-%m-%d')
|
|
# #coin.blocksize = blocksize
|
|
# #coin.transactions = txs
|
|
# #coin.revenue = revenue
|
|
# #coin.fee = fees
|
|
# #coin.inflation = inflation
|
|
# #coin.hashrate = hashrate
|
|
# #coin.difficulty = difficulty
|
|
# #coin.stocktoflow = stocktoflow
|
|
# coin.priceusd = priceusd
|
|
# coin.pricebtc = pricebtc
|
|
# #coin.supply = supply
|
|
# coin.save()
|
|
|
|
# # print('Success: ' + str(success))
|
|
# # print('Errors: ' + str(errors))
|
|
# # print('Name: ' + coin.name)
|
|
# # print('Date: ' + str(coin.date))
|
|
# # print('Blocksize: ' + str(coin.blocksize))
|
|
# # print('Transactions: ' + str(coin.transactions))
|
|
# # print('Revenue: ' + str(coin.revenue))
|
|
# # print('Fees: ' + str(coin.fee))
|
|
# # print('Inflation: ' + str(coin.inflation))
|
|
# # print('Hashrate: ' + str(coin.hashrate))
|
|
# # print('Stocktoflow: ' + str(coin.stocktoflow))
|
|
# # print('Priceusd: ' + str(coin.priceusd))
|
|
# # print('Pricebtc: ' + str(coin.pricebtc))
|
|
# # print('Supply: ' + str(coin.supply))
|
|
# except:
|
|
# return False
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social and coins data
|
|
####################################################################################
|
|
async def update_others_data(date):
|
|
with open("settings.json") as file:
|
|
data = json.load(file)
|
|
file.close()
|
|
|
|
url_btc = data["metrics_provider"][0]["metrics_url_new"] + "btc/" + date
|
|
url_dash = data["metrics_provider"][0]["metrics_url_new"] + "dash/" + date
|
|
url_grin = data["metrics_provider"][0]["metrics_url_new"] + "grin/" + date
|
|
url_zec = data["metrics_provider"][0]["metrics_url_new"] + "zec/" + date
|
|
actions = []
|
|
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# reddit data
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
|
|
# coinmetrics data
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "btc", url_btc)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "dash", url_dash)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "grin", url_grin)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "zec", url_zec)))
|
|
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=False)))
|
|
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=True)))
|
|
|
|
try:
|
|
await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social and coins data
|
|
####################################################################################
|
|
async def update_social_data(symbol):
|
|
|
|
actions = []
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# reddit data
|
|
actions.append(asyncio.ensure_future(get_social_data(session, "Monero")))
|
|
actions.append(asyncio.ensure_future(get_social_data(session, "Bitcoin")))
|
|
actions.append(
|
|
asyncio.ensure_future(get_social_data(session, "Cryptocurrency"))
|
|
)
|
|
|
|
try:
|
|
await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get p2pool and p2poolmini data and then save to google sheets
|
|
####################################################################################
|
|
async def get_p2pool_data(session, mini):
|
|
today = date.today()
|
|
yesterday = date.today() - timedelta(1)
|
|
try:
|
|
p2pool_stat = P2Pool.objects.filter(mini=mini).get(date=today)
|
|
if p2pool_stat.percentage > 0:
|
|
update = False
|
|
else:
|
|
p2pool_stat.delete()
|
|
try:
|
|
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
|
|
if coin.hashrate > 0:
|
|
update = True
|
|
else:
|
|
update = False
|
|
except Exception:
|
|
update = False
|
|
except Exception:
|
|
try:
|
|
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
|
|
if coin.hashrate > 0:
|
|
update = True
|
|
else:
|
|
update = False
|
|
except Exception:
|
|
update = False
|
|
|
|
if update:
|
|
p2pool_stat = P2Pool()
|
|
p2pool_stat.date = today
|
|
|
|
if not (mini):
|
|
async with session.get("https://p2pool.io/api/pool/stats") as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
|
|
p2pool_stat.percentage = (
|
|
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
|
|
)
|
|
p2pool_stat.miners = data["pool_statistics"]["miners"]
|
|
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
|
|
p2pool_stat.totalblocksfound = data["pool_statistics"][
|
|
"totalBlocksFound"
|
|
]
|
|
p2pool_stat.mini = False
|
|
p2pool_stat.save()
|
|
print("p2pool saved!")
|
|
gc = pygsheets.authorize(
|
|
service_file="service_account_credentials.json"
|
|
)
|
|
sh = gc.open("zcash_bitcoin")
|
|
wks = sh.worksheet_by_title("p2pool")
|
|
values_mat = wks.get_values(
|
|
start=(3, 1), end=(9999, 3), returnas="matrix"
|
|
)
|
|
k = len(values_mat)
|
|
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
|
|
if date_aux < date_aux2:
|
|
cell = "F" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalblocksfound)
|
|
cell = "E" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalhashes)
|
|
cell = "D" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.percentage)
|
|
cell = "C" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.hashrate)
|
|
cell = "B" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.miners)
|
|
cell = "A" + str(k + 3)
|
|
wks.update_value(
|
|
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
|
|
)
|
|
print("spreadsheet updated")
|
|
else:
|
|
print("spreadsheet already with the latest data")
|
|
return data
|
|
else:
|
|
async with session.get("https://p2pool.io/mini/api/pool/stats") as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
|
|
p2pool_stat.percentage = (
|
|
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
|
|
)
|
|
p2pool_stat.miners = data["pool_statistics"]["miners"]
|
|
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
|
|
p2pool_stat.totalblocksfound = data["pool_statistics"][
|
|
"totalBlocksFound"
|
|
]
|
|
p2pool_stat.mini = True
|
|
p2pool_stat.save()
|
|
print("p2pool_mini saved!")
|
|
gc = pygsheets.authorize(
|
|
service_file="service_account_credentials.json"
|
|
)
|
|
sh = gc.open("zcash_bitcoin")
|
|
wks = sh.worksheet_by_title("p2poolmini")
|
|
values_mat = wks.get_values(
|
|
start=(3, 1), end=(9999, 3), returnas="matrix"
|
|
)
|
|
k = len(values_mat)
|
|
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
|
|
if date_aux < date_aux2:
|
|
cell = "F" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalblocksfound)
|
|
cell = "E" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalhashes)
|
|
cell = "D" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.percentage)
|
|
cell = "C" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.hashrate)
|
|
cell = "B" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.miners)
|
|
cell = "A" + str(k + 3)
|
|
wks.update_value(
|
|
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
|
|
)
|
|
print("spreadsheet updated")
|
|
else:
|
|
print("spreadsheet already with the latest data")
|
|
return data
|
|
else:
|
|
return False
|
|
return True
|