Kumi
a9f1a887e9
Some checks are pending
Docker / build (push) Waiting to run
Updated file paths to use `BASE_DIR` from Django settings instead of hardcoded paths. This change enhances flexibility and simplifies configuration management as file paths are now dynamically resolved relative to the project base directory.
509 lines
21 KiB
Python
509 lines
21 KiB
Python
import aiohttp
|
|
import asyncio
|
|
import json
|
|
from .synchronous import data_prep_posts, data_prep_comments, update_rank, update_dominance
|
|
import datetime
|
|
from datetime import date, timedelta
|
|
from .models import Coin, Social, P2Pool
|
|
import requests
|
|
import pygsheets
|
|
from django.conf import settings
|
|
|
|
BASE_DIR = settings.BASE_DIR
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get block data from xmrchain
|
|
####################################################################################
|
|
async def get_block_data(session, block: str):
|
|
url = "https://localmonero.co/blocks/api/get_block_data/" + block
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "localmonero"
|
|
if res.status < 299:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get network data from xmrchain
|
|
####################################################################################
|
|
async def get_network_data(session, block: str):
|
|
url = "https://xmrchain.net/api/networkinfo"
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "xmrchain"
|
|
if res.status < 299:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get coinmarketcap data for price USD and BTC
|
|
####################################################################################
|
|
async def get_coinmarketcap_data(session, symbol: str, convert: str):
|
|
with open(BASE_DIR / "settings.json") as file:
|
|
data = json.load(file)
|
|
file.close()
|
|
|
|
url = data["metrics_provider"][0]["price_url_old"] + symbol
|
|
parameters = {
|
|
"convert": convert,
|
|
}
|
|
headers = {
|
|
"Accepts": "application/json",
|
|
data["metrics_provider"][0]["api_key_name"]: data["metrics_provider"][0][
|
|
"api_key_value"
|
|
],
|
|
}
|
|
|
|
async with session.get(url, headers=headers, params=parameters) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data["provider"] = "coinmarketcap"
|
|
if res.status < 299:
|
|
try:
|
|
if data["data"][symbol.upper()]["cmc_rank"]:
|
|
data["success"] = True
|
|
else:
|
|
data["success"] = False
|
|
except Exception:
|
|
data["success"] = False
|
|
else:
|
|
data["success"] = False
|
|
return data
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get coinmetrics data for any coin inside URL
|
|
####################################################################################
|
|
async def get_coin_data(session, symbol, url):
|
|
update = True
|
|
count = 0
|
|
|
|
while update:
|
|
async with session.get(url) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data_aux = data["data"]
|
|
for item in data_aux:
|
|
day, hour = str(item["time"]).split("T")
|
|
day = datetime.datetime.strptime(day, "%Y-%m-%d")
|
|
day = datetime.datetime.strftime(day, "%Y-%m-%d")
|
|
try:
|
|
coin = Coin.objects.filter(name=symbol).get(date=day)
|
|
except Exception:
|
|
coin = Coin()
|
|
|
|
try:
|
|
coin.name = symbol
|
|
coin.date = day
|
|
try:
|
|
coin.priceusd = float(item["PriceUSD"])
|
|
except Exception:
|
|
coin.priceusd = 0
|
|
try:
|
|
coin.pricebtc = float(item["PriceBTC"])
|
|
except Exception:
|
|
coin.pricebtc = 0
|
|
try:
|
|
coin.inflation = float(item["IssContPctAnn"])
|
|
coin.stocktoflow = (100 / coin.inflation) ** 1.65
|
|
except Exception:
|
|
coin.inflation = 0
|
|
coin.stocktoflow = 0
|
|
try:
|
|
coin.supply = float(item["SplyCur"])
|
|
except Exception:
|
|
coin.supply = 0
|
|
try:
|
|
coin.fee = float(item["FeeTotNtv"])
|
|
except Exception:
|
|
coin.fee = 0
|
|
try:
|
|
coin.revenue = float(item["RevNtv"])
|
|
except Exception:
|
|
coin.revenue = 0
|
|
try:
|
|
coin.hashrate = float(item["HashRate"])
|
|
except Exception:
|
|
coin.hashrate = 0
|
|
try:
|
|
coin.transactions = float(item["TxCnt"])
|
|
except Exception:
|
|
coin.transactions = 0
|
|
coin.save()
|
|
count += 1
|
|
except Exception:
|
|
pass
|
|
try:
|
|
url = data["next_page_url"]
|
|
update = True
|
|
except Exception:
|
|
update = False
|
|
break
|
|
return count
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social metrics from reddit
|
|
####################################################################################
|
|
async def get_social_data(session, symbol):
|
|
yesterday = datetime.datetime.strftime(date.today() - timedelta(1), "%Y-%m-%d")
|
|
try:
|
|
social = Social.objects.filter(name=symbol).get(date=yesterday)
|
|
except Exception:
|
|
url = "https://www.reddit.com/r/" + symbol + "/about.json"
|
|
|
|
async with session.get(
|
|
url, headers={"User-agent": "Checking new social data"}
|
|
) as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
data = data["data"]
|
|
|
|
social = Social()
|
|
social.name = symbol
|
|
social.date = yesterday
|
|
social.subscriberCount = data["subscribers"]
|
|
|
|
timestamp1 = int(
|
|
datetime.datetime.timestamp(
|
|
datetime.datetime.strptime(yesterday, "%Y-%m-%d")
|
|
)
|
|
)
|
|
timestamp2 = int(timestamp1 - 7200)
|
|
limit = 1000
|
|
filters = []
|
|
data = data_prep_posts(symbol, timestamp2, timestamp1, filters, limit)
|
|
social.postsPerHour = len(data) / 2
|
|
data = data_prep_comments(symbol, timestamp2, timestamp1, filters, limit)
|
|
social.commentsPerHour = len(data) / 2
|
|
social.save()
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get whole xmr data calling coinmarketcap and xmrchain
|
|
####################################################################################
|
|
async def update_xmr_data(yesterday, coin):
|
|
# TODO: Why is this all commented out?
|
|
name = coin.name # noqa: F841
|
|
# Coin.objects.filter(name=coin.name).filter(date=yesterday).delete()
|
|
|
|
url = "https://localmonero.co/blocks/api/get_stats"
|
|
response = requests.get(url)
|
|
data = json.loads(response.text)
|
|
height = int(data["height"]) # noqa: F841
|
|
difficulty = int(data["difficulty"]) # noqa: F841
|
|
hashrate = int(data["hashrate"]) # noqa: F841
|
|
supply = int(data["total_emission"]) # noqa: F841
|
|
blocksize = 0 # noqa: F841
|
|
actions = []
|
|
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# for count in range(1, 1400):
|
|
# block = str(height - count)
|
|
# actions.append(asyncio.ensure_future(get_block_data(session, block)))
|
|
actions.append(
|
|
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "USD"))
|
|
)
|
|
actions.append(
|
|
asyncio.ensure_future(get_coinmarketcap_data(session, "xmr", "BTC"))
|
|
)
|
|
|
|
try:
|
|
responses = await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
errors = 0
|
|
success = 0 # noqa: F841
|
|
txs = 0 # noqa: F841
|
|
revenue = 0 # noqa: F841
|
|
fees = 0 # noqa: F841
|
|
priceusd = 0 # noqa: F841
|
|
pricebtc = 0
|
|
for response in responses:
|
|
if response:
|
|
try:
|
|
# if response['provider'] == 'localmonero':
|
|
# date_aux = response['block_header']['timestamp']
|
|
# if date_aux == yesterday:
|
|
# try:
|
|
# blocksize += int(response['block_header']['size'])
|
|
# for tx in response['block_header']['txs']:
|
|
# if tx['coinbase']:
|
|
# revenue += int(tx['xmr_outputs'])
|
|
# else:
|
|
# txs += 1
|
|
# fees += int(tx['tx_fee'])
|
|
# revenue += int(tx['tx_fee'])
|
|
# success += 1
|
|
# except:
|
|
# errors += 1
|
|
|
|
if response["provider"] == "coinmarketcap":
|
|
try:
|
|
# priceusd = float(response['data']['XMR']['quote']['USD']['price'])
|
|
update_rank(response)
|
|
update_dominance(response)
|
|
except Exception:
|
|
try:
|
|
pricebtc = float( # noqa: F841
|
|
response["data"]["XMR"]["quote"]["BTC"]["price"]
|
|
)
|
|
except Exception:
|
|
errors += 1
|
|
except Exception:
|
|
errors += 1
|
|
else:
|
|
errors += 1
|
|
|
|
# blocksize = blocksize/success
|
|
# revenue = float(revenue)/10**12
|
|
# fees = float(fees)/10**12
|
|
# inflation = 100*365*(revenue)/float(coin.supply)
|
|
# stocktoflow = (100/inflation)**1.65
|
|
# supply = coin.supply + revenue
|
|
|
|
# try:
|
|
# coin = Coin.objects.filter(name='xmr').get(date=yesterday)
|
|
# coin.name = name
|
|
# coin.date = datetime.datetime.strptime(yesterday, '%Y-%m-%d')
|
|
# coin.date = datetime.datetime.strftime(coin.date, '%Y-%m-%d')
|
|
# #coin.blocksize = blocksize
|
|
# #coin.transactions = txs
|
|
# #coin.revenue = revenue
|
|
# #coin.fee = fees
|
|
# #coin.inflation = inflation
|
|
# #coin.hashrate = hashrate
|
|
# #coin.difficulty = difficulty
|
|
# #coin.stocktoflow = stocktoflow
|
|
# coin.priceusd = priceusd
|
|
# coin.pricebtc = pricebtc
|
|
# #coin.supply = supply
|
|
# coin.save()
|
|
|
|
# # print('Success: ' + str(success))
|
|
# # print('Errors: ' + str(errors))
|
|
# # print('Name: ' + coin.name)
|
|
# # print('Date: ' + str(coin.date))
|
|
# # print('Blocksize: ' + str(coin.blocksize))
|
|
# # print('Transactions: ' + str(coin.transactions))
|
|
# # print('Revenue: ' + str(coin.revenue))
|
|
# # print('Fees: ' + str(coin.fee))
|
|
# # print('Inflation: ' + str(coin.inflation))
|
|
# # print('Hashrate: ' + str(coin.hashrate))
|
|
# # print('Stocktoflow: ' + str(coin.stocktoflow))
|
|
# # print('Priceusd: ' + str(coin.priceusd))
|
|
# # print('Pricebtc: ' + str(coin.pricebtc))
|
|
# # print('Supply: ' + str(coin.supply))
|
|
# except:
|
|
# return False
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social and coins data
|
|
####################################################################################
|
|
async def update_others_data(date):
|
|
with open(BASE_DIR / "settings.json") as file:
|
|
data = json.load(file)
|
|
file.close()
|
|
|
|
url_btc = data["metrics_provider"][0]["metrics_url_new"] + "btc/" + date
|
|
url_dash = data["metrics_provider"][0]["metrics_url_new"] + "dash/" + date
|
|
url_grin = data["metrics_provider"][0]["metrics_url_new"] + "grin/" + date
|
|
url_zec = data["metrics_provider"][0]["metrics_url_new"] + "zec/" + date
|
|
actions = []
|
|
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# reddit data
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
|
|
# actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
|
|
# coinmetrics data
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "btc", url_btc)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "dash", url_dash)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "grin", url_grin)))
|
|
actions.append(asyncio.ensure_future(get_coin_data(session, "zec", url_zec)))
|
|
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=False)))
|
|
actions.append(asyncio.ensure_future(get_p2pool_data(session, mini=True)))
|
|
|
|
try:
|
|
await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get social and coins data
|
|
####################################################################################
|
|
async def update_social_data(symbol):
|
|
|
|
actions = []
|
|
my_timeout = aiohttp.ClientTimeout(total=10, sock_connect=10, sock_read=10)
|
|
client_args = dict(trust_env=True, timeout=my_timeout)
|
|
|
|
async with aiohttp.ClientSession(**client_args) as session:
|
|
# reddit data
|
|
actions.append(asyncio.ensure_future(get_social_data(session, "Monero")))
|
|
actions.append(asyncio.ensure_future(get_social_data(session, "Bitcoin")))
|
|
actions.append(
|
|
asyncio.ensure_future(get_social_data(session, "Cryptocurrency"))
|
|
)
|
|
|
|
try:
|
|
await asyncio.gather(*actions, return_exceptions=True)
|
|
except asyncio.exceptions.TimeoutError:
|
|
print("Timeout!")
|
|
|
|
return True
|
|
|
|
|
|
####################################################################################
|
|
# Asynchronous get p2pool and p2poolmini data and then save to google sheets
|
|
####################################################################################
|
|
async def get_p2pool_data(session, mini):
|
|
today = date.today()
|
|
yesterday = date.today() - timedelta(1)
|
|
try:
|
|
p2pool_stat = P2Pool.objects.filter(mini=mini).get(date=today)
|
|
if p2pool_stat.percentage > 0:
|
|
update = False
|
|
else:
|
|
p2pool_stat.delete()
|
|
try:
|
|
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
|
|
if coin.hashrate > 0:
|
|
update = True
|
|
else:
|
|
update = False
|
|
except Exception:
|
|
update = False
|
|
except Exception:
|
|
try:
|
|
coin = Coin.objects.filter(name="xmr").get(date=yesterday)
|
|
if coin.hashrate > 0:
|
|
update = True
|
|
else:
|
|
update = False
|
|
except Exception:
|
|
update = False
|
|
|
|
if update:
|
|
p2pool_stat = P2Pool()
|
|
p2pool_stat.date = today
|
|
|
|
if not (mini):
|
|
async with session.get("https://p2pool.io/api/pool/stats") as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
|
|
p2pool_stat.percentage = (
|
|
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
|
|
)
|
|
p2pool_stat.miners = data["pool_statistics"]["miners"]
|
|
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
|
|
p2pool_stat.totalblocksfound = data["pool_statistics"][
|
|
"totalBlocksFound"
|
|
]
|
|
p2pool_stat.mini = False
|
|
p2pool_stat.save()
|
|
print("p2pool saved!")
|
|
gc = pygsheets.authorize(
|
|
service_file="service_account_credentials.json"
|
|
)
|
|
sh = gc.open("zcash_bitcoin")
|
|
wks = sh.worksheet_by_title("p2pool")
|
|
values_mat = wks.get_values(
|
|
start=(3, 1), end=(9999, 3), returnas="matrix"
|
|
)
|
|
k = len(values_mat)
|
|
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
|
|
if date_aux < date_aux2:
|
|
cell = "F" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalblocksfound)
|
|
cell = "E" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalhashes)
|
|
cell = "D" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.percentage)
|
|
cell = "C" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.hashrate)
|
|
cell = "B" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.miners)
|
|
cell = "A" + str(k + 3)
|
|
wks.update_value(
|
|
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
|
|
)
|
|
print("spreadsheet updated")
|
|
else:
|
|
print("spreadsheet already with the latest data")
|
|
return data
|
|
else:
|
|
async with session.get("https://p2pool.io/mini/api/pool/stats") as res:
|
|
data = await res.read()
|
|
data = json.loads(data)
|
|
p2pool_stat.hashrate = data["pool_statistics"]["hashRate"]
|
|
p2pool_stat.percentage = (
|
|
100 * data["pool_statistics"]["hashRate"] / coin.hashrate
|
|
)
|
|
p2pool_stat.miners = data["pool_statistics"]["miners"]
|
|
p2pool_stat.totalhashes = data["pool_statistics"]["totalHashes"]
|
|
p2pool_stat.totalblocksfound = data["pool_statistics"][
|
|
"totalBlocksFound"
|
|
]
|
|
p2pool_stat.mini = True
|
|
p2pool_stat.save()
|
|
print("p2pool_mini saved!")
|
|
gc = pygsheets.authorize(
|
|
service_file="service_account_credentials.json"
|
|
)
|
|
sh = gc.open("zcash_bitcoin")
|
|
wks = sh.worksheet_by_title("p2poolmini")
|
|
values_mat = wks.get_values(
|
|
start=(3, 1), end=(9999, 3), returnas="matrix"
|
|
)
|
|
k = len(values_mat)
|
|
date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d")
|
|
date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d")
|
|
if date_aux < date_aux2:
|
|
cell = "F" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalblocksfound)
|
|
cell = "E" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.totalhashes)
|
|
cell = "D" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.percentage)
|
|
cell = "C" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.hashrate)
|
|
cell = "B" + str(k + 3)
|
|
wks.update_value(cell, p2pool_stat.miners)
|
|
cell = "A" + str(k + 3)
|
|
wks.update_value(
|
|
cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d")
|
|
)
|
|
print("spreadsheet updated")
|
|
else:
|
|
print("spreadsheet already with the latest data")
|
|
return data
|
|
else:
|
|
return False
|
|
return True
|