From 302c590f1c841a3c332a3c4ad15044f018ec2323 Mon Sep 17 00:00:00 2001 From: Kumi Date: Wed, 13 Nov 2024 12:05:11 +0100 Subject: [PATCH] refactor(sync): switch to local excel file for updates Replaced pygsheets-based Google Sheets handling with pandas to read and write to local Excel files. Updated exception handling to include `TypeError`. This change enhances performance and reliability by avoiding network dependencies and directly interacting with local files. Fixes issues with unauthorized access and inefficient data handling. --- app/charts/synchronous.py | 96 ++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 51 deletions(-) diff --git a/app/charts/synchronous.py b/app/charts/synchronous.py index c2396c8..9791fa2 100644 --- a/app/charts/synchronous.py +++ b/app/charts/synchronous.py @@ -17,6 +17,10 @@ from requests import Session from requests.exceptions import ConnectionError, Timeout, TooManyRedirects import pygsheets import pytz +from django.conf import settings + +DATA_FILE = settings.DATA_FILE +BASE_DIR = settings.BASE_DIR #################################################################################### # Reddit api @@ -87,7 +91,7 @@ def get_history_function(symbol, start_time=None, end_time=None): blocksize = 0 difficulty = 0 - with open("settings.json") as file: + with open(BASE_DIR / "settings.json") as file: data = json.load(file) file.close() @@ -145,19 +149,19 @@ def get_history_function(symbol, start_time=None, end_time=None): try: coin.priceusd = float(item["PriceUSD"]) priceusd = coin.priceusd - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.priceusd = priceusd try: coin.pricebtc = float(item["PriceBTC"]) pricebtc = coin.pricebtc - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.pricebtc = pricebtc try: coin.inflation = float(item["IssContPctAnn"]) coin.stocktoflow = (100 / coin.inflation) ** 1.65 inflation = coin.inflation stocktoflow = coin.stocktoflow - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.inflation = inflation coin.stocktoflow = stocktoflow try: @@ -170,43 +174,43 @@ def get_history_function(symbol, start_time=None, end_time=None): else: coin.supply = float(item["SplyCur"]) supply = coin.supply - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.supply = supply try: coin.fee = float(item["FeeTotNtv"]) fee = coin.fee - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.fee = fee try: coin.revenue = float(item["RevNtv"]) revenue = coin.revenue - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.revenue = revenue try: coin.hashrate = float(item["HashRate"]) hashrate = coin.hashrate - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.hashrate = hashrate try: coin.transactions = float(item["TxCnt"]) transactions = coin.transactions - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.transactions = transactions try: coin.blocksize = float(item["BlkSizeMeanByte"]) blocksize = coin.blocksize - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.blocksize = blocksize try: coin.difficulty = float(item["DiffLast"]) difficulty = coin.difficulty - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): coin.difficulty = difficulty coin.save() count += 1 print(coin.name + " " + str(coin.date) + " " + str(item["SplyCur"])) - except (ValueError, KeyError): + except (ValueError, KeyError, TypeError): pass try: url = ( @@ -338,7 +342,7 @@ def get_binance_withdrawal(symbol): # Get latest price data for Monero def get_latest_price(symbol): - with open("settings.json") as file: + with open(BASE_DIR / "settings.json") as file: data = json.load(file) url = data["metrics_provider"][0]["price_url_old"] + symbol @@ -858,31 +862,26 @@ def update_p2pool(): p2pool_stat.save() print("p2pool saved!") - gc = pygsheets.authorize(service_file="service_account_credentials.json") - sh = gc.open("zcash_bitcoin") - wks = sh.worksheet_by_title("p2pool") - - values_mat = wks.get_values(start=(3, 1), end=(9999, 3), returnas="matrix") + df = pd.read_excel(DATA_FILE, engine="odf", sheet_name="p2pool") + start_row, end_row = 2, 9999 + start_col, end_col = 0, 2 + values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy() k = len(values_mat) date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d") date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d") date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d") if date_aux < date_aux2: - cell = "F" + str(k + 3) - wks.update_value(cell, p2pool_stat.totalblocksfound) - cell = "E" + str(k + 3) - wks.update_value(cell, p2pool_stat.totalhashes) - cell = "D" + str(k + 3) - wks.update_value(cell, p2pool_stat.percentage) - cell = "C" + str(k + 3) - wks.update_value(cell, p2pool_stat.hashrate) - cell = "B" + str(k + 3) - wks.update_value(cell, p2pool_stat.miners) - cell = "A" + str(k + 3) - wks.update_value( - cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d") - ) + values_mat[k][5] = p2pool_stat.totalblocksfound + values_mat[k][4] = p2pool_stat.totalhashes + values_mat[k][3] = p2pool_stat.percentage + values_mat[k][2] = p2pool_stat.hashrate + values_mat[k][1] = p2pool_stat.miners + values_mat[k][0] = datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d") + + df.iloc[start_row:end_row, start_col:end_col] = values_mat + df.to_excel(DATA_FILE, sheet_name="p2pool", index=False) + print("spreadsheet updated") else: print("spreadsheet already with the latest data") @@ -939,31 +938,26 @@ def update_p2pool(): p2pool_stat.save() print("p2pool_mini saved!") - gc = pygsheets.authorize(service_file="service_account_credentials.json") - sh = gc.open("zcash_bitcoin") - wks = sh.worksheet_by_title("p2poolmini") - - values_mat = wks.get_values(start=(3, 1), end=(9999, 3), returnas="matrix") + df = pd.read_excel(DATA_FILE, engine="odf", sheet_name="p2poolmini") + start_row, end_row = 2, 9999 + start_col, end_col = 0, 2 + values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy() k = len(values_mat) date_aux = datetime.datetime.strptime(values_mat[k - 1][0], "%Y-%m-%d") date_aux2 = datetime.datetime.strftime(date.today(), "%Y-%m-%d") date_aux2 = datetime.datetime.strptime(date_aux2, "%Y-%m-%d") if date_aux < date_aux2: - cell = "F" + str(k + 3) - wks.update_value(cell, p2pool_stat.totalblocksfound) - cell = "E" + str(k + 3) - wks.update_value(cell, p2pool_stat.totalhashes) - cell = "D" + str(k + 3) - wks.update_value(cell, p2pool_stat.percentage) - cell = "C" + str(k + 3) - wks.update_value(cell, p2pool_stat.hashrate) - cell = "B" + str(k + 3) - wks.update_value(cell, p2pool_stat.miners) - cell = "A" + str(k + 3) - wks.update_value( - cell, datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d") - ) + values_mat[k][5] = p2pool_stat.totalblocksfound + values_mat[k][4] = p2pool_stat.totalhashes + values_mat[k][3] = p2pool_stat.percentage + values_mat[k][2] = p2pool_stat.hashrate + values_mat[k][1] = p2pool_stat.miners + values_mat[k][0] = datetime.datetime.strftime(p2pool_stat.date, "%Y-%m-%d") + + df.iloc[start_row:end_row, start_col:end_col] = values_mat + df.to_excel(DATA_FILE, sheet_name="p2poolmini", index=False) + print("spreadsheet updated") else: print("spreadsheet already with the latest data")