refactor: switch to pandas for data extraction

Replaced pygsheets with pandas and removed dependency on Google Sheets for data extraction, leveraging local spreadsheet files instead. This improves reliability and simplifies setup by avoiding external API dependencies. Updated code to handle data conversion and string checks ensuring data consistency. Adjusted settings to accommodate new input file.

app/charts/views.py

@@ -1,6 +1,7 @@
 from django.shortcuts import render
 from django.http import HttpResponseRedirect
 from django.urls import reverse
+from django.conf import settings
 import json
 from .models import (
     Coin,
@@ -21,7 +22,6 @@ import pandas as pd
 from . import asynchronous
 from . import synchronous
 from operator import truediv
-import pygsheets
 from django.contrib.auth.decorators import login_required
 from django.contrib.staticfiles.storage import staticfiles_storage
 from charts.synchronous import get_history_function
@@ -32,6 +32,12 @@ from charts.synchronous import get_history_function
 #   Set some parameters
 ####################################################################################
 locale.setlocale(locale.LC_ALL, "en_US.utf8")  # TODO: Why is this here?
+DATA_FILE = settings.DATA_FILE
+
+try:
+    locale.format
+except AttributeError:
+    locale.format = locale.format_string
 
 ####################################################################################
 #   Useful functions for admins
@@ -145,12 +151,13 @@ def get_history(request, symbol, start_time=None, end_time=None):
 def load_rank(request, symbol):
     if not request.user.is_superuser:
         return render(request, "users/error.html")
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet8")
+
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet8", engine="odf")
+    start_row, end_row = 2, 9999
+    start_col, end_col = 0, 2
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     count = 0
-    values_mat = wks.get_values(start=(3, 1), end=(9999, 2), returnas="matrix")
     print(len(values_mat))
     Rank.objects.all().delete()
 
@@ -181,21 +188,44 @@ def load_p2pool(request):
         return render(request, "users/error.html")
 
     count = 0
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("p2pool")
-    values_mat = wks.get_values(start=(3, 1), end=(9999, 6), returnas="matrix")
+
+    df = pd.read_excel(DATA_FILE, sheet_name="p2pool", engine="odf")
+    start_row, end_row = 2, 9999
+    start_col, end_col = 0, 6
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
+
     P2Pool.objects.all().delete()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][1]:
             p2pool_stat = P2Pool()
             p2pool_stat.date = values_mat[k][0]
-            p2pool_stat.miners = float(values_mat[k][1].replace(",", "."))
-            p2pool_stat.hashrate = float(values_mat[k][2].replace(",", "."))
-            p2pool_stat.percentage = float(values_mat[k][3].replace(",", "."))
-            p2pool_stat.totalhashes = float(values_mat[k][4].replace(",", "."))
-            p2pool_stat.totalblocksfound = float(values_mat[k][5].replace(",", "."))
+
+            p2pool_stat.miners = (
+                float(values_mat[k][1].replace(",", "."))
+                if isinstance(values_mat[k][1], str)
+                else values_mat[k][1]
+            )
+            p2pool_stat.hashrate = (
+                float(values_mat[k][2].replace(",", "."))
+                if isinstance(values_mat[k][2], str)
+                else values_mat[k][2]
+            )
+            p2pool_stat.percentage = (
+                float(values_mat[k][3].replace(",", "."))
+                if isinstance(values_mat[k][3], str)
+                else values_mat[k][3]
+            )
+            p2pool_stat.totalhashes = (
+                float(values_mat[k][4].replace(",", "."))
+                if isinstance(values_mat[k][4], str)
+                else values_mat[k][4]
+            )
+            p2pool_stat.totalblocksfound = (
+                float(values_mat[k][5].replace(",", "."))
+                if isinstance(values_mat[k][5], str)
+                else values_mat[k][1]
+            )
             p2pool_stat.mini = False
             p2pool_stat.save()
             count += 1
@@ -203,18 +233,40 @@ def load_p2pool(request):
         else:
             break
 
-    wks = sh.worksheet_by_title("p2poolmini")
-    values_mat = wks.get_values(start=(3, 1), end=(9999, 6), returnas="matrix")
+    df = pd.read_excel(DATA_FILE, sheet_name="p2poolmini", engine="odf")
+    start_row, end_row = 2, 9999
+    start_col, end_col = 0, 6
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][1]:
             p2pool_stat = P2Pool()
             p2pool_stat.date = values_mat[k][0]
-            p2pool_stat.miners = float(values_mat[k][1].replace(",", "."))
-            p2pool_stat.hashrate = float(values_mat[k][2].replace(",", "."))
-            p2pool_stat.percentage = float(values_mat[k][3].replace(",", "."))
-            p2pool_stat.totalhashes = float(values_mat[k][4].replace(",", "."))
-            p2pool_stat.totalblocksfound = float(values_mat[k][5].replace(",", "."))
+            p2pool_stat.miners = (
+                float(values_mat[k][1].replace(",", "."))
+                if isinstance(values_mat[k][1], str)
+                else values_mat[k][1]
+            )
+            p2pool_stat.hashrate = (
+                float(values_mat[k][2].replace(",", "."))
+                if isinstance(values_mat[k][2], str)
+                else values_mat[k][2]
+            )
+            p2pool_stat.percentage = (
+                float(values_mat[k][3].replace(",", "."))
+                if isinstance(values_mat[k][3], str)
+                else values_mat[k][3]
+            )
+            p2pool_stat.totalhashes = (
+                float(values_mat[k][4].replace(",", "."))
+                if isinstance(values_mat[k][4], str)
+                else values_mat[k][4]
+            )
+            p2pool_stat.totalblocksfound = (
+                float(values_mat[k][5].replace(",", "."))
+                if isinstance(values_mat[k][5], str)
+                else values_mat[k][5]
+            )
             p2pool_stat.mini = True
             p2pool_stat.save()
             count += 1
@@ -233,12 +285,13 @@ def load_p2pool(request):
 def load_dominance(request, symbol):
     if not request.user.is_superuser:
         return render(request, "users/error.html")
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet7")
+
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet7", engine="odf")
+    start_row, end_row = 2, 9999
+    start_col, end_col = 0, 2
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     count = 0
-    values_mat = wks.get_values(start=(3, 1), end=(9999, 2), returnas="matrix")
     Dominance.objects.all().delete()
 
     for k in range(0, len(values_mat)):
@@ -246,7 +299,11 @@ def load_dominance(request, symbol):
             dominance = Dominance()
             dominance.name = symbol
             dominance.date = values_mat[k][0]
-            dominance.dominance = float(values_mat[k][1].replace(",", "."))
+            dominance.dominance = (
+                float(values_mat[k][1].replace(",", "."))
+                if isinstance(values_mat[k][1], str)
+                else values_mat[k][1]
+            )
             if not (dominance.dominance) and not (dominance.date):
                 break
             else:
@@ -1449,9 +1506,9 @@ async def index(request):
         synchronous.update_database(yesterday, yesterday)
 
     if True:
-        enabled = synchronous.get_binance_withdrawal( # TODO: And... then... what? # noqa: F841
+        enabled = synchronous.get_binance_withdrawal(  # TODO: And... then... what? # noqa: F841
             "Monero"
-        )  
+        )
 
     return HttpResponseRedirect(reverse("charts:sfmodel"))
 
@@ -2120,7 +2177,9 @@ def pricelin(request):
         if reward < 0.6 * (10**12):
             reward = 0.6 * (10**12)
         supply += int(720 * reward)
-        stock = (100 / (100 * reward * 720 * 365 / supply)) ** 1.65 # TODO: Seems unused... # noqa: F841
+        stock = (
+            100 / (100 * reward * 720 * 365 / supply)
+        ) ** 1.65  # TODO: Seems unused... # noqa: F841
 
     now_price = "$" + locale.format("%.2f", now_price, grouping=True)
     now_sf = "$" + locale.format("%.2f", now_sf, grouping=True)
@@ -4374,12 +4433,14 @@ def shielded(request):
     values = []
     values2 = []
     values3 = []
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet1")
+
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet1", engine="odf")
+    start_row, end_row = 2, 9999
+    start_col, end_col = 0, 5
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
+
     dominance = 0
     monthly = 0
-    values_mat = wks.get_values(start=(3, 1), end=(999, 5), returnas="matrix")
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][3]:
@@ -5053,10 +5114,11 @@ def dread_subscribers(request):
     data2 = []
     now_xmr = 0
     now_btc = 0
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet6")
-    values_mat = wks.get_values(start=(3, 1), end=(99, 3), returnas="matrix")
+
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet6", engine="odf")
+    start_row, end_row = 2, 99
+    start_col, end_col = 0, 2
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][2]:
@@ -5100,11 +5162,10 @@ def coincards(request):
     now_xmr = 0
     now_btc = 0
 
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet2")
-
-    values_mat = wks.get_values(start=(3, 1), end=(99, 5), returnas="matrix")
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet2", engine="odf")
+    start_row, end_row = 2, 99
+    start_col, end_col = 0, 5
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][2]:
@@ -5160,11 +5221,10 @@ def merchants(request):
     now_btc = 0
     now_eth = 0
 
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet3")
-
-    values_mat = wks.get_values(start=(3, 1), end=(99, 8), returnas="matrix")
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet3", engine="odf")
+    start_row, end_row = 2, 99
+    start_col, end_col = 0, 8
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][2]:
@@ -5234,11 +5294,10 @@ def merchants_increase(request):
     now_btc = 0
     now_eth = 0
 
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet4")
-
-    values_mat = wks.get_values(start=(3, 1), end=(99, 8), returnas="matrix")
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet4", engine="odf")
+    start_row, end_row = 2, 99
+    start_col, end_col = 0, 8
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][2]:
@@ -5308,11 +5367,10 @@ def merchants_percentage(request):
     now_btc = 0
     now_eth = 0
 
-    gc = pygsheets.authorize(service_file="service_account_credentials.json")
-    sh = gc.open("zcash_bitcoin")
-    wks = sh.worksheet_by_title("Sheet5")
-
-    values_mat = wks.get_values(start=(3, 1), end=(99, 8), returnas="matrix")
+    df = pd.read_excel(DATA_FILE, sheet_name="Sheet5", engine="odf")
+    start_row, end_row = 2, 99
+    start_col, end_col = 0, 8
+    values_mat = df.iloc[start_row:end_row, start_col:end_col].to_numpy()
 
     for k in range(0, len(values_mat)):
         if values_mat[k][0] and values_mat[k][2]:

app/moneropro/settings.py

@@ -48,7 +48,7 @@ ALLOWED_HOSTS = [
 if "MONEROPRO_DEV_HOST" in os.environ:
     ALLOWED_HOSTS.append(os.environ["MONEROPRO_DEV_HOST"])
 
-CORS_ALLOWED_ORIGINS = [f"http://{host}" for host in ALLOWED_HOSTS] + [
+CSRF_TRUSTED_ORIGINS = [f"http://{host}" for host in ALLOWED_HOSTS] + [
     f"https://{host}" for host in ALLOWED_HOSTS
 ]
 
@@ -108,6 +108,8 @@ DATABASES = {
     }
 }
 
+DATA_FILE = BASE_DIR / "data" / "zcash_bitcoin.ods"
+
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
 

requirements.txt

@@ -24,6 +24,7 @@ httplib2==0.20.4
 multidict==6.1.0
 numpy
 oauthlib==3.2.2
+odfpy<2.0.0
 pandas==2.2.3
 protobuf==3.20.1 # TODO: 5.28.3
 psaw==0.1.0 # TODO: Unmaintained, recommends switch to https://github.com/mattpodolak/pmaw