From a9e8459641a4bac2a6c26def741b1e91432fee0d Mon Sep 17 00:00:00 2001 From: anon Date: Fri, 23 Sep 2022 17:38:20 -0300 Subject: [PATCH] fixed social --- charts/asynchronous.py | 68 ++++++++++++++++++++++++++++-------------- charts/synchronous.py | 23 +++++++------- charts/views.py | 28 +++++++++++++++-- moneropro/settings.py | 4 +-- 4 files changed, 85 insertions(+), 38 deletions(-) diff --git a/charts/asynchronous.py b/charts/asynchronous.py index 477c6a8..3a55f67 100644 --- a/charts/asynchronous.py +++ b/charts/asynchronous.py @@ -140,39 +140,32 @@ async def get_coin_data(session, symbol, url): # Asynchronous get social metrics from reddit #################################################################################### async def get_social_data(session, symbol): - date_now = datetime.datetime.strftime(date.today(), '%Y-%m-%d') - socials = Social.objects.filter(name=symbol).filter(date=date_now) - - if not(socials): - print('getting new data') + yesterday = datetime.datetime.strftime(date.today()-timedelta(1), '%Y-%m-%d') + try: + social = Social.objects.filter(name=symbol).get(date=yesterday) + except: url = 'https://www.reddit.com/r/'+ symbol +'/about.json' async with session.get(url, headers={'User-agent': 'Checking new social data'}) as res: data = await res.read() data = json.loads(data) data = data['data'] - subscribers = data['subscribers'] + social = Social() social.name = symbol - social.date = date_now - social.subscriberCount = subscribers + social.date = yesterday + social.subscriberCount = data['subscribers'] - date_aux = date.today() - date_aux = datetime.datetime.strftime(date_aux, '%Y-%m-%d') - date_aux = datetime.datetime.strptime(date_aux, '%Y-%m-%d') - timestamp1 = int(datetime.datetime.timestamp(date_aux)) - - timestamp2 = int(timestamp1 - 86400) - limit = 2000 + timestamp1 = int(datetime.datetime.timestamp(datetime.datetime.strptime(yesterday, '%Y-%m-%d'))) + timestamp2 = int(timestamp1 - 7200) + limit = 1000 filters = [] data = data_prep_posts(symbol, timestamp2, timestamp1, filters, limit) - social.postsPerHour = len(data)/24 - - timestamp2 = int(timestamp1 - 7200) - limit = 2000 + social.postsPerHour = len(data)/2 data = data_prep_comments(symbol, timestamp2, timestamp1, filters, limit) social.commentsPerHour = len(data)/2 social.save() + print(str(social.name) + ' - ' + str(social.date) + ' - ' + str(social.subscriberCount) + ' - ' + str(social.commentsPerHour) + ' - ' + str(social.postsPerHour) ) return True #################################################################################### @@ -323,9 +316,9 @@ async def update_others_data(date): async with aiohttp.ClientSession(**client_args) as session: # reddit data - actions.append(asyncio.ensure_future(get_social_data(session, 'Monero'))) - actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin'))) - actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency'))) + #actions.append(asyncio.ensure_future(get_social_data(session, 'Monero'))) + #actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin'))) + #actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency'))) # coinmetrics data actions.append(asyncio.ensure_future(get_coin_data(session, 'btc', url_btc))) actions.append(asyncio.ensure_future(get_coin_data(session, 'dash', url_dash))) @@ -338,7 +331,36 @@ async def update_others_data(date): await asyncio.gather(*actions, return_exceptions=True) except asyncio.exceptions.TimeoutError: print('Timeout!') - update_database(date, date) + + return True + +#################################################################################### +# Asynchronous get social and coins data +#################################################################################### +async def update_social_data(symbol): + + actions = [] + my_timeout = aiohttp.ClientTimeout( + total=10, + sock_connect=10, + sock_read=10 + ) + client_args = dict( + trust_env=True, + timeout=my_timeout + ) + + async with aiohttp.ClientSession(**client_args) as session: + # reddit data + actions.append(asyncio.ensure_future(get_social_data(session, 'Monero'))) + actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin'))) + actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency'))) + + try: + await asyncio.gather(*actions, return_exceptions=True) + except asyncio.exceptions.TimeoutError: + print('Timeout!') + return True #################################################################################### diff --git a/charts/synchronous.py b/charts/synchronous.py index 19efd84..fc9a9b3 100644 --- a/charts/synchronous.py +++ b/charts/synchronous.py @@ -215,11 +215,11 @@ def update_rank(data=None): # Load Reddit api to check if there are new followers def check_new_social(symbol): - date_now = datetime.datetime.strftime(date.today(), '%Y-%m-%d') - socials = Social.objects.filter(name=symbol).filter(date=date_now) + yesterday = datetime.datetime.strftime(date.today()-timedelta(1), '%Y-%m-%d') + socials = Social.objects.filter(name=symbol).filter(date=yesterday) if not(socials): - print('getting new data') + print('getting new data - ' + str(symbol)) request = 'https://www.reddit.com/r/'+ symbol +'/about.json' response = requests.get(request, headers = {'User-agent': 'Checking new social data'}) data = json.loads(response.content) @@ -227,7 +227,7 @@ def check_new_social(symbol): subscribers = data['subscribers'] social = Social() social.name = symbol - social.date = date_now + social.date = yesterday social.subscriberCount = subscribers date_aux = date.today() @@ -292,18 +292,19 @@ def update_database(date_from=None, date_to=None): count_aux = 0 found = False + print(date_aux) while count_aux < 100 and not(found): - try: - date_aux2 = date_aux - timedelta(count_aux) - social_btc = Social.objects.filter(name='Bitcoin').get(date=date_aux2) - social_xmr = Social.objects.filter(name='Monero').get(date=date_aux2) - social_crypto = Social.objects.filter(name='CryptoCurrency').get(date=date_aux2) + date_aux3 = date_aux - timedelta(count_aux) + social_btc = Social.objects.filter(name='Bitcoin').get(date=date_aux3) + social_xmr = Social.objects.filter(name='Monero').get(date=date_aux3) + social_crypto = Social.objects.filter(name='CryptoCurrency').get(date=date_aux3) found = True - except: - count_aux += 1 found = False + count_aux += 1 + + print(date_aux3) except: return count diff --git a/charts/views.py b/charts/views.py index cfe5a95..264e927 100644 --- a/charts/views.py +++ b/charts/views.py @@ -925,8 +925,9 @@ async def index(request): date_aux = datetime.datetime.strftime(date.today() - timedelta(2), '%Y-%m-%d') update_xmr = False update_btc = False + update_socials = False - if now > 1: + if now > 1 and now < 5: try: coin_xmr = Coin.objects.filter(name='xmr').get(date=yesterday) if coin_xmr: @@ -945,7 +946,23 @@ async def index(request): print('no xmr found yesterday - 2') update_xmr = True - if now > 5: + if now > 3 and now < 5: + try: + social_xmr = list(Social.objects.filter(name='Monero').filter(date=yesterday))[0] + social_btc = list(Social.objects.filter(name='Bitcoin').filter(date=yesterday))[0] + social_crypto = list(Social.objects.filter(name='Cryptocurrency').filter(date=yesterday))[0] + + if social_btc and social_xmr and social_crypto: + print('socials found yesterday') + update_socials = False + else: + print('no socials found yesterday - 1') + update_socials = True + except: + print('no socials found yesterday - 2') + update_socials = True + + if now > 5 and now < 12: try: coin_btc = list(Coin.objects.filter(name='btc').filter(date=yesterday))[0] coin_zec = list(Coin.objects.filter(name='zec').filter(date=yesterday))[0] @@ -969,6 +986,7 @@ async def index(request): except: print('no coins found yesterday - 2') update_btc = True + try: coin_xmr = Coin.objects.filter(name='xmr').get(date=date_aux) except: @@ -977,6 +995,12 @@ async def index(request): if update_xmr: await asynchronous.update_xmr_data(yesterday, coin_xmr) + if update_socials: + synchronous.check_new_social('Bitcoin') + synchronous.check_new_social('Monero') + synchronous.check_new_social('Cryptocurrency') + #await asynchronous.update_social_data(yesterday) + if update_btc: await asynchronous.update_others_data(yesterday) synchronous.update_database(yesterday, yesterday) diff --git a/moneropro/settings.py b/moneropro/settings.py index 6b8403f..a15ec5a 100644 --- a/moneropro/settings.py +++ b/moneropro/settings.py @@ -28,8 +28,8 @@ STATICFILES_DIRS = [ SECRET_KEY = 'dafsdasdfasfa7sdfs9d7das7f9as7df8ad43525fsffdssdg455$$###77saf9df79as79s' # SECURITY WARNING: don't run with debug turned on in production! -#DEBUG = True -DEBUG = False +DEBUG = True +#DEBUG = False ALLOWED_HOSTS = ['80.78.22.34', 'www.moneroj.net', 'localhost', '127.0.0.1', 'moneroj.net', 'moneroj5xq4ttg4ec7e5secqdyw5mcovzvfvlq6i7omv353i6mnexlqd.onion']