fixed social

charts/asynchronous.py

@@ -140,39 +140,32 @@ async def get_coin_data(session, symbol, url):
 #   Asynchronous get social metrics from reddit
 #################################################################################### 
 async def get_social_data(session, symbol):
-    date_now = datetime.datetime.strftime(date.today(), '%Y-%m-%d')
+    yesterday = datetime.datetime.strftime(date.today()-timedelta(1), '%Y-%m-%d')
-    socials = Social.objects.filter(name=symbol).filter(date=date_now)
+    try:
-
+        social = Social.objects.filter(name=symbol).get(date=yesterday)
-    if not(socials):
+    except:
-        print('getting new data')
         url = 'https://www.reddit.com/r/'+ symbol +'/about.json'
 
         async with session.get(url, headers={'User-agent': 'Checking new social data'}) as res:
             data = await res.read()
             data = json.loads(data) 
             data = data['data']
-            subscribers = data['subscribers']
+
             social = Social()
             social.name = symbol
-            social.date = date_now
+            social.date = yesterday
-            social.subscriberCount = subscribers
+            social.subscriberCount = data['subscribers']
 
-            date_aux = date.today()
+            timestamp1 = int(datetime.datetime.timestamp(datetime.datetime.strptime(yesterday, '%Y-%m-%d')))
-            date_aux = datetime.datetime.strftime(date_aux, '%Y-%m-%d')
+            timestamp2 = int(timestamp1 - 7200)
-            date_aux = datetime.datetime.strptime(date_aux, '%Y-%m-%d')
+            limit = 1000
-            timestamp1 = int(datetime.datetime.timestamp(date_aux))
-
-            timestamp2 = int(timestamp1 - 86400)
-            limit = 2000
             filters = []
             data = data_prep_posts(symbol, timestamp2, timestamp1, filters, limit)
-            social.postsPerHour = len(data)/24
+            social.postsPerHour = len(data)/2
-
-            timestamp2 = int(timestamp1 - 7200)
-            limit = 2000
             data = data_prep_comments(symbol, timestamp2, timestamp1, filters, limit)
             social.commentsPerHour = len(data)/2
             social.save()
+            print(str(social.name) + ' - ' + str(social.date) + ' - ' + str(social.subscriberCount) + ' - ' + str(social.commentsPerHour) + ' - ' + str(social.postsPerHour) ) 
     return True
 
 ####################################################################################
@@ -323,9 +316,9 @@ async def update_others_data(date):
 
     async with aiohttp.ClientSession(**client_args) as session:  
         # reddit data
-        actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
+        #actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
-        actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
+        #actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
-        actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
+        #actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
         # coinmetrics data
         actions.append(asyncio.ensure_future(get_coin_data(session, 'btc', url_btc)))
         actions.append(asyncio.ensure_future(get_coin_data(session, 'dash', url_dash)))
@@ -338,7 +331,36 @@ async def update_others_data(date):
             await asyncio.gather(*actions, return_exceptions=True)
         except asyncio.exceptions.TimeoutError:
             print('Timeout!')
-        update_database(date, date)
+
+    return True
+
+####################################################################################
+#   Asynchronous get social and coins data
+#################################################################################### 
+async def update_social_data(symbol):
+
+    actions = []
+    my_timeout = aiohttp.ClientTimeout(
+        total=10,
+        sock_connect=10, 
+        sock_read=10 
+    )
+    client_args = dict(
+        trust_env=True,
+        timeout=my_timeout
+    )
+
+    async with aiohttp.ClientSession(**client_args) as session:  
+        # reddit data
+        actions.append(asyncio.ensure_future(get_social_data(session, 'Monero')))
+        actions.append(asyncio.ensure_future(get_social_data(session, 'Bitcoin')))
+        actions.append(asyncio.ensure_future(get_social_data(session, 'Cryptocurrency')))
+        
+        try:
+            await asyncio.gather(*actions, return_exceptions=True)
+        except asyncio.exceptions.TimeoutError:
+            print('Timeout!')
+
     return True
 
 ####################################################################################

charts/synchronous.py

@@ -215,11 +215,11 @@ def update_rank(data=None):
 
 # Load Reddit api to check if there are new followers
 def check_new_social(symbol):
-    date_now = datetime.datetime.strftime(date.today(), '%Y-%m-%d')
+    yesterday = datetime.datetime.strftime(date.today()-timedelta(1), '%Y-%m-%d')
-    socials = Social.objects.filter(name=symbol).filter(date=date_now)
+    socials = Social.objects.filter(name=symbol).filter(date=yesterday)
 
     if not(socials):
-        print('getting new data')
+        print('getting new data - ' + str(symbol))
         request = 'https://www.reddit.com/r/'+ symbol +'/about.json'
         response = requests.get(request, headers = {'User-agent': 'Checking new social data'})
         data = json.loads(response.content)    
@@ -227,7 +227,7 @@ def check_new_social(symbol):
         subscribers = data['subscribers']
         social = Social()
         social.name = symbol
-        social.date = date_now
+        social.date = yesterday
         social.subscriberCount = subscribers
 
         date_aux = date.today()
@@ -292,18 +292,19 @@ def update_database(date_from=None, date_to=None):
 
             count_aux = 0
             found = False
+            print(date_aux)
             while count_aux < 100 and not(found):
-
                 try:
-                    date_aux2 = date_aux - timedelta(count_aux)
+                    date_aux3 = date_aux - timedelta(count_aux)
-                    social_btc = Social.objects.filter(name='Bitcoin').get(date=date_aux2)
+                    social_btc = Social.objects.filter(name='Bitcoin').get(date=date_aux3)
-                    social_xmr = Social.objects.filter(name='Monero').get(date=date_aux2)
+                    social_xmr = Social.objects.filter(name='Monero').get(date=date_aux3)
-                    social_crypto = Social.objects.filter(name='CryptoCurrency').get(date=date_aux2)
+                    social_crypto = Social.objects.filter(name='CryptoCurrency').get(date=date_aux3)
                     found = True
-
                 except:
-                    count_aux += 1
                     found = False
+                count_aux += 1
+            
+            print(date_aux3)
         except:
             return count
 

charts/views.py

@@ -925,8 +925,9 @@ async def index(request):
     date_aux = datetime.datetime.strftime(date.today() - timedelta(2), '%Y-%m-%d')
     update_xmr = False
     update_btc = False
+    update_socials = False
 
-    if now > 1:
+    if now > 1 and now < 5:
         try:
             coin_xmr = Coin.objects.filter(name='xmr').get(date=yesterday)
             if coin_xmr:
@@ -945,7 +946,23 @@ async def index(request):
             print('no xmr found yesterday - 2')
             update_xmr = True
 
-    if now > 5:
+    if now > 3 and now < 5:
+        try:
+            social_xmr = list(Social.objects.filter(name='Monero').filter(date=yesterday))[0]
+            social_btc = list(Social.objects.filter(name='Bitcoin').filter(date=yesterday))[0]
+            social_crypto = list(Social.objects.filter(name='Cryptocurrency').filter(date=yesterday))[0]
+
+            if social_btc and social_xmr and social_crypto:
+                print('socials found yesterday')
+                update_socials = False
+            else:
+                print('no socials found yesterday - 1')
+                update_socials = True
+        except:
+            print('no socials found yesterday - 2')
+            update_socials = True  
+
+    if now > 5 and now < 12:
         try:
             coin_btc = list(Coin.objects.filter(name='btc').filter(date=yesterday))[0]
             coin_zec = list(Coin.objects.filter(name='zec').filter(date=yesterday))[0]
@@ -969,6 +986,7 @@ async def index(request):
         except:
             print('no coins found yesterday - 2')
             update_btc = True  
+
     try:
         coin_xmr = Coin.objects.filter(name='xmr').get(date=date_aux)
     except:
@@ -977,6 +995,12 @@ async def index(request):
     if update_xmr:
         await asynchronous.update_xmr_data(yesterday, coin_xmr)
 
+    if update_socials:
+        synchronous.check_new_social('Bitcoin')
+        synchronous.check_new_social('Monero')
+        synchronous.check_new_social('Cryptocurrency')
+        #await asynchronous.update_social_data(yesterday)
+
     if update_btc:
         await asynchronous.update_others_data(yesterday)
         synchronous.update_database(yesterday, yesterday)

moneropro/settings.py

@@ -28,8 +28,8 @@ STATICFILES_DIRS = [
 SECRET_KEY = 'dafsdasdfasfa7sdfs9d7das7f9as7df8ad43525fsffdssdg455$$###77saf9df79as79s'
 
 # SECURITY WARNING: don't run with debug turned on in production!
-#DEBUG = True 
+DEBUG = True 
-DEBUG = False
+#DEBUG = False
 
 ALLOWED_HOSTS = ['80.78.22.34', 'www.moneroj.net', 'localhost', '127.0.0.1', 'moneroj.net', 'moneroj5xq4ttg4ec7e5secqdyw5mcovzvfvlq6i7omv353i6mnexlqd.onion']