From efcb31ad2080009e5a73705084e7b1ede3ead7cd Mon Sep 17 00:00:00 2001 From: Klaus-Uwe Mitterer Date: Tue, 21 Feb 2017 02:27:14 +0100 Subject: [PATCH] Add fillerfilter file for hooks in filler --- dbtools/fillerfilter.py | 47 ++++++++++++++++++++++++++++++++++++++ filler.py | 50 ++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 23 deletions(-) create mode 100644 dbtools/fillerfilter.py diff --git a/dbtools/fillerfilter.py b/dbtools/fillerfilter.py new file mode 100644 index 0000000..5eadb9d --- /dev/null +++ b/dbtools/fillerfilter.py @@ -0,0 +1,47 @@ +""" +This file allows you to add your own hooks to the filler.py script without +having to mess around with the code. +""" + +def tweetFilter(tweet): + """ + Code to be executed when a new tweet is found. + + :param tweet: tweepy.Status object for the new tweet + :return: True if the tweet should be stored in the database, else False + """ + + return True + +def messageFilter(message, incoming): + """ + Code to be executed when a new incoming or outgoing message is found. + + :param message: tweepy.DirectMessage object for the new message + :param incoming: True if the message is incoming (not sent by us), else False + :return: True if the message should be stored in the database, else False + """ + + return True + +def followerFilter(follower, current): + """ + Code to be executed when a follower is gained or lost. + + :param follower: Integer ID of the following account + :param current: True if they started following, False if they unfollowed + :return: True if the following/unfollowing action should be stored, else False + """ + + return True + +def followingFilter(following, current): + """ + Code to be executed when we start or stop following another account. + + :param following: Integer ID of the account we follow/stopped following + :param current: True if we started following, False if we unfollowed + :return: True if the following/unfollowing action should be stored, else False + """ + + return True diff --git a/filler.py b/filler.py index 2f71e72..999c893 100755 --- a/filler.py +++ b/filler.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import argparse, dbtools, setuptools, time, twitools +import argparse, dbtools, dbtools.fillerfilter, setuptools, time, twitools def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twitools.twObject()): query = "from:" + user @@ -15,14 +15,16 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000" text = setuptools.unescapeText(status.text) - try: - db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") - except: - pass - db.commit() + if dbtools.fillerfilter.tweetFilter(status): + try: + db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") + except: + pass - last = status.id - tw_counter = tw_counter + 1 + last = status.id + tw_counter = tw_counter + 1 + + db.commit() return tw_counter, last, savepoint @@ -33,20 +35,22 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()): new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True) for m in new_messages: - try: - db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) - mcount += 1 - except: - pass + if dbtools.fillerfilter.messageFilter(m, True): + try: + db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) + mcount += 1 + except: + pass for m in new_out_messages: - try: - db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) - mcount += 1 - except: - pass + if dbtools.fillerfilter.messageFilter(m, False): + try: + db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) + mcount += 1 + except: + pass - db.commit() + db.commit() return mcount, savepoint or 0, db.getLatestMessage @@ -61,14 +65,14 @@ def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) return 0,0 for follower in new: - if follower not in current: + if follower not in current and dbtools.fillerfilter.followerFilter(follower, True): db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (str(follower), int(time.time()))) db.commit() print("New follower: %s" % (twitools.getNameByID(follower) if not firstrun else follower)) gained += 1 for follower in current: - if follower not in new: + if follower not in new and dbtools.fillerfilter.followerFilter(follower, False): db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(follower))) db.commit() print("Lost follower: %s" % twitools.getNameByID(follower)) @@ -87,14 +91,14 @@ def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) return 0,0 for following in new: - if following not in current: + if following not in current and dbtools.fillerfilter.followingFilter(following, True): db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (str(following), int(time.time()))) db.commit() print("You started following: %s" % (str(following) if not firstrun else following)) gained += 1 for following in current: - if following not in new: + if following not in new and dbtools.fillerfilter.followingFilter(following, False): db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(following))) db.commit() print("You no longer follow: %s" % twitools.getNameByID(following))