Implement following/follower monitoring
This commit is contained in:
parent
2ebe1e10ae
commit
4255596ecf
4 changed files with 52 additions and 3 deletions
|
@ -50,6 +50,9 @@ class dbObject:
|
||||||
def executeQuery(self, query):
|
def executeQuery(self, query):
|
||||||
return self.cur.execute(query)
|
return self.cur.execute(query)
|
||||||
|
|
||||||
|
def getAll(self):
|
||||||
|
return self.cur.fetchall()
|
||||||
|
|
||||||
def getNext(self):
|
def getNext(self):
|
||||||
return self.cur.fetchone()
|
return self.cur.fetchone()
|
||||||
|
|
||||||
|
@ -71,6 +74,17 @@ class dbObject:
|
||||||
self.executeQuery("SELECT %s(SUBSTR(timestamp,0,11)) FROM tweets" % mode)
|
self.executeQuery("SELECT %s(SUBSTR(timestamp,0,11)) FROM tweets" % mode)
|
||||||
return setuptools.getDate(str(self.getNext()["%s(SUBSTR(timestamp,0,11))" % mode]))
|
return setuptools.getDate(str(self.getNext()["%s(SUBSTR(timestamp,0,11))" % mode]))
|
||||||
|
|
||||||
|
def getFollowers(db):
|
||||||
|
db.executeQuery("SELECT id FROM followers;")
|
||||||
|
for i in db.getAll():
|
||||||
|
yield i[0]
|
||||||
|
|
||||||
|
def getFollowing(db):
|
||||||
|
db.executeQuery("SELECT id FROM following;")
|
||||||
|
for i in db.getAll():
|
||||||
|
yield i[0]
|
||||||
|
|
||||||
|
|
||||||
def getLatestMessage(db):
|
def getLatestMessage(db):
|
||||||
db.executeQuery("SELECT max(id) FROM messages")
|
db.executeQuery("SELECT max(id) FROM messages")
|
||||||
try:
|
try:
|
||||||
|
|
34
filler.py
34
filler.py
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import dbtools, setuptools, twitools
|
import dbtools, setuptools, time, twitools
|
||||||
|
|
||||||
def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twitools.twObject()):
|
def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twitools.twObject()):
|
||||||
query = "from:" + user
|
query = "from:" + user
|
||||||
|
@ -52,8 +52,40 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
|
|
||||||
return mcount, savepoint or 0, db.getLatestMessage()
|
return mcount, savepoint or 0, db.getLatestMessage()
|
||||||
|
|
||||||
|
def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
|
current = db.getFollowers()
|
||||||
|
new = twitools.getNamesByIDs(twitools.getFollowerIDs())
|
||||||
|
|
||||||
|
for follower in new:
|
||||||
|
if follower not in current:
|
||||||
|
db.executeQuery("INSERT INTO followers VALUES('%s', %i, NULL)" % (follower, int(time.time())))
|
||||||
|
|
||||||
|
for follower in current:
|
||||||
|
if follower not in new:
|
||||||
|
db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` IS NULL" % (int(time.time()), follower))
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
|
current = db.getFollowing()
|
||||||
|
new = twitools.getNamesByIDs(twitools.getFollowingIDs())
|
||||||
|
|
||||||
|
for following in new:
|
||||||
|
if following not in current:
|
||||||
|
db.executeQuery("INSERT INTO following VALUES('%s', %i, NULL)" % (following, int(time.time())))
|
||||||
|
|
||||||
|
for following in current:
|
||||||
|
if following not in new:
|
||||||
|
db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = %s AND `until` IS NULL" % (int(time.time()), following))
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
count, last, first = getTweets()
|
count, last, first = getTweets()
|
||||||
print("Stored %i tweets after %i until %i." % (count, first, last))
|
print("Stored %i tweets after %i until %i." % (count, first, last))
|
||||||
count, last, first = getMessages()
|
count, last, first = getMessages()
|
||||||
print("Stored %i messages after %i until %i." % (count, first, last))
|
print("Stored %i messages after %i until %i." % (count, first, last))
|
||||||
|
getFollowers()
|
||||||
|
print("Processed followers.")
|
||||||
|
getFollowing()
|
||||||
|
print("Processed following.")
|
||||||
|
|
4
setup.py
4
setup.py
|
@ -51,8 +51,8 @@ else:
|
||||||
if not db.isInitialized():
|
if not db.isInitialized():
|
||||||
db.executeQuery("CREATE TABLE tweets(`tweet_id` INTEGER NOT NULL, `in_reply_to_status_id` TEXT, `in_reply_to_user_id` TEXT, `timestamp` TEXT, `source` TEXT, `text` TEXT, `retweeted_status_id` TEXT, `retweeted_status_user_id` TEXT, `retweeted_status_timestamp` TEXT, `expanded_urls` TEXT, PRIMARY KEY(tweet_id));")
|
db.executeQuery("CREATE TABLE tweets(`tweet_id` INTEGER NOT NULL, `in_reply_to_status_id` TEXT, `in_reply_to_user_id` TEXT, `timestamp` TEXT, `source` TEXT, `text` TEXT, `retweeted_status_id` TEXT, `retweeted_status_user_id` TEXT, `retweeted_status_timestamp` TEXT, `expanded_urls` TEXT, PRIMARY KEY(tweet_id));")
|
||||||
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` INTEGER, `recipient_id` INTEGER, `created_at` TEXT, PRIMARY KEY(id));")
|
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` INTEGER, `recipient_id` INTEGER, `created_at` TEXT, PRIMARY KEY(id));")
|
||||||
db.executeQuery("CREATE TABLE followers(`id` INTEGER NOT NULL, `since` INTEGER, `until` INTEGER);")
|
db.executeQuery("CREATE TABLE followers(`id` TEXT NOT NULL, `since` INTEGER NOT NULL, `until` INTEGER);")
|
||||||
db.executeQuery("CREATE TABLE following(`id` INTEGER NOT NULL, `since` INTEGER, `until` INTEGER);")
|
db.executeQuery("CREATE TABLE following(`id` TEXT NOT NULL, `since` INTEGER NOT NULL, `until` INTEGER);")
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
db.closeConnection()
|
db.closeConnection()
|
||||||
|
|
|
@ -22,6 +22,9 @@ def getFollowerIDs(two=twObject()):
|
||||||
''' Returns 5,000 follower IDs at most '''
|
''' Returns 5,000 follower IDs at most '''
|
||||||
return two.api.followers_ids(screen_name=twObject().whoami())
|
return two.api.followers_ids(screen_name=twObject().whoami())
|
||||||
|
|
||||||
|
def getFollowingIDs(two=twObject()):
|
||||||
|
return two.api.friends_ids(screen_name=twObject().whoami())
|
||||||
|
|
||||||
def getNamesByIDs(fids=getFollowerIDs(), two=twObject()):
|
def getNamesByIDs(fids=getFollowerIDs(), two=twObject()):
|
||||||
for page in setuptools.paginate(fids, 100):
|
for page in setuptools.paginate(fids, 100):
|
||||||
followers = two.api.lookup_users(user_ids=page)
|
followers = two.api.lookup_users(user_ids=page)
|
||||||
|
|
Loading…
Reference in a new issue