kumi/twitools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Store followers/followings as user IDs rather than handles. Store handles in separate table.

Browse source
This commit is contained in:
Klaus-Uwe Mitterer 2017-01-30 03:04:51 +01:00
parent c10f7d53f9
commit 701afaef45
3 changed files with 44 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
dbtools/__init__.py
View file

@ -77,12 +77,12 @@ class dbObject:
def getFollowers(db):
db.executeQuery("SELECT id FROM followers WHERE `until` = 0;")
for i in db.getAll():
yield i[0]
yield int(i[0])
def getFollowing(db):
db.executeQuery("SELECT id FROM following WHERE `until` = 0;")
for i in db.getAll():
yield i[0]
yield int(i[0])
def getLatestMessage(db):
@ -99,6 +99,15 @@ class dbObject:
except:
return 0
def matchNameID(db, name, id):
db.executeQuery("SELECT COUNT(*) FROM names WHERE id = '%s' AND name = '%s' AND until = 0;" % (id, name))
try:
if int(db.getNext()[0]) != 0:
return True
except:
pass
return False
def dbHelper():
if setuptools.dbtype() == SQLITE:
return dbObject(dbtype=SQLITE, path=setuptools.dbpath())

47
filler.py
View file

@ -48,13 +48,11 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
db.commit()
return mcount, savepoint or 0, db.getLatestMessage()
return mcount, savepoint or 0, db.getLatestMessage
def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
""" Get handles of users we are following. :param db: Database object to be used. :param two: Twitter object to be used. :param firstrun: Must be set to True if the function is executed for the first time. Defaults to False. :return: Returns the number of gained and lost followings in a list (gained, lost). """
current = list(db.getFollowers())
new = list(twitools.getNamesByIDs(twitools.getFollowerIDs()))
new = list(twitools.getFollowerIDs())
gained = 0
lost = 0
@ -64,31 +62,23 @@ def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False)
for follower in new:
if follower not in current:
db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (follower, int(time.time())))
print("New follower: %s" % follower)
db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (str(follower), int(time.time())))
db.commit()
print("New follower: %s" % (twitools.getNameByID(follower) if not firstrun else follower))
gained += 1
for follower in current:
if follower not in new:
db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), follower))
print("Lost follower: %s" % follower)
lost += 1
db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(follower)))
db.commit()
print("Lost follower: %s" % twitools.getNameByID(follower))
lost += 1
return gained, lost
def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
"""
Get handles of users we are following.
:param db: Database object to be used.
:param two: Twitter object to be used.
:param firstrun: Must be set to True if the function is executed for the first time. Defaults to False.
:return: Returns the number of gained and lost followings in a list (gained, lost).
"""
current = list(db.getFollowing())
new = list(twitools.getNamesByIDs(twitools.getFollowingIDs()))
new = list(twitools.getFollowingIDs())
gained = 0
lost = 0
@ -98,20 +88,29 @@ def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False)
for following in new:
if following not in current:
db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (following, int(time.time())))
print("You started following: %s" % following)
db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (str(following), int(time.time())))
db.commit()
print("You started following: %s" % (str(following) if not firstrun else following))
gained += 1
for following in current:
if following not in new:
db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), following))
print("You no longer follow: %s" % following)
db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(following)))
db.commit()
print("You no longer follow: %s" % twitools.getNameByID(following))
lost += 1
db.commit()
return gained, lost
def getNames(db = dbtools.dbHelper(), two = twitools.twObject()):
for user in twitools.getNamesByIDs(list(set(list(db.getFollowing()) + list(db.getFollowers())))):
if not db.matchNameID(user["name"], user["id"]):
db.executeQuery("UPDATE names SET `until` = %i WHERE `id` = '%s' AND `name` = '%s';" % (int(time.time()), str(user["id"]), str(user["name"])))
db.executeQuery("INSERT INTO names VALUES('%s', '%s', %i, 0)" % (str(user["id"]), str(user["name"]), int(time.time())))
db.commit()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--first", help="first run: ignore empty databases", action="store_true")
@ -125,3 +124,5 @@ if __name__ == "__main__":
print("Gained %i followers, lost %i." % (gained, lost))
gained, lost = getFollowing(db, firstrun=args.first)
print("Started following %i, stopped following %i." % (gained, lost))
getNames(db)
print("Stored handles of following/followers.")

12
twitools/__init__.py
View file

@ -26,13 +26,19 @@ class twObject:
def getFollowerIDs(two=twObject()):
''' Returns 5,000 follower IDs at most '''
return two.api.followers_ids(screen_name=twObject().whoami())
for id in list(two.api.followers_ids(screen_name=twObject().whoami())):
yield int(id)
def getFollowingIDs(two=twObject()):
return two.api.friends_ids(screen_name=twObject().whoami())
for id in list(two.api.friends_ids(screen_name=twObject().whoami())):
yield int(id)
def getNameByID(uid, two=twObject()):
return two.api.get_user(uid).screen_name
def getNamesByIDs(fids=getFollowerIDs(), two=twObject()):
for page in setuptools.paginate(fids, 100):
followers = two.api.lookup_users(user_ids=page)
for follower in followers:
yield follower.screen_name
yield {"id": follower.id, "name": follower.screen_name}