twitools/filler.py

#!/usr/bin/env python3

import tools

import html.parser, os

def getSavepoint(db):
 db.executeQuery("SELECT max(tweet_id) FROM tweets")
 try:
  return int(db.getNext()[0])
 except:
  print("No tweets stored yet.")
  return 0

def unescapeText(text):
 return html.parser.HTMLParser().unescape(text).replace("'","''")

def fill(dbpath=tools.config.dbpath, user=tools.config.user, two=tools.twObject()):
 query = "from:" + user

 db = tools.dbHelper(dbpath)

 savepoint = getSavepoint(db)
 last = savepoint

 timeline = two.search(query, savepoint)

 tw_counter = 0

 for status in timeline:
  timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
  text = unescapeText(status.text)

  db.executeQuery("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
  db.commit()

  last = status.id
  tw_counter = tw_counter + 1

 db.closeConnection()

 return tw_counter, last, savepoint

if __name__ == "__main__":
 count, last, first = fill()
 print("Stored %i tweets after %i until %i." % (count, first, last))