twitools/filler.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import html.parser, sqlite3, time, tweepy, os

user = "Username"
cke = "Consumer Key"
cse = "Consumer Secret"
ato = "Access Token"
ase = "Access Secret"

search = "from:" + user

database_filename = "Database.db"

sql_conn = sqlite3.connect(database_filename)
cur = sql_conn.cursor()

try:
 cur.execute("SELECT max(tweet_id) FROM tweets")
except:
 print("Please create a database using ./makedb.py or ./csvdb.py before trying to populate it.")

try:
 savepoint = int(cur.fetchone()[0])
except:
 savepoint = 0

auth = tweepy.OAuthHandler(cke, cse)
auth.set_access_token(ato, ase)
api = tweepy.API(auth)

timelineIterator = list(tweepy.Cursor(api.search, q=search, since_id=savepoint).items())

timeline = []

for status in timelineIterator:
 timeline.append(status)

timeline.reverse()

tw_counter = 0

for status in timeline:
 print("(%(date)s) %(name)s: %(message)s\n" % \
  { "date" : status.created_at,
    "name" : status.author.screen_name.encode('utf-8'),
    "message" : status.text.encode('utf-8') })

 timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
 text = html.parser.HTMLParser().unescape(status.text).replace("'", "''")

 cur.execute("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
 tw_counter = tw_counter + 1

sql_conn.commit()
sql_conn.close()

print("Finished. %d Tweets stored" % (tw_counter))