From 010f9e56b6b67331ec58276308b0cd85b1b6eb09 Mon Sep 17 00:00:00 2001
From: Klaus-Uwe Mitterer <git@klaus-uwe.me>
Date: Wed, 18 Mar 2015 21:04:10 +0100
Subject: [PATCH] First part of Weckbot, filling the database with tweets

---
 getter.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 makedb.py | 20 ++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100755 getter.py
 create mode 100755 makedb.py

diff --git a/getter.py b/getter.py
new file mode 100755
index 0000000..a55514c
--- /dev/null
+++ b/getter.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import HTMLParser, tweepy, os, sqlite3, datetime, dateutil.parser, time
+
+user = "Twitter Handle (without @)"
+cke = "Consumer Key"
+cse = "Consumer Secret"
+ato = "Access Token"
+ase = "Access Secret"
+
+database_filename = "Database.db"
+
+search = "@" + user
+
+auth = tweepy.OAuthHandler(cke, cse)
+auth.set_access_token(ato, ase)
+api = tweepy.API(auth)
+
+sql_conn = sqlite3.connect(database_filename)
+cur = sql_conn.cursor()
+
+try:
+ cur.execute("SELECT max(tweet_id) FROM tweets")
+ savepoint = int(cur.fetchone()[0])
+except:
+ savepoint = ""
+ print "No savepoint found. Trying to get as many results as possible."
+
+while True:
+ timeline = tweepy.Cursor(api.search, q=search, since_id=savepoint).items()
+
+ for status in timeline:
+  if status.text.find("@" + user) > 1:
+   continue
+  text = status.text.encode("UTF-8").split(" ")[1]
+  sender = status.user.screen_name.encode("UTF-8")
+  twid = int(status.id)
+  try:
+   if "-" in text:
+    date = dateutil.parser.parse(text.split("-")[0])
+    comment = HTMLParser.HTMLParser().unescape(text.split("-")[1])
+   else:
+    date = dateutil.parser.parse(text) 
+  except ValueError:
+   api.update_status("@%s Sorry, ich verstehe deinen Tweet nicht... :(" % sender, twid)
+   continue
+
+  cur.execute("INSERT INTO tweets VALUES(%i,'%s','%s','%s')" % (twid,date.strftime("%Y-%m-%dT%H:%M:%S"),sender,comment))
+
+  savepoint = twid
+  time.sleep(60)
diff --git a/makedb.py b/makedb.py
new file mode 100755
index 0000000..1388fc3
--- /dev/null
+++ b/makedb.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os, sqlite3, sys
+
+try:
+ file = sys.argv[1]
+except IndexError:
+ file = "Database.db"
+
+if os.path.isfile(file):
+ os.remove(file)
+
+conn = sqlite3.connect(file)
+curs = conn.cursor()
+
+curs.execute("CREATE TABLE tweets(tweet_id numeric, timestamp text, sender text, comment text);")
+
+conn.commit()
+conn.close()