Fixed filler dependencies
This commit is contained in:
parent
e0abdf0a56
commit
367b4e7864
3 changed files with 21 additions and 23 deletions
|
@ -4,6 +4,9 @@ SQLITE = 0
|
||||||
MYSQL = 1
|
MYSQL = 1
|
||||||
MARIADB = MYSQL
|
MARIADB = MYSQL
|
||||||
|
|
||||||
|
MIN = 0
|
||||||
|
MAX = 1
|
||||||
|
|
||||||
class dbObject:
|
class dbObject:
|
||||||
|
|
||||||
# --------------------------------------------- Initialization -------------------------------------------------
|
# --------------------------------------------- Initialization -------------------------------------------------
|
||||||
|
@ -56,9 +59,17 @@ class dbObject:
|
||||||
except:
|
except:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def getFLDate(self, val = 0):
|
def getFLDate(self, val = MIN):
|
||||||
if val == 0:
|
if val == MIN:
|
||||||
mode = "MIN"
|
mode = "MIN"
|
||||||
else:
|
else:
|
||||||
mode = "MAX"
|
mode = "MAX"
|
||||||
return getDate(str(list(self.executeQuery("SELECT %s(SUBSTR(timestamp,0,11)) FROM tweets" % mode))[0][0]))
|
return getDate(str(list(self.executeQuery("SELECT %s(SUBSTR(timestamp,0,11)) FROM tweets" % mode))[0][0]))
|
||||||
|
|
||||||
|
def getLatestTweet(db):
|
||||||
|
db.executeQuery("SELECT max(tweet_id) FROM tweets")
|
||||||
|
try:
|
||||||
|
return int(db.getNext()[0])
|
||||||
|
except:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
24
filler.py
24
filler.py
|
@ -1,26 +1,10 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import tools
|
import dbtools, tools
|
||||||
|
|
||||||
import html.parser, os
|
def fill(db=dbtools.dbObject(), user=tools.user(), two=tools.twObject()):
|
||||||
|
|
||||||
def getSavepoint(db):
|
|
||||||
db.executeQuery("SELECT max(tweet_id) FROM tweets")
|
|
||||||
try:
|
|
||||||
return int(db.getNext()[0])
|
|
||||||
except:
|
|
||||||
print("No tweets stored yet.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def unescapeText(text):
|
|
||||||
return html.parser.HTMLParser().unescape(text).replace("'","''")
|
|
||||||
|
|
||||||
def fill(dbpath=tools.dbpath(), user=tools.user(), two=tools.twObject()):
|
|
||||||
query = "from:" + user
|
query = "from:" + user
|
||||||
|
savepoint = db.getLatestTweet()
|
||||||
db = tools.dbHelper(dbpath)
|
|
||||||
|
|
||||||
savepoint = getSavepoint(db)
|
|
||||||
last = savepoint
|
last = savepoint
|
||||||
|
|
||||||
timeline = two.search(query, savepoint)
|
timeline = two.search(query, savepoint)
|
||||||
|
@ -29,7 +13,7 @@ def fill(dbpath=tools.dbpath(), user=tools.user(), two=tools.twObject()):
|
||||||
|
|
||||||
for status in timeline:
|
for status in timeline:
|
||||||
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
||||||
text = unescapeText(status.text)
|
text = tools.unescapeText(status.text)
|
||||||
|
|
||||||
db.executeQuery("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
db.executeQuery("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import configparser, csv, datetime, itertools, os, sqlite3, sys, tweepy
|
import configparser, csv, datetime, html.parser, itertools, os, sqlite3, sys, tweepy
|
||||||
|
|
||||||
class SetupException(Exception):
|
class SetupException(Exception):
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
|
@ -158,3 +158,6 @@ def printCSV(inlist):
|
||||||
writer = csv.writer(sys.stdout)
|
writer = csv.writer(sys.stdout)
|
||||||
writer.writerows(inlist)
|
writer.writerows(inlist)
|
||||||
|
|
||||||
|
def unescapeText(text):
|
||||||
|
return html.parser.HTMLParser().unescape(text).replace("'","''")
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue