kumi/twitools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove duplicated code in getMessages(), implement media download for messages

Browse source
This commit is contained in:
Klaus-Uwe Mitterer 2017-02-27 00:56:37 +01:00
parent c015769950
commit fd1d1d5bac
1 changed files with 7 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
filler.py
View file

@ -4,7 +4,7 @@ import argparse, dbtools, dbtools.fillerfilter, requests, setuptools, time, twit
def downloadMedia(url, tid, mid): def downloadMedia(url, tid, mid):
remote = requests.get(url, stream=True) remote = requests.get(url, stream=True)
filename = "media/%i_%i.%s" % (int(tid), int(mid), url.split(".")[-1]) filename = "media/%s_%i.%s" % (str(tid), int(mid), url.split(".")[-1])
with open(filename, 'wb') as outfile: with open(filename, 'wb') as outfile:
for chunk in remote.iter_content(chunk_size=1024): for chunk in remote.iter_content(chunk_size=1024):
@ -50,7 +50,7 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
new_messages = two.api.direct_messages(since_id=savepoint, count=200, full_text=True) new_messages = two.api.direct_messages(since_id=savepoint, count=200, full_text=True)
new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True) new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True)
for m in new_messages: for m in (new_messages + new_out_messages):
if dbtools.fillerfilter.messageFilter(m, True): if dbtools.fillerfilter.messageFilter(m, True):
try: try:
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
@ -58,13 +58,11 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
except: except:
pass pass
for m in new_out_messages: if 'media' in m.entities:
if dbtools.fillerfilter.messageFilter(m, False): mid = 0
try: for med in m.entities['media']:
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) downloadMedia(med['media_url'], status.id, mid)
mcount += 1 mid += 1
except:
pass
db.commit() db.commit()