kumi/twitools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove duplicated code in getMessages(), implement media download for messages

Browse source
This commit is contained in:
Klaus-Uwe Mitterer 2017-02-27 00:56:37 +01:00
parent 92c898a20d
commit 4755215271
1 changed files with 7 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
filler.py
View file

@ -4,7 +4,7 @@ import argparse, dbtools, dbtools.fillerfilter, requests, setuptools, time, twit
def downloadMedia(url, tid, mid):
remote = requests.get(url, stream=True)
filename = "media/%i_%i.%s" % (int(tid), int(mid), url.split(".")[-1])
filename = "media/%s_%i.%s" % (str(tid), int(mid), url.split(".")[-1])
with open(filename, 'wb') as outfile:
for chunk in remote.iter_content(chunk_size=1024):
@ -50,7 +50,7 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
new_messages = two.api.direct_messages(since_id=savepoint, count=200, full_text=True)
new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True)
for m in new_messages:
for m in (new_messages + new_out_messages):
if dbtools.fillerfilter.messageFilter(m, True):
try:
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
@ -58,13 +58,11 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
except:
pass
for m in new_out_messages:
if dbtools.fillerfilter.messageFilter(m, False):
try:
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
mcount += 1
except:
pass
if 'media' in m.entities:
mid = 0
for med in m.entities['media']:
downloadMedia(med['media_url'], status.id, mid)
mid += 1
db.commit()