Fix streaming after Twitter API changes
This commit is contained in:
parent
f4c4dbf06c
commit
c9e77afe0e
5 changed files with 103 additions and 22 deletions
|
@ -1,4 +1,4 @@
|
||||||
import ast, dbtools, html, io, logging, moviepy.editor, os, PIL.Image, random, re, setuptools, string, bottools.strings, sys, telegram.ext, telegram, time, twitools, bottools.streaming, urllib.request, tweepy
|
import ast, dbtools, html, io, logging, moviepy.editor, os, PIL.Image, random, re, setuptools, string, bottools.strings, sys, telegram.ext, telegram, time, twitools, bottools.streaming, bottools.thread, urllib.request, tweepy
|
||||||
|
|
||||||
def getTwo(message):
|
def getTwo(message):
|
||||||
try:
|
try:
|
||||||
|
@ -307,11 +307,14 @@ def explicitTweet(bot, update, args, reply = None):
|
||||||
else:
|
else:
|
||||||
out = mobj
|
out = mobj
|
||||||
|
|
||||||
status = two.api.update_with_media(filename, ' '.join(args), reply, file=out)
|
status = two.api.update_with_media(filename, ' '.join(args), in_reply_to_status_id=reply, file=out)
|
||||||
out.close()
|
out.close()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
status = two.tweet(' '.join(args), reply)
|
try:
|
||||||
|
status = two.tweet(' '.join(args), reply)
|
||||||
|
except Exception as e:
|
||||||
|
logging.exception(e)
|
||||||
|
|
||||||
bottools.methods.tweetMessage(status, update.message.chat_id, bot)
|
bottools.methods.tweetMessage(status, update.message.chat_id, bot)
|
||||||
|
|
||||||
|
@ -328,7 +331,7 @@ def reply(bot, update, args):
|
||||||
if not ("@%s" % sender.strip("@")) in [("@%s" % a.strip("@")) for a in args]:
|
if not ("@%s" % sender.strip("@")) in [("@%s" % a.strip("@")) for a in args]:
|
||||||
mentions = []
|
mentions = []
|
||||||
|
|
||||||
for m in re.split('[^\w@]+', otweet.text):
|
for m in re.split('[^\w@]+', otweet.full_text):
|
||||||
try:
|
try:
|
||||||
if m[0] == "@" and m[0].strip() != "":
|
if m[0] == "@" and m[0].strip() != "":
|
||||||
mentions += [m]
|
mentions += [m]
|
||||||
|
@ -464,6 +467,11 @@ def tweet(bot, update):
|
||||||
def tweetMessage(status, cid, bot, force = False, callback = None, notified = None):
|
def tweetMessage(status, cid, bot, force = False, callback = None, notified = None):
|
||||||
db = dbtools.dbHelper()
|
db = dbtools.dbHelper()
|
||||||
|
|
||||||
|
try:
|
||||||
|
text = status.text
|
||||||
|
except:
|
||||||
|
text = status.full_text
|
||||||
|
|
||||||
if not (force or callback):
|
if not (force or callback):
|
||||||
try:
|
try:
|
||||||
two = twitools.twoBotHelper(cid)
|
two = twitools.twoBotHelper(cid)
|
||||||
|
@ -485,7 +493,7 @@ def tweetMessage(status, cid, bot, force = False, callback = None, notified = No
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
i = int(callback.message.text.split()[1].strip(":"))
|
i = int(text.split()[1].strip(":"))
|
||||||
|
|
||||||
buttons = []
|
buttons = []
|
||||||
header = None
|
header = None
|
||||||
|
@ -515,12 +523,15 @@ def tweetMessage(status, cid, bot, force = False, callback = None, notified = No
|
||||||
if status.in_reply_to_status_id:
|
if status.in_reply_to_status_id:
|
||||||
buttons += [telegram.InlineKeyboardButton("View Thread", callback_data = "/thread %i" % i)]
|
buttons += [telegram.InlineKeyboardButton("View Thread", callback_data = "/thread %i" % i)]
|
||||||
|
|
||||||
rmu = telegram.InlineKeyboardMarkup(makeMenu(buttons, header=header))
|
try:
|
||||||
|
rmu = telegram.InlineKeyboardMarkup(makeMenu(buttons, header=header))
|
||||||
|
except Exception as e:
|
||||||
|
logging.exception(e)
|
||||||
|
|
||||||
if callback:
|
if callback:
|
||||||
bot.editMessageReplyMarkup(chat_id=callback.message.chat_id, message_id=callback.message.message_id, reply_markup=rmu)
|
bot.editMessageReplyMarkup(chat_id=callback.message.chat_id, message_id=callback.message.message_id, reply_markup=rmu)
|
||||||
else:
|
else:
|
||||||
bot.sendMessage(chat_id = cid or callback.message.chat_id, text = "Tweet %i:\n%s (@%s) at %s:\n%s" % (i, status.author.name, status.author.screen_name, status.created_at, html.unescape(status.text)), reply_markup=rmu)
|
bot.sendMessage(chat_id = cid or callback.message.chat_id, text = "Tweet %i:\n%s (@%s) at %s:\n%s" % (i, status.author.name, status.author.screen_name, status.created_at, html.unescape(text)), reply_markup=rmu)
|
||||||
|
|
||||||
def trends(bot, update, args):
|
def trends(bot, update, args):
|
||||||
try:
|
try:
|
||||||
|
@ -639,9 +650,10 @@ def timeline(bot, update, args = [10]):
|
||||||
mentionstreams = {}
|
mentionstreams = {}
|
||||||
|
|
||||||
def makeStream(bot, cid, ato, ase):
|
def makeStream(bot, cid, ato, ase):
|
||||||
two = twitools.twObject(ato=ato, ase=ase)
|
# two = twitools.twObject(ato=ato, ase=ase)
|
||||||
stream = tweepy.Stream(auth = two.auth, listener = bottools.streaming.BotStreamListener(bot, cid, ato))
|
# stream = tweepy.Stream(auth = two.auth, listener = bottools.streaming.BotStreamListener(bot, cid, ato))
|
||||||
stream.filter(track=["@%s" % two.whoami().strip("@")], async=True)
|
# stream.filter(track=["%s" % two.whoami().strip("@")], async=True)
|
||||||
|
stream = bottools.thread.BotStreamListener(bot, cid, ato)
|
||||||
return stream
|
return stream
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -688,10 +700,18 @@ def isadmin(message):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def restart(bot, update):
|
def restart(bot, update):
|
||||||
|
global mentionstreams
|
||||||
|
|
||||||
if bottools.methods.isadmin(update.message):
|
if bottools.methods.isadmin(update.message):
|
||||||
update.message.reply_text(bottools.strings.restart)
|
update.message.reply_text(bottools.strings.restart)
|
||||||
time.sleep(0.5)
|
|
||||||
|
while len(mentionstreams) > 0:
|
||||||
|
so = mentionstreams.pop(list(mentionstreams)[0])
|
||||||
|
so.disconnect()
|
||||||
|
so.join()
|
||||||
|
|
||||||
os.execl(sys.executable, sys.executable, *sys.argv)
|
os.execl(sys.executable, sys.executable, *sys.argv)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
bottools.methods.unknown(bot, update)
|
bottools.methods.unknown(bot, update)
|
||||||
|
|
||||||
|
|
15
bottools/streaming.py
Normal file
15
bottools/streaming.py
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
import bottools.methods, dbtools, html, tweepy
|
||||||
|
|
||||||
|
class BotStreamListener(tweepy.StreamListener):
|
||||||
|
def __init__(self, bot, cid, ato, *args, **kwargs):
|
||||||
|
tweepy.StreamListener.__init__(self, *args, **kwargs)
|
||||||
|
self.bot = bot
|
||||||
|
self.cid = cid
|
||||||
|
self.ato = ato
|
||||||
|
|
||||||
|
def on_status(self, status):
|
||||||
|
bottools.methods.tweetMessage(status, self.cid, self.bot, notified = self.ato)
|
||||||
|
|
||||||
|
def on_error(self, status):
|
||||||
|
if status == 420:
|
||||||
|
return False
|
41
bottools/thread.py
Normal file
41
bottools/thread.py
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
import bottools.methods, dbtools, logging, threading, time, twitools
|
||||||
|
|
||||||
|
class BotStreamThread(threading.Thread):
|
||||||
|
def __init__(self, bot, cid, ato, searchstring = None):
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
|
||||||
|
self.bot = bot
|
||||||
|
self.cid = cid
|
||||||
|
self.ato = ato
|
||||||
|
|
||||||
|
self.ase = dbtools.dbHelper().aseByAto(ato)
|
||||||
|
self.two = twitools.twObject(ato=ato, ase=self.ase)
|
||||||
|
|
||||||
|
self.searchstring = searchstring or "@%s" % self.two.whoami().strip("@")
|
||||||
|
|
||||||
|
self.lm = self.two.getLastMention()
|
||||||
|
self.stop = False
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while not self.stop:
|
||||||
|
try:
|
||||||
|
for status in self.two.getMentions(self.lm):
|
||||||
|
try:
|
||||||
|
bottools.methods.tweetMessage(status, self.cid, self.bot, notified = self.ato)
|
||||||
|
self.lm = status.id
|
||||||
|
except Exception as e:
|
||||||
|
logging.exception(e)
|
||||||
|
except Exception as e:
|
||||||
|
logging.exception(e)
|
||||||
|
|
||||||
|
for i in range(0, 89):
|
||||||
|
if not self.stop:
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
def disconnect(self):
|
||||||
|
self.stop = True
|
||||||
|
|
||||||
|
def BotStreamListener(bot, cid, ato, searchstring = None):
|
||||||
|
obj = bottools.thread.BotStreamThread(bot, cid, ato, searchstring)
|
||||||
|
obj.start()
|
||||||
|
return obj
|
13
filler.py
13
filler.py
|
@ -6,18 +6,20 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit
|
||||||
savepoint = db.getLatestTweet() + 1
|
savepoint = db.getLatestTweet() + 1
|
||||||
last = savepoint
|
last = savepoint
|
||||||
|
|
||||||
timeline = two.api.user_timeline(screen_name = user, since_id = savepoint, include_rts = True)
|
timeline = two.api.user_timeline(screen_name = user, since_id = savepoint, include_rts = True, tweet_mode = "extended")
|
||||||
|
|
||||||
tw_counter = 0
|
tw_counter = 0
|
||||||
|
|
||||||
for status in timeline:
|
for status in timeline:
|
||||||
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
||||||
text = setuptools.unescapeText(status.text)
|
text = setuptools.unescapeText(status.full_text)
|
||||||
|
|
||||||
if filters.filler.tweetFilter(status):
|
if filters.filler.tweetFilter(status):
|
||||||
try:
|
try:
|
||||||
db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
||||||
except:
|
db.commit()
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
print("Failed to insert %s into database." % str(status.id))
|
print("Failed to insert %s into database." % str(status.id))
|
||||||
|
|
||||||
if 'media' in status.entities:
|
if 'media' in status.entities:
|
||||||
|
@ -32,8 +34,6 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit
|
||||||
last = status.id
|
last = status.id
|
||||||
tw_counter = tw_counter + 1
|
tw_counter = tw_counter + 1
|
||||||
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
return tw_counter, last, savepoint
|
return tw_counter, last, savepoint
|
||||||
|
|
||||||
def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
|
@ -46,6 +46,7 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
if filters.filler.messageFilter(m, True):
|
if filters.filler.messageFilter(m, True):
|
||||||
try:
|
try:
|
||||||
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
|
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
|
||||||
|
db.commit()
|
||||||
mcount += 1
|
mcount += 1
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
@ -56,8 +57,6 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
||||||
httptools.downloadMedia(med['media_url'], "m%i" % int(m.id), mid)
|
httptools.downloadMedia(med['media_url'], "m%i" % int(m.id), mid)
|
||||||
mid += 1
|
mid += 1
|
||||||
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
return mcount, savepoint or 0, db.getLatestMessage
|
return mcount, savepoint or 0, db.getLatestMessage
|
||||||
|
|
||||||
def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
|
def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
|
||||||
|
|
|
@ -14,7 +14,7 @@ class twObject:
|
||||||
self.api.destroy_status(id)
|
self.api.destroy_status(id)
|
||||||
|
|
||||||
def search(self, query, savepoint = 0):
|
def search(self, query, savepoint = 0):
|
||||||
tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint, include_entities=True).items())
|
tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint, result_type="recent", include_entities=True, tweet_mode="extended").items())
|
||||||
tweets.reverse()
|
tweets.reverse()
|
||||||
return tweets
|
return tweets
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ class twObject:
|
||||||
return self.auth.get_username()
|
return self.auth.get_username()
|
||||||
|
|
||||||
def tweet(self, text, reply = 0):
|
def tweet(self, text, reply = 0):
|
||||||
return self.api.update_status(text, reply)
|
return self.api.update_status(text, in_reply_to_status_id = reply)
|
||||||
|
|
||||||
def authenticate(self):
|
def authenticate(self):
|
||||||
try:
|
try:
|
||||||
|
@ -44,7 +44,13 @@ class twObject:
|
||||||
return self.auth.access_token, self.auth.access_token_secret
|
return self.auth.access_token, self.auth.access_token_secret
|
||||||
|
|
||||||
def getTweet(self, tid):
|
def getTweet(self, tid):
|
||||||
return self.api.get_status(tid)
|
return self.api.get_status(tid, tweet_mode="extended")
|
||||||
|
|
||||||
|
def getLastMention(self):
|
||||||
|
return self.getMentions()[-1].id
|
||||||
|
|
||||||
|
def getMentions(self, since = 0):
|
||||||
|
return self.search("@%s" % self.whoami().strip("@"), since)
|
||||||
|
|
||||||
def getFollowerIDs(section = setuptools.TWITTER):
|
def getFollowerIDs(section = setuptools.TWITTER):
|
||||||
''' Returns 5,000 follower IDs at most '''
|
''' Returns 5,000 follower IDs at most '''
|
||||||
|
@ -65,7 +71,7 @@ def getNamesByIDs(fids=getFollowerIDs(), section = setuptools.TWITTER):
|
||||||
yield {"id": follower.id, "name": follower.screen_name}
|
yield {"id": follower.id, "name": follower.screen_name}
|
||||||
|
|
||||||
def getTweet(tid, section = setuptools.TWITTER):
|
def getTweet(tid, section = setuptools.TWITTER):
|
||||||
return twoHelper(section).api.get_status(tid)
|
return twoHelper(section).getTweet(tid)
|
||||||
|
|
||||||
def twoHelper(section = setuptools.TWITTER):
|
def twoHelper(section = setuptools.TWITTER):
|
||||||
try:
|
try:
|
||||||
|
|
Loading…
Reference in a new issue