twitools/markov.py
2017-12-06 18:20:13 +01:00

70 lines
1.6 KiB
Python
Executable file
Raw Permalink Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
import dbtools
import setuptools
import twitools
import filters.markov
import argparse
import datetime
import html
import markovify
import multiprocessing
import nltk
import operator
import os
import random
import re
import string
import sys
import time
class Possy(markovify.NewlineText):
def word_split(self, sentence):
words = re.split(self.word_split_pattern, sentence)
words = [ "::".join(tag) for tag in nltk.pos_tag(words) ]
return words
def word_join(self, words):
sentence = " ".join(word.split("::")[0] for word in words)
return sentence
def sanitizeText(text):
split = text.split()
try:
if text[0] == "@" or text[1] == "@":
if split[1][0] not in string.ascii_lowercase:
return sanitizeText(" ".join(split[1:]))
if split[-1][0] == "@" and text[-1] in (string.digits + string.ascii_letters + "_"):
return sanitizeText(" ".join(split[:-1]))
if text[:4] == "RT @":
return sanitizeText(text.partition(":")[2])
except:
return ""
return text
def getText(db = dbtools.dbHelper()):
text = ""
for string in db.executeQuery('SELECT text FROM tweets;'):
text += sanitizeText(string[0]) + "\n"
return html.unescape("".join([s for s in text.strip().splitlines(True) if s.strip()]))
def markovifyText(text):
return Possy(text).make_short_sentence(270).replace("@", "@")
def worker():
text = markovifyText(getText())
outtext = filters.markov.textFilter(text)
if outtext:
filters.markov.tweetFilter(twitools.tweet(outtext, section = setuptools.MARKOV), text)
if __name__ == "__main__":
p = multiprocessing.Process(target=worker)
p.start()
time.sleep(20*60)
if p.is_alive():
p.terminate()
p.join()