Check in stuff primarily copied over from twitools
This commit is contained in:
parent
a2869d70c9
commit
e21fcc1a8c
4 changed files with 365 additions and 0 deletions
79
dbtools/__init__.py
Normal file
79
dbtools/__init__.py
Normal file
|
@ -0,0 +1,79 @@
|
|||
import setuptools
|
||||
import sqlite3, pymysql, pymysql.cursors
|
||||
|
||||
SQLITE = 0
|
||||
MYSQL = 1
|
||||
MARIADB = MYSQL
|
||||
|
||||
MIN = 0
|
||||
MAX = 1
|
||||
|
||||
class dbObject:
|
||||
|
||||
# --------------------------------------------- Initialization -------------------------------------------------
|
||||
|
||||
def initMySQL(self, host, port, user, pwd, db):
|
||||
self.conn = pymysql.connect(host = host, port = port, user = user, password = pwd, db = db, charset = "utf8mb4", cursorclass = pymysql.cursors.DictCursor)
|
||||
self.cur = self.conn.cursor()
|
||||
self.dbtype = MYSQL
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.user = user
|
||||
self.pwd = pwd
|
||||
self.db = db
|
||||
|
||||
def initSQLite(self, path):
|
||||
self.conn = sqlite3.connect(path)
|
||||
self.cur = self.conn.cursor()
|
||||
self.dbtype = SQLITE
|
||||
self.path = path
|
||||
|
||||
def __init__(self, dbtype = SQLITE, path = None, host = None, port = None, user = None, pwd = None, db = None):
|
||||
|
||||
if dbtype == SQLITE:
|
||||
self.initSQLite(path or 'Database.db')
|
||||
|
||||
elif dbtype == MYSQL:
|
||||
self.initMySQL(host or 'localhost', port or 3306, user, pwd, db)
|
||||
|
||||
else:
|
||||
raise ValueError("Unknown database type %s." % str(dbtype))
|
||||
|
||||
# ---------------------------------------------- No more initialization ----------------------------------------
|
||||
|
||||
def closeConnection(self):
|
||||
return self.conn.close()
|
||||
|
||||
def commit(self):
|
||||
return self.conn.commit()
|
||||
|
||||
def executeQuery(self, query):
|
||||
return self.cur.execute(query)
|
||||
|
||||
def getAll(self):
|
||||
return self.cur.fetchall()
|
||||
|
||||
def getNext(self):
|
||||
return self.cur.fetchone()
|
||||
|
||||
def isInitialized(self):
|
||||
try:
|
||||
self.executeQuery("SELECT * FROM messages")
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def getLatestMessage(db):
|
||||
db.executeQuery("SELECT max(id) FROM messages")
|
||||
try:
|
||||
return int(db.getNext()[0])
|
||||
except:
|
||||
return 0
|
||||
|
||||
def dbHelper():
|
||||
if setuptools.dbtype() == SQLITE:
|
||||
return dbObject(dbtype=SQLITE, path=setuptools.dbpath())
|
||||
elif setuptools.dbtype() == MYSQL:
|
||||
return dbObject(dbtype=MYSQL, host=setuptools.dbhost(), user=setuptools.dbuser(), pwd=setuptools.dbpass(), db=setuptools.dbname())
|
||||
else:
|
||||
raise setuptools.SetupException()
|
88
messages.py
Executable file
88
messages.py
Executable file
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
import urllib.request, urllib.error, urllib.parse, time, os
|
||||
import setuptools
|
||||
|
||||
def status(driver):
|
||||
if "/main/login.php" not in driver.page_source:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def loadPage(url,period=5,init=False, driver=driver):
|
||||
if not (init or status(driver)):
|
||||
login()
|
||||
driver.get(url)
|
||||
time.sleep(period)
|
||||
|
||||
def loginHandler(user = setuptools.user, password = setuptools.password, driver = driver):
|
||||
loadPage("http://www.planetromeo.com/",10,True,driver)
|
||||
|
||||
curfield = driver.find_element_by_name("username")
|
||||
curfield.send_keys(user)
|
||||
|
||||
curfield = driver.find_element_by_name("passwort")
|
||||
curfield.send_keys(password)
|
||||
|
||||
curfield.send_keys(Keys.RETURN)
|
||||
time.sleep(10)
|
||||
|
||||
return status()
|
||||
|
||||
class LoginError(Exception):
|
||||
pass
|
||||
|
||||
def login():
|
||||
if not (status() or loginHandler()):
|
||||
raise LoginError("Login failed.")
|
||||
return True
|
||||
|
||||
def messageID(url):
|
||||
return url.split("?")[1]
|
||||
|
||||
def messageHandler(mid, driver):
|
||||
loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver=driver)
|
||||
|
||||
try:
|
||||
links = driver.find_elements_by_partial_link_text('pix/popup.php/')
|
||||
for link in links:
|
||||
phototools.processURL(link.get_attribute('href'),vid)
|
||||
except NoSuchElementException as e:
|
||||
pass
|
||||
|
||||
def pageHandler(driver):
|
||||
webpage = driver.page_source
|
||||
links = BeautifulSoup(webpage).findAll('a')
|
||||
count = 0
|
||||
|
||||
for l in links:
|
||||
url = l['href']
|
||||
if "/msg/?id=" in url:
|
||||
count += 1
|
||||
mid = messageID(url)
|
||||
if mid <= dbtools.getLatestMessage()
|
||||
return False
|
||||
messageHandler(mid, driver)
|
||||
|
||||
if count == 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def siteHandler(p = 0, driver = driver):
|
||||
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?seite=" + str(p), driver=driver)
|
||||
if pageHandler(driver):
|
||||
siteHandler(p+1, driver)
|
||||
|
||||
if __name__ == "__main__":
|
||||
driver = webdriver.Firefox()
|
||||
if login(driver):
|
||||
siteHandler(driver=driver)
|
||||
|
||||
print("KTHXBAI")
|
||||
|
||||
driver.close()
|
77
setup.py
Executable file
77
setup.py
Executable file
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import configparser, os.path, tweepy, dbtools, getpass
|
||||
|
||||
if os.path.isfile("config.cfg"):
|
||||
print("config.cfg already exists. Please remove it before running this script.")
|
||||
exit(1)
|
||||
|
||||
config = configparser.RawConfigParser()
|
||||
|
||||
config.add_section('Database')
|
||||
|
||||
print('''Romeotools will use a database for certain tasks. You can use a file or a MySQL database for this purpose.
|
||||
|
||||
If you wish to use a MySQL database, you will need the credentials for it. If you don't know what any of that means, stick with the default value and just press Enter.
|
||||
''')
|
||||
|
||||
dbtype = input("Database type: %i (file), %i (MySQL) [%i]: " % (dbtools.SQLITE, dbtools.MYSQL, dbtools.SQLITE))
|
||||
print()
|
||||
|
||||
try:
|
||||
dbtype = int(dbtype)
|
||||
except:
|
||||
pass
|
||||
|
||||
if dbtype == dbtools.MYSQL:
|
||||
dbhost = input("MySQL host [localhost]: ") or "localhost"
|
||||
dbuser = input("MySQL username [twitools]: ") or "twitools"
|
||||
dbpass = getpass.getpass("MySQL password (not echoed!): ")
|
||||
dbname = input("MySQL database name [twitools]: ") or "twitools"
|
||||
print()
|
||||
|
||||
config.set('Database', 'type', dbtype)
|
||||
config.set('Database', 'host', dbhost)
|
||||
config.set('Database', 'user', dbuser)
|
||||
config.set('Database', 'pass', dbpass)
|
||||
config.set('Database', 'name', dbname)
|
||||
|
||||
else:
|
||||
dbtype = dbtools.SQLITE
|
||||
dbpath = input("Name of the database file [Database.db]: ") or "Database.db"
|
||||
print()
|
||||
|
||||
config.set('Database', 'type', dbtype)
|
||||
config.set('Database', 'path', dbpath)
|
||||
|
||||
if dbtype == dbtools.MYSQL:
|
||||
db = dbtools.dbObject(dbtype=dbtype, host=dbhost, user=dbuser, pwd=dbpass, db=dbname)
|
||||
else:
|
||||
db = dbtools.dbObject(dbtype=dbtype, path=dbpath)
|
||||
if not db.isInitialized():
|
||||
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));")
|
||||
db.commit()
|
||||
|
||||
db.closeConnection()
|
||||
|
||||
config.add_section("Romeo")
|
||||
|
||||
print('''In the next step, we'll get you connected to PlanetRomeo. For this, we
|
||||
will need your username and password. Please note that these will be stored on
|
||||
your hard disk in plain text. Sadly PlanetRomeo doesn't offer a better way for
|
||||
third party applications to authenticate...
|
||||
''')
|
||||
|
||||
unam = input("Username: ")
|
||||
pwrd = getpass.getpass("Password (not echoed back!): ")
|
||||
print()
|
||||
|
||||
config.set("Romeo", "user", unam)
|
||||
config.set("Romeo", "pass", pwrd)
|
||||
|
||||
print("Seems like everything worked out fine. Let's write that config file...")
|
||||
|
||||
with open('config.cfg', 'wt') as cfg:
|
||||
config.write(cfg)
|
||||
|
||||
print("We're all done. You can now use Twitools. Have fun!")
|
121
setuptools/__init__.py
Normal file
121
setuptools/__init__.py
Normal file
|
@ -0,0 +1,121 @@
|
|||
import configparser, csv, datetime, html.parser, itertools, os, sqlite3, sys
|
||||
|
||||
class SetupException(Exception):
|
||||
def __str__(self):
|
||||
return "Seems like config.cfg has not been created yet or contains serious errors. Run setup.py to create it."
|
||||
|
||||
def getSetting(section, setting, path = "config.cfg"):
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(path)
|
||||
return config.get(section, setting)
|
||||
|
||||
def dbtype():
|
||||
try:
|
||||
return int(getSetting("Database", "type"))
|
||||
except:
|
||||
return 0 # for SQLite3
|
||||
|
||||
### Must only be called AFTER dbtype()! ###
|
||||
|
||||
def dbhost():
|
||||
try:
|
||||
return getSetting("Database", "host")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def dbuser():
|
||||
try:
|
||||
return getSetting("Database", "user")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def dbpass():
|
||||
try:
|
||||
return getSetting("Database", "pass")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def dbname():
|
||||
try:
|
||||
return getSetting("Database", "name")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def dbpath():
|
||||
try:
|
||||
return getSetting("Database", "path")
|
||||
except:
|
||||
return SetupException()
|
||||
|
||||
###
|
||||
|
||||
def user():
|
||||
try:
|
||||
return getSetting("Romeo", "user")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def password():
|
||||
try:
|
||||
return getSetting("Romeo", "pass")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def dbCheck(db, create = False):
|
||||
if (not create and dbInitialized(db)) or (create and not dbInitialized(db)):
|
||||
return True
|
||||
if create:
|
||||
raise ValueError("Provided database file " + db.path + " is already initialized. Remove it manually before trying to recreate it.")
|
||||
raise ValueError("Provided database file " + db.path + " is not initialized. Create it using makedb.py or csvdb.py")
|
||||
|
||||
|
||||
def dbHelper(path, create = False):
|
||||
db = dbObject(path)
|
||||
dbCheck(db, create)
|
||||
return db
|
||||
|
||||
def dbInitialized(db):
|
||||
return db.isInitialized()
|
||||
|
||||
def fileExists(path):
|
||||
return os.path.isfile(path)
|
||||
|
||||
def getDate(date):
|
||||
try:
|
||||
return datetime.datetime.strptime(date, '%Y-%m-%d')
|
||||
except ValueError:
|
||||
raise ValueError("Dates must be in YYYY-MM-DD format.")
|
||||
|
||||
def paginate(iterable, page_size):
|
||||
while True:
|
||||
i1, i2 = itertools.tee(iterable)
|
||||
iterable, page = (itertools.islice(i1, page_size, None), list(itertools.islice(i2, page_size)))
|
||||
if len(page) == 0:
|
||||
break
|
||||
yield page
|
||||
|
||||
def parseArgs(argv):
|
||||
args = []
|
||||
path = None
|
||||
nextpath = False
|
||||
|
||||
for a in argv[1:]:
|
||||
if nextpath:
|
||||
path = a
|
||||
nextpath = False
|
||||
elif a == "-f":
|
||||
if path != None:
|
||||
raise ValueError("You can only pass one database file.")
|
||||
nextpath = True
|
||||
else:
|
||||
args += [a]
|
||||
|
||||
return args, path
|
||||
|
||||
def printCSV(inlist):
|
||||
writer = csv.writer(sys.stdout)
|
||||
writer.writerows(inlist)
|
||||
|
||||
def unescapeText(text):
|
||||
return html.parser.HTMLParser().unescape(text).replace("'","''")
|
||||
|
Loading…
Reference in a new issue