Handle photos, correct counter
This commit is contained in:
parent
c53ffa66dc
commit
ce1fec74ab
2 changed files with 43 additions and 26 deletions
68
handler.py
68
handler.py
|
@ -7,10 +7,6 @@ from selenium.common.exceptions import NoSuchElementException
|
||||||
import multiprocessing, urllib.request, urllib.error, urllib.parse, time, os
|
import multiprocessing, urllib.request, urllib.error, urllib.parse, time, os
|
||||||
import dbtools, phototools, setuptools
|
import dbtools, phototools, setuptools
|
||||||
|
|
||||||
currentRun = []
|
|
||||||
rec = 0
|
|
||||||
snt = 0
|
|
||||||
|
|
||||||
def status(driver):
|
def status(driver):
|
||||||
if "/main/login.php" not in driver.page_source:
|
if "/main/login.php" not in driver.page_source:
|
||||||
return True
|
return True
|
||||||
|
@ -49,8 +45,23 @@ def login(driver):
|
||||||
def messageID(url):
|
def messageID(url):
|
||||||
return url.split("=")[-1]
|
return url.split("=")[-1]
|
||||||
|
|
||||||
|
def quickshareHandler(driver, url, sender):
|
||||||
|
nurl = "https://www.planetromeo.com/" + url if "planetromeo.com" not in url else url
|
||||||
|
loadPage(nurl)
|
||||||
|
juha = BeautifulSoup(driver.page_source, "html5lib")
|
||||||
|
|
||||||
|
try:
|
||||||
|
links = juha.findAll("a")
|
||||||
|
for link in links:
|
||||||
|
try:
|
||||||
|
purl = "https://www.planetromeo.com/" + link["data-pic"] if "planetromeo.com" not in link["data-pic"] else link["data-pic"]
|
||||||
|
phototools.processURL(purl, sender, shutup=True)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
def messageHandler(sender, recipient, mid, date, driver, mode = 0, db = dbtools.dbHelper()):
|
def messageHandler(sender, recipient, mid, date, driver, mode = 0, db = dbtools.dbHelper()):
|
||||||
global currentRun
|
|
||||||
if mode == 0:
|
if mode == 0:
|
||||||
loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver)
|
loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver)
|
||||||
else:
|
else:
|
||||||
|
@ -66,17 +77,20 @@ def messageHandler(sender, recipient, mid, date, driver, mode = 0, db = dbtools.
|
||||||
for link in links:
|
for link in links:
|
||||||
if "/pix/popup.php/" in link["href"]:
|
if "/pix/popup.php/" in link["href"]:
|
||||||
phototools.processURL(link["href"], sender)
|
phototools.processURL(link["href"], sender)
|
||||||
|
try:
|
||||||
|
db.executeQuery("INSERT INTO photos(mid, pid) VALUES('%s', '%s');" % (mid, phototools.parseurl(link["href"]).split('/')[-1]))
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
if "/quickshare/" in link["href"]:
|
||||||
|
quickshareHandler(driver, link["href"], sender)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
currentRun += [mid]
|
def pageHandler(driver, db = dbtools.dbHelper(), ret = {"rec": 0,"snt": 0}):
|
||||||
|
mode = 0
|
||||||
def pageHandler(driver, db = dbtools.dbHelper()):
|
abort = True
|
||||||
global currentRun, rec, snt
|
|
||||||
count = 0
|
|
||||||
juha = BeautifulSoup(driver.page_source, "html5lib")
|
juha = BeautifulSoup(driver.page_source, "html5lib")
|
||||||
|
|
||||||
mode = 0
|
|
||||||
if "sent" in driver.current_url:
|
if "sent" in driver.current_url:
|
||||||
mode = 1
|
mode = 1
|
||||||
|
|
||||||
|
@ -88,45 +102,47 @@ def pageHandler(driver, db = dbtools.dbHelper()):
|
||||||
mid = messageID(data[2].find("a")["href"])
|
mid = messageID(data[2].find("a")["href"])
|
||||||
date = data[3].string
|
date = data[3].string
|
||||||
if not db.checkID(mid):
|
if not db.checkID(mid):
|
||||||
|
abort = False
|
||||||
if mode == 0:
|
if mode == 0:
|
||||||
messageHandler(user, setuptools.user(), mid, date, driver, mode, db)
|
messageHandler(user, setuptools.user(), mid, date, driver, mode, db)
|
||||||
rec += 1
|
ret["rec"] += 1
|
||||||
else:
|
else:
|
||||||
messageHandler(setuptools.user(), user, mid, date, driver, mode, db)
|
messageHandler(setuptools.user(), user, mid, date, driver, mode, db)
|
||||||
snt += 1
|
ret["snt"] += 1
|
||||||
count += 1
|
|
||||||
except IndexError:
|
except IndexError:
|
||||||
pass
|
pass
|
||||||
except IndexError:
|
except IndexError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if count == 0:
|
if abort:
|
||||||
return False
|
return False
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
return True
|
def siteHandler(driver, mode = 0, ret = {"rec": 0,"snt": 0}, p = 0, db = dbtools.dbHelper()):
|
||||||
|
|
||||||
def siteHandler(driver, mode = 0, p = 0, db = dbtools.dbHelper()):
|
|
||||||
if mode == 0:
|
if mode == 0:
|
||||||
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?view=all&seite=" + str(p), driver)
|
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?view=all&seite=" + str(p), driver)
|
||||||
else:
|
else:
|
||||||
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?view=sent&seite=" + str(p), driver)
|
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?view=sent&seite=" + str(p), driver)
|
||||||
if pageHandler(driver, db):
|
|
||||||
siteHandler(driver, mode, p+1, db)
|
|
||||||
|
|
||||||
def mainHandler():
|
if pageHandler(driver, db, ret):
|
||||||
siteHandler(driver, db=db)
|
siteHandler(driver, mode, ret, p+1, db)
|
||||||
siteHandler(driver, 1, db=db)
|
|
||||||
|
def mainHandler(ret = {"rec": 0,"snt": 0}):
|
||||||
|
siteHandler(driver, 0, ret, db=db)
|
||||||
|
siteHandler(driver, 1, ret, db=db)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
db = dbtools.dbHelper()
|
db = dbtools.dbHelper()
|
||||||
driver = webdriver.Firefox()
|
driver = webdriver.Firefox()
|
||||||
|
ret = {"rec": 0,"snt": 0}
|
||||||
if loginHandler(driver):
|
if loginHandler(driver):
|
||||||
p = multiprocessing.Process(target=mainHandler)
|
p = multiprocessing.Process(target=mainHandler, args=([ret]))
|
||||||
p.start()
|
p.start()
|
||||||
p.join(1500)
|
p.join(1500)
|
||||||
if p.is_alive():
|
if p.is_alive():
|
||||||
p.terminate()
|
p.terminate()
|
||||||
p.join()
|
p.join()
|
||||||
driver.close()
|
driver.close()
|
||||||
print("Received", rec, "messages")
|
print("Received", ret["rec"], "messages")
|
||||||
print("Sent", snt, "messages")
|
print("Sent", ret["snt"], "messages")
|
||||||
|
|
1
setup.py
1
setup.py
|
@ -50,6 +50,7 @@ else:
|
||||||
db = dbtools.dbObject(dbtype=dbtype, path=dbpath)
|
db = dbtools.dbObject(dbtype=dbtype, path=dbpath)
|
||||||
if not db.isInitialized():
|
if not db.isInitialized():
|
||||||
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));")
|
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));")
|
||||||
|
db.executeQuery("CREATE TABLE photos(`mid` TEXT, `pid` TEXT);")
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
db.closeConnection()
|
db.closeConnection()
|
||||||
|
|
Loading…
Reference in a new issue