From f8bc40b4e47b9dccb243bbd901075c40a11b9583 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Fri, 15 Jul 2016 17:02:59 +0200 Subject: [PATCH] introducing automatic purging of expired pastes, triggered by default at least 5 minutes apart, deleting a maximum of 10 pastes - resolves #3 --- lib/configuration.php | 5 ++ lib/model.php | 17 +++++++ lib/privatebin.php | 1 + lib/privatebin/abstract.php | 29 +++++++++++ lib/privatebin/data.php | 87 ++++++++++++++++++++++++++++++++ lib/privatebin/db.php | 28 ++++++++++- lib/purgelimiter.php | 99 +++++++++++++++++++++++++++++++++++++ tst/configuration.php | 1 + tst/privatebin.php | 29 ++++++++--- tst/privatebin/data.php | 33 +++++++++++++ tst/privatebin/db.php | 42 ++++++++++++++++ tst/privatebinWithDb.php | 9 ++-- tst/purgelimiter.php | 36 ++++++++++++++ 13 files changed, 404 insertions(+), 12 deletions(-) create mode 100644 lib/purgelimiter.php create mode 100644 tst/purgelimiter.php diff --git a/lib/configuration.php b/lib/configuration.php index 458c6462..18a6a401 100644 --- a/lib/configuration.php +++ b/lib/configuration.php @@ -70,6 +70,11 @@ class configuration 'header' => null, 'dir' => 'data', ), + 'purge' => array( + 'limit' => 300, + 'batchsize' => 10, + 'dir' => 'data', + ), 'model' => array( 'class' => 'privatebin_data', ), diff --git a/lib/model.php b/lib/model.php index 7ca63174..ba727b0f 100644 --- a/lib/model.php +++ b/lib/model.php @@ -35,6 +35,7 @@ class model * Factory constructor. * * @param configuration $conf + * @return void */ public function __construct(configuration $conf) { @@ -54,8 +55,24 @@ class model return $paste; } + /** + * Checks if a purge is necessary and triggers it if yes. + * + * @return void + */ + public function purge() + { + purgelimiter::setConfiguration($this->_conf); + if (purgelimiter::canPurge()) + { + $this->_getStore()->purge($this->_conf->getKey('batchsize', 'purge')); + } + } + /** * Gets, and creates if neccessary, a store object + * + * @return privatebin_abstract */ private function _getStore() { diff --git a/lib/privatebin.php b/lib/privatebin.php index 9279c613..f47e96c0 100644 --- a/lib/privatebin.php +++ b/lib/privatebin.php @@ -264,6 +264,7 @@ class privatebin // The user posts a standard paste. else { + $this->_model->purge(); $paste = $this->_model->getPaste(); try { $paste->setData($data); diff --git a/lib/privatebin/abstract.php b/lib/privatebin/abstract.php index 28b3eb04..2a0f24dc 100644 --- a/lib/privatebin/abstract.php +++ b/lib/privatebin/abstract.php @@ -123,6 +123,35 @@ abstract class privatebin_abstract */ abstract public function existsComment($pasteid, $parentid, $commentid); + /** + * Returns up to batch size number of paste ids that have expired + * + * @access protected + * @param int $batchsize + * @return array + */ + abstract protected function _getExpiredPastes($batchsize); + + /** + * Perform a purge of old pastes, at most the given batchsize is deleted. + * + * @access public + * @param int $batchsize + * @return void + */ + public function purge($batchsize) + { + if ($batchsize < 1) return; + $pastes = $this->_getExpiredPastes($batchsize); + if (count($pastes)) + { + foreach ($pastes as $pasteid) + { + $this->delete($pasteid); + } + } + } + /** * Get next free slot for comment from postdate. * diff --git a/lib/privatebin/data.php b/lib/privatebin/data.php index 2fdc54bf..55f27c67 100644 --- a/lib/privatebin/data.php +++ b/lib/privatebin/data.php @@ -210,6 +210,67 @@ class privatebin_data extends privatebin_abstract ); } + /** + * Returns up to batch size number of paste ids that have expired + * + * @access private + * @param int $batchsize + * @return array + */ + protected function _getExpiredPastes($batchsize) + { + $pastes = array(); + $firstLevel = array_filter( + scandir(self::$_dir), + array('self', '_isFirstLevelDir') + ); + if (count($firstLevel) > 0) + { + // try at most 10 times the $batchsize pastes before giving up + for ($i = 0, $max = $batchsize * 10; $i < $max; ++$i) + { + $firstKey = array_rand($firstLevel); + $secondLevel = array_filter( + scandir(self::$_dir . $firstLevel[$firstKey]), + array('self', '_isSecondLevelDir') + ); + + // skip this folder in the next checks if it is empty + if (count($secondLevel) == 0) + { + unset($firstLevel[$firstKey]); + continue; + } + + $secondKey = array_rand($secondLevel); + $path = self::$_dir . $firstLevel[$firstKey] . '/' . $secondLevel[$secondKey]; + if (!is_dir($path)) continue; + $thirdLevel = array_filter( + scandir($path), + array('model_paste', 'isValidId') + ); + if (count($thirdLevel) == 0) continue; + $thirdKey = array_rand($thirdLevel); + $pasteid = $thirdLevel[$thirdKey]; + if (in_array($pasteid, $pastes)) continue; + + if ($this->exists($pasteid)) + { + $data = $this->read($pasteid); + if ( + property_exists($data->meta, 'expire_date') && + $data->meta->expire_date < time() + ) + { + $pastes[] = $pasteid; + if (count($pastes) >= $batchsize) break; + } + } + } + } + return $pastes; + } + /** * initialize privatebin * @@ -266,4 +327,30 @@ class privatebin_data extends privatebin_abstract { return self::_dataid2path($dataid) . $dataid . '.discussion/'; } + + /** + * Check that the given element is a valid first level directory. + * + * @access private + * @static + * @param string $element + * @return bool + */ + private static function _isFirstLevelDir($element) + { + return self::_isSecondLevelDir($element) && is_dir(self::$_dir . '/' . $element); + } + + /** + * Check that the given element is a valid second level directory. + * + * @access private + * @static + * @param string $element + * @return bool + */ + private static function _isSecondLevelDir($element) + { + return (bool) preg_match('/^[a-f0-9]{2}$/', $element); + } } diff --git a/lib/privatebin/db.php b/lib/privatebin/db.php index 373919a9..89ff7847 100644 --- a/lib/privatebin/db.php +++ b/lib/privatebin/db.php @@ -302,7 +302,7 @@ class privatebin_db extends privatebin_abstract * Test if a paste exists. * * @access public - * @param string $dataid + * @param string $pasteid * @return void */ public function exists($pasteid) @@ -381,7 +381,7 @@ class privatebin_db extends privatebin_abstract * Test if a comment exists. * * @access public - * @param string $dataid + * @param string $pasteid * @param string $parentid * @param string $commentid * @return void @@ -395,6 +395,30 @@ class privatebin_db extends privatebin_abstract ); } + /** + * Returns up to batch size number of paste ids that have expired + * + * @access private + * @param int $batchsize + * @return array + */ + protected function _getExpiredPastes($batchsize) + { + $pastes = array(); + $rows = self::_select( + 'SELECT dataid FROM ' . self::_sanitizeIdentifier('paste') . + ' WHERE expiredate < ? LIMIT ?', array(time(), $batchsize) + ); + if (count($rows)) + { + foreach ($rows as $row) + { + $pastes[] = $row['dataid']; + } + } + return $pastes; + } + /** * execute a statement * diff --git a/lib/purgelimiter.php b/lib/purgelimiter.php new file mode 100644 index 00000000..cc0c976a --- /dev/null +++ b/lib/purgelimiter.php @@ -0,0 +1,99 @@ +getKey('limit', 'purge')); + self::setPath($conf->getKey('dir', 'purge')); + } + + /** + * check if the purge can be performed + * + * @access public + * @static + * @throws Exception + * @return bool + */ + public static function canPurge() + { + // disable limits if set to less then 1 + if (self::$_limit < 1) return true; + + $file = 'purge_limiter.php'; + $now = time(); + if (!self::_exists($file)) + { + self::_store( + $file, + '= $now) + { + $result = false; + } + else + { + $result = true; + self::_store( + $file, + '_options = configuration::getDefaults(); $this->_options['model_options']['dir'] = PATH . $this->_options['model_options']['dir']; $this->_options['traffic']['dir'] = PATH . $this->_options['traffic']['dir']; + $this->_options['purge']['dir'] = PATH . $this->_options['purge']['dir']; $this->_minimalConfig = '[main]' . PHP_EOL . '[model]' . PHP_EOL . '[model_options]'; } diff --git a/tst/privatebin.php b/tst/privatebin.php index ce822474..bf2ccc00 100644 --- a/tst/privatebin.php +++ b/tst/privatebin.php @@ -184,7 +184,11 @@ class privatebinTest extends PHPUnit_Framework_TestCase public function testCreateInvalidTimelimit() { $this->reset(); - $_POST = helper::getPaste(); + $options = parse_ini_file(CONF, true); + $options['traffic']['limit'] = 0; + helper::confBackup(); + helper::createIniFile(CONF, $options); + $_POST = helper::getPaste(array('expire' => 25)); $_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest'; $_SERVER['REQUEST_METHOD'] = 'POST'; $_SERVER['REMOTE_ADDR'] = '::1'; @@ -193,8 +197,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase new privatebin; $content = ob_get_contents(); $response = json_decode($content, true); - $this->assertEquals(1, $response['status'], 'outputs error status'); - $this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data'); + $this->assertEquals(0, $response['status'], 'outputs status'); + $this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data'); + $paste = $this->_model->read($response['id']); + $this->assertEquals( + hash_hmac('sha256', $response['id'], $paste->meta->salt), + $response['deletetoken'], + 'outputs valid delete token' + ); } /** @@ -228,11 +238,10 @@ class privatebinTest extends PHPUnit_Framework_TestCase $this->reset(); $options = parse_ini_file(CONF, true); $options['traffic']['header'] = 'X_FORWARDED_FOR'; - $options['traffic']['limit'] = 100; helper::confBackup(); helper::createIniFile(CONF, $options); $_POST = helper::getPaste(); - $_SERVER['HTTP_X_FORWARDED_FOR'] = '::1'; + $_SERVER['HTTP_X_FORWARDED_FOR'] = '::2'; $_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest'; $_SERVER['REQUEST_METHOD'] = 'POST'; $_SERVER['REMOTE_ADDR'] = '::1'; @@ -240,8 +249,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase new privatebin; $content = ob_get_contents(); $response = json_decode($content, true); - $this->assertEquals(1, $response['status'], 'outputs error status'); - $this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data'); + $this->assertEquals(0, $response['status'], 'outputs status'); + $this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data'); + $paste = $this->_model->read($response['id']); + $this->assertEquals( + hash_hmac('sha256', $response['id'], $paste->meta->salt), + $response['deletetoken'], + 'outputs valid delete token' + ); } /** diff --git a/tst/privatebin/data.php b/tst/privatebin/data.php index 6c728fe0..0372b5ac 100644 --- a/tst/privatebin/data.php +++ b/tst/privatebin/data.php @@ -63,4 +63,37 @@ class privatebin_dataTest extends PHPUnit_Framework_TestCase $this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId())); } + public function testPurge() + { + $expired = helper::getPaste(array('expire_date' => 1344803344)); + $paste = helper::getPaste(array('expire_date' => time() + 3600)); + $keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z'); + $ids = array(); + foreach ($keys as $key) + { + $ids[$key] = substr(md5($key), 0, 16); + $this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist"); + if (in_array($key, array('x', 'y', 'z'))) + { + $this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste"); + } + else + { + $this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste"); + } + $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it"); + } + $this->_model->purge(10); + foreach ($ids as $key => $id) + { + if (in_array($key, array('x', 'y', 'z'))) + { + $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge"); + } + else + { + $this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged"); + } + } + } } diff --git a/tst/privatebin/db.php b/tst/privatebin/db.php index a622bb1c..2c8d3b57 100644 --- a/tst/privatebin/db.php +++ b/tst/privatebin/db.php @@ -16,6 +16,12 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase $this->_model = privatebin_db::getInstance($this->_options); } + public function tearDown() + { + /* Tear Down Routine */ + if (is_dir(PATH . 'data')) helper::rmdir(PATH . 'data'); + } + public function testDatabaseBasedDataStoreWorks() { $this->_model->delete(helper::getPasteId()); @@ -62,6 +68,41 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase $this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId())); } + public function testPurge() + { + $this->_model->delete(helper::getPasteId()); + $expired = helper::getPaste(array('expire_date' => 1344803344)); + $paste = helper::getPaste(array('expire_date' => time() + 3600)); + $keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z'); + $ids = array(); + foreach ($keys as $key) + { + $ids[$key] = substr(md5($key), 0, 16); + $this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist"); + if (in_array($key, array('x', 'y', 'z'))) + { + $this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste"); + } + else + { + $this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste"); + } + $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it"); + } + $this->_model->purge(10); + foreach ($ids as $key => $id) + { + if (in_array($key, array('x', 'y', 'z'))) + { + $this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge"); + } + else + { + $this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged"); + } + } + } + /** * @expectedException PDOException */ @@ -185,6 +226,7 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase public function testTableUpgrade() { + mkdir(PATH . 'data'); $path = PATH . 'data/db-test.sq3'; @unlink($path); $this->_options['dsn'] = 'sqlite:' . $path; diff --git a/tst/privatebinWithDb.php b/tst/privatebinWithDb.php index 8fcebc91..bd2d85ab 100644 --- a/tst/privatebinWithDb.php +++ b/tst/privatebinWithDb.php @@ -4,7 +4,6 @@ require_once 'privatebin.php'; class privatebinWithDbTest extends privatebinTest { private $_options = array( - 'dsn' => 'sqlite:../data/tst.sq3', 'usr' => null, 'pwd' => null, 'opt' => array( @@ -13,11 +12,15 @@ class privatebinWithDbTest extends privatebinTest ), ); + private $_path; + public function setUp() { /* Setup Routine */ + $this->_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data'; + if(!is_dir($this->_path)) mkdir($this->_path); + $this->_options['dsn'] = 'sqlite:' . $this->_path . '/tst.sq3'; $this->_model = privatebin_db::getInstance($this->_options); - serversalt::setPath(PATH . 'data'); $this->reset(); } @@ -25,7 +28,7 @@ class privatebinWithDbTest extends privatebinTest { /* Tear Down Routine */ parent::tearDown(); - @unlink('../data/tst.sq3'); + helper::rmdir($this->_path); } public function reset() diff --git a/tst/purgelimiter.php b/tst/purgelimiter.php new file mode 100644 index 00000000..d2f8d8be --- /dev/null +++ b/tst/purgelimiter.php @@ -0,0 +1,36 @@ +_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data'; + if(!is_dir($this->_path)) mkdir($this->_path); + purgelimiter::setPath($this->_path); + } + + public function tearDown() + { + /* Tear Down Routine */ + helper::rmdir($this->_path); + } + + public function testLimit() + { + // initialize it + purgelimiter::canPurge(); + + // try setting it + purgelimiter::setLimit(1); + $this->assertEquals(false, purgelimiter::canPurge()); + sleep(2); + $this->assertEquals(true, purgelimiter::canPurge()); + + // disable it + purgelimiter::setLimit(0); + purgelimiter::canPurge(); + $this->assertEquals(true, purgelimiter::canPurge()); + } +}