blitzwing/bintzwing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

introducing automatic purging of expired pastes, triggered by default at least 5 minutes apart, deleting a maximum of 10 pastes - resolves #3

Browse source
This commit is contained in:
El RIDO 2016-07-15 17:02:59 +02:00
parent 4d10fd9690
commit f8bc40b4e4
13 changed files with 404 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/configuration.php
View file

@ -70,6 +70,11 @@ class configuration
'header' => null,
'dir' => 'data',
),
'purge' => array(
'limit' => 300,
'batchsize' => 10,
'dir' => 'data',
),
'model' => array(
'class' => 'privatebin_data',
),

17
lib/model.php
View file

@ -35,6 +35,7 @@ class model
* Factory constructor.
*
* @param configuration $conf
* @return void
*/
public function __construct(configuration $conf)
{
@ -54,8 +55,24 @@ class model
return $paste;
}
/**
* Checks if a purge is necessary and triggers it if yes.
*
* @return void
*/
public function purge()
{
purgelimiter::setConfiguration($this->_conf);
if (purgelimiter::canPurge())
{
$this->_getStore()->purge($this->_conf->getKey('batchsize', 'purge'));
}
}
/**
* Gets, and creates if neccessary, a store object
*
* @return privatebin_abstract
*/
private function _getStore()
{

1
lib/privatebin.php
View file

@ -264,6 +264,7 @@ class privatebin
// The user posts a standard paste.
else
{
$this->_model->purge();
$paste = $this->_model->getPaste();
try {
$paste->setData($data);

29
lib/privatebin/abstract.php
View file

@ -123,6 +123,35 @@ abstract class privatebin_abstract
*/
abstract public function existsComment($pasteid, $parentid, $commentid);
/**
* Returns up to batch size number of paste ids that have expired
*
* @access protected
* @param int $batchsize
* @return array
*/
abstract protected function _getExpiredPastes($batchsize);
/**
* Perform a purge of old pastes, at most the given batchsize is deleted.
*
* @access public
* @param int $batchsize
* @return void
*/
public function purge($batchsize)
{
if ($batchsize < 1) return;
$pastes = $this->_getExpiredPastes($batchsize);
if (count($pastes))
{
foreach ($pastes as $pasteid)
{
$this->delete($pasteid);
}
}
}
/**
* Get next free slot for comment from postdate.
*

87
lib/privatebin/data.php
View file

@ -210,6 +210,67 @@ class privatebin_data extends privatebin_abstract
);
}
/**
* Returns up to batch size number of paste ids that have expired
*
* @access private
* @param int $batchsize
* @return array
*/
protected function _getExpiredPastes($batchsize)
{
$pastes = array();
$firstLevel = array_filter(
scandir(self::$_dir),
array('self', '_isFirstLevelDir')
);
if (count($firstLevel) > 0)
{
// try at most 10 times the $batchsize pastes before giving up
for ($i = 0, $max = $batchsize * 10; $i < $max; ++$i)
{
$firstKey = array_rand($firstLevel);
$secondLevel = array_filter(
scandir(self::$_dir . $firstLevel[$firstKey]),
array('self', '_isSecondLevelDir')
);
// skip this folder in the next checks if it is empty
if (count($secondLevel) == 0)
{
unset($firstLevel[$firstKey]);
continue;
}
$secondKey = array_rand($secondLevel);
$path = self::$_dir . $firstLevel[$firstKey] . '/' . $secondLevel[$secondKey];
if (!is_dir($path)) continue;
$thirdLevel = array_filter(
scandir($path),
array('model_paste', 'isValidId')
);
if (count($thirdLevel) == 0) continue;
$thirdKey = array_rand($thirdLevel);
$pasteid = $thirdLevel[$thirdKey];
if (in_array($pasteid, $pastes)) continue;
if ($this->exists($pasteid))
{
$data = $this->read($pasteid);
if (
property_exists($data->meta, 'expire_date') &&
$data->meta->expire_date < time()
)
{
$pastes[] = $pasteid;
if (count($pastes) >= $batchsize) break;
}
}
}
}
return $pastes;
}
/**
* initialize privatebin
*
@ -266,4 +327,30 @@ class privatebin_data extends privatebin_abstract
{
return self::_dataid2path($dataid) . $dataid . '.discussion/';
}
/**
* Check that the given element is a valid first level directory.
*
* @access private
* @static
* @param string $element
* @return bool
*/
private static function _isFirstLevelDir($element)
{
return self::_isSecondLevelDir($element) && is_dir(self::$_dir . '/' . $element);
}
/**
* Check that the given element is a valid second level directory.
*
* @access private
* @static
* @param string $element
* @return bool
*/
private static function _isSecondLevelDir($element)
{
return (bool) preg_match('/^[a-f0-9]{2}$/', $element);
}
}

28
lib/privatebin/db.php
View file

@ -302,7 +302,7 @@ class privatebin_db extends privatebin_abstract
* Test if a paste exists.
*
* @access public
* @param string $dataid
* @param string $pasteid
* @return void
*/
public function exists($pasteid)
@ -381,7 +381,7 @@ class privatebin_db extends privatebin_abstract
* Test if a comment exists.
*
* @access public
* @param string $dataid
* @param string $pasteid
* @param string $parentid
* @param string $commentid
* @return void
@ -395,6 +395,30 @@ class privatebin_db extends privatebin_abstract
);
}
/**
* Returns up to batch size number of paste ids that have expired
*
* @access private
* @param int $batchsize
* @return array
*/
protected function _getExpiredPastes($batchsize)
{
$pastes = array();
$rows = self::_select(
'SELECT dataid FROM ' . self::_sanitizeIdentifier('paste') .
' WHERE expiredate < ? LIMIT ?', array(time(), $batchsize)
);
if (count($rows))
{
foreach ($rows as $row)
{
$pastes[] = $row['dataid'];
}
}
return $pastes;
}
/**
* execute a statement
*

99
lib/purgelimiter.php Normal file
View file

@ -0,0 +1,99 @@
<?php
/**
* PrivateBin
*
* a zero-knowledge paste bin
*
* @link https://github.com/PrivateBin/PrivateBin
* @copyright 2012 Sébastien SAUVAGE (sebsauvage.net)
* @license http://www.opensource.org/licenses/zlib-license.php The zlib/libpng License
* @version 0.22
*/
/**
* purgelimiter
*
* Handles purge limiting, so purging is not triggered to often.
*/
class purgelimiter extends persistence
{
/**
* time limit in seconds, defaults to 300s
*
* @access private
* @static
* @var int
*/
private static $_limit = 300;
/**
* set the time limit in seconds
*
* @access public
* @static
* @param int $limit
* @return void
*/
public static function setLimit($limit)
{
self::$_limit = $limit;
}
/**
* set configuration options of the traffic limiter
*
* @access public
* @static
* @param configuration $conf
* @return void
*/
public static function setConfiguration(configuration $conf)
{
self::setLimit($conf->getKey('limit', 'purge'));
self::setPath($conf->getKey('dir', 'purge'));
}
/**
* check if the purge can be performed
*
* @access public
* @static
* @throws Exception
* @return bool
*/
public static function canPurge()
{
// disable limits if set to less then 1
if (self::$_limit < 1) return true;
$file = 'purge_limiter.php';
$now = time();
if (!self::_exists($file))
{
self::_store(
$file,
'<?php' . PHP_EOL .
'$GLOBALS[\'purge_limiter\'] = ' . $now . ';' . PHP_EOL
);
}
$path = self::getPath($file);
require $path;
$pl = $GLOBALS['purge_limiter'];
if ($pl + self::$_limit >= $now)
{
$result = false;
}
else
{
$result = true;
self::_store(
$file,
'<?php' . PHP_EOL .
'$GLOBALS[\'purge_limiter\'] = ' . $now . ';' . PHP_EOL
);
}
return $result;
}
}

1
tst/configuration.php
View file

@ -12,6 +12,7 @@ class configurationTest extends PHPUnit_Framework_TestCase
$this->_options = configuration::getDefaults();
$this->_options['model_options']['dir'] = PATH . $this->_options['model_options']['dir'];
$this->_options['traffic']['dir'] = PATH . $this->_options['traffic']['dir'];
$this->_options['purge']['dir'] = PATH . $this->_options['purge']['dir'];
$this->_minimalConfig = '[main]' . PHP_EOL . '[model]' . PHP_EOL . '[model_options]';
}

29
tst/privatebin.php
View file

@ -184,7 +184,11 @@ class privatebinTest extends PHPUnit_Framework_TestCase
public function testCreateInvalidTimelimit()
{
$this->reset();
$_POST = helper::getPaste();
$options = parse_ini_file(CONF, true);
$options['traffic']['limit'] = 0;
helper::confBackup();
helper::createIniFile(CONF, $options);
$_POST = helper::getPaste(array('expire' => 25));
$_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest';
$_SERVER['REQUEST_METHOD'] = 'POST';
$_SERVER['REMOTE_ADDR'] = '::1';
@ -193,8 +197,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase
new privatebin;
$content = ob_get_contents();
$response = json_decode($content, true);
$this->assertEquals(1, $response['status'], 'outputs error status');
$this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data');
$this->assertEquals(0, $response['status'], 'outputs status');
$this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data');
$paste = $this->_model->read($response['id']);
$this->assertEquals(
hash_hmac('sha256', $response['id'], $paste->meta->salt),
$response['deletetoken'],
'outputs valid delete token'
);
}
/**
@ -228,11 +238,10 @@ class privatebinTest extends PHPUnit_Framework_TestCase
$this->reset();
$options = parse_ini_file(CONF, true);
$options['traffic']['header'] = 'X_FORWARDED_FOR';
$options['traffic']['limit'] = 100;
helper::confBackup();
helper::createIniFile(CONF, $options);
$_POST = helper::getPaste();
$_SERVER['HTTP_X_FORWARDED_FOR'] = '::1';
$_SERVER['HTTP_X_FORWARDED_FOR'] = '::2';
$_SERVER['HTTP_X_REQUESTED_WITH'] = 'JSONHttpRequest';
$_SERVER['REQUEST_METHOD'] = 'POST';
$_SERVER['REMOTE_ADDR'] = '::1';
@ -240,8 +249,14 @@ class privatebinTest extends PHPUnit_Framework_TestCase
new privatebin;
$content = ob_get_contents();
$response = json_decode($content, true);
$this->assertEquals(1, $response['status'], 'outputs error status');
$this->assertFalse($this->_model->exists(helper::getPasteId()), 'paste exists after posting data');
$this->assertEquals(0, $response['status'], 'outputs status');
$this->assertTrue($this->_model->exists($response['id']), 'paste exists after posting data');
$paste = $this->_model->read($response['id']);
$this->assertEquals(
hash_hmac('sha256', $response['id'], $paste->meta->salt),
$response['deletetoken'],
'outputs valid delete token'
);
}
/**

33
tst/privatebin/data.php
View file

@ -63,4 +63,37 @@ class privatebin_dataTest extends PHPUnit_Framework_TestCase
$this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId()));
}
public function testPurge()
{
$expired = helper::getPaste(array('expire_date' => 1344803344));
$paste = helper::getPaste(array('expire_date' => time() + 3600));
$keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z');
$ids = array();
foreach ($keys as $key)
{
$ids[$key] = substr(md5($key), 0, 16);
$this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist");
if (in_array($key, array('x', 'y', 'z')))
{
$this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste");
}
else
{
$this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste");
}
$this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it");
}
$this->_model->purge(10);
foreach ($ids as $key => $id)
{
if (in_array($key, array('x', 'y', 'z')))
{
$this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge");
}
else
{
$this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged");
}
}
}
}

42
tst/privatebin/db.php
View file

@ -16,6 +16,12 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
$this->_model = privatebin_db::getInstance($this->_options);
}
public function tearDown()
{
/* Tear Down Routine */
if (is_dir(PATH . 'data')) helper::rmdir(PATH . 'data');
}
public function testDatabaseBasedDataStoreWorks()
{
$this->_model->delete(helper::getPasteId());
@ -62,6 +68,41 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
$this->assertEquals(json_decode(json_encode($original)), $this->_model->read(helper::getPasteId()));
}
public function testPurge()
{
$this->_model->delete(helper::getPasteId());
$expired = helper::getPaste(array('expire_date' => 1344803344));
$paste = helper::getPaste(array('expire_date' => time() + 3600));
$keys = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'x', 'y', 'z');
$ids = array();
foreach ($keys as $key)
{
$ids[$key] = substr(md5($key), 0, 16);
$this->assertFalse($this->_model->exists($ids[$key]), "paste $key does not yet exist");
if (in_array($key, array('x', 'y', 'z')))
{
$this->assertTrue($this->_model->create($ids[$key], $paste), "store $key paste");
}
else
{
$this->assertTrue($this->_model->create($ids[$key], $expired), "store $key paste");
}
$this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after storing it");
}
$this->_model->purge(10);
foreach ($ids as $key => $id)
{
if (in_array($key, array('x', 'y', 'z')))
{
$this->assertTrue($this->_model->exists($ids[$key]), "paste $key exists after purge");
}
else
{
$this->assertFalse($this->_model->exists($ids[$key]), "paste $key was purged");
}
}
}
/**
* @expectedException PDOException
*/
@ -185,6 +226,7 @@ class privatebin_dbTest extends PHPUnit_Framework_TestCase
public function testTableUpgrade()
{
mkdir(PATH . 'data');
$path = PATH . 'data/db-test.sq3';
@unlink($path);
$this->_options['dsn'] = 'sqlite:' . $path;

9
tst/privatebinWithDb.php
View file

@ -4,7 +4,6 @@ require_once 'privatebin.php';
class privatebinWithDbTest extends privatebinTest
{
private $_options = array(
'dsn' => 'sqlite:../data/tst.sq3',
'usr' => null,
'pwd' => null,
'opt' => array(
@ -13,11 +12,15 @@ class privatebinWithDbTest extends privatebinTest
),
);
private $_path;
public function setUp()
{
/* Setup Routine */
$this->_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data';
if(!is_dir($this->_path)) mkdir($this->_path);
$this->_options['dsn'] = 'sqlite:' . $this->_path . '/tst.sq3';
$this->_model = privatebin_db::getInstance($this->_options);
serversalt::setPath(PATH . 'data');
$this->reset();
}
@ -25,7 +28,7 @@ class privatebinWithDbTest extends privatebinTest
{
/* Tear Down Routine */
parent::tearDown();
@unlink('../data/tst.sq3');
helper::rmdir($this->_path);
}
public function reset()

36
tst/purgelimiter.php Normal file
View file

@ -0,0 +1,36 @@
<?php
class purgelimiterTest extends PHPUnit_Framework_TestCase
{
private $_path;
public function setUp()
{
/* Setup Routine */
$this->_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'privatebin_data';
if(!is_dir($this->_path)) mkdir($this->_path);
purgelimiter::setPath($this->_path);
}
public function tearDown()
{
/* Tear Down Routine */
helper::rmdir($this->_path);
}
public function testLimit()
{
// initialize it
purgelimiter::canPurge();
// try setting it
purgelimiter::setLimit(1);
$this->assertEquals(false, purgelimiter::canPurge());
sleep(2);
$this->assertEquals(true, purgelimiter::canPurge());
// disable it
purgelimiter::setLimit(0);
purgelimiter::canPurge();
$this->assertEquals(true, purgelimiter::canPurge());
}
}