Download Tar archives from playlists

classes/PlaylistArchiveStream.php

@@ -0,0 +1,200 @@
+<?php
+/**
+ * PlaylistArchiveStream class.
+ */
+
+namespace Alltube;
+
+use Barracuda\ArchiveStream\TarArchive;
+
+/**
+ * Class used to create a Tar archive from playlists and stream it to the browser.
+ *
+ * @link http://php.net/manual/en/class.streamwrapper.php
+ */
+class PlaylistArchiveStream extends TarArchive
+{
+    /**
+     * Files to add in the archive.
+     *
+     * @var array[]
+     */
+    private $files;
+
+    /**
+     * Stream used to store data before it is sent to the browser.
+     *
+     * @var resource
+     */
+    private $buffer;
+
+    /**
+     * Guzzle client.
+     *
+     * @var \GuzzleHttp\Client
+     */
+    private $client;
+
+    /**
+     * VideoDownload instance.
+     *
+     * @var VideoDownload
+     */
+    private $download;
+
+    /**
+     * Current file position in $files array.
+     *
+     * @var int
+     */
+    private $curFile = 0;
+
+    /**
+     * Video format to download.
+     *
+     * @var string
+     */
+    private $format;
+
+    /**
+     * PlaylistArchiveStream constructor.
+     */
+    public function __construct()
+    {
+        $this->client = new \GuzzleHttp\Client();
+        $this->download = new VideoDownload();
+    }
+
+    /**
+     * Add data to the archive.
+     *
+     * @param mixed $data Data
+     *
+     * @return void
+     */
+    protected function send($data)
+    {
+        $pos = ftell($this->buffer);
+        fwrite($this->buffer, $data);
+        fseek($this->buffer, $pos);
+    }
+
+    /**
+     * Called when fopen() is used on the stream.
+     *
+     * @param string $path Playlist path (should be playlist://url1;url2;.../format)
+     * @param string $mode Stream mode
+     *
+     * @return bool
+     */
+    public function stream_open($path)
+    {
+        $this->format = ltrim(parse_url($path, PHP_URL_PATH), '/');
+        $this->buffer = fopen('php://temp', 'r+');
+        foreach (explode(';', parse_url($path, PHP_URL_HOST)) as $url) {
+            $this->files[] = [
+                'url'        => urldecode($url),
+                'headersSent'=> false,
+                'complete'   => false,
+                'stream'     => null,
+            ];
+        }
+
+        return true;
+    }
+
+    /**
+     * Called when fwrite() is used on the stream.
+     *
+     * @return int
+     */
+    public function stream_write()
+    {
+        //We don't support writing to a stream
+        return 0;
+    }
+
+    /**
+     * Called when fstat() is used on the stream.
+     *
+     * @return array
+     */
+    public function stream_stat()
+    {
+        //We need this so Slim won't try to get the size of the stream
+        return [
+            'mode'=> 0010000,
+        ];
+    }
+
+    /**
+     * Called when ftell() is used on the stream.
+     *
+     * @return int
+     */
+    public function stream_tell()
+    {
+        return ftell($this->buffer);
+    }
+
+    /**
+     * Called when fseek() is used on the stream.
+     *
+     * @param int $offset Offset
+     *
+     * @return bool
+     */
+    public function stream_seek($offset)
+    {
+        return fseek($this->buffer, $offset) == 0;
+    }
+
+    /**
+     * Called when feof() is used on the stream.
+     *
+     * @return bool
+     */
+    public function stream_eof()
+    {
+        foreach ($this->files as $file) {
+            if (!$file['complete']) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Called when fread() is used on the stream.
+     *
+     * @param int $count Number of bytes to read
+     *
+     * @return mixed
+     */
+    public function stream_read($count)
+    {
+        if (!$this->files[$this->curFile]['headersSent']) {
+            $urls = $this->download->getUrl($this->files[$this->curFile]['url'], $this->format);
+            $response = $this->client->request('GET', $urls[0], ['stream' => true]);
+
+            $contentLengthHeaders = $response->getHeader('Content-Length');
+            $this->init_file_stream_transfer(
+                $this->download->getFilename($this->files[$this->curFile]['url'], $this->format),
+                $contentLengthHeaders[0]
+            );
+
+            $this->files[$this->curFile]['headersSent'] = true;
+            $this->files[$this->curFile]['stream'] = $response->getBody();
+        } elseif (!$this->files[$this->curFile]['stream']->eof()) {
+            $this->stream_file_part($this->files[$this->curFile]['stream']->read($count));
+        } elseif (!$this->files[$this->curFile]['complete']) {
+            $this->complete_file_stream();
+            $this->files[$this->curFile]['complete'] = true;
+        } elseif (isset($this->files[$this->curFile])) {
+            $this->curFile += 1;
+        }
+
+        return fread($this->buffer, $count);
+    }
+}

classes/VideoDownload.php

@@ -364,4 +364,23 @@ class VideoDownload
     {
         return popen($this->getRtmpProcess($video)->getCommandLine(), 'r');
     }
+
+    /**
+     * Get a Tar stream containing every video in the playlist piped through the server.
+     *
+     * @param string $video  Video object returned by youtube-dl
+     * @param string $format Requested format
+     *
+     * @return Response HTTP response
+     */
+    public function getPlaylistArchiveStream($video, $format)
+    {
+        $playlistItems = [];
+        foreach ($video->entries as $entry) {
+            $playlistItems[] = urlencode($entry->url);
+        }
+        $stream = fopen('playlist://'.implode(';', $playlistItems).'/'.$format, 'r');
+
+        return $stream;
+    }
 }

composer.json

@@ -13,7 +13,8 @@
     "ptachoire/process-builder-chain": "~1.2.0",
     "rudloff/smarty-plugin-noscheme": "~0.1.0",
     "guzzlehttp/guzzle": "~6.2.0",
-    "aura/session": "~2.1.0"
+    "aura/session": "~2.1.0",
+    "barracudanetworks/archivestream-php": "~1.0.5"
   },
   "require-dev": {
     "symfony/var-dumper": "~3.2.0",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "18563e36d487153b121da01e2cf040db",
+    "content-hash": "6f968a7d8b884758ea4655dd80f0697d",
     "packages": [
         {
             "name": "aura/session",
@@ -68,6 +68,46 @@
             ],
             "time": "2016-10-03T20:28:32+00:00"
         },
+        {
+            "name": "barracudanetworks/archivestream-php",
+            "version": "1.0.5",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/barracudanetworks/ArchiveStream-php.git",
+                "reference": "1bf98097d1e9b137fd40081f26abb0a17b097ef7"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/barracudanetworks/ArchiveStream-php/zipball/1bf98097d1e9b137fd40081f26abb0a17b097ef7",
+                "reference": "1bf98097d1e9b137fd40081f26abb0a17b097ef7",
+                "shasum": ""
+            },
+            "require": {
+                "ext-gmp": "*",
+                "ext-mbstring": "*",
+                "php": ">=5.1.2"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Barracuda\\ArchiveStream\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "description": "A library for dynamically streaming dynamic tar or zip files without the need to have the complete file stored on the server.",
+            "homepage": "https://github.com/barracudanetworks/ArchiveStream-php",
+            "keywords": [
+                "archive",
+                "php",
+                "stream",
+                "tar",
+                "zip"
+            ],
+            "time": "2017-01-13T14:52:38+00:00"
+        },
         {
             "name": "container-interop/container-interop",
             "version": "1.2.0",

controllers/FrontController.php

@@ -317,7 +317,16 @@ class FrontController
     private function getStream($url, $format, Response $response, Request $request, $password = null)
     {
         $video = $this->download->getJSON($url, $format, $password);
-        if ($video->protocol == 'rtmp') {
+        if (isset($video->entries)) {
+            $stream = $this->download->getPlaylistArchiveStream($video, $format);
+            $response = $response->withHeader('Content-Type', 'application/x-tar');
+            $response = $response->withHeader(
+                'Content-Disposition',
+                'attachment; filename="'.$video->title.'.tar"'
+            );
+
+            return $response->withBody(new Stream($stream));
+        } elseif ($video->protocol == 'rtmp') {
             $stream = $this->download->getRtmpStream($video);
             $response = $response->withHeader('Content-Type', 'video/'.$video->ext);
             if ($request->isGet()) {
@@ -426,6 +435,10 @@ class FrontController
                 $this->sessionSegment->getFlash($url)
             );
         } else {
+            if (empty($videoUrls[0])) {
+                throw new \Exception("Can't find URL of video");
+            }
+
             return $response->withRedirect($videoUrls[0]);
         }
     }

index.php

@@ -3,6 +3,7 @@
 require_once __DIR__.'/vendor/autoload.php';
 use Alltube\Config;
 use Alltube\Controller\FrontController;
+use Alltube\PlaylistArchiveStream;
 use Alltube\UglyRouter;
 use Alltube\ViewFactory;
 use Slim\App;
@@ -12,6 +13,8 @@ if (isset($_SERVER['REQUEST_URI']) && strpos($_SERVER['REQUEST_URI'], '/index.ph
     die;
 }
 
+stream_wrapper_register('playlist', PlaylistArchiveStream::class);
+
 $app = new App();
 $container = $app->getContainer();
 $config = Config::getInstance();

templates/playlist.tpl

@@ -6,6 +6,9 @@
     <a href="{$video->webpage_url}">
 {$video->title}</a></i>{/if} playlist:
 </p>
+{if $config->stream}
+    <a href="{path_for name="redirect"}?url={$video->webpage_url}" class="downloadBtn">Download everything</a>
+{/if}
 {foreach $video->entries as $video}
     <div class="playlist-entry">
         <h3><a target="_blank" href="{strip}

tests/FrontControllerTest.php

@@ -453,4 +453,29 @@ class FrontControllerTest extends \PHPUnit_Framework_TestCase
     {
         $this->assertRequestIsServerError('redirect', ['url'=>'http://example.com/foo']);
     }
+
+    /**
+     * Test the redirect() function with an video that returns an empty URL.
+     * This can be caused by trying to redirect to a playlist.
+     *
+     * @return void
+     */
+    public function testRedirectWithEmptyUrl()
+    {
+        $this->assertRequestIsServerError('redirect', ['url'=>'https://www.youtube.com/playlist?list=PLgdySZU6KUXL_8Jq5aUkyNV7wCa-4wZsC']);
+    }
+
+    /**
+     * Test the redirect() function with a playlist stream.
+     *
+     * @return void
+     */
+    public function testRedirectWithPlaylist()
+    {
+        $this->assertRequestIsOk(
+            'redirect',
+            ['url'=> 'https://www.youtube.com/playlist?list=PLgdySZU6KUXL_8Jq5aUkyNV7wCa-4wZsC'],
+            new Config(['stream'=>true])
+        );
+    }
 }

tests/PlaylistArchiveStreamTest.php

@@ -0,0 +1,98 @@
+<?php
+/**
+ * PlaylistArchiveStreamTest class.
+ */
+
+namespace Alltube\Test;
+
+use Alltube\PlaylistArchiveStream;
+
+/**
+ * Unit tests for the ViewFactory class.
+ */
+class PlaylistArchiveStreamTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * Prepare tests.
+     */
+    protected function setUp()
+    {
+        $this->stream = new PlaylistArchiveStream();
+    }
+
+    /**
+     * Test the stream_open() function.
+     *
+     * @return void
+     */
+    public function testStreamOpen()
+    {
+        $this->assertTrue($this->stream->stream_open('playlist://foo', 'r'));
+    }
+
+    /**
+     * Test the stream_write() function.
+     *
+     * @return void
+     */
+    public function testStreamWrite()
+    {
+        $this->assertEquals(0, $this->stream->stream_write());
+    }
+
+    /**
+     * Test the stream_stat() function.
+     *
+     * @return void
+     */
+    public function testStreamStat()
+    {
+        $this->assertEquals(['mode'=>4096], $this->stream->stream_stat());
+    }
+
+    /**
+     * Test the stream_tell() function.
+     *
+     * @return void
+     */
+    public function testStreamTell()
+    {
+        $this->stream->stream_open('playlist://foo', 'r');
+        $this->assertInternalType('int', $this->stream->stream_tell());
+    }
+
+    /**
+     * Test the stream_seek() function.
+     *
+     * @return void
+     */
+    public function testStreamSeek()
+    {
+        $this->stream->stream_open('playlist://foo', 'r');
+        $this->assertInternalType('bool', $this->stream->stream_seek(3));
+    }
+
+    /**
+     * Test the stream_read() function.
+     *
+     * @return void
+     */
+    public function testStreamRead()
+    {
+        $this->stream->stream_open('playlist://BaW_jenozKc;BaW_jenozKc/worst', 'r');
+        while (!$this->stream->stream_eof()) {
+            $this->assertLessThanOrEqual(8192, strlen($this->stream->stream_read(8192)));
+        }
+    }
+
+    /**
+     * Test the stream_eof() function.
+     *
+     * @return void
+     */
+    public function testStreamEof()
+    {
+        $this->stream->stream_open('playlist://foo', 'r');
+        $this->assertFalse($this->stream->stream_eof(3));
+    }
+}

tests/VideoDownloadTest.php

@@ -480,4 +480,15 @@ class VideoDownloadTest extends \PHPUnit_Framework_TestCase
         $video = $download->getJSON($url, $format);
         $download->getM3uStream($video);
     }
+
+    /**
+     * Test getPlaylistArchiveStream function without avconv.
+     *
+     * @return void
+     */
+    public function testGetPlaylistArchiveStream()
+    {
+        $video = $this->download->getJSON('https://www.youtube.com/playlist?list=PLgdySZU6KUXL_8Jq5aUkyNV7wCa-4wZsC', 'best');
+        $this->assertStream($this->download->getPlaylistArchiveStream($video, 'best'));
+    }
 }

tests/bootstrap.php

@@ -2,6 +2,7 @@
 /**
  * File used to bootstrap tests.
  */
+use Alltube\PlaylistArchiveStream;
 
 /**
  * Composer autoload.
@@ -9,3 +10,5 @@
 require_once __DIR__.'/../vendor/autoload.php';
 
 session_start();
+
+stream_wrapper_register('playlist', PlaylistArchiveStream::class);