opkg: vfork external gzip command to uncompress data
Opkg's builtin decompression code is unsuitable to process nested archives as it uses a single shared state and relies on undefined seek behaviour for pipes. Rework the extraction logic to use the external gzip command as I/O filter for decompressing data and remove the builtin inflate code entirely. This shrinks the final opkg binary by about 4KB and results in less runtime memory consumption due to efficient use of vfork() and less copy-on-write operations in the forked child. Rework by Felix: create a thread that relays data to the gzip process instead of using a fragile poll loop Signed-off-by: Jo-Philipp Wich <jo@mein.io> Signed-off-by: Felix Fietkau <nbd@nbd.name>
This commit is contained in:
Jo-Philipp Wich
Felix Fietkau
parent
dc5f496a0d
commit
e1d1c31890
4 changed files with 746 additions and 23 deletions
|
|
@ -11,7 +11,7 @@ include $(INCLUDE_DIR)/version.mk
|
||||||
include $(INCLUDE_DIR)/feeds.mk
|
include $(INCLUDE_DIR)/feeds.mk
|
||||||
|
|
||||||
PKG_NAME:=opkg
|
PKG_NAME:=opkg
|
||||||
PKG_RELEASE:=15
|
PKG_RELEASE:=16
|
||||||
|
|
||||||
PKG_SOURCE_PROTO:=git
|
PKG_SOURCE_PROTO:=git
|
||||||
PKG_SOURCE_URL:=http://git.yoctoproject.org/git/opkg
|
PKG_SOURCE_URL:=http://git.yoctoproject.org/git/opkg
|
||||||
|
|
|
||||||
719
package/system/opkg/patches/070-use_external_gzip.patch
Normal file
719
package/system/opkg/patches/070-use_external_gzip.patch
Normal file
|
|
@ -0,0 +1,719 @@
|
||||||
|
--- a/libbb/unarchive.c
|
||||||
|
+++ b/libbb/unarchive.c
|
||||||
|
@@ -28,6 +28,7 @@
|
||||||
|
#include <libgen.h>
|
||||||
|
|
||||||
|
#include "libbb.h"
|
||||||
|
+#include "gzip.h"
|
||||||
|
|
||||||
|
#define CONFIG_FEATURE_TAR_OLDGNU_COMPATABILITY 1
|
||||||
|
#define CONFIG_FEATURE_TAR_GNU_EXTENSIONS
|
||||||
|
@@ -39,38 +40,15 @@ static char *linkname = NULL;
|
||||||
|
|
||||||
|
off_t archive_offset;
|
||||||
|
|
||||||
|
-#define SEEK_BUF 4096
|
||||||
|
static ssize_t
|
||||||
|
-seek_by_read(FILE* fd, size_t len)
|
||||||
|
-{
|
||||||
|
- ssize_t cc, total = 0;
|
||||||
|
- char buf[SEEK_BUF];
|
||||||
|
-
|
||||||
|
- while (len) {
|
||||||
|
- cc = fread(buf, sizeof(buf[0]),
|
||||||
|
- len > SEEK_BUF ? SEEK_BUF : len,
|
||||||
|
- fd);
|
||||||
|
-
|
||||||
|
- total += cc;
|
||||||
|
- len -= cc;
|
||||||
|
-
|
||||||
|
- if(feof(fd) || ferror(fd))
|
||||||
|
- break;
|
||||||
|
- }
|
||||||
|
- return total;
|
||||||
|
-}
|
||||||
|
-
|
||||||
|
-static void
|
||||||
|
-seek_sub_file(FILE *fd, const int count)
|
||||||
|
+seek_forward(struct gzip_handle *zh, ssize_t len)
|
||||||
|
{
|
||||||
|
- archive_offset += count;
|
||||||
|
+ ssize_t slen = gzip_seek(zh, len);
|
||||||
|
|
||||||
|
- /* Do not use fseek() on a pipe. It may fail with ESPIPE, leaving the
|
||||||
|
- * stream at an undefined location.
|
||||||
|
- */
|
||||||
|
- seek_by_read(fd, count);
|
||||||
|
+ if (slen == len)
|
||||||
|
+ archive_offset += len;
|
||||||
|
|
||||||
|
- return;
|
||||||
|
+ return slen;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@@ -87,7 +65,7 @@ seek_sub_file(FILE *fd, const int count)
|
||||||
|
* trailing '/' or else the last dir will be assumed to be the file prefix
|
||||||
|
*/
|
||||||
|
static char *
|
||||||
|
-extract_archive(FILE *src_stream, FILE *out_stream,
|
||||||
|
+extract_archive(struct gzip_handle *src_stream, FILE *out_stream,
|
||||||
|
const file_header_t *file_entry, const int function,
|
||||||
|
const char *prefix,
|
||||||
|
int *err)
|
||||||
|
@@ -129,14 +107,14 @@ extract_archive(FILE *src_stream, FILE *
|
||||||
|
|
||||||
|
if (function & extract_to_stream) {
|
||||||
|
if (S_ISREG(file_entry->mode)) {
|
||||||
|
- *err = copy_file_chunk(src_stream, out_stream, file_entry->size);
|
||||||
|
+ *err = gzip_copy(src_stream, out_stream, file_entry->size);
|
||||||
|
archive_offset += file_entry->size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (function & extract_one_to_buffer) {
|
||||||
|
if (S_ISREG(file_entry->mode)) {
|
||||||
|
buffer = (char *) xmalloc(file_entry->size + 1);
|
||||||
|
- fread(buffer, 1, file_entry->size, src_stream);
|
||||||
|
+ gzip_read(src_stream, buffer, file_entry->size);
|
||||||
|
buffer[file_entry->size] = '\0';
|
||||||
|
archive_offset += file_entry->size;
|
||||||
|
goto cleanup;
|
||||||
|
@@ -156,7 +134,7 @@ extract_archive(FILE *src_stream, FILE *
|
||||||
|
*err = -1;
|
||||||
|
error_msg("%s not created: newer or same age file exists", file_entry->name);
|
||||||
|
}
|
||||||
|
- seek_sub_file(src_stream, file_entry->size);
|
||||||
|
+ seek_forward(src_stream, file_entry->size);
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -185,11 +163,11 @@ extract_archive(FILE *src_stream, FILE *
|
||||||
|
} else {
|
||||||
|
if ((dst_stream = wfopen(full_name, "w")) == NULL) {
|
||||||
|
*err = -1;
|
||||||
|
- seek_sub_file(src_stream, file_entry->size);
|
||||||
|
+ seek_forward(src_stream, file_entry->size);
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
archive_offset += file_entry->size;
|
||||||
|
- *err = copy_file_chunk(src_stream, dst_stream, file_entry->size);
|
||||||
|
+ *err = gzip_copy(src_stream, dst_stream, file_entry->size);
|
||||||
|
fclose(dst_stream);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
@@ -250,7 +228,7 @@ extract_archive(FILE *src_stream, FILE *
|
||||||
|
/* If we arent extracting data we have to skip it,
|
||||||
|
* if data size is 0 then then just do it anyway
|
||||||
|
* (saves testing for it) */
|
||||||
|
- seek_sub_file(src_stream, file_entry->size);
|
||||||
|
+ seek_forward(src_stream, file_entry->size);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* extract_list and extract_verbose_list can be used in conjunction
|
||||||
|
@@ -274,8 +252,8 @@ cleanup:
|
||||||
|
}
|
||||||
|
|
||||||
|
static char *
|
||||||
|
-unarchive(FILE *src_stream, FILE *out_stream,
|
||||||
|
- file_header_t *(*get_headers)(FILE *),
|
||||||
|
+unarchive(struct gzip_handle *src_stream, FILE *out_stream,
|
||||||
|
+ file_header_t *(*get_headers)(struct gzip_handle *),
|
||||||
|
void (*free_headers)(file_header_t *),
|
||||||
|
const int extract_function,
|
||||||
|
const char *prefix,
|
||||||
|
@@ -329,7 +307,7 @@ unarchive(FILE *src_stream, FILE *out_st
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/* seek past the data entry */
|
||||||
|
- seek_sub_file(src_stream, file_entry->size);
|
||||||
|
+ seek_forward(src_stream, file_entry->size);
|
||||||
|
}
|
||||||
|
free_headers(file_entry);
|
||||||
|
}
|
||||||
|
@@ -337,108 +315,9 @@ unarchive(FILE *src_stream, FILE *out_st
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
-static file_header_t *
|
||||||
|
-get_header_ar(FILE *src_stream)
|
||||||
|
-{
|
||||||
|
- file_header_t *typed;
|
||||||
|
- union {
|
||||||
|
- char raw[60];
|
||||||
|
- struct {
|
||||||
|
- char name[16];
|
||||||
|
- char date[12];
|
||||||
|
- char uid[6];
|
||||||
|
- char gid[6];
|
||||||
|
- char mode[8];
|
||||||
|
- char size[10];
|
||||||
|
- char magic[2];
|
||||||
|
- } formated;
|
||||||
|
- } ar;
|
||||||
|
- static char *ar_long_names;
|
||||||
|
-
|
||||||
|
- if (fread(ar.raw, 1, 60, src_stream) != 60) {
|
||||||
|
- return(NULL);
|
||||||
|
- }
|
||||||
|
- archive_offset += 60;
|
||||||
|
- /* align the headers based on the header magic */
|
||||||
|
- if ((ar.formated.magic[0] != '`') || (ar.formated.magic[1] != '\n')) {
|
||||||
|
- /* some version of ar, have an extra '\n' after each data entry,
|
||||||
|
- * this puts the next header out by 1 */
|
||||||
|
- if (ar.formated.magic[1] != '`') {
|
||||||
|
- error_msg("Invalid magic");
|
||||||
|
- return(NULL);
|
||||||
|
- }
|
||||||
|
- /* read the next char out of what would be the data section,
|
||||||
|
- * if its a '\n' then it is a valid header offset by 1*/
|
||||||
|
- archive_offset++;
|
||||||
|
- if (fgetc(src_stream) != '\n') {
|
||||||
|
- error_msg("Invalid magic");
|
||||||
|
- return(NULL);
|
||||||
|
- }
|
||||||
|
- /* fix up the header, we started reading 1 byte too early */
|
||||||
|
- /* raw_header[60] wont be '\n' as it should, but it doesnt matter */
|
||||||
|
- memmove(ar.raw, &ar.raw[1], 59);
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- typed = (file_header_t *) xcalloc(1, sizeof(file_header_t));
|
||||||
|
-
|
||||||
|
- typed->size = (size_t) atoi(ar.formated.size);
|
||||||
|
- /* long filenames have '/' as the first character */
|
||||||
|
- if (ar.formated.name[0] == '/') {
|
||||||
|
- if (ar.formated.name[1] == '/') {
|
||||||
|
- /* If the second char is a '/' then this entries data section
|
||||||
|
- * stores long filename for multiple entries, they are stored
|
||||||
|
- * in static variable long_names for use in future entries */
|
||||||
|
- ar_long_names = (char *) xrealloc(ar_long_names, typed->size);
|
||||||
|
- fread(ar_long_names, 1, typed->size, src_stream);
|
||||||
|
- archive_offset += typed->size;
|
||||||
|
- /* This ar entries data section only contained filenames for other records
|
||||||
|
- * they are stored in the static ar_long_names for future reference */
|
||||||
|
- return (get_header_ar(src_stream)); /* Return next header */
|
||||||
|
- } else if (ar.formated.name[1] == ' ') {
|
||||||
|
- /* This is the index of symbols in the file for compilers */
|
||||||
|
- seek_sub_file(src_stream, typed->size);
|
||||||
|
- return (get_header_ar(src_stream)); /* Return next header */
|
||||||
|
- } else {
|
||||||
|
- /* The number after the '/' indicates the offset in the ar data section
|
||||||
|
- (saved in variable long_name) that conatains the real filename */
|
||||||
|
- if (!ar_long_names) {
|
||||||
|
- error_msg("Cannot resolve long file name");
|
||||||
|
- return (NULL);
|
||||||
|
- }
|
||||||
|
- typed->name = xstrdup(ar_long_names + atoi(&ar.formated.name[1]));
|
||||||
|
- }
|
||||||
|
- } else {
|
||||||
|
- /* short filenames */
|
||||||
|
- typed->name = xcalloc(1, 16);
|
||||||
|
- strncpy(typed->name, ar.formated.name, 16);
|
||||||
|
- }
|
||||||
|
- typed->name[strcspn(typed->name, " /")]='\0';
|
||||||
|
-
|
||||||
|
- /* convert the rest of the now valid char header to its typed struct */
|
||||||
|
- parse_mode(ar.formated.mode, &typed->mode);
|
||||||
|
- typed->mtime = atoi(ar.formated.date);
|
||||||
|
- typed->uid = atoi(ar.formated.uid);
|
||||||
|
- typed->gid = atoi(ar.formated.gid);
|
||||||
|
-
|
||||||
|
- return(typed);
|
||||||
|
-}
|
||||||
|
-
|
||||||
|
-static void
|
||||||
|
-free_header_ar(file_header_t *ar_entry)
|
||||||
|
-{
|
||||||
|
- if (ar_entry == NULL)
|
||||||
|
- return;
|
||||||
|
-
|
||||||
|
- free(ar_entry->name);
|
||||||
|
- if (ar_entry->link_name)
|
||||||
|
- free(ar_entry->link_name);
|
||||||
|
-
|
||||||
|
- free(ar_entry);
|
||||||
|
-}
|
||||||
|
-
|
||||||
|
|
||||||
|
static file_header_t *
|
||||||
|
-get_header_tar(FILE *tar_stream)
|
||||||
|
+get_header_tar(struct gzip_handle *tar_stream)
|
||||||
|
{
|
||||||
|
union {
|
||||||
|
unsigned char raw[512];
|
||||||
|
@@ -467,10 +346,10 @@ get_header_tar(FILE *tar_stream)
|
||||||
|
long sum = 0;
|
||||||
|
|
||||||
|
if (archive_offset % 512 != 0) {
|
||||||
|
- seek_sub_file(tar_stream, 512 - (archive_offset % 512));
|
||||||
|
+ seek_forward(tar_stream, 512 - (archive_offset % 512));
|
||||||
|
}
|
||||||
|
|
||||||
|
- if (fread(tar.raw, 1, 512, tar_stream) != 512) {
|
||||||
|
+ if (gzip_read(tar_stream, tar.raw, 512) != 512) {
|
||||||
|
/* Unfortunately its common for tar files to have all sorts of
|
||||||
|
* trailing garbage, fail silently */
|
||||||
|
// error_msg("Couldnt read header");
|
||||||
|
@@ -557,7 +436,7 @@ get_header_tar(FILE *tar_stream)
|
||||||
|
# ifdef CONFIG_FEATURE_TAR_GNU_EXTENSIONS
|
||||||
|
case 'L': {
|
||||||
|
longname = xmalloc(tar_entry->size + 1);
|
||||||
|
- if(fread(longname, tar_entry->size, 1, tar_stream) != 1)
|
||||||
|
+ if(gzip_read(tar_stream, longname, tar_entry->size) != tar_entry->size)
|
||||||
|
return NULL;
|
||||||
|
longname[tar_entry->size] = '\0';
|
||||||
|
archive_offset += tar_entry->size;
|
||||||
|
@@ -566,7 +445,7 @@ get_header_tar(FILE *tar_stream)
|
||||||
|
}
|
||||||
|
case 'K': {
|
||||||
|
linkname = xmalloc(tar_entry->size + 1);
|
||||||
|
- if(fread(linkname, tar_entry->size, 1, tar_stream) != 1)
|
||||||
|
+ if(gzip_read(tar_stream, linkname, tar_entry->size) != tar_entry->size)
|
||||||
|
return NULL;
|
||||||
|
linkname[tar_entry->size] = '\0';
|
||||||
|
archive_offset += tar_entry->size;
|
||||||
|
@@ -642,6 +521,9 @@ deb_extract(const char *package_filename
|
||||||
|
char *ared_file = NULL;
|
||||||
|
char ar_magic[8];
|
||||||
|
int gz_err;
|
||||||
|
+ struct gzip_handle tar_outer, tar_inner;
|
||||||
|
+ file_header_t *tar_header;
|
||||||
|
+ ssize_t len;
|
||||||
|
|
||||||
|
*err = 0;
|
||||||
|
|
||||||
|
@@ -672,111 +554,44 @@ deb_extract(const char *package_filename
|
||||||
|
/* set the buffer size */
|
||||||
|
setvbuf(deb_stream, NULL, _IOFBF, 0x8000);
|
||||||
|
|
||||||
|
- /* check ar magic */
|
||||||
|
- fread(ar_magic, 1, 8, deb_stream);
|
||||||
|
-
|
||||||
|
- if (strncmp(ar_magic,"!<arch>",7) == 0) {
|
||||||
|
- archive_offset = 8;
|
||||||
|
+ memset(&tar_outer, 0, sizeof(tar_outer));
|
||||||
|
+ tar_outer.file = deb_stream;
|
||||||
|
+ gzip_exec(&tar_outer, NULL);
|
||||||
|
|
||||||
|
- while ((ar_header = get_header_ar(deb_stream)) != NULL) {
|
||||||
|
- if (strcmp(ared_file, ar_header->name) == 0) {
|
||||||
|
- int gunzip_pid = 0;
|
||||||
|
- FILE *uncompressed_stream;
|
||||||
|
- /* open a stream of decompressed data */
|
||||||
|
- uncompressed_stream = gz_open(deb_stream, &gunzip_pid);
|
||||||
|
- if (uncompressed_stream == NULL) {
|
||||||
|
- *err = -1;
|
||||||
|
- goto cleanup;
|
||||||
|
- }
|
||||||
|
+ /* walk through outer tar file to find ared_file */
|
||||||
|
+ while ((tar_header = get_header_tar(&tar_outer)) != NULL) {
|
||||||
|
+ int name_offset = 0;
|
||||||
|
+ if (strncmp(tar_header->name, "./", 2) == 0)
|
||||||
|
+ name_offset = 2;
|
||||||
|
|
||||||
|
- archive_offset = 0;
|
||||||
|
- output_buffer = unarchive(uncompressed_stream,
|
||||||
|
- out_stream, get_header_tar,
|
||||||
|
- free_header_tar,
|
||||||
|
- extract_function, prefix,
|
||||||
|
- file_list, err);
|
||||||
|
- fclose(uncompressed_stream);
|
||||||
|
- gz_err = gz_close(gunzip_pid);
|
||||||
|
- if (gz_err)
|
||||||
|
- *err = -1;
|
||||||
|
- free_header_ar(ar_header);
|
||||||
|
- break;
|
||||||
|
- }
|
||||||
|
- if (fseek(deb_stream, ar_header->size, SEEK_CUR) == -1) {
|
||||||
|
- opkg_perror(ERROR, "Couldn't fseek into %s", package_filename);
|
||||||
|
- *err = -1;
|
||||||
|
- free_header_ar(ar_header);
|
||||||
|
- goto cleanup;
|
||||||
|
- }
|
||||||
|
- free_header_ar(ar_header);
|
||||||
|
- }
|
||||||
|
- goto cleanup;
|
||||||
|
- } else if (strncmp(ar_magic, "\037\213", 2) == 0) {
|
||||||
|
- /* it's a gz file, let's assume it's an opkg */
|
||||||
|
- int unzipped_opkg_pid;
|
||||||
|
- FILE *unzipped_opkg_stream;
|
||||||
|
- file_header_t *tar_header;
|
||||||
|
- archive_offset = 0;
|
||||||
|
- if (fseek(deb_stream, 0, SEEK_SET) == -1) {
|
||||||
|
- opkg_perror(ERROR, "Couldn't fseek into %s", package_filename);
|
||||||
|
- *err = -1;
|
||||||
|
- goto cleanup;
|
||||||
|
- }
|
||||||
|
- unzipped_opkg_stream = gz_open(deb_stream, &unzipped_opkg_pid);
|
||||||
|
- if (unzipped_opkg_stream == NULL) {
|
||||||
|
- *err = -1;
|
||||||
|
- goto cleanup;
|
||||||
|
- }
|
||||||
|
+ if (strcmp(ared_file, tar_header->name+name_offset) == 0) {
|
||||||
|
+ memset(&tar_inner, 0, sizeof(tar_inner));
|
||||||
|
+ tar_inner.gzip = &tar_outer;
|
||||||
|
+ gzip_exec(&tar_inner, NULL);
|
||||||
|
|
||||||
|
- /* walk through outer tar file to find ared_file */
|
||||||
|
- while ((tar_header = get_header_tar(unzipped_opkg_stream)) != NULL) {
|
||||||
|
- int name_offset = 0;
|
||||||
|
- if (strncmp(tar_header->name, "./", 2) == 0)
|
||||||
|
- name_offset = 2;
|
||||||
|
- if (strcmp(ared_file, tar_header->name+name_offset) == 0) {
|
||||||
|
- int gunzip_pid = 0;
|
||||||
|
- FILE *uncompressed_stream;
|
||||||
|
- /* open a stream of decompressed data */
|
||||||
|
- uncompressed_stream = gz_open(unzipped_opkg_stream, &gunzip_pid);
|
||||||
|
- if (uncompressed_stream == NULL) {
|
||||||
|
- *err = -1;
|
||||||
|
- goto cleanup;
|
||||||
|
- }
|
||||||
|
- archive_offset = 0;
|
||||||
|
+ archive_offset = 0;
|
||||||
|
|
||||||
|
- output_buffer = unarchive(uncompressed_stream,
|
||||||
|
- out_stream,
|
||||||
|
- get_header_tar,
|
||||||
|
- free_header_tar,
|
||||||
|
- extract_function,
|
||||||
|
- prefix,
|
||||||
|
- file_list,
|
||||||
|
- err);
|
||||||
|
+ output_buffer = unarchive(&tar_inner,
|
||||||
|
+ out_stream,
|
||||||
|
+ get_header_tar,
|
||||||
|
+ free_header_tar,
|
||||||
|
+ extract_function,
|
||||||
|
+ prefix,
|
||||||
|
+ file_list,
|
||||||
|
+ err);
|
||||||
|
|
||||||
|
- free_header_tar(tar_header);
|
||||||
|
- fclose(uncompressed_stream);
|
||||||
|
- gz_err = gz_close(gunzip_pid);
|
||||||
|
- if (gz_err)
|
||||||
|
- *err = -1;
|
||||||
|
- break;
|
||||||
|
- }
|
||||||
|
- seek_sub_file(unzipped_opkg_stream, tar_header->size);
|
||||||
|
free_header_tar(tar_header);
|
||||||
|
+ gzip_close(&tar_inner);
|
||||||
|
+ break;
|
||||||
|
}
|
||||||
|
- fclose(unzipped_opkg_stream);
|
||||||
|
- gz_err = gz_close(unzipped_opkg_pid);
|
||||||
|
- if (gz_err)
|
||||||
|
- *err = -1;
|
||||||
|
|
||||||
|
- goto cleanup;
|
||||||
|
- } else {
|
||||||
|
- *err = -1;
|
||||||
|
- error_msg("%s: invalid magic", package_filename);
|
||||||
|
+ seek_forward(&tar_outer, tar_header->size);
|
||||||
|
+ free_header_tar(tar_header);
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
- if (deb_stream)
|
||||||
|
- fclose(deb_stream);
|
||||||
|
+ gzip_close(&tar_outer);
|
||||||
|
+
|
||||||
|
if (file_list)
|
||||||
|
free(file_list);
|
||||||
|
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/libbb/gzip.h
|
||||||
|
@@ -0,0 +1,41 @@
|
||||||
|
+/*
|
||||||
|
+ * Copyright (C) 2016 Jo-Philipp Wich <jo@mein.io>
|
||||||
|
+ *
|
||||||
|
+ * Zlib decrompression utility routines.
|
||||||
|
+ *
|
||||||
|
+ * This program is free software; you can redistribute it and/or modify
|
||||||
|
+ * it under the terms of the GNU General Public License as published by
|
||||||
|
+ * the Free Software Foundation; either version 2 of the License, or
|
||||||
|
+ * (at your option) any later version.
|
||||||
|
+ *
|
||||||
|
+ * This program is distributed in the hope that it will be useful,
|
||||||
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
+ * GNU Library General Public License for more details.
|
||||||
|
+ *
|
||||||
|
+ * You should have received a copy of the GNU General Public License
|
||||||
|
+ * along with this program; if not, write to the Free Software
|
||||||
|
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
+ */
|
||||||
|
+
|
||||||
|
+#include <stdio.h>
|
||||||
|
+#include <signal.h>
|
||||||
|
+#include <pthread.h>
|
||||||
|
+
|
||||||
|
+struct gzip_handle {
|
||||||
|
+ FILE *file;
|
||||||
|
+ struct gzip_handle *gzip;
|
||||||
|
+
|
||||||
|
+ pid_t pid;
|
||||||
|
+ int rfd, wfd;
|
||||||
|
+ struct sigaction pipe_sa;
|
||||||
|
+ pthread_t thread;
|
||||||
|
+};
|
||||||
|
+
|
||||||
|
+int gzip_exec(struct gzip_handle *zh, const char *filename);
|
||||||
|
+ssize_t gzip_read(struct gzip_handle *zh, char *buf, ssize_t len);
|
||||||
|
+ssize_t gzip_copy(struct gzip_handle *zh, FILE *out, ssize_t len);
|
||||||
|
+int gzip_close(struct gzip_handle *zh);
|
||||||
|
+FILE *gzip_fdopen(struct gzip_handle *zh, const char *filename);
|
||||||
|
+
|
||||||
|
+#define gzip_seek(zh, len) gzip_copy(zh, NULL, len)
|
||||||
|
--- a/libbb/Makefile.am
|
||||||
|
+++ b/libbb/Makefile.am
|
||||||
|
@@ -4,9 +4,8 @@ ALL_CFLAGS=-g -O -Wall -DHOST_CPU_STR=\"
|
||||||
|
|
||||||
|
noinst_LIBRARIES = libbb.a
|
||||||
|
|
||||||
|
-libbb_a_SOURCES = gz_open.c \
|
||||||
|
+libbb_a_SOURCES = \
|
||||||
|
libbb.h \
|
||||||
|
- unzip.c \
|
||||||
|
wfopen.c \
|
||||||
|
unarchive.c \
|
||||||
|
copy_file.c \
|
||||||
|
@@ -20,7 +19,8 @@ libbb_a_SOURCES = gz_open.c \
|
||||||
|
parse_mode.c \
|
||||||
|
time_string.c \
|
||||||
|
all_read.c \
|
||||||
|
- mode_string.c
|
||||||
|
+ mode_string.c \
|
||||||
|
+ gzip.c
|
||||||
|
|
||||||
|
libbb_la_CFLAGS = $(ALL_CFLAGS)
|
||||||
|
#libbb_la_LDFLAGS = -static
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/libbb/gzip.c
|
||||||
|
@@ -0,0 +1,208 @@
|
||||||
|
+/*
|
||||||
|
+ * Copyright (C) 2016 Jo-Philipp Wich <jo@mein.io>
|
||||||
|
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
|
||||||
|
+ *
|
||||||
|
+ * Zlib decrompression utility routines.
|
||||||
|
+ *
|
||||||
|
+ * This program is free software; you can redistribute it and/or modify
|
||||||
|
+ * it under the terms of the GNU General Public License as published by
|
||||||
|
+ * the Free Software Foundation; either version 2 of the License, or
|
||||||
|
+ * (at your option) any later version.
|
||||||
|
+ *
|
||||||
|
+ * This program is distributed in the hope that it will be useful,
|
||||||
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
+ * GNU Library General Public License for more details.
|
||||||
|
+ *
|
||||||
|
+ * You should have received a copy of the GNU General Public License
|
||||||
|
+ * along with this program; if not, write to the Free Software
|
||||||
|
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
+ */
|
||||||
|
+
|
||||||
|
+#include <string.h>
|
||||||
|
+#include <errno.h>
|
||||||
|
+#include <fcntl.h>
|
||||||
|
+#include <unistd.h>
|
||||||
|
+#include <poll.h>
|
||||||
|
+#include <stdlib.h>
|
||||||
|
+#include <sys/stat.h>
|
||||||
|
+#include <sys/wait.h>
|
||||||
|
+
|
||||||
|
+#include "gzip.h"
|
||||||
|
+
|
||||||
|
+static void
|
||||||
|
+to_devnull(int fd)
|
||||||
|
+{
|
||||||
|
+ int devnull = open("/dev/null", fd ? O_WRONLY : O_RDONLY);
|
||||||
|
+
|
||||||
|
+ if (devnull >= 0)
|
||||||
|
+ dup2(devnull, fd);
|
||||||
|
+
|
||||||
|
+ if (devnull > STDERR_FILENO)
|
||||||
|
+ close(devnull);
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+void *
|
||||||
|
+gzip_thread(void *ptr)
|
||||||
|
+{
|
||||||
|
+ struct gzip_handle *zh = ptr;
|
||||||
|
+ char buf[4096];
|
||||||
|
+ int len, ret;
|
||||||
|
+
|
||||||
|
+ while (1) {
|
||||||
|
+ if (zh->file)
|
||||||
|
+ len = fread(buf, 1, sizeof(buf), zh->file);
|
||||||
|
+ else if (zh->gzip)
|
||||||
|
+ len = gzip_read(zh->gzip, buf, sizeof(buf));
|
||||||
|
+
|
||||||
|
+ if (len <= 0)
|
||||||
|
+ break;
|
||||||
|
+
|
||||||
|
+ do {
|
||||||
|
+ ret = write(zh->wfd, buf, len);
|
||||||
|
+ } while (ret == -1 && errno == EINTR);
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ close(zh->wfd);
|
||||||
|
+ zh->wfd = -1;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+int
|
||||||
|
+gzip_exec(struct gzip_handle *zh, const char *filename)
|
||||||
|
+{
|
||||||
|
+ int rpipe[2] = { -1, -1 }, wpipe[2] = { -1, -1 };
|
||||||
|
+ struct sigaction pipe_sa = { .sa_handler = SIG_IGN };
|
||||||
|
+
|
||||||
|
+ zh->rfd = -1;
|
||||||
|
+ zh->wfd = -1;
|
||||||
|
+
|
||||||
|
+ if (sigaction(SIGPIPE, &pipe_sa, &zh->pipe_sa) < 0)
|
||||||
|
+ return -1;
|
||||||
|
+
|
||||||
|
+ if (pipe(rpipe) < 0)
|
||||||
|
+ return -1;
|
||||||
|
+
|
||||||
|
+ if (!filename && pipe(wpipe) < 0) {
|
||||||
|
+ close(rpipe[0]);
|
||||||
|
+ close(rpipe[1]);
|
||||||
|
+ return -1;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ zh->pid = vfork();
|
||||||
|
+
|
||||||
|
+ switch (zh->pid) {
|
||||||
|
+ case -1:
|
||||||
|
+ return -1;
|
||||||
|
+
|
||||||
|
+ case 0:
|
||||||
|
+ to_devnull(STDERR_FILENO);
|
||||||
|
+
|
||||||
|
+ if (filename) {
|
||||||
|
+ to_devnull(STDIN_FILENO);
|
||||||
|
+ }
|
||||||
|
+ else {
|
||||||
|
+ dup2(wpipe[0], STDIN_FILENO);
|
||||||
|
+ close(wpipe[0]);
|
||||||
|
+ close(wpipe[1]);
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ dup2(rpipe[1], STDOUT_FILENO);
|
||||||
|
+ close(rpipe[0]);
|
||||||
|
+ close(rpipe[1]);
|
||||||
|
+
|
||||||
|
+ execlp("gzip", "gzip", "-d", "-c", filename, NULL);
|
||||||
|
+ exit(-1);
|
||||||
|
+
|
||||||
|
+ default:
|
||||||
|
+ zh->rfd = rpipe[0];
|
||||||
|
+ zh->wfd = wpipe[1];
|
||||||
|
+
|
||||||
|
+ fcntl(zh->rfd, F_SETFD, fcntl(zh->rfd, F_GETFD) | FD_CLOEXEC);
|
||||||
|
+ close(rpipe[1]);
|
||||||
|
+
|
||||||
|
+ if (zh->wfd >= 0) {
|
||||||
|
+ fcntl(zh->wfd, F_SETFD, fcntl(zh->wfd, F_GETFD) | FD_CLOEXEC);
|
||||||
|
+ close(wpipe[0]);
|
||||||
|
+ pthread_create(&zh->thread, NULL, gzip_thread, zh);
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return 0;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+ssize_t
|
||||||
|
+gzip_read(struct gzip_handle *zh, char *buf, ssize_t len)
|
||||||
|
+{
|
||||||
|
+ ssize_t ret;
|
||||||
|
+
|
||||||
|
+ do {
|
||||||
|
+ ret = read(zh->rfd, buf, len);
|
||||||
|
+ } while (ret == -1 && errno != EINTR);
|
||||||
|
+
|
||||||
|
+ return ret;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+ssize_t
|
||||||
|
+gzip_copy(struct gzip_handle *zh, FILE *out, ssize_t len)
|
||||||
|
+{
|
||||||
|
+ char buf[4096];
|
||||||
|
+ ssize_t rlen, total = 0;
|
||||||
|
+
|
||||||
|
+ while (len > 0) {
|
||||||
|
+ rlen = gzip_read(zh, buf,
|
||||||
|
+ (len > sizeof(buf)) ? sizeof(buf) : len);
|
||||||
|
+
|
||||||
|
+ if (rlen <= 0)
|
||||||
|
+ break;
|
||||||
|
+
|
||||||
|
+ if (out != NULL) {
|
||||||
|
+ if (fwrite(buf, 1, rlen, out) != rlen)
|
||||||
|
+ break;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ len -= rlen;
|
||||||
|
+ total += rlen;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return total;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+FILE *
|
||||||
|
+gzip_fdopen(struct gzip_handle *zh, const char *filename)
|
||||||
|
+{
|
||||||
|
+ memset(zh, 0, sizeof(*zh));
|
||||||
|
+
|
||||||
|
+ if (!filename || gzip_exec(zh, filename) < 0)
|
||||||
|
+ return NULL;
|
||||||
|
+
|
||||||
|
+ fcntl(zh->rfd, F_SETFL, fcntl(zh->rfd, F_GETFL) & ~O_NONBLOCK);
|
||||||
|
+
|
||||||
|
+ return fdopen(zh->rfd, "r");
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+int
|
||||||
|
+gzip_close(struct gzip_handle *zh)
|
||||||
|
+{
|
||||||
|
+ int code = -1;
|
||||||
|
+
|
||||||
|
+ if (zh->rfd >= 0)
|
||||||
|
+ close(zh->rfd);
|
||||||
|
+
|
||||||
|
+ if (zh->wfd >= 0)
|
||||||
|
+ close(zh->wfd);
|
||||||
|
+
|
||||||
|
+ if (zh->pid > 0) {
|
||||||
|
+ kill(zh->pid, SIGKILL);
|
||||||
|
+ waitpid(zh->pid, &code, 0);
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if (zh->file)
|
||||||
|
+ fclose(zh->file);
|
||||||
|
+
|
||||||
|
+ if (zh->thread)
|
||||||
|
+ pthread_join(zh->thread, NULL);
|
||||||
|
+
|
||||||
|
+ sigaction(SIGPIPE, &zh->pipe_sa, NULL);
|
||||||
|
+
|
||||||
|
+ return WIFEXITED(code) ? WEXITSTATUS(code) : -1;
|
||||||
|
+}
|
||||||
|
--- a/src/Makefile.am
|
||||||
|
+++ b/src/Makefile.am
|
||||||
|
@@ -3,4 +3,4 @@ bin_PROGRAMS = opkg-cl
|
||||||
|
|
||||||
|
opkg_cl_SOURCES = opkg-cl.c
|
||||||
|
opkg_cl_LDADD = $(top_builddir)/libopkg/libopkg.a \
|
||||||
|
- $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS)
|
||||||
|
+ $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) -lpthread
|
||||||
|
--- a/tests/Makefile.am
|
||||||
|
+++ b/tests/Makefile.am
|
||||||
|
@@ -16,7 +16,7 @@ noinst_PROGRAMS = libopkg_test
|
||||||
|
#opkg_active_list_test_SOURCES = opkg_active_list_test.c
|
||||||
|
#opkg_active_list_test_CFLAGS = $(ALL_CFLAGS) -I$(top_srcdir)
|
||||||
|
|
||||||
|
-libopkg_test_LDADD = $(top_builddir)/libopkg/libopkg.a $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS)
|
||||||
|
+libopkg_test_LDADD = $(top_builddir)/libopkg/libopkg.a $(top_builddir)/libbb/libbb.a $(CURL_LIBS) $(GPGME_LIBS) $(OPENSSL_LIBS) $(PATHFINDER_LIBS) -lpthread
|
||||||
|
libopkg_test_SOURCE = libopkg_test.c
|
||||||
|
libopkg_test_LDFLAGS = -static
|
||||||
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
- FILE *in, *out;
|
- FILE *in, *out;
|
||||||
- struct _curl_cb_data cb_data;
|
- struct _curl_cb_data cb_data;
|
||||||
- char *tmp_file_name = NULL;
|
- char *tmp_file_name = NULL;
|
||||||
|
-
|
||||||
- sprintf_alloc(&tmp_file_name, "%s/%s.gz", tmp,
|
- sprintf_alloc(&tmp_file_name, "%s/%s.gz", tmp,
|
||||||
- src->name);
|
- src->name);
|
||||||
-
|
-
|
||||||
|
|
@ -26,7 +26,7 @@
|
||||||
- err = opkg_download(url, tmp_file_name,
|
- err = opkg_download(url, tmp_file_name,
|
||||||
- (curl_progress_func) curl_progress_cb,
|
- (curl_progress_func) curl_progress_cb,
|
||||||
- &cb_data, 0);
|
- &cb_data, 0);
|
||||||
-
|
|
||||||
- if (err == 0) {
|
- if (err == 0) {
|
||||||
- opkg_msg(INFO, "Inflating %s...\n",
|
- opkg_msg(INFO, "Inflating %s...\n",
|
||||||
- tmp_file_name);
|
- tmp_file_name);
|
||||||
|
|
@ -87,34 +87,38 @@
|
||||||
opkg_msg(NOTICE, "Updated list of available packages in %s.\n",
|
opkg_msg(NOTICE, "Updated list of available packages in %s.\n",
|
||||||
--- a/libopkg/pkg_hash.c
|
--- a/libopkg/pkg_hash.c
|
||||||
+++ b/libopkg/pkg_hash.c
|
+++ b/libopkg/pkg_hash.c
|
||||||
@@ -102,12 +102,18 @@ pkg_hash_add_from_file(const char *file_
|
@@ -29,6 +29,7 @@
|
||||||
pkg_src_t *src, pkg_dest_t *dest, int is_status_file)
|
#include "sprintf_alloc.h"
|
||||||
{
|
#include "file_util.h"
|
||||||
pkg_t *pkg;
|
#include "libbb/libbb.h"
|
||||||
- FILE *fp;
|
+#include "libbb/gzip.h"
|
||||||
+ FILE *fp, *fp_c = NULL;
|
|
||||||
|
void
|
||||||
|
pkg_hash_init(void)
|
||||||
|
@@ -106,8 +107,15 @@ pkg_hash_add_from_file(const char *file_
|
||||||
char *buf;
|
char *buf;
|
||||||
const size_t len = 4096;
|
const size_t len = 4096;
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
+ int pid;
|
+ struct gzip_handle zh;
|
||||||
|
|
||||||
fp = fopen(file_name, "r");
|
|
||||||
+ if (fp && src && src->gzip) {
|
|
||||||
+ fp_c = fp;
|
|
||||||
+ fp = gz_open(fp_c, &pid);
|
|
||||||
+ }
|
|
||||||
+
|
+
|
||||||
|
+ if (src && src->gzip) {
|
||||||
|
+ fp = gzip_fdopen(&zh, file_name);
|
||||||
|
+ }
|
||||||
|
+ else {
|
||||||
|
+ fp = fopen(file_name, "r");
|
||||||
|
+ }
|
||||||
|
|
||||||
|
- fp = fopen(file_name, "r");
|
||||||
if (fp == NULL) {
|
if (fp == NULL) {
|
||||||
opkg_perror(ERROR, "Failed to open %s", file_name);
|
opkg_perror(ERROR, "Failed to open %s", file_name);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -154,6 +160,10 @@ pkg_hash_add_from_file(const char *file_
|
@@ -155,6 +163,9 @@ pkg_hash_add_from_file(const char *file_
|
||||||
|
|
||||||
free(buf);
|
free(buf);
|
||||||
fclose(fp);
|
fclose(fp);
|
||||||
+ if (fp_c) {
|
|
||||||
+ fclose(fp_c);
|
|
||||||
+ gz_close(pid);
|
|
||||||
+ }
|
|
||||||
|
|
||||||
|
+ if (src && src->gzip)
|
||||||
|
+ gzip_close(&zh);
|
||||||
|
+
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -769,7 +769,7 @@
|
||||||
#include "pkg.h"
|
#include "pkg.h"
|
||||||
#include "opkg_message.h"
|
#include "opkg_message.h"
|
||||||
#include "pkg_vec.h"
|
#include "pkg_vec.h"
|
||||||
@@ -183,40 +182,6 @@ pkg_hash_load_feeds(void)
|
@@ -184,40 +183,6 @@ pkg_hash_load_feeds(void)
|
||||||
lists_dir = conf->restrict_to_default_dest ?
|
lists_dir = conf->restrict_to_default_dest ?
|
||||||
conf->default_dest->lists_dir : conf->lists_dir;
|
conf->default_dest->lists_dir : conf->lists_dir;
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue