KumiSystems/openwrtv4
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

include/download.mk: generate reproducable SCM tarballs

Browse source 
Apply a number of changes to the tarball generation in order to produce
identical files on different systems:

1) Use an explicit `gzip -cn` to avoid storing file mtime in the gzip header
2) Instruct `tar` to unconditionally use uid and gid 0 for archive members
3) Instruct `tar` to sort archive members by file name
4) For SCMs that do not preserve file modification times like Git or Mercurial,
   use the date of the last commit to the repository and pass it as `--mtime`
   value to `tar`

After these changes, locally produced tarballs generated from SCM checkouts
should be identical on any system, simplifying the mirroring of cache archives.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>
This commit is contained in:
Jo-Philipp Wich 2016-10-04 18:33:55 +02:00
parent 8462ec3134
commit 5d86dc791e
1 changed files with 9 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
include/download.mk
View file

@ -33,9 +33,9 @@ $(strip \
endef endef
# code for creating tarballs from cvs/svn/git/bzr/hg/darcs checkouts - useful for mirror support # code for creating tarballs from cvs/svn/git/bzr/hg/darcs checkouts - useful for mirror support
dl_pack/bz2=$(TAR) cjf $(1) $(2) dl_pack/bz2=$(TAR) --owner=0 --group=0 --sort=name $$$${TAR_TIMESTAMP:+--mtime="$$$$TAR_TIMESTAMP"} -cjf $(1) $(2)
dl_pack/gz=$(TAR) czf $(1) $(2) dl_pack/gz=$(TAR) --owner=0 --group=0 --sort=name $$$${TAR_TIMESTAMP:+--mtime="$$$$TAR_TIMESTAMP"} -c $(2) | gzip -nc > $(1)
dl_pack/xz=$(TAR) c $(2) | xz -zc > $(1) dl_pack/xz=$(TAR) --owner=0 --group=0 --sort=name $$$${TAR_TIMESTAMP:+--mtime="$$$$TAR_TIMESTAMP"} -c $(2) | xz -zc > $(1)
dl_pack/unknown=echo "ERROR: Unknown pack format for file $(1)"; false dl_pack/unknown=echo "ERROR: Unknown pack format for file $(1)"; false
define dl_pack define dl_pack
$(if $(dl_pack/$(call ext,$(1))),$(dl_pack/$(call ext,$(1))),$(dl_pack/unknown)) $(if $(dl_pack/$(call ext,$(1))),$(dl_pack/$(call ext,$(1))),$(dl_pack/unknown))
@ -79,6 +79,7 @@ define DownloadMethod/svn
svn export --non-interactive --trust-server-cert -r$(VERSION) $(URL) $(SUBDIR) || \ svn export --non-interactive --trust-server-cert -r$(VERSION) $(URL) $(SUBDIR) || \
svn export --non-interactive -r$(VERSION) $(URL) $(SUBDIR) ) && \ svn export --non-interactive -r$(VERSION) $(URL) $(SUBDIR) ) && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
export TAR_TIMESTAMP="" && \
$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \ $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \ mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \
rm -rf $(SUBDIR); \ rm -rf $(SUBDIR); \
@ -96,6 +97,7 @@ define DownloadMethod/git
(cd $(SUBDIR) && git checkout $(VERSION) && \ (cd $(SUBDIR) && git checkout $(VERSION) && \
git submodule update --init --recursive) && \ git submodule update --init --recursive) && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
export TAR_TIMESTAMP=`cd $(SUBDIR) && git log -1 --format='@%ct'` && \
rm -rf $(SUBDIR)/.git && \ rm -rf $(SUBDIR)/.git && \
$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \ $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \ mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \
@ -110,8 +112,9 @@ define DownloadMethod/bzr
cd $(TMP_DIR)/dl && \ cd $(TMP_DIR)/dl && \
rm -rf $(SUBDIR) && \ rm -rf $(SUBDIR) && \
[ \! -d $(SUBDIR) ] && \ [ \! -d $(SUBDIR) ] && \
bzr export -r$(VERSION) $(SUBDIR) $(URL) && \ bzr export --per-file-timestamps -r$(VERSION) $(SUBDIR) $(URL) && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
export TAR_TIMESTAMP="" && \
$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \ $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \ mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \
rm -rf $(SUBDIR); \ rm -rf $(SUBDIR); \
@ -126,6 +129,7 @@ define DownloadMethod/hg
rm -rf $(SUBDIR) && \ rm -rf $(SUBDIR) && \
[ \! -d $(SUBDIR) ] && \ [ \! -d $(SUBDIR) ] && \
hg clone -r $(VERSION) $(URL) $(SUBDIR) && \ hg clone -r $(VERSION) $(URL) $(SUBDIR) && \
export TAR_TIMESTAMP=`cd $(SUBDIR) && hg log --template '@{date}' -l 1` && \
find $(SUBDIR) -name .hg | xargs rm -rf && \ find $(SUBDIR) -name .hg | xargs rm -rf && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \ $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
@ -142,6 +146,7 @@ define DownloadMethod/darcs
rm -rf $(SUBDIR) && \ rm -rf $(SUBDIR) && \
[ \! -d $(SUBDIR) ] && \ [ \! -d $(SUBDIR) ] && \
darcs get -t $(VERSION) $(URL) $(SUBDIR) && \ darcs get -t $(VERSION) $(URL) $(SUBDIR) && \
export TAR_TIMESTAMP=`cd $(SUBDIR) && LC_ALL=C darcs log --last 1 | sed -ne 's!^Date: \+!!p'` && \
find $(SUBDIR) -name _darcs | xargs rm -rf && \ find $(SUBDIR) -name _darcs | xargs rm -rf && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
$(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \ $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \