|
@@ -0,0 +1,642 @@
|
|
|
|
+# HG changeset patch
|
|
|
|
+# User Simon Fraser <sfraser@mozilla.com>
|
|
|
|
+# Date 1591914190 0
|
|
|
|
+# Node ID 325d0a84cf461865d509b5aa0dd9a03e0dfa0fd7
|
|
|
|
+# Parent fbc2bd478088d38532afadc47b6ee910b20f04e7
|
|
|
|
+Bug 1643625 Clear up references to MAR_OLD_FORMAT in packaging tools r=nthomas
|
|
|
|
+
|
|
|
|
+I think we don't need these checks any more.
|
|
|
|
+
|
|
|
|
+Differential Revision: https://phabricator.services.mozilla.com/D78480
|
|
|
|
+
|
|
|
|
+diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
|
|
|
|
+--- a/python/mozbuild/mozbuild/mach_commands.py
|
|
|
|
++++ b/python/mozbuild/mozbuild/mach_commands.py
|
|
|
|
+@@ -1159,31 +1159,27 @@ class Repackage(MachCommandBase):
|
|
|
|
+ @SubCommand('repackage', 'mar',
|
|
|
|
+ description='Repackage into complete MAR file')
|
|
|
|
+ @CommandArgument('--input', '-i', type=str, required=True,
|
|
|
|
+ help='Input filename')
|
|
|
|
+ @CommandArgument('--mar', type=str, required=True,
|
|
|
|
+ help='Mar binary path')
|
|
|
|
+ @CommandArgument('--output', '-o', type=str, required=True,
|
|
|
|
+ help='Output filename')
|
|
|
|
+- @CommandArgument('--format', type=str, default='lzma',
|
|
|
|
+- choices=('lzma', 'bz2'),
|
|
|
|
+- help='Mar format')
|
|
|
|
+ @CommandArgument('--arch', type=str, required=True,
|
|
|
|
+ help='The archtecture you are building.')
|
|
|
|
+ @CommandArgument('--mar-channel-id', type=str,
|
|
|
|
+ help='Mar channel id')
|
|
|
|
+- def repackage_mar(self, input, mar, output, format, arch, mar_channel_id):
|
|
|
|
++ def repackage_mar(self, input, mar, output, arch, mar_channel_id):
|
|
|
|
+ from mozbuild.repackaging.mar import repackage_mar
|
|
|
|
+ repackage_mar(
|
|
|
|
+ self.topsrcdir,
|
|
|
|
+ input,
|
|
|
|
+ mar,
|
|
|
|
+ output,
|
|
|
|
+- format,
|
|
|
|
+ arch=arch,
|
|
|
|
+ mar_channel_id=mar_channel_id,
|
|
|
|
+ )
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ @CommandProvider
|
|
|
|
+ class CreateMachEnvironment(MachCommandBase):
|
|
|
|
+ """Create the mach virtualenvs."""
|
|
|
|
+diff --git a/python/mozbuild/mozbuild/repackaging/mar.py b/python/mozbuild/mozbuild/repackaging/mar.py
|
|
|
|
+--- a/python/mozbuild/mozbuild/repackaging/mar.py
|
|
|
|
++++ b/python/mozbuild/mozbuild/repackaging/mar.py
|
|
|
|
+@@ -22,17 +22,17 @@ from mozbuild.util import (
|
|
|
|
+ _BCJ_OPTIONS = {
|
|
|
|
+ 'x86': ['--x86'],
|
|
|
|
+ 'x86_64': ['--x86'],
|
|
|
|
+ 'aarch64': [],
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ def repackage_mar(
|
|
|
|
+- topsrcdir, package, mar, output, mar_format="lzma", arch=None, mar_channel_id=None
|
|
|
|
++ topsrcdir, package, mar, output, arch=None, mar_channel_id=None
|
|
|
|
+ ):
|
|
|
|
+ if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
|
|
|
|
+ raise Exception("Package file %s is not a valid .zip or .tar file." % package)
|
|
|
|
+ if arch and arch not in _BCJ_OPTIONS:
|
|
|
|
+ raise Exception("Unknown architecture {}, available architectures: {}".format(
|
|
|
|
+ arch, list(_BCJ_OPTIONS.keys())))
|
|
|
|
+
|
|
|
|
+ ensureParentDir(output)
|
|
|
|
+@@ -61,18 +61,16 @@ def repackage_mar(
|
|
|
|
+
|
|
|
|
+ make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')
|
|
|
|
+
|
|
|
|
+ env = os.environ.copy()
|
|
|
|
+ env['MOZ_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version')
|
|
|
|
+ env['MAR'] = mozpath.normpath(mar)
|
|
|
|
+ if arch:
|
|
|
|
+ env['BCJ_OPTIONS'] = ' '.join(_BCJ_OPTIONS[arch])
|
|
|
|
+- if mar_format == 'bz2':
|
|
|
|
+- env['MAR_OLD_FORMAT'] = '1'
|
|
|
|
+ if mar_channel_id:
|
|
|
|
+ env['MAR_CHANNEL_ID'] = mar_channel_id
|
|
|
|
+ # The Windows build systems have xz installed but it isn't in the path
|
|
|
|
+ # like it is on Linux and Mac OS X so just use the XZ env var so the mar
|
|
|
|
+ # generation scripts can find it.
|
|
|
|
+ xz_path = mozpath.join(topsrcdir, 'xz/xz.exe')
|
|
|
|
+ if os.path.exists(xz_path):
|
|
|
|
+ env['XZ'] = mozpath.normpath(xz_path)
|
|
|
|
+diff --git a/tools/update-packaging/change_mar_compression.pl b/tools/update-packaging/change_mar_compression.pl
|
|
|
|
+deleted file mode 100644
|
|
|
|
+--- a/tools/update-packaging/change_mar_compression.pl
|
|
|
|
++++ /dev/null
|
|
|
|
+@@ -1,208 +0,0 @@
|
|
|
|
+-#!/usr/bin/perl -w
|
|
|
|
+-# This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
+-# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
+-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
+-
|
|
|
|
+-#
|
|
|
|
+-# This tool extracts a mar file, changes the compression of the files contained
|
|
|
|
+-# by the mar file either from bzip2 to lzma or lzma to bzip2, and then recreates
|
|
|
|
+-# the mar file. The script determines whether the files are compressed with
|
|
|
|
+-# bzip2 or lzma to determine what compression should be used. The permissions
|
|
|
|
+-# of the files will be retained as long as the script is run on a system such
|
|
|
|
+-# as Linux or Mac OS X but not on Windows. The mar file will be created with the
|
|
|
|
+-# same MAR channel and Product version information as the original mar file.
|
|
|
|
+-# If a mar signature is required the converted mar files will need to be
|
|
|
|
+-# re-signed afterwards.
|
|
|
|
+-# Author: Robert Strong
|
|
|
|
+-#
|
|
|
|
+-
|
|
|
|
+-# -----------------------------------------------------------------------------
|
|
|
|
+-# By default just assume that these tools exist in our path
|
|
|
|
+-
|
|
|
|
+-use Getopt::Std;
|
|
|
|
+-use Cwd 'abs_path';
|
|
|
|
+-use File::Basename;
|
|
|
|
+-
|
|
|
|
+-$|++;
|
|
|
|
+-
|
|
|
|
+-my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $FILES, $CHANNEL, $VERSION, $REPLACE, $archive, $tmparchive, @marentries, @marfiles);
|
|
|
|
+-
|
|
|
|
+-if (defined($ENV{"MAR"})) {
|
|
|
|
+- $MAR = $ENV{"MAR"};
|
|
|
|
+-}
|
|
|
|
+-else {
|
|
|
|
+- $MAR = "mar";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-if (defined($ENV{"BZIP2"})) {
|
|
|
|
+- $BZIP2 = $ENV{"BZIP2"};
|
|
|
|
+-}
|
|
|
|
+-else {
|
|
|
|
+- $BZIP2 = "bzip2";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-if (defined($ENV{"XZ"})) {
|
|
|
|
+- $XZ = $ENV{"XZ"};
|
|
|
|
+-}
|
|
|
|
+-else {
|
|
|
|
+- $XZ = "xz";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-sub print_usage
|
|
|
|
+-{
|
|
|
|
+- print "Usage: change_mar_compression.pl [OPTIONS] ARCHIVE\n";
|
|
|
|
+- print "\n";
|
|
|
|
+- print "The ARCHIVE will be recreated using either bzip2 for mar file contents that\n";
|
|
|
|
+- print "are compressed with lzma or lzma for mar file contents that are compressed\n";
|
|
|
|
+- print "with bzip2. The new mar file that is created will not be signed but all other\n";
|
|
|
|
+- print "attributes that should be retained will be retained.\n";
|
|
|
|
+- print "\n";
|
|
|
|
+- print "Options:\n";
|
|
|
|
+- print " -h show this help text\n";
|
|
|
|
+- print " -r replace the original mar file with the new mar file\n";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-my %opts;
|
|
|
|
+-getopts("hr", \%opts);
|
|
|
|
+-
|
|
|
|
+-if (defined($opts{'h'}) || scalar(@ARGV) != 1) {
|
|
|
|
+- print_usage();
|
|
|
|
+- exit 1;
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-if ($opts{'r'}) {
|
|
|
|
+- $REPLACE = 1;
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-$archive = $ARGV[0];
|
|
|
|
+-@marentries = `"$MAR" -T "$archive"`;
|
|
|
|
+-$? && die("Couldn't run \"$MAR\" -t");
|
|
|
|
+-
|
|
|
|
+-system("$MAR -x \"$archive\"") == 0 ||
|
|
|
|
+- die "Couldn't run $MAR -x";
|
|
|
|
+-
|
|
|
|
+-open(my $testfilename, "updatev3.manifest") or die $!;
|
|
|
|
+-binmode($testfilename);
|
|
|
|
+-read($testfilename, my $bytes, 3);
|
|
|
|
+-if ($bytes eq "BZh") {
|
|
|
|
+- $MAR_OLD_TO_NEW = 1;
|
|
|
|
+- print "Converting mar file from bzip2 to lzma compression\n";
|
|
|
|
+-} else {
|
|
|
|
+- undef $MAR_OLD_TO_NEW;
|
|
|
|
+- print "Converting mar file from lzma to bzip2 compression\n";
|
|
|
|
+-}
|
|
|
|
+-close $testfilename;
|
|
|
|
+-
|
|
|
|
+-print "\n";
|
|
|
|
+-
|
|
|
|
+-# The channel is the 4th line of the output
|
|
|
|
+-shift @marentries;
|
|
|
|
+-shift @marentries;
|
|
|
|
+-shift @marentries;
|
|
|
|
+-$CHANNEL = substr($marentries[0], 24, -1);
|
|
|
|
+-print "MAR channel name: " . $CHANNEL . "\n";
|
|
|
|
+-
|
|
|
|
+-# The version is the 5th line of the output
|
|
|
|
+-shift @marentries;
|
|
|
|
+-$VERSION = substr($marentries[0], 23, -1);
|
|
|
|
+-print "Product version: " . $VERSION . "\n";
|
|
|
|
+-
|
|
|
|
+-# The file entries start on the 8th line of the output
|
|
|
|
+-shift @marentries;
|
|
|
|
+-shift @marentries;
|
|
|
|
+-shift @marentries;
|
|
|
|
+-
|
|
|
|
+-print "\n";
|
|
|
|
+-# Decompress the extracted files
|
|
|
|
+-foreach (@marentries) {
|
|
|
|
+- tr/\n\r//d;
|
|
|
|
+- my @splits = split(/\t/,$_);
|
|
|
|
+- my $file = $splits[2];
|
|
|
|
+-
|
|
|
|
+- print "Decompressing: " . $file . "\n";
|
|
|
|
+- if ($MAR_OLD_TO_NEW) {
|
|
|
|
+- system("mv \"$file\" \"$file.bz2\"") == 0 ||
|
|
|
|
+- print "\n" && die "Couldn't mv \"$file\"";
|
|
|
|
+- system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
|
|
|
|
+- print "\n" && die "Couldn't decompress \"$file\"";
|
|
|
|
+- }
|
|
|
|
+- else {
|
|
|
|
+- system("mv \"$file\" \"$file.xz\"") == 0 ||
|
|
|
|
+- print "\n" && die "Couldn't mv \"$file\"";
|
|
|
|
+- system("\"$XZ\" -d \"$file.xz\"") == 0 ||
|
|
|
|
+- print "\n" && die "Couldn't decompress \"$file\"";
|
|
|
|
+- }
|
|
|
|
+-}
|
|
|
|
+-print "All files decompressed\n";
|
|
|
|
+-
|
|
|
|
+-print "\n";
|
|
|
|
+-# Compress the files in the requested format
|
|
|
|
+-$FILES = "";
|
|
|
|
+-foreach (@marentries) {
|
|
|
|
+- tr/\n\r//d;
|
|
|
|
+- my @splits = split(/\t/,$_);
|
|
|
|
+- my $mod = $splits[1];
|
|
|
|
+- my $file = $splits[2];
|
|
|
|
+-
|
|
|
|
+- print "Compressing: " . $file . "\n";
|
|
|
|
+- if ($MAR_OLD_TO_NEW) {
|
|
|
|
+- system("\"$XZ\" --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout \"$file\" > \"$file.xz\"") == 0 ||
|
|
|
|
+- die "Couldn't compress \"$file\"";
|
|
|
|
+- system("mv \"$file.xz\" \"$file\"") == 0 ||
|
|
|
|
+- die "Couldn't mv \"$file.xz\"";
|
|
|
|
+- }
|
|
|
|
+- else {
|
|
|
|
+- system("\"$BZIP2\" -z9 \"$file\"") == 0 ||
|
|
|
|
+- die "Couldn't compress \"$file\"";
|
|
|
|
+- system("mv \"$file.bz2\" \"$file\"") == 0 ||
|
|
|
|
+- die "Couldn't mv \"$file.bz2\"";
|
|
|
|
+- }
|
|
|
|
+- $FILES = $FILES . "\"$file\" ";
|
|
|
|
+- chmod oct($mod), $file;
|
|
|
|
+-}
|
|
|
|
+-print "All files compressed\n";
|
|
|
|
+-
|
|
|
|
+-my $filesuffix = ".bz";
|
|
|
|
+-if ($MAR_OLD_TO_NEW) {
|
|
|
|
+- $filesuffix = ".xz";
|
|
|
|
+-}
|
|
|
|
+-$tmparchive = $archive . $filesuffix;
|
|
|
|
+-
|
|
|
|
+-system("$MAR -H $CHANNEL -V $VERSION -c \"$tmparchive\" $FILES") == 0 ||
|
|
|
|
+- die "Couldn't run $MAR -c";
|
|
|
|
+-
|
|
|
|
+-if ($REPLACE) {
|
|
|
|
+- print "\n";
|
|
|
|
+- print "Replacing mar file with the converted mar file\n";
|
|
|
|
+- unlink $archive;
|
|
|
|
+- system("mv \"$tmparchive\" \"$archive\"") == 0 ||
|
|
|
|
+- die "Couldn't mv \"$tmparchive\"";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-print "\n";
|
|
|
|
+-print "Removing extracted files\n";
|
|
|
|
+-foreach (@marentries) {
|
|
|
|
+- tr/\n\r//d;
|
|
|
|
+- my @splits = split(/\t/,$_);
|
|
|
|
+- my $file = $splits[2];
|
|
|
|
+-
|
|
|
|
+- unlink $file;
|
|
|
|
+- my $dirpath = $file;
|
|
|
|
+- while (1) {
|
|
|
|
+- if (index($dirpath, '/') < 0) {
|
|
|
|
+- last;
|
|
|
|
+- }
|
|
|
|
+- $dirpath = substr($dirpath, 0, rindex($dirpath, '/'));
|
|
|
|
+- rmdir($dirpath);
|
|
|
|
+- if (-d $dirpath) {
|
|
|
|
+- last;
|
|
|
|
+- }
|
|
|
|
+- }
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+-print "\n";
|
|
|
|
+-if ($MAR_OLD_TO_NEW) {
|
|
|
|
+- print "Finished converting mar file from bzip2 to lzma compression\n";
|
|
|
|
+-} else {
|
|
|
|
+- print "Finished converting mar file from lzma to bzip2 compression\n";
|
|
|
|
+-}
|
|
|
|
+diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh
|
|
|
|
+--- a/tools/update-packaging/common.sh
|
|
|
|
++++ b/tools/update-packaging/common.sh
|
|
|
|
+@@ -9,39 +9,34 @@
|
|
|
|
+ #
|
|
|
|
+
|
|
|
|
+ # -----------------------------------------------------------------------------
|
|
|
|
+ QUIET=0
|
|
|
|
+
|
|
|
|
+ # By default just assume that these tools exist on our path
|
|
|
|
+ MAR=${MAR:-mar}
|
|
|
|
+ MBSDIFF=${MBSDIFF:-mbsdiff}
|
|
|
|
+-if [[ -z "${MAR_OLD_FORMAT}" ]]; then
|
|
|
|
+- XZ=${XZ:-xz}
|
|
|
|
++XZ=${XZ:-xz}
|
|
|
|
++$XZ --version > /dev/null 2>&1
|
|
|
|
++if [ $? -ne 0 ]; then
|
|
|
|
++ # If $XZ is not set and not found on the path then this is probably
|
|
|
|
++ # running on a windows buildbot. Some of the Windows build systems have
|
|
|
|
++ # xz.exe in topsrcdir/xz/. Look in the places this would be in both a
|
|
|
|
++ # mozilla-central and comm-central build.
|
|
|
|
++ XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
|
|
|
|
+ $XZ --version > /dev/null 2>&1
|
|
|
|
+ if [ $? -ne 0 ]; then
|
|
|
|
+- # If $XZ is not set and not found on the path then this is probably
|
|
|
|
+- # running on a windows buildbot. Some of the Windows build systems have
|
|
|
|
+- # xz.exe in topsrcdir/xz/. Look in the places this would be in both a
|
|
|
|
+- # mozilla-central and comm-central build.
|
|
|
|
+- XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
|
|
|
|
++ XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
|
|
|
|
+ $XZ --version > /dev/null 2>&1
|
|
|
|
+ if [ $? -ne 0 ]; then
|
|
|
|
+- XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
|
|
|
|
+- $XZ --version > /dev/null 2>&1
|
|
|
|
+- if [ $? -ne 0 ]; then
|
|
|
|
+- echo "xz was not found on this system!"
|
|
|
|
+- echo "exiting"
|
|
|
|
+- exit 1
|
|
|
|
+- fi
|
|
|
|
++ echo "xz was not found on this system!"
|
|
|
|
++ echo "exiting"
|
|
|
|
++ exit 1
|
|
|
|
+ fi
|
|
|
|
+ fi
|
|
|
|
+-else
|
|
|
|
+- MAR_OLD_FORMAT=1
|
|
|
|
+- BZIP2=${BZIP2:-bzip2}
|
|
|
|
+ fi
|
|
|
|
+
|
|
|
|
+ # -----------------------------------------------------------------------------
|
|
|
|
+ # Helper routines
|
|
|
|
+
|
|
|
|
+ notice() {
|
|
|
|
+ echo "$*" 1>&2
|
|
|
|
+ }
|
|
|
|
+diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh
|
|
|
|
+--- a/tools/update-packaging/make_full_update.sh
|
|
|
|
++++ b/tools/update-packaging/make_full_update.sh
|
|
|
|
+@@ -96,38 +96,29 @@ for ((i=0; $i<$num_files; i=$i+1)); do
|
|
|
|
+ make_add_instruction "$f" "$updatemanifestv2" "" 1
|
|
|
|
+ fi
|
|
|
|
+ else
|
|
|
|
+ make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
|
|
|
+ fi
|
|
|
|
+
|
|
|
|
+ dir=$(dirname "$f")
|
|
|
|
+ mkdir -p "$workdir/$dir"
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
|
|
|
|
+ copy_perm "$targetdir/$f" "$workdir/$f"
|
|
|
|
+
|
|
|
|
+ targetfiles="$targetfiles \"$f\""
|
|
|
|
+ done
|
|
|
|
+
|
|
|
|
+ # Append remove instructions for any dead files.
|
|
|
|
+ notice ""
|
|
|
|
+ notice "Adding file and directory remove instructions from file 'removed-files'"
|
|
|
|
+ append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
|
|
|
|
+
|
|
|
|
+-if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
|
|
|
+- $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
|
|
|
|
+-else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
|
|
|
+-fi
|
|
|
|
++$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
|
|
|
|
++$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
|
|
|
+
|
|
|
|
+ mar_command="$mar_command -C \"$workdir\" -c output.mar"
|
|
|
|
+ eval "$mar_command $targetfiles"
|
|
|
|
+ mv -f "$workdir/output.mar" "$archive"
|
|
|
|
+
|
|
|
|
+ # cleanup
|
|
|
|
+ rm -fr "$workdir"
|
|
|
|
+
|
|
|
|
+diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh
|
|
|
|
+--- a/tools/update-packaging/make_incremental_update.sh
|
|
|
|
++++ b/tools/update-packaging/make_incremental_update.sh
|
|
|
|
+@@ -169,35 +169,27 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
|
|
|
|
+ f="${oldfiles[$i]}"
|
|
|
|
+
|
|
|
|
+ # If this file exists in the new directory as well, then check if it differs.
|
|
|
|
+ if [ -f "$newdir/$f" ]; then
|
|
|
|
+
|
|
|
|
+ if check_for_add_if_not_update "$f"; then
|
|
|
|
+ # The full workdir may not exist yet, so create it if necessary.
|
|
|
|
+ mkdir -p `dirname "$workdir/$f"`
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+ copy_perm "$newdir/$f" "$workdir/$f"
|
|
|
|
+ make_add_if_not_instruction "$f" "$updatemanifestv3"
|
|
|
|
+ archivefiles="$archivefiles \"$f\""
|
|
|
|
+ continue 1
|
|
|
|
+ fi
|
|
|
|
+
|
|
|
|
+ if check_for_forced_update "$requested_forced_updates" "$f"; then
|
|
|
|
+ # The full workdir may not exist yet, so create it if necessary.
|
|
|
|
+ mkdir -p `dirname "$workdir/$f"`
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+ copy_perm "$newdir/$f" "$workdir/$f"
|
|
|
|
+ make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
|
|
|
|
+ archivefiles="$archivefiles \"$f\""
|
|
|
|
+ continue 1
|
|
|
|
+ fi
|
|
|
|
+
|
|
|
|
+ if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
|
|
|
|
+ # Compute both the compressed binary diff and the compressed file, and
|
|
|
|
+@@ -215,54 +207,31 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
|
|
|
|
+ # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
|
|
|
|
+ # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
|
|
|
|
+ #
|
|
|
|
+ # Note: patches are bzipped or xz stashed in funsize to gain more speed
|
|
|
|
+
|
|
|
|
+ # if service is not enabled then default to old behavior
|
|
|
|
+ if [ -z "$MBSDIFF_HOOK" ]; then
|
|
|
|
+ $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -z9 "$workdir/$f.patch"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
|
|
|
+ else
|
|
|
|
+ # if service enabled then check patch existence for retrieval
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
|
|
|
|
+- verbose_notice "file \"$f\" found in funsize, diffing skipped"
|
|
|
|
+- else
|
|
|
|
+- # if not found already - compute it and cache it for future use
|
|
|
|
+- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
|
|
|
+- $BZIP2 -z9 "$workdir/$f.patch"
|
|
|
|
+- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
|
|
|
|
+- fi
|
|
|
|
++ if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
|
|
|
|
++ verbose_notice "file \"$f\" found in funsize, diffing skipped"
|
|
|
|
+ else
|
|
|
|
+- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
|
|
|
|
+- verbose_notice "file \"$f\" found in funsize, diffing skipped"
|
|
|
|
+- else
|
|
|
|
+- # if not found already - compute it and cache it for future use
|
|
|
|
+- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
|
|
|
+- $XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
|
|
|
+- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
|
|
|
|
+- fi
|
|
|
|
++ # if not found already - compute it and cache it for future use
|
|
|
|
++ $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
|
|
|
++ $XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
|
|
|
++ $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
|
|
|
|
+ fi
|
|
|
|
+ fi
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+ copy_perm "$newdir/$f" "$workdir/$f"
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- patchfile="$workdir/$f.patch.bz2"
|
|
|
|
+- else
|
|
|
|
+- patchfile="$workdir/$f.patch.xz"
|
|
|
|
+- fi
|
|
|
|
++ patchfile="$workdir/$f.patch.xz"
|
|
|
|
+ patchsize=$(get_file_size "$patchfile")
|
|
|
|
+ fullsize=$(get_file_size "$workdir/$f")
|
|
|
|
+
|
|
|
|
+ if [ $patchsize -lt $fullsize ]; then
|
|
|
|
+ make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
|
|
|
+ mv -f "$patchfile" "$workdir/$f.patch"
|
|
|
|
+ rm -f "$workdir/$f"
|
|
|
|
+ archivefiles="$archivefiles \"$f.patch\""
|
|
|
|
+@@ -293,21 +262,17 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
|
|
|
|
+ if [ "$f" = "${oldfiles[j]}" ]; then
|
|
|
|
+ continue 2
|
|
|
|
+ fi
|
|
|
|
+ done
|
|
|
|
+
|
|
|
|
+ dir=$(dirname "$workdir/$f")
|
|
|
|
+ mkdir -p "$dir"
|
|
|
|
+
|
|
|
|
+- if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+- fi
|
|
|
|
++ $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
|
|
|
+ copy_perm "$newdir/$f" "$workdir/$f"
|
|
|
|
+
|
|
|
|
+ if check_for_add_if_not_update "$f"; then
|
|
|
|
+ make_add_if_not_instruction "$f" "$updatemanifestv3"
|
|
|
|
+ else
|
|
|
|
+ make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
|
|
|
+ fi
|
|
|
|
+
|
|
|
|
+@@ -338,23 +303,18 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
|
|
|
|
+ # If this dir doesn't exist in the new directory remove it.
|
|
|
|
+ if [ ! -d "$newdir/$f" ]; then
|
|
|
|
+ verbose_notice " rmdir $f/"
|
|
|
|
+ echo "rmdir \"$f/\"" >> $updatemanifestv2
|
|
|
|
+ echo "rmdir \"$f/\"" >> $updatemanifestv3
|
|
|
|
+ fi
|
|
|
|
+ done
|
|
|
|
+
|
|
|
|
+-if [[ -n $MAR_OLD_FORMAT ]]; then
|
|
|
|
+- $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
|
|
|
+- $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
|
|
|
|
+-else
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
|
|
|
|
+- $XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
|
|
|
+-fi
|
|
|
|
++$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
|
|
|
|
++$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
|
|
|
+
|
|
|
|
+ mar_command="$mar_command -C \"$workdir\" -c output.mar"
|
|
|
|
+ eval "$mar_command $archivefiles"
|
|
|
|
+ mv -f "$workdir/output.mar" "$archive"
|
|
|
|
+
|
|
|
|
+ # cleanup
|
|
|
|
+ rm -fr "$workdir"
|
|
|
|
+
|
|
|
|
+diff --git a/tools/update-packaging/unwrap_full_update.pl b/tools/update-packaging/unwrap_full_update.pl
|
|
|
|
+--- a/tools/update-packaging/unwrap_full_update.pl
|
|
|
|
++++ b/tools/update-packaging/unwrap_full_update.pl
|
|
|
|
+@@ -8,32 +8,25 @@
|
|
|
|
+ # Author: Benjamin Smedberg
|
|
|
|
+ #
|
|
|
|
+
|
|
|
|
+ # -----------------------------------------------------------------------------
|
|
|
|
+ # By default just assume that these tools exist on our path
|
|
|
|
+
|
|
|
|
+ use Getopt::Std;
|
|
|
|
+
|
|
|
|
+-my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles);
|
|
|
|
++my ($MAR, $XZ, $archive, @marentries, @marfiles);
|
|
|
|
+
|
|
|
|
+ if (defined($ENV{"MAR"})) {
|
|
|
|
+ $MAR = $ENV{"MAR"};
|
|
|
|
+ }
|
|
|
|
+ else {
|
|
|
|
+ $MAR = "mar";
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+-if (defined($ENV{"BZIP2"})) {
|
|
|
|
+- $BZIP2 = $ENV{"BZIP2"};
|
|
|
|
+-}
|
|
|
|
+-else {
|
|
|
|
+- $BZIP2 = "bzip2";
|
|
|
|
+-}
|
|
|
|
+-
|
|
|
|
+ if (defined($ENV{"XZ"})) {
|
|
|
|
+ $XZ = $ENV{"XZ"};
|
|
|
|
+ }
|
|
|
|
+ else {
|
|
|
|
+ if (system("xz --version > /dev/null 2>&1") != 0) {
|
|
|
|
+ # Some of the Windows build systems have xz.exe in topsrcdir/xz/.
|
|
|
|
+ my $xzwinpath = __FILE__;
|
|
|
|
+ $xzwinpath = substr($xzwinpath, 0, rindex($xzwinpath, '/'));
|
|
|
|
+@@ -79,45 +72,24 @@ if (defined($opts{'h'}) || scalar(@ARGV)
|
|
|
|
+
|
|
|
|
+ $archive = $ARGV[0];
|
|
|
|
+ @marentries = `"$MAR" -t "$archive"`;
|
|
|
|
+ $? && die("Couldn't run \"$MAR\" -t");
|
|
|
|
+
|
|
|
|
+ system($MAR, "-x", $archive) == 0 ||
|
|
|
|
+ die "Couldn't run $MAR -x";
|
|
|
|
+
|
|
|
|
+-# Try to determine if the mar file contains bzip2 compressed files and if not
|
|
|
|
+-# assume that the mar file contains lzma compressed files. The updatev3.manifest
|
|
|
|
+-# file is checked since a valid mar file must have this file in the root path.
|
|
|
|
+-open(my $testfilename, "updatev3.manifest") or die $!;
|
|
|
|
+-binmode($testfilename);
|
|
|
|
+-read($testfilename, my $bytes, 3);
|
|
|
|
+-if ($bytes eq "BZh") {
|
|
|
|
+- $MAR_OLD_FORMAT = 1;
|
|
|
|
+-} else {
|
|
|
|
+- undef $MAR_OLD_FORMAT;
|
|
|
|
+-}
|
|
|
|
+-close $testfilename;
|
|
|
|
+-
|
|
|
|
+ shift @marentries;
|
|
|
|
+
|
|
|
|
+ foreach (@marentries) {
|
|
|
|
+ tr/\n\r//d;
|
|
|
|
+ my @splits = split(/\t/,$_);
|
|
|
|
+ my $file = $splits[2];
|
|
|
|
+
|
|
|
|
+ print "Decompressing: " . $file . "\n";
|
|
|
|
+- if ($MAR_OLD_FORMAT) {
|
|
|
|
+- system("mv", $file, "$file.bz2") == 0 ||
|
|
|
|
+- die "Couldn't mv \"$file\"";
|
|
|
|
+- system($BZIP2, "-d", "$file.bz2") == 0 ||
|
|
|
|
+- die "Couldn't decompress \"$file\"";
|
|
|
|
+- }
|
|
|
|
+- else {
|
|
|
|
+- system("mv", $file, "$file.xz") == 0 ||
|
|
|
|
+- die "Couldn't mv \"$file\"";
|
|
|
|
+- system($XZ, "-d", "$file.xz") == 0 ||
|
|
|
|
+- die "Couldn't decompress \"$file\"";
|
|
|
|
+- }
|
|
|
|
++ system("mv", $file, "$file.xz") == 0 ||
|
|
|
|
++ die "Couldn't mv \"$file\"";
|
|
|
|
++ system($XZ, "-d", "$file.xz") == 0 ||
|
|
|
|
++ die "Couldn't decompress \"$file\"";
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ print "Finished\n";
|
|
|
|
+
|