more mach backports

mozilla-release/patches/1377445-2no3-89a1.patch

@@ -2,7 +2,7 @@
 # User Mike Hommey <mh+mozilla@glandium.org>
 # Date 1618883823 0
 # Node ID 0ce69d9f596b3910876f4838c5b04250a553b9c4
-# Parent  696d754ce884ee7ba8e0babefba383f4f93ec93c
+# Parent  8a5f94f0c1a1858913ad485c208862dc331b4750
 Bug 1377445 - Remove build dependencies on gtk+2. r=firefox-build-system-reviewers,mhentges
 
 Differential Revision: https://phabricator.services.mozilla.com/D111997
@@ -119,7 +119,7 @@ diff --git a/old-configure.in b/old-configure.in
  dnl Set various checks
  dnl ========================================================
  MISSING_X=
-@@ -1788,21 +1787,16 @@ if test "$COMPILE_ENVIRONMENT"; then
+@@ -1755,21 +1754,16 @@ if test "$COMPILE_ENVIRONMENT"; then
      dnl GDK_VERSION_MIN_REQUIRED is not set here as GDK3 deprecated warnings
      dnl are suppressed by widget/gtk/compat-gtk3/gdk/gdkversionmacros.h.
      AC_DEFINE_UNQUOTED(GDK_VERSION_MAX_ALLOWED,$GDK_VERSION_MAX_ALLOWED)
@@ -144,8 +144,8 @@ diff --git a/old-configure.in b/old-configure.in
 diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
 --- a/python/mozboot/mozboot/archlinux.py
 +++ b/python/mozboot/mozboot/archlinux.py
-@@ -40,17 +40,16 @@ class ArchlinuxBootstrapper(
-         'python',  # This is Python 3 on Arch.
+@@ -35,17 +35,16 @@ class ArchlinuxBootstrapper(
+         'python-pip',
          'unzip',
          'zip',
      ]
@@ -165,7 +165,7 @@ diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archli
 diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py
 --- a/python/mozboot/mozboot/centosfedora.py
 +++ b/python/mozboot/mozboot/centosfedora.py
-@@ -39,18 +39,16 @@ class CentOSFedoraBootstrapper(NasmInsta
+@@ -32,18 +32,16 @@ class CentOSFedoraBootstrapper(
          self.browser_group_packages = [
              'GNOME Software Development',
          ]
@@ -187,7 +187,7 @@ diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/cen
 diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
 --- a/python/mozboot/mozboot/debian.py
 +++ b/python/mozboot/mozboot/debian.py
-@@ -60,17 +60,16 @@ class DebianBootstrapper(NasmInstall, No
+@@ -54,17 +54,16 @@ class DebianBootstrapper(
      # These are common packages for building Firefox for Desktop
      # (browser) for all Debian-derived distros (such as Ubuntu).
      BROWSER_COMMON_PACKAGES = [

mozilla-release/patches/1390693-8-57a1.patch

@@ -0,0 +1,249 @@
+# HG changeset patch
+# User Gregory Szorc <gps@mozilla.com>
+# Date 1503598341 25200
+# Node ID c0b6cbecee95bae5bfc409fc5e48efd0adbdbdde
+# Parent  447b597581ae9bb3d7d3203b2c32bf8fc182e5b3
+Bug 1390693 - Upload docs to project and version specific locations; r=dustin
+
+Previously, we uploaded the main Firefox tree docs to /.
+
+In reality, there are multiple Sphinx projects in the repo. In
+addition, it is sometimes desirable to access docs for an older
+version of Firefox.
+
+In this commit, we add support for specifying the S3 key prefix
+for uploads. Then we change the upload code to upload to multiple
+locations:
+
+* <project>/latest (always)
+* <project>/<version> (if a version is defined in the Sphinx config)
+* / (for the main Sphinx docs project)
+
+For the Firefox docs, ``version`` corresponds to a sanitized value from
+``milestone.txt``. Currently, it resolves to ``57.0``.
+
+While we're here, we add support for declaring an alternate project
+name in the Sphinx conf.py file. If ``moz_project_name`` is defined,
+we use that as the project name. For Firefox, we set it to ``main``.
+This means our paths (local and uploaded) are now ``main`` instead of
+``Mozilla_Source_Tree_Docs``. That's much more pleasant.
+
+MozReview-Commit-ID: 8Gl6l2m6uU4
+
+diff --git a/tools/docs/conf.py b/tools/docs/conf.py
+--- a/tools/docs/conf.py
++++ b/tools/docs/conf.py
+@@ -89,8 +89,10 @@ else:
+     # environment handles this otherwise.
+     import sphinx_rtd_theme
+     html_theme = 'sphinx_rtd_theme'
+     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+ 
+ 
+ html_static_path = ['_static']
+ htmlhelp_basename = 'MozillaTreeDocs'
++
++moz_project_name = 'main'
+diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
+--- a/tools/docs/mach_commands.py
++++ b/tools/docs/mach_commands.py
+@@ -68,35 +68,34 @@ class Documentation(MachCommandBase):
+         for path in what:
+             path = os.path.normpath(os.path.abspath(path))
+             docdir = self._find_doc_dir(path)
+ 
+             if not docdir:
+                 failed.append((path, 'could not find docs at this location'))
+                 continue
+ 
+-            # find project name to use as a namespace within `outdir`
+-            project = self._find_project_name(docdir)
+-            savedir = os.path.join(format_outdir, project)
++            props = self._project_properties(docdir)
++            savedir = os.path.join(format_outdir, props['project'])
+ 
+             args = [
+                 'sphinx',
+                 '-b', format,
+                 docdir,
+                 savedir,
+             ]
+             result = sphinx.build_main(args)
+             if result != 0:
+                 failed.append((path, 'sphinx return code %d' % result))
+             else:
+                 generated.append(savedir)
+ 
+             if archive:
+                 archive_path = os.path.join(outdir,
+-                                            '%s.tar.gz' %  project)
++                                            '%s.tar.gz' % props['project'])
+                 moztreedocs.create_tarball(archive_path, savedir)
+                 print('Archived to %s' % archive_path)
+ 
+             index_path = os.path.join(savedir, 'index.html')
+             if not http and auto_open and os.path.isfile(index_path):
+                 webbrowser.open(index_path)
+ 
+         if generated:
+@@ -112,24 +111,33 @@ class Documentation(MachCommandBase):
+             if len(addr) != 2:
+                 return die('invalid address: %s' % http)
+ 
+             httpd = mozhttpd.MozHttpd(host=addr[0], port=addr[1],
+                                       docroot=format_outdir)
+             print('listening on %s:%d' % addr)
+             httpd.start(block=True)
+ 
+-    def _find_project_name(self, path):
++    def _project_properties(self, path):
+         import imp
+         path = os.path.join(path, 'conf.py')
+         with open(path, 'r') as fh:
+             conf = imp.load_module('doc_conf', fh, path,
+                                    ('.py', 'r', imp.PY_SOURCE))
+ 
+-        return conf.project.replace(' ', '_')
++        # Prefer the Mozilla project name, falling back to Sphinx's
++        # default variable if it isn't defined.
++        project = getattr(conf, 'moz_project_name', None)
++        if not project:
++            project = conf.project.replace(' ', '_')
++
++        return {
++            'project': project,
++            'version': getattr(conf, 'version', None)
++        }
+ 
+     def _find_doc_dir(self, path):
+         search_dirs = ('doc', 'docs')
+         for d in search_dirs:
+             p = os.path.join(path, d)
+             if os.path.isfile(os.path.join(p, 'conf.py')):
+                 return p
+ 
+diff --git a/tools/docs/mach_commands.py.1390693-8.later b/tools/docs/mach_commands.py.1390693-8.later
+new file mode 100644
+--- /dev/null
++++ b/tools/docs/mach_commands.py.1390693-8.later
+@@ -0,0 +1,63 @@
++--- mach_commands.py
+++++ mach_commands.py
++@@ -61,17 +61,17 @@ class Documentation(MachCommandBase):
++ 
++             if archive:
++                 archive_path = os.path.join(outdir,
++                                             '%s.tar.gz' % props['project'])
++                 moztreedocs.create_tarball(archive_path, savedir)
++                 print('Archived to %s' % archive_path)
++ 
++             if upload:
++-                self._s3_upload(savedir)
+++                self._s3_upload(savedir, props['project'], props['version'])
++ 
++             index_path = os.path.join(savedir, 'index.html')
++             if not http and auto_open and os.path.isfile(index_path):
++                 webbrowser.open(index_path)
++ 
++         if generated:
++             print('\nGenerated documentation:\n%s\n' % '\n'.join(generated))
++ 
++@@ -108,21 +108,38 @@ class Documentation(MachCommandBase):
++ 
++     def _find_doc_dir(self, path):
++         search_dirs = ('doc', 'docs')
++         for d in search_dirs:
++             p = os.path.join(path, d)
++             if os.path.isfile(os.path.join(p, 'conf.py')):
++                 return p
++ 
++-    def _s3_upload(self, root):
+++    def _s3_upload(self, root, project, version=None):
++         self.virtualenv_manager.install_pip_package('boto3==1.4.4')
++ 
++         from moztreedocs import distribution_files
++         from moztreedocs.upload import s3_upload
++-        files = distribution_files(root)
++-        s3_upload(files)
+++
+++        # Files are uploaded to multiple locations:
+++        #
+++        # <project>/latest
+++        # <project>/<version>
+++        #
+++        # This allows multiple projects and versions to be stored in the
+++        # S3 bucket.
+++
+++        files = list(distribution_files(root))
+++
+++        s3_upload(files, key_prefix='%s/latest' % project)
+++        if version:
+++            s3_upload(files, key_prefix='%s/%s' % (project, version))
+++
+++        # Until we redirect / to main/latest, upload the main docs
+++        # to the root.
+++        if project == 'main':
+++            s3_upload(files)
++ 
++ 
++ def die(msg, exit_code=1):
++     msg = '%s: %s' % (sys.argv[0], msg)
++     print(msg, file=sys.stderr)
++     return exit_code
+diff --git a/tools/docs/moztreedocs/upload.py.1390693-8.later b/tools/docs/moztreedocs/upload.py.1390693-8.later
+new file mode 100644
+--- /dev/null
++++ b/tools/docs/moztreedocs/upload.py.1390693-8.later
+@@ -0,0 +1,52 @@
++--- upload.py
+++++ upload.py
++@@ -7,17 +7,25 @@ from __future__ import absolute_import, 
++ import io
++ import mimetypes
++ import os
++ 
++ import boto3
++ import requests
++ 
++ 
++-def s3_upload(files):
+++def s3_upload(files, key_prefix=None):
+++    """Upload files to an S3 bucket.
+++
+++    ``files`` is an iterable of ``(path, BaseFile)`` (typically from a
+++    mozpack Finder).
+++
+++    Keys in the bucket correspond to source filenames. If ``key_prefix`` is
+++    defined, key names will be ``<key_prefix>/<path>``.
+++    """
++     region = 'us-west-2'
++     level = os.environ.get('MOZ_SCM_LEVEL', '1')
++     bucket = {
++         '1': 'gecko-docs.mozilla.org-l1',
++         '2': 'gecko-docs.mozilla.org-l2',
++         '3': 'gecko-docs.mozilla.org',
++     }[level]
++     secrets_url = 'http://taskcluster/secrets/v1/secret/'
++@@ -43,13 +51,20 @@ def s3_upload(files):
++ 
++     for path, f in files:
++         content_type, content_encoding = mimetypes.guess_type(path)
++         extra_args = {}
++         if content_type:
++             extra_args['ContentType'] = content_type
++         if content_encoding:
++             extra_args['ContentEncoding'] = content_encoding
++-        print('uploading', path)
+++
+++        if key_prefix:
+++            key = '%s/%s' % (key_prefix, path)
+++        else:
+++            key = path
+++
+++        print('uploading %s to %s' % (path, key))
+++
++         # The file types returned by mozpack behave like file objects. But they
++         # don't accept an argument to read(). So we wrap in a BytesIO.
++-        s3.upload_fileobj(io.BytesIO(f.read()), bucket, path,
+++        s3.upload_fileobj(io.BytesIO(f.read()), bucket, key,
++                           ExtraArgs=extra_args)

mozilla-release/patches/1402154-59a1.patch

@@ -0,0 +1,36 @@
+# HG changeset patch
+# User Tom Prince <mozilla@hocat.ca>
+# Date 1511204212 25200
+# Node ID d6a95c825821c9611f2224914b0e9c04ae0ad482
+# Parent  7ef5b783fbfb9284d6964258e450765699e8bd69
+Bug 1402154: Allow specifying an absolute path to mount sphinx docs. r=gps
+
+MozReview-Commit-ID: 8OLCtwg8zXc
+
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -40,18 +40,21 @@ class SphinxManager(object):
+         config = fakeconfig(self._topsrcdir)
+         reader = BuildReader(config)
+ 
+         for path, name, key, value in reader.find_sphinx_variables():
+             reldir = os.path.dirname(path)
+ 
+             if name == 'SPHINX_TREES':
+                 assert key
+-                self.add_tree(os.path.join(reldir, value),
+-                    os.path.join(reldir, key))
++                if key.startswith('/'):
++                    key = key[1:]
++                else:
++                    key = os.path.join(reldir, key)
++                self.add_tree(os.path.join(reldir, value), key)
+ 
+             if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
+                 self.add_python_package_dir(os.path.join(reldir, value))
+ 
+     def add_tree(self, source_dir, dest_dir):
+         """Add a directory from where docs should be sourced."""
+         if dest_dir in self._trees:
+             raise Exception('%s has already been registered as a destination.'

mozilla-release/patches/1410424-2-61a1.patch

@@ -2,7 +2,7 @@
 # User Andrew Halberstadt <ahalberstadt@mozilla.com>
 # Date 1523025031 14400
 # Node ID 15a5e48d01a51dc1b493074443af4021f4133d91
-# Parent  62f28df89663e8b1f179a18225d97c2413a147c2
+# Parent  92b0248b34a61b51e849ca2b45bbb3fc60cb60f6
 Bug 1410424 - [docs] Remove ability to specify multiple doc paths at the same time r=mshal
 
 This removes the ability to specify multiple doc paths at the same time with
@@ -21,7 +21,7 @@ MozReview-Commit-ID: GXEZJSgLpgF
 diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
 --- a/tools/docs/mach_commands.py
 +++ b/tools/docs/mach_commands.py
-@@ -22,94 +22,82 @@ here = os.path.abspath(os.path.dirname(_
+@@ -22,93 +22,81 @@ here = os.path.abspath(os.path.dirname(_
  
  
  @CommandProvider
@@ -86,12 +86,10 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
 +            return die('failed to generate documentation:\n'
 +                       '%s: could not find docs at this location' % path)
  
--            # find project name to use as a namespace within `outdir`
--            project = self._find_project_name(docdir)
--            savedir = os.path.join(format_outdir, project)
-+        # find project name to use as a namespace within `outdir`
-+        project = self._find_project_name(docdir)
-+        savedir = os.path.join(format_outdir, project)
+-            props = self._project_properties(docdir)
+-            savedir = os.path.join(format_outdir, props['project'])
++        props = self._project_properties(docdir)
++        savedir = os.path.join(format_outdir, props['project'])
  
 -            args = [
 -                'sphinx',
@@ -119,12 +117,12 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
  
 -            if archive:
 -                archive_path = os.path.join(outdir,
--                                            '%s.tar.gz' %  project)
+-                                            '%s.tar.gz' % props['project'])
 -                moztreedocs.create_tarball(archive_path, savedir)
 -                print('Archived to %s' % archive_path)
 +        if archive:
 +            archive_path = os.path.join(outdir,
-+                                        '%s.tar.gz' %  project)
++                                        '%s.tar.gz' % props['project'])
 +            moztreedocs.create_tarball(archive_path, savedir)
 +            print('Archived to %s' % archive_path)
  

mozilla-release/patches/1410424-3-61a1.patch

@@ -2,7 +2,7 @@
 # User Andrew Halberstadt <ahalberstadt@mozilla.com>
 # Date 1523026376 14400
 # Node ID d34cf7a17b3bed597b611413fb73ad7f0f8a5d1d
-# Parent  380b3651554215108becebe3a78d5011b52db074
+# Parent  79386fa49ac657f45b191353a7e65a812ba88c0a
 Bug 1410424 - [docs] Support live reloading with |mach doc| r=mshal
 
 This changes the default to opening a livereload webserver after doc generation
@@ -29,7 +29,7 @@ MozReview-Commit-ID: FQecuePM0zZ
 diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
 --- a/tools/docs/mach_commands.py
 +++ b/tools/docs/mach_commands.py
-@@ -1,114 +1,124 @@
+@@ -1,113 +1,123 @@
  # This Source Code Form is subject to the terms of the Mozilla Public
  # License, v. 2.0. If a copy of the MPL was not distributed with this
  # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
@@ -113,9 +113,8 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
              return die('failed to generate documentation:\n'
                         '%s: could not find docs at this location' % path)
  
-         # find project name to use as a namespace within `outdir`
-         project = self._find_project_name(docdir)
-         savedir = os.path.join(format_outdir, project)
+         props = self._project_properties(docdir)
+         savedir = os.path.join(format_outdir, props['project'])
  
 -        args = [
 -            'sphinx',
@@ -134,7 +133,7 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
  
          if archive:
              archive_path = os.path.join(outdir,
-                                         '%s.tar.gz' %  project)
+                                         '%s.tar.gz' % props['project'])
              moztreedocs.create_tarball(archive_path, savedir)
              print('Archived to %s' % archive_path)
  
@@ -158,16 +157,16 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
 +            port = int(port)
 +        except ValueError:
 +            return die('invalid address: %s' % http)
-+
-+        server = Server()
-+        server.watch(docdir, run_sphinx)
-+        server.serve(host=host, port=port, root=savedir,
-+                     open_url_delay=0.1 if auto_open else None)
  
 -            httpd = mozhttpd.MozHttpd(host=addr[0], port=addr[1],
 -                                      docroot=format_outdir)
 -            print('listening on %s:%d' % addr)
 -            httpd.start(block=True)
++        server = Server()
++        server.watch(docdir, run_sphinx)
++        server.serve(host=host, port=port, root=savedir,
++                     open_url_delay=0.1 if auto_open else None)
++
 +    def _run_sphinx(self, docdir, savedir, fmt='html'):
 +        import sphinx
 +        args = [
@@ -178,7 +177,7 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
 +        ]
 +        return sphinx.build_main(args)
  
-     def _find_project_name(self, path):
+     def _project_properties(self, path):
          import imp
          path = os.path.join(path, 'conf.py')
          with open(path, 'r') as fh:

mozilla-release/patches/1437526-62a1.patch

@@ -3,7 +3,7 @@
 # Date 1527862741 14400
 #      Fri Jun 01 10:19:01 2018 -0400
 # Node ID eb122eb10aad7702541251873ccfc74c5c405d2f
-# Parent  6c4e0432a8ebe62506e8fab1685620b0f964ad0b
+# Parent  d3553ed3132395c98e854bdb983cb77358555fcb
 Bug 1437526 - [docs] Upgrade doc dependencies to their latest versions, r=davehunt
 
 This upgrades sphinx to version 1.7.5, which contained a couple backwards
@@ -241,7 +241,7 @@ new file mode 100644
 diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
 --- a/tools/docs/moztreedocs/__init__.py
 +++ b/tools/docs/moztreedocs/__init__.py
-@@ -73,31 +73,31 @@ class SphinxManager(object):
+@@ -88,31 +88,31 @@ class _SphinxManager(object):
          app.info('Staging static documentation')
          self._synchronize_docs()
          app.info('Generating Python API documentation')
@@ -249,12 +249,12 @@ diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init_
  
      def _generate_python_api_docs(self):
          """Generate Python API doc files."""
-         out_dir = os.path.join(self._docs_dir, 'python')
+         out_dir = os.path.join(self.staging_dir, 'python')
 -        base_args = ['sphinx', '--no-toc', '-o', out_dir]
 +        base_args = ['--no-toc', '-o', out_dir]
  
-         for p in sorted(self._python_package_dirs):
-             full = os.path.join(self._topsrcdir, p)
+         for p in sorted(self.python_package_dirs):
+             full = os.path.join(self.topsrcdir, p)
  
              finder = FileFinder(full)
              dirs = {os.path.dirname(f[0]) for f in finder.find('**')}
@@ -271,7 +271,7 @@ diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init_
      def _synchronize_docs(self):
          m = InstallManifest()
  
-         m.add_link(self._conf_py_path, 'conf.py')
+         m.add_link(self.conf_py_path, 'conf.py')
  
-         for dest, source in sorted(self._trees.items()):
-             source_dir = os.path.join(self._topsrcdir, source)
+         for dest, source in sorted(self.trees.items()):
+             source_dir = os.path.join(self.topsrcdir, source)

mozilla-release/patches/1454640-1-62a1.patch

@@ -0,0 +1,166 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1524068264 14400
+# Node ID 3783bfd78a0fa374c1a5984b54a20f399709a1c9
+# Parent  f4df3abb96c19d29cde7df570992d4c2c2408c34
+Bug 1454640 - [moztreedocs] Move 'create_tarball' into a package submodule r=mshal
+
+These two functions are typically only used by CI for packaging/uploading the
+documentation. This is a minor re-organiztion for clarity.
+
+MozReview-Commit-ID: 62UhQhSSkOs
+
+diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
+--- a/tools/docs/mach_commands.py
++++ b/tools/docs/mach_commands.py
+@@ -48,19 +48,19 @@ class Documentation(MachCommandBase):
+             which.which('jsdoc')
+         except which.WhichError:
+             return die('jsdoc not found - please install from npm.')
+ 
+         self._activate_virtualenv()
+         self.virtualenv_manager.install_pip_requirements(
+             os.path.join(here, 'requirements.txt'), quiet=True)
+ 
+-        import moztreedocs
+         import webbrowser
+         from livereload import Server
++        from moztreedocs.package import create_tarball
+ 
+         outdir = outdir or os.path.join(self.topobjdir, 'docs')
+         format_outdir = os.path.join(outdir, fmt)
+ 
+         path = path or os.path.join(self.topsrcdir, 'tools')
+         path = os.path.normpath(os.path.abspath(path))
+ 
+         docdir = self._find_doc_dir(path)
+@@ -77,17 +77,17 @@ class Documentation(MachCommandBase):
+             return die('failed to generate documentation:\n'
+                        '%s: sphinx return code %d' % (path, result))
+         else:
+             print('\nGenerated documentation:\n%s' % savedir)
+ 
+         if archive:
+             archive_path = os.path.join(outdir,
+                                         '%s.tar.gz' % props['project'])
+-            moztreedocs.create_tarball(archive_path, savedir)
++            create_tarball(archive_path, savedir)
+             print('Archived to %s' % archive_path)
+ 
+         if not serve:
+             index_path = os.path.join(savedir, 'index.html')
+             if auto_open and os.path.isfile(index_path):
+                 webbrowser.open(index_path)
+             return
+ 
+diff --git a/tools/docs/mach_commands.py.1454640-1.later b/tools/docs/mach_commands.py.1454640-1.later
+new file mode 100644
+--- /dev/null
++++ b/tools/docs/mach_commands.py.1454640-1.later
+@@ -0,0 +1,21 @@
++--- mach_commands.py
+++++ mach_commands.py
++@@ -142,17 +142,17 @@ class Documentation(MachCommandBase):
++         for d in search_dirs:
++             p = os.path.join(path, d)
++             if os.path.isfile(os.path.join(p, 'conf.py')):
++                 return p
++ 
++     def _s3_upload(self, root, project, version=None):
++         self.virtualenv_manager.install_pip_package('boto3==1.4.4')
++ 
++-        from moztreedocs import distribution_files
+++        from moztreedocs.package import distribution_files
++         from moztreedocs.upload import s3_upload
++ 
++         # Files are uploaded to multiple locations:
++         #
++         # <project>/latest
++         # <project>/<version>
++         #
++         # This allows multiple projects and versions to be stored in the
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -2,17 +2,16 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import unicode_literals
+ 
+ import os
+ 
+ from mozbuild.frontend.reader import BuildReader
+-from mozpack.archive import create_tar_gz_from_files
+ from mozpack.copier import FileCopier
+ from mozpack.files import FileFinder
+ from mozpack.manifests import InstallManifest
+ 
+ import sphinx
+ import sphinx.apidoc
+ 
+ 
+@@ -123,28 +122,8 @@ class SphinxManager(object):
+ 
+         packages = [os.path.basename(p) for p in self._python_package_dirs]
+         packages = ['python/%s' % p for p in packages]
+         packages = '\n   '.join(sorted(packages))
+         data = data.format(indexes=indexes, python_packages=packages)
+ 
+         with open(os.path.join(self._docs_dir, 'index.rst'), 'wb') as fh:
+             fh.write(data)
+-
+-
+-def distribution_files(root):
+-    """Find all files suitable for distributing.
+-
+-    Given the path to generated Sphinx documentation, returns an iterable
+-    of (path, BaseFile) for files that should be archived, uploaded, etc.
+-    Paths are relative to given root directory.
+-    """
+-    finder = FileFinder(root, ignore=('_staging', '_venv'))
+-    return finder.find('**')
+-
+-
+-def create_tarball(filename, root):
+-    """Create a tar.gz archive of docs in a directory."""
+-    files = dict(distribution_files(root))
+-
+-    with open(filename, 'wb') as fh:
+-        create_tar_gz_from_files(fh, files, filename=os.path.basename(filename),
+-                                 compresslevel=6)
+diff --git a/tools/docs/moztreedocs/package.py b/tools/docs/moztreedocs/package.py
+new file mode 100644
+--- /dev/null
++++ b/tools/docs/moztreedocs/package.py
+@@ -0,0 +1,30 @@
++# This Source Code Form is subject to the terms of the Mozilla Public
++# License, v. 2.0. If a copy of the MPL was not distributed with this
++# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
++
++from __future__ import absolute_import, unicode_literals
++
++import os
++
++from mozpack.archive import create_tar_gz_from_files
++from mozpack.files import FileFinder
++
++
++def distribution_files(root):
++    """Find all files suitable for distributing.
++
++    Given the path to generated Sphinx documentation, returns an iterable
++    of (path, BaseFile) for files that should be archived, uploaded, etc.
++    Paths are relative to given root directory.
++    """
++    finder = FileFinder(root, ignore=('_staging', '_venv'))
++    return finder.find('**')
++
++
++def create_tarball(filename, root):
++    """Create a tar.gz archive of docs in a directory."""
++    files = dict(distribution_files(root))
++
++    with open(filename, 'wb') as fh:
++        create_tar_gz_from_files(fh, files, filename=os.path.basename(filename),
++                                 compresslevel=6)

mozilla-release/patches/1454640-2-62a1.patch

@@ -0,0 +1,213 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1524070615 14400
+# Node ID 50bb4134a2e45dd02d44a0425116d0ef8c613f14
+# Parent  33243ed2901b85b1f8e88e851af9a285e700cb2c
+Bug 1454640 - [docs] Use a single SphinxManager instance across all rebuilds r=mshal
+
+In the mozbuild.sphinx extension, we create a new SphinxManager instance each
+time. However this isn't ideal now that we can rebuild the docs within the same
+interpreter using the livereload server.
+
+This makes use of a singleton so that we can share state not only between
+multiple invocations of sphinx-build, but also with the mach command. This will
+be taken advantage of more heavily in future commits in this series.
+
+MozReview-Commit-ID: 7ERYeN5BPeI
+
+diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py
+--- a/python/mozbuild/mozbuild/sphinx.py
++++ b/python/mozbuild/mozbuild/sphinx.py
+@@ -164,37 +164,32 @@ class MozbuildSymbols(Directive):
+         # into the parser for conversion. We don't even emit ourselves, so
+         # there's no record of us.
+         self.state_machine.insert_input(format_module(module), fname)
+ 
+         return []
+ 
+ 
+ def setup(app):
++    from mozbuild.virtualenv import VirtualenvManager
++    from moztreedocs import manager
++
+     app.add_directive('mozbuildsymbols', MozbuildSymbols)
+ 
+     # Unlike typical Sphinx installs, our documentation is assembled from
+     # many sources and staged in a common location. This arguably isn't a best
+     # practice, but it was the easiest to implement at the time.
+     #
+     # Here, we invoke our custom code for staging/generating all our
+     # documentation.
+-    from moztreedocs import SphinxManager
+-
+-    topsrcdir = app.config._raw_config['topsrcdir']
+-    manager = SphinxManager(topsrcdir,
+-        os.path.join(topsrcdir, 'tools', 'docs'),
+-        app.outdir)
+     manager.generate_docs(app)
+-
+-    app.srcdir = os.path.join(app.outdir, '_staging')
++    app.srcdir = manager.staging_dir
+ 
+     # We need to adjust sys.path in order for Python API docs to get generated
+     # properly. We leverage the in-tree virtualenv for this.
+-    from mozbuild.virtualenv import VirtualenvManager
+-
++    topsrcdir = manager.topsrcdir
+     ve = VirtualenvManager(topsrcdir,
+         os.path.join(topsrcdir, 'dummy-objdir'),
+         os.path.join(app.outdir, '_venv'),
+         sys.stderr,
+         os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+     ve.ensure()
+     ve.activate()
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -1,52 +1,59 @@
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import unicode_literals
+ 
+ import os
+ 
++from mozbuild.base import MozbuildObject
+ from mozbuild.frontend.reader import BuildReader
+ from mozpack.copier import FileCopier
+ from mozpack.files import FileFinder
+ from mozpack.manifests import InstallManifest
+ 
+ import sphinx
+ import sphinx.apidoc
+ 
++here = os.path.abspath(os.path.dirname(__file__))
++build = MozbuildObject.from_environment(cwd=here)
+ 
+-class SphinxManager(object):
++MAIN_DOC_PATH = os.path.join(build.topsrcdir, 'tools', 'docs')
++
++
++class _SphinxManager(object):
+     """Manages the generation of Sphinx documentation for the tree."""
+ 
+-    def __init__(self, topsrcdir, main_path, output_dir):
+-        self._topsrcdir = topsrcdir
+-        self._output_dir = output_dir
+-        self._docs_dir = os.path.join(output_dir, '_staging')
+-        self._conf_py_path = os.path.join(main_path, 'conf.py')
+-        self._index_path = os.path.join(main_path, 'index.rst')
++    def __init__(self, topsrcdir, main_path):
++        self.topsrcdir = topsrcdir
++        self.conf_py_path = os.path.join(main_path, 'conf.py')
++        self.index_path = os.path.join(main_path, 'index.rst')
++
+         self._trees = {}
+         self._python_package_dirs = set()
+ 
++        # Instance variables that get set in self.generate_docs()
++        self.staging_dir = None
++
+     def read_build_config(self):
+         """Read the active build config and add docs to this instance."""
+ 
+         # Reading the Sphinx variables doesn't require a full build context.
+         # Only define the parts we need.
+         class fakeconfig(object):
+             def __init__(self, topsrcdir):
+                 self.topsrcdir = topsrcdir
+ 
+-        config = fakeconfig(self._topsrcdir)
++        config = fakeconfig(self.topsrcdir)
+         reader = BuildReader(config)
+ 
+         for path, name, key, value in reader.find_sphinx_variables():
+             reldir = os.path.dirname(path)
+-
+             if name == 'SPHINX_TREES':
+                 assert key
+                 if key.startswith('/'):
+                     key = key[1:]
+                 else:
+                     key = os.path.join(reldir, key)
+                 self.add_tree(os.path.join(reldir, value), key)
+ 
+@@ -65,65 +72,70 @@ class SphinxManager(object):
+         """Add a directory containing Python packages.
+ 
+         Added directories will have Python API docs generated automatically.
+         """
+         self._python_package_dirs.add(source_dir)
+ 
+     def generate_docs(self, app):
+         """Generate/stage documentation."""
++        self.staging_dir = os.path.join(app.outdir, '_staging')
++
+         app.info('Reading Sphinx metadata from build configuration')
+         self.read_build_config()
+         app.info('Staging static documentation')
+         self._synchronize_docs()
+         app.info('Generating Python API documentation')
+         self._generate_python_api_docs()
+ 
+     def _generate_python_api_docs(self):
+         """Generate Python API doc files."""
+-        out_dir = os.path.join(self._docs_dir, 'python')
++        out_dir = os.path.join(self.staging_dir, 'python')
+         base_args = ['sphinx', '--no-toc', '-o', out_dir]
+ 
+         for p in sorted(self._python_package_dirs):
+-            full = os.path.join(self._topsrcdir, p)
++            full = os.path.join(self.topsrcdir, p)
+ 
+             finder = FileFinder(full)
+             dirs = {os.path.dirname(f[0]) for f in finder.find('**')}
+ 
+             excludes = {d for d in dirs if d.endswith('test')}
+ 
+             args = list(base_args)
+             args.append(full)
+             args.extend(excludes)
+ 
+             sphinx.apidoc.main(args)
+ 
+     def _synchronize_docs(self):
+         m = InstallManifest()
+ 
+-        m.add_link(self._conf_py_path, 'conf.py')
++        m.add_link(self.conf_py_path, 'conf.py')
+ 
+         for dest, source in sorted(self._trees.items()):
+-            source_dir = os.path.join(self._topsrcdir, source)
++            source_dir = os.path.join(self.topsrcdir, source)
+             for root, dirs, files in os.walk(source_dir):
+                 for f in files:
+                     source_path = os.path.join(root, f)
+                     rel_source = source_path[len(source_dir) + 1:]
+ 
+                     m.add_link(source_path, os.path.join(dest, rel_source))
+ 
+         copier = FileCopier()
+         m.populate_registry(copier)
+-        copier.copy(self._docs_dir)
++        copier.copy(self.staging_dir)
+ 
+-        with open(self._index_path, 'rb') as fh:
++        with open(self.index_path, 'rb') as fh:
+             data = fh.read()
+ 
+         indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
+         indexes = '\n   '.join(indexes)
+ 
+         packages = [os.path.basename(p) for p in self._python_package_dirs]
+         packages = ['python/%s' % p for p in packages]
+         packages = '\n   '.join(sorted(packages))
+         data = data.format(indexes=indexes, python_packages=packages)
+ 
+-        with open(os.path.join(self._docs_dir, 'index.rst'), 'wb') as fh:
++        with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
+             fh.write(data)
++
++
++manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH)

mozilla-release/patches/1454640-3-62a1.patch

@@ -0,0 +1,199 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1524078070 14400
+# Node ID 611810b4fd66218b014a0e85c8790b7a2e3e53d4
+# Parent  e5d9919c44af5adcf658fe71582c223a59bb8ca1
+Bug 1454640 - [docs] Memoize the result of processing sphinx moz.build variables r=mshal
+
+Now that we can rebuild docs with the liveserver, there are some optimizations
+we should make. One of those is processing the sphinx moz.build variables. This
+patch makes sure we don't re-process moz.build if we've already done so in a
+previous rebuild.
+
+MozReview-Commit-ID: 2AIr1KeAPQV
+
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -3,100 +3,96 @@
+ # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import unicode_literals
+ 
+ import os
+ 
+ from mozbuild.base import MozbuildObject
+ from mozbuild.frontend.reader import BuildReader
++from mozbuild.util import memoize
+ from mozpack.copier import FileCopier
+ from mozpack.files import FileFinder
+ from mozpack.manifests import InstallManifest
+ 
+ import sphinx
+ import sphinx.apidoc
+ 
+ here = os.path.abspath(os.path.dirname(__file__))
+ build = MozbuildObject.from_environment(cwd=here)
+ 
+ MAIN_DOC_PATH = os.path.join(build.topsrcdir, 'tools', 'docs')
+ 
+ 
++@memoize
++def read_build_config(docdir):
++    """Read the active build config and return the relevant doc paths.
++
++    The return value is cached so re-generating with the same docdir won't
++    invoke the build system a second time."""
++    trees = {}
++    python_package_dirs = set()
++
++    # Reading the Sphinx variables doesn't require a full build context.
++    # Only define the parts we need.
++    class fakeconfig(object):
++        topsrcdir = build.topsrcdir
++
++    reader = BuildReader(fakeconfig())
++    for path, name, key, value in reader.find_sphinx_variables():
++        reldir = os.path.join(os.path.dirname(path), value)
++
++        if name == 'SPHINX_TREES':
++            assert key
++            if key.startswith('/'):
++                key = key[1:]
++            else:
++                key = os.path.join(reldir, key)
++
++            if key in trees:
++                raise Exception('%s has already been registered as a destination.' % key)
++            trees[key] = reldir
++
++        if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
++            python_package_dirs.add(reldir)
++
++    return trees, python_package_dirs
++
++
+ class _SphinxManager(object):
+     """Manages the generation of Sphinx documentation for the tree."""
+ 
+     def __init__(self, topsrcdir, main_path):
+         self.topsrcdir = topsrcdir
+         self.conf_py_path = os.path.join(main_path, 'conf.py')
+         self.index_path = os.path.join(main_path, 'index.rst')
+ 
+-        self._trees = {}
+-        self._python_package_dirs = set()
+-
+         # Instance variables that get set in self.generate_docs()
+         self.staging_dir = None
+-
+-    def read_build_config(self):
+-        """Read the active build config and add docs to this instance."""
+-
+-        # Reading the Sphinx variables doesn't require a full build context.
+-        # Only define the parts we need.
+-        class fakeconfig(object):
+-            def __init__(self, topsrcdir):
+-                self.topsrcdir = topsrcdir
+-
+-        config = fakeconfig(self.topsrcdir)
+-        reader = BuildReader(config)
+-
+-        for path, name, key, value in reader.find_sphinx_variables():
+-            reldir = os.path.dirname(path)
+-            if name == 'SPHINX_TREES':
+-                assert key
+-                if key.startswith('/'):
+-                    key = key[1:]
+-                else:
+-                    key = os.path.join(reldir, key)
+-                self.add_tree(os.path.join(reldir, value), key)
+-
+-            if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
+-                self.add_python_package_dir(os.path.join(reldir, value))
+-
+-    def add_tree(self, source_dir, dest_dir):
+-        """Add a directory from where docs should be sourced."""
+-        if dest_dir in self._trees:
+-            raise Exception('%s has already been registered as a destination.'
+-                            % dest_dir)
+-
+-        self._trees[dest_dir] = source_dir
+-
+-    def add_python_package_dir(self, source_dir):
+-        """Add a directory containing Python packages.
+-
+-        Added directories will have Python API docs generated automatically.
+-        """
+-        self._python_package_dirs.add(source_dir)
++        self.trees = None
++        self.python_package_dirs = None
+ 
+     def generate_docs(self, app):
+         """Generate/stage documentation."""
+         self.staging_dir = os.path.join(app.outdir, '_staging')
+ 
+         app.info('Reading Sphinx metadata from build configuration')
+-        self.read_build_config()
++        self.trees, self.python_package_dirs = read_build_config(app.srcdir)
++
+         app.info('Staging static documentation')
+         self._synchronize_docs()
+         app.info('Generating Python API documentation')
+         self._generate_python_api_docs()
+ 
+     def _generate_python_api_docs(self):
+         """Generate Python API doc files."""
+         out_dir = os.path.join(self.staging_dir, 'python')
+         base_args = ['sphinx', '--no-toc', '-o', out_dir]
+ 
+-        for p in sorted(self._python_package_dirs):
++        for p in sorted(self.python_package_dirs):
+             full = os.path.join(self.topsrcdir, p)
+ 
+             finder = FileFinder(full)
+             dirs = {os.path.dirname(f[0]) for f in finder.find('**')}
+ 
+             excludes = {d for d in dirs if d.endswith('test')}
+ 
+             args = list(base_args)
+@@ -105,36 +101,36 @@ class _SphinxManager(object):
+ 
+             sphinx.apidoc.main(args)
+ 
+     def _synchronize_docs(self):
+         m = InstallManifest()
+ 
+         m.add_link(self.conf_py_path, 'conf.py')
+ 
+-        for dest, source in sorted(self._trees.items()):
++        for dest, source in sorted(self.trees.items()):
+             source_dir = os.path.join(self.topsrcdir, source)
+             for root, dirs, files in os.walk(source_dir):
+                 for f in files:
+                     source_path = os.path.join(root, f)
+                     rel_source = source_path[len(source_dir) + 1:]
+ 
+                     m.add_link(source_path, os.path.join(dest, rel_source))
+ 
+         copier = FileCopier()
+         m.populate_registry(copier)
+         copier.copy(self.staging_dir)
+ 
+         with open(self.index_path, 'rb') as fh:
+             data = fh.read()
+ 
+-        indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
++        indexes = ['%s/index' % p for p in sorted(self.trees.keys())]
+         indexes = '\n   '.join(indexes)
+ 
+-        packages = [os.path.basename(p) for p in self._python_package_dirs]
++        packages = [os.path.basename(p) for p in self.python_package_dirs]
+         packages = ['python/%s' % p for p in packages]
+         packages = '\n   '.join(sorted(packages))
+         data = data.format(indexes=indexes, python_packages=packages)
+ 
+         with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
+             fh.write(data)
+ 
+ 

mozilla-release/patches/1454640-4-62a1.patch

@@ -2,7 +2,7 @@
 # User Andrew Halberstadt <ahalberstadt@mozilla.com>
 # Date 1523980297 14400
 # Node ID 1381b6cea12d903b6154a6c09a4cc0603984b95e
-# Parent  9b3ec1b1ee73c2e1dbf1346f97c4da2c58a4b28e
+# Parent  724bb7b1528f336425dfdc2c6a95baea7cc7b696
 Bug 1454640 - [mozbuild] Ability to find sphinx variables relevant to a given path r=mshal
 
 The current mechanism for reading SPHINX variables assumes we always want to
@@ -79,46 +79,42 @@ diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbu
  
              tree = ast.parse(source, full)
              Visitor().visit(tree)
-diff --git a/tools/docs/moztreedocs/__init__.py.1454640.later b/tools/docs/moztreedocs/__init__.py.1454640.later
-new file mode 100644
---- /dev/null
-+++ b/tools/docs/moztreedocs/__init__.py.1454640.later
-@@ -0,0 +1,38 @@
-+--- __init__.py
-++++ __init__.py
-+@@ -26,26 +26,34 @@ MAIN_DOC_PATH = os.path.join(build.topsr
-+ def read_build_config(docdir):
-+     """Read the active build config and return the relevant doc paths.
-+ 
-+     The return value is cached so re-generating with the same docdir won't
-+     invoke the build system a second time."""
-+     trees = {}
-+     python_package_dirs = set()
-+ 
-++    is_main = docdir == MAIN_DOC_PATH
-++    relevant_mozbuild_path = None if is_main else docdir
-++
-+     # Reading the Sphinx variables doesn't require a full build context.
-+     # Only define the parts we need.
-+     class fakeconfig(object):
-+         topsrcdir = build.topsrcdir
-+ 
-+     reader = BuildReader(fakeconfig())
-+-    for path, name, key, value in reader.find_sphinx_variables():
-++    for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
-+         reldir = os.path.join(os.path.dirname(path), value)
-+ 
-+         if name == 'SPHINX_TREES':
-++            # If we're building a subtree, only process that specific subtree.
-++            absdir = os.path.join(build.topsrcdir, reldir)
-++            if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
-++                continue
-++
-+             assert key
-+             if key.startswith('/'):
-+                 key = key[1:]
-+             else:
-+                 key = os.path.join(reldir, key)
-+ 
-+             if key in trees:
-+                 raise Exception('%s has already been registered as a destination.' % key)
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -26,26 +26,34 @@ MAIN_DOC_PATH = os.path.join(build.topsr
+ def read_build_config(docdir):
+     """Read the active build config and return the relevant doc paths.
+ 
+     The return value is cached so re-generating with the same docdir won't
+     invoke the build system a second time."""
+     trees = {}
+     python_package_dirs = set()
+ 
++    is_main = docdir == MAIN_DOC_PATH
++    relevant_mozbuild_path = None if is_main else docdir
++
+     # Reading the Sphinx variables doesn't require a full build context.
+     # Only define the parts we need.
+     class fakeconfig(object):
+         topsrcdir = build.topsrcdir
+ 
+     reader = BuildReader(fakeconfig())
+-    for path, name, key, value in reader.find_sphinx_variables():
++    for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
+         reldir = os.path.join(os.path.dirname(path), value)
+ 
+         if name == 'SPHINX_TREES':
++            # If we're building a subtree, only process that specific subtree.
++            absdir = os.path.join(build.topsrcdir, reldir)
++            if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
++                continue
++
+             assert key
+             if key.startswith('/'):
+                 key = key[1:]
+             else:
+                 key = os.path.join(reldir, key)
+ 
+             if key in trees:
+                 raise Exception('%s has already been registered as a destination.' % key)

mozilla-release/patches/1454640-5-62a1.patch

@@ -0,0 +1,166 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1524086291 14400
+# Node ID 155a531ddcdd4cf2e1ae4f7ad1fd0e44808a2b51
+# Parent  c6c3bcff42680fdfb82f2fd1debcc2b92fd9dac8
+Bug 1454640 - [docs] Always build docs with the tools/docs/conf.py r=mshal
+
+Previously, running |mach doc <subtree>| would use whatever conf.py file
+happened to live in the subtree. For example, running:
+
+./mach doc tools/lint
+
+Would build with tools/lint/docs/conf.py. This is bad because it means the
+generated docs will look different from the docs that eventually will be
+published to firefox-source-docs.mozilla.com.
+
+This patch makes sure we always use tools/docs/conf.py for building, even when
+only generating a subtree.
+
+Furthermore, this sets things up such that when you modify a file, only the
+subtree containing the modified file will be re-generated.  This cuts down
+rebuild times from ~2 minutes to ~20 seconds.
+
+There is one caveat. When rebuilding a subtree, the index of other trees will
+be overwritten in that particular subtree. I couldn't figure out anyway around
+this. This tradeoff for *much* faster rebuild times seems worth it.
+
+MozReview-Commit-ID: Ly88mvHKpo7
+
+diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
+--- a/tools/docs/mach_commands.py
++++ b/tools/docs/mach_commands.py
+@@ -19,16 +19,21 @@ from mozbuild.base import MachCommandBas
+ 
+ here = os.path.abspath(os.path.dirname(__file__))
+ 
+ 
+ @CommandProvider
+ class Documentation(MachCommandBase):
+     """Helps manage in-tree documentation."""
+ 
++    def __init__(self, context):
++        super(Documentation, self).__init__(context)
++
++        self._manager = None
++
+     @Command('doc', category='devenv',
+              description='Generate and serve documentation from the tree.')
+     @CommandArgument('path', default=None, metavar='DIRECTORY', nargs='?',
+                      help='Path to documentation to build and display.')
+     @CommandArgument('--format', default='html', dest='fmt',
+                      help='Documentation format to write.')
+     @CommandArgument('--outdir', default=None, metavar='DESTINATION',
+                      help='Where to write output.')
+@@ -53,31 +58,28 @@ class Documentation(MachCommandBase):
+         self.virtualenv_manager.install_pip_requirements(
+             os.path.join(here, 'requirements.txt'), quiet=True)
+ 
+         import webbrowser
+         from livereload import Server
+         from moztreedocs.package import create_tarball
+ 
+         outdir = outdir or os.path.join(self.topobjdir, 'docs')
+-        format_outdir = os.path.join(outdir, fmt)
++        savedir = os.path.join(outdir, fmt)
+ 
+         path = path or os.path.join(self.topsrcdir, 'tools')
+         path = os.path.normpath(os.path.abspath(path))
+ 
+         docdir = self._find_doc_dir(path)
+         if not docdir:
+             return die('failed to generate documentation:\n'
+                        '%s: could not find docs at this location' % path)
+ 
+         props = self._project_properties(docdir)
+-        savedir = os.path.join(format_outdir, props['project'])
+-
+-        run_sphinx = partial(self._run_sphinx, docdir, savedir, fmt)
+-        result = run_sphinx()
++        result = self._run_sphinx(docdir, savedir, fmt=fmt)
+         if result != 0:
+             return die('failed to generate documentation:\n'
+                        '%s: sphinx return code %d' % (path, result))
+         else:
+             print('\nGenerated documentation:\n%s' % savedir)
+ 
+         if archive:
+             archive_path = os.path.join(outdir,
+@@ -95,52 +97,71 @@ class Documentation(MachCommandBase):
+         # will cause a re-build and refresh of the browser (if open).
+         try:
+             host, port = http.split(':', 1)
+             port = int(port)
+         except ValueError:
+             return die('invalid address: %s' % http)
+ 
+         server = Server()
+-        server.watch(docdir, run_sphinx)
++
++        sphinx_trees = self.manager.trees or {savedir: docdir}
++        for dest, src in sphinx_trees.items():
++            run_sphinx = partial(self._run_sphinx, src, savedir, fmt=fmt)
++            server.watch(src, run_sphinx)
+         server.serve(host=host, port=port, root=savedir,
+                      open_url_delay=0.1 if auto_open else None)
+ 
+-    def _run_sphinx(self, docdir, savedir, fmt='html'):
++    def _run_sphinx(self, docdir, savedir, config=None, fmt='html'):
+         import sphinx
++        config = config or self.manager.conf_py_path
+         args = [
+             'sphinx',
+             '-b', fmt,
++            '-c', os.path.dirname(config),
+             docdir,
+             savedir,
+         ]
+         return sphinx.build_main(args)
+ 
++    @property
++    def manager(self):
++        if not self._manager:
++            from moztreedocs import manager
++            self._manager = manager
++        return self._manager
++
+     def _project_properties(self, path):
+         import imp
+-        path = os.path.join(path, 'conf.py')
++        path = os.path.normpath(self.manager.conf_py_path)
+         with open(path, 'r') as fh:
+             conf = imp.load_module('doc_conf', fh, path,
+                                    ('.py', 'r', imp.PY_SOURCE))
+ 
+         # Prefer the Mozilla project name, falling back to Sphinx's
+         # default variable if it isn't defined.
+         project = getattr(conf, 'moz_project_name', None)
+         if not project:
+             project = conf.project.replace(' ', '_')
+ 
+         return {
+             'project': project,
+             'version': getattr(conf, 'version', None)
+         }
+ 
+     def _find_doc_dir(self, path):
+-        search_dirs = ('doc', 'docs')
+-        for d in search_dirs:
++        if os.path.isfile(path):
++            return
++
++        valid_doc_dirs = ('doc', 'docs')
++        if os.path.basename(path) in valid_doc_dirs:
++            return path
++
++        for d in valid_doc_dirs:
+             p = os.path.join(path, d)
+-            if os.path.isfile(os.path.join(p, 'conf.py')):
++            if os.path.isdir(p):
+                 return p
+ 
+ 
+ def die(msg, exit_code=1):
+     msg = '%s: %s' % (sys.argv[0], msg)
+     print(msg, file=sys.stderr)
+     return exit_code

mozilla-release/patches/1454640-6-62a1.patch

@@ -0,0 +1,142 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1524087070 14400
+# Node ID a71038f16918df5b43dd9813f238b23fd703b6ba
+# Parent  b23c22c84b249ede7f8f8ee62b5f160577bd67be
+Bug 1454640 - [docs] Lazy load the package and version properties r=mshal on a CLOSED TREE
+
+We no longer store the docs under a project name (since all the docs are now
+built using the root conf.py). This mean the name and version are only used for
+packaging and uploading, which typically is only used in CI.
+
+This allows us to lazy load the package name and version, so we only read the
+conf.py when we need to.
+
+MozReview-Commit-ID: DV5Jxrbskoh
+
+diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
+--- a/tools/docs/mach_commands.py
++++ b/tools/docs/mach_commands.py
+@@ -23,16 +23,18 @@ here = os.path.abspath(os.path.dirname(_
+ @CommandProvider
+ class Documentation(MachCommandBase):
+     """Helps manage in-tree documentation."""
+ 
+     def __init__(self, context):
+         super(Documentation, self).__init__(context)
+ 
+         self._manager = None
++        self._project = None
++        self._version = None
+ 
+     @Command('doc', category='devenv',
+              description='Generate and serve documentation from the tree.')
+     @CommandArgument('path', default=None, metavar='DIRECTORY', nargs='?',
+                      help='Path to documentation to build and display.')
+     @CommandArgument('--format', default='html', dest='fmt',
+                      help='Documentation format to write.')
+     @CommandArgument('--outdir', default=None, metavar='DESTINATION',
+@@ -68,27 +70,25 @@ class Documentation(MachCommandBase):
+         path = path or os.path.join(self.topsrcdir, 'tools')
+         path = os.path.normpath(os.path.abspath(path))
+ 
+         docdir = self._find_doc_dir(path)
+         if not docdir:
+             return die('failed to generate documentation:\n'
+                        '%s: could not find docs at this location' % path)
+ 
+-        props = self._project_properties(docdir)
+         result = self._run_sphinx(docdir, savedir, fmt=fmt)
+         if result != 0:
+             return die('failed to generate documentation:\n'
+                        '%s: sphinx return code %d' % (path, result))
+         else:
+             print('\nGenerated documentation:\n%s' % savedir)
+ 
+         if archive:
+-            archive_path = os.path.join(outdir,
+-                                        '%s.tar.gz' % props['project'])
++            archive_path = os.path.join(outdir, '%s.tar.gz' % self.project)
+             create_tarball(archive_path, savedir)
+             print('Archived to %s' % archive_path)
+ 
+         if not serve:
+             index_path = os.path.join(savedir, 'index.html')
+             if auto_open and os.path.isfile(index_path):
+                 webbrowser.open(index_path)
+             return
+@@ -124,33 +124,43 @@ class Documentation(MachCommandBase):
+ 
+     @property
+     def manager(self):
+         if not self._manager:
+             from moztreedocs import manager
+             self._manager = manager
+         return self._manager
+ 
+-    def _project_properties(self, path):
++    def _read_project_properties(self):
+         import imp
+         path = os.path.normpath(self.manager.conf_py_path)
+         with open(path, 'r') as fh:
+             conf = imp.load_module('doc_conf', fh, path,
+                                    ('.py', 'r', imp.PY_SOURCE))
+ 
+         # Prefer the Mozilla project name, falling back to Sphinx's
+         # default variable if it isn't defined.
+         project = getattr(conf, 'moz_project_name', None)
+         if not project:
+             project = conf.project.replace(' ', '_')
+ 
+-        return {
+-            'project': project,
+-            'version': getattr(conf, 'version', None)
+-        }
++        self._project = project,
++        self._version = getattr(conf, 'version', None)
++
++    @property
++    def project(self):
++        if not self._project:
++            self._read_project_properties()
++        return self._project
++
++    @property
++    def version(self):
++        if not self._version:
++            self._read_project_properties()
++        return self._version
+ 
+     def _find_doc_dir(self, path):
+         if os.path.isfile(path):
+             return
+ 
+         valid_doc_dirs = ('doc', 'docs')
+         if os.path.basename(path) in valid_doc_dirs:
+             return path
+diff --git a/tools/docs/mach_commands.py.1454640-6.later b/tools/docs/mach_commands.py.1454640-6.later
+new file mode 100644
+--- /dev/null
++++ b/tools/docs/mach_commands.py.1454640-6.later
+@@ -0,0 +1,21 @@
++--- mach_commands.py
+++++ mach_commands.py
++@@ -70,17 +70,17 @@ class Documentation(MachCommandBase):
++             print('\nGenerated documentation:\n%s' % savedir)
++ 
++         if archive:
++             archive_path = os.path.join(outdir, '%s.tar.gz' % self.project)
++             create_tarball(archive_path, savedir)
++             print('Archived to %s' % archive_path)
++ 
++         if upload:
++-            self._s3_upload(savedir, props['project'], props['version'])
+++            self._s3_upload(savedir, self.project, self.version)
++ 
++         if not serve:
++             index_path = os.path.join(savedir, 'index.html')
++             if auto_open and os.path.isfile(index_path):
++                 webbrowser.open(index_path)
++             return
++ 
++         # Create livereload server. Any files modified in the specified docdir

mozilla-release/patches/1474746-1-63a1.patch

@@ -0,0 +1,59 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1531320753 0
+# Node ID abc9aece7e635a408a9bb853fc8b19b25b313c4b
+# Parent  57e16f5c0cd315bb1336997af81e046d538ec449
+Bug 1474746 - [docs] Fix firefox-source-docs url regression by removing redundant "docs" directory; r=gps
+
+This fixes a regression from bug 1454640 where urls had an extra 'docs' path inserted into them, e.g:
+toolkit/components/telemetry/telemetry/index.html
+
+became:
+toolkit/components/telemetry/docs/telemetry/index.html
+
+Differential Revision: https://phabricator.services.mozilla.com/D2079
+
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -36,36 +36,36 @@ def read_build_config(docdir):
+ 
+     # Reading the Sphinx variables doesn't require a full build context.
+     # Only define the parts we need.
+     class fakeconfig(object):
+         topsrcdir = build.topsrcdir
+ 
+     reader = BuildReader(fakeconfig())
+     for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
+-        reldir = os.path.join(os.path.dirname(path), value)
++        reldir = os.path.dirname(path)
+ 
+         if name == 'SPHINX_TREES':
+             # If we're building a subtree, only process that specific subtree.
+-            absdir = os.path.join(build.topsrcdir, reldir)
++            absdir = os.path.join(build.topsrcdir, reldir, value)
+             if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
+                 continue
+ 
+             assert key
+             if key.startswith('/'):
+                 key = key[1:]
+             else:
+                 key = os.path.join(reldir, key)
+ 
+             if key in trees:
+                 raise Exception('%s has already been registered as a destination.' % key)
+-            trees[key] = reldir
++            trees[key] = os.path.join(reldir, value)
+ 
+         if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
+-            python_package_dirs.add(reldir)
++            python_package_dirs.add(os.path.join(reldir, value))
+ 
+     return trees, python_package_dirs
+ 
+ 
+ class _SphinxManager(object):
+     """Manages the generation of Sphinx documentation for the tree."""
+ 
+     def __init__(self, topsrcdir, main_path):

mozilla-release/patches/1474746-2-63a1.patch

@@ -0,0 +1,46 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1531324881 0
+# Node ID 2f80a6085cbba56b3cb01392decc59ce1dcb8fae
+# Parent  20a79a803fac51edaa6f98552cfc75d6dce434b7
+Bug 1474746 - [docs] Normalize keys from MOZ_SPHINX_TREES; r=gps
+
+This will allow developers to use '.' as the key, e.g:
+MOZ_SPHINX_TREES['.'] = 'docs'
+
+This will give consumers the ability to remove redundancies from
+their urls. For example, the telemetry docs currently have:
+MOZ_SPHINX_TREES['telemetry'] = 'docs'
+
+This results in a url like:
+https://firefox-source-docs.mozilla.org/main/latest/toolkit/components/telemetry/telemetry/index.html
+
+If they changed their key to '.' instead, the new url would become:
+https://firefox-source-docs.mozilla.org/main/latest/toolkit/components/telemetry/index.html
+
+Depends on D2079.
+
+Differential Revision: https://phabricator.services.mozilla.com/D2080
+
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -48,17 +48,17 @@ def read_build_config(docdir):
+             absdir = os.path.join(build.topsrcdir, reldir, value)
+             if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
+                 continue
+ 
+             assert key
+             if key.startswith('/'):
+                 key = key[1:]
+             else:
+-                key = os.path.join(reldir, key)
++                key = os.path.normpath(os.path.join(reldir, key))
+ 
+             if key in trees:
+                 raise Exception('%s has already been registered as a destination.' % key)
+             trees[key] = os.path.join(reldir, value)
+ 
+         if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
+             python_package_dirs.add(os.path.join(reldir, value))
+ 

mozilla-release/patches/1482675-80a1.patch

@@ -0,0 +1,51 @@
+# HG changeset patch
+# User Karl Dubost <kdubost@mozilla.com>
+# Date 1593562315 0
+# Node ID f9fdbef12cd8644cba53967161a3f9d3fdcb1c56
+# Parent  e18d1313e70681b569e9b9d4e269a29a7d0c56cc
+Bug 1482675 - Adds no-system-changes option to bootstrap. r=rstewart
+
+Differential Revision: https://phabricator.services.mozilla.com/D78855
+
+diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
+--- a/python/mozboot/bin/bootstrap.py
++++ b/python/mozboot/bin/bootstrap.py
+@@ -165,16 +165,19 @@ def main(args):
+     parser.add_option('--vcs', dest='vcs', default=None,
+                       help='VCS (hg or git) to use for downloading the source code, '
+                       'instead of using the default interactive prompt.')
+     parser.add_option('--no-interactive', dest='no_interactive', action='store_true',
+                       help='Answer yes to any (Y/n) interactive prompts.')
+     parser.add_option('--debug', dest='debug', action='store_true',
+                       help='Print extra runtime information useful for debugging and '
+                       'bug reports.')
++    parser.add_option('--no-system-changes', dest='no_system_changes', action='store_true',
++                      help='Only executes actions that leave the system '
++                      'configuration alone.')
+ 
+     options, leftover = parser.parse_args(args)
+ 
+     try:
+         try:
+             cls = ensure_environment(options.repo_url, options.repo_rev,
+                                      options.repo_type)
+         except Exception as e:
+@@ -185,17 +188,17 @@ def main(args):
+             if options.debug:
+                 # Raise full tracebacks during debugging and for bug reporting.
+                 raise
+ 
+             print(e)
+             return 1
+ 
+         dasboot = cls(choice=options.application_choice, no_interactive=options.no_interactive,
+-                      vcs=options.vcs)
++                      vcs=options.vcs, no_system_changes=options.no_system_changes)
+         dasboot.bootstrap()
+ 
+         return 0
+     finally:
+         if TEMPDIR is not None:
+             shutil.rmtree(TEMPDIR)
+ 
+ 

mozilla-release/patches/1542963-2-69a1.patch

@@ -2,7 +2,7 @@
 # User Justin Wood <Callek@gmail.com>
 # Date 1559053320 0
 # Node ID 41e1409393169e979098e02645f3dfe190b19ef2
-# Parent  fac47b019c02d0e004f55f3b0b6953a655f81d41
+# Parent  84746eabe178d37c8ee442c619fe3ded3416e249
 Bug 1542963 - run './mach lint ... --fix' on mozbuild/mozbuild, undoes some black changes. r=glandium
 
 Lint python/mozbuild/{mozbuild,mozpack}.
@@ -661,7 +661,7 @@ diff --git a/python/mozbuild/mozbuild/action/process_install_manifest.py b/pytho
 diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py
 --- a/python/mozbuild/mozbuild/action/test_archive.py
 +++ b/python/mozbuild/mozbuild/action/test_archive.py
-@@ -503,17 +503,17 @@ if buildconfig.substs.get('MOZ_ASAN') an
+@@ -514,17 +514,17 @@ if buildconfig.substs.get('commtopsrcdir
  # Verify nothing sneaks into ARCHIVE_FILES without a corresponding exclusion
  # rule in the "common" archive.
  for k, v in ARCHIVE_FILES.items():
@@ -733,17 +733,11 @@ diff --git a/python/mozbuild/mozbuild/action/xpccheck.py b/python/mozbuild/mozbu
 +        name = os.path.basename(f)
 +        if name.endswith('.in'):
 +            name = name[:-3]
-+
-+        if not name.endswith('.js'):
-+            continue
  
 -    if not name.endswith('.js'):
 -      continue
-+        found = False
-+        for test in initests:
-+            if os.path.join(os.path.abspath(directory), name) == test['path']:
-+                found = True
-+                break
++        if not name.endswith('.js'):
++            continue
  
 -    found = False
 -    for test in initests:
@@ -754,6 +748,12 @@ diff --git a/python/mozbuild/mozbuild/action/xpccheck.py b/python/mozbuild/mozbu
 -    if not found:
 -      print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (name, os.path.join(directory, 'xpcshell.ini'))
 -      sys.exit(1)
++        found = False
++        for test in initests:
++            if os.path.join(os.path.abspath(directory), name) == test['path']:
++                found = True
++                break
++
 +        if not found:
 +            print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (
 +                name, os.path.join(directory, 'xpcshell.ini'))
@@ -1681,9 +1681,10 @@ diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/m
 +            workspace_settings_dir, 'org.eclipse.core.resources.prefs')
          with open(core_resources_prefs_path, 'wb') as fh:
 -            fh.write(STATIC_CORE_RESOURCES_PREFS);
-+            fh.write(STATIC_CORE_RESOURCES_PREFS)
- 
+-
 -        core_runtime_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.core.runtime.prefs')
++            fh.write(STATIC_CORE_RESOURCES_PREFS)
++
 +        core_runtime_prefs_path = os.path.join(
 +            workspace_settings_dir, 'org.eclipse.core.runtime.prefs')
          with open(core_runtime_prefs_path, 'wb') as fh:
@@ -1721,9 +1722,10 @@ diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/m
              # so we need add those now:
              cdt_core_prefs += FORMATTER_SETTINGS
 -            fh.write(cdt_core_prefs);
-+            fh.write(cdt_core_prefs)
- 
+-
 -        editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs");
++            fh.write(cdt_core_prefs)
++
 +        editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs")
          with open(editor_prefs_path, 'wb') as fh:
 -            fh.write(EDITOR_SETTINGS);
@@ -3498,7 +3500,7 @@ diff --git a/python/mozbuild/mozbuild/compilation/codecomplete.py b/python/mozbu
 diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py
 --- a/python/mozbuild/mozbuild/compilation/database.py
 +++ b/python/mozbuild/mozbuild/compilation/database.py
-@@ -165,18 +165,18 @@ class CompileDBBackend(CommonBackend):
+@@ -171,18 +171,18 @@ class CompileDBBackend(CommonBackend):
          '.mm': 'CXXFLAGS',
      }
  
@@ -4779,7 +4781,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
          # Copy state from parent.
          for p in parent.source_stack:
-@@ -588,16 +590,17 @@ class PathMeta(type):
+@@ -575,16 +577,17 @@ class PathMeta(type):
              if value.startswith('!'):
                  cls = ObjDirPath
              elif value.startswith('%'):
@@ -4797,7 +4799,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
        - '/topsrcdir/relative/paths'
        - 'srcdir/relative/paths'
        - '!/topobjdir/relative/paths'
-@@ -654,16 +657,17 @@ class Path(ContextDerivedValue, unicode)
+@@ -641,16 +644,17 @@ class Path(ContextDerivedValue, unicode)
  
      @memoized_property
      def target_basename(self):
@@ -4815,7 +4817,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          super(SourcePath, self).__init__(context, value)
  
          if value.startswith('/'):
-@@ -694,69 +698,74 @@ class SourcePath(Path):
+@@ -681,69 +685,74 @@ class SourcePath(Path):
  class RenamedSourcePath(SourcePath):
      """Like SourcePath, but with a different base name when installed.
  
@@ -4891,7 +4893,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          def __getitem__(self, name):
              name = self.normalize(name)
              return super(_TypedListWithItems, self).__getitem__(name)
-@@ -880,60 +889,63 @@ def ContextDerivedTypedHierarchicalStrin
+@@ -867,60 +876,63 @@ def ContextDerivedTypedHierarchicalStrin
              child = self._children.get(name)
              if not child:
                  child = self._children[name] = _TypedListWithItems(
@@ -4957,7 +4959,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
      'script': unicode,
      'inputs': list,
      'force': bool,
-@@ -983,38 +995,38 @@ class Files(SubContext):
+@@ -970,38 +982,38 @@ class Files(SubContext):
      The difference in behavior between ``*`` and ``**`` is only evident if
      a pattern follows the ``*`` or ``**``. A pattern ending with ``*`` is
      greedy. ``**`` is needed when you need an additional pattern after the
@@ -4999,7 +5001,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
              with Files('runtests.py'):
                 IMPACTED_TESTS.files += [
-@@ -1052,17 +1064,17 @@ class Files(SubContext):
+@@ -1039,17 +1051,17 @@ class Files(SubContext):
                  IMPACTED_TESTS.flavors += [
                      'mochitest',
                  ]
@@ -5018,7 +5020,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
              do not schedule them, aside from those described in a Files
              subcontext.  For example, py-lint tasks need not be scheduled for
              most changes, but should be scheduled when any Python file changes.
-@@ -1208,76 +1220,76 @@ SUBCONTEXTS = {cls.__name__: cls for cls
+@@ -1195,76 +1207,76 @@ SUBCONTEXTS = {cls.__name__: cls for cls
  # This defines the set of mutable global variables.
  #
  # Each variable is a tuple of:
@@ -5105,7 +5107,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          If the optional ``script`` attribute is not present on an entry, it
          is assumed that rules for generating the file are present in
          the associated Makefile.in.
-@@ -1318,17 +1330,17 @@ VARIABLES = {
+@@ -1305,17 +1317,17 @@ VARIABLES = {
  
          When the ``force`` attribute is present, the file is generated every
          build, regardless of whether it is stale.  This is special to the
@@ -5124,7 +5126,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          a string-literal in the program, the value needs to have
          double-quotes.
  
-@@ -1346,73 +1358,73 @@ VARIABLES = {
+@@ -1333,73 +1345,73 @@ VARIABLES = {
             DEFINES.update({
                 'NS_NO_XPCOM': True,
                 'MOZ_EXTENSIONS_DB_SCHEMA': 15,
@@ -5204,7 +5206,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          ``FINAL_TARGET_FILES``. For a build with ``--enable-ui-locale``,
          the file will be taken from ``$LOCALE_SRCDIR``, with the leading
          ``en-US`` removed. For a l10n repack of an en-US build, the file
-@@ -1444,24 +1456,24 @@ VARIABLES = {
+@@ -1431,24 +1443,24 @@ VARIABLES = {
          ``toolkit/locales/en-US/foo.js`` and
          ``toolkit/locales/en-US/things/*.ini`` to ``$(DIST)/bin/foo`` in an
          en-US build, and in a build of a different locale (or a repack),
@@ -5231,7 +5233,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          1. The function in the Python script will be passed an additional keyword argument `locale`
             which provides the locale in use, i.e. ``en-US``.
          2. The ``inputs`` list may contain paths to files that will be taken from the locale
-@@ -1475,248 +1487,248 @@ VARIABLES = {
+@@ -1462,248 +1474,248 @@ VARIABLES = {
          In addition, ``LOCALIZED_GENERATED_FILES`` can use the special substitutions ``{AB_CD}``
          and ``{AB_rCD}`` in their output paths.  ``{AB_CD}`` expands to the current locale during
          multi-locale builds and single-locale repacks and ``{AB_rCD}`` expands to an
@@ -5515,7 +5517,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          subdirectory they should be exported to. For example, to export
          ``foo.h`` to the top-level directory, and ``bar.h`` to ``mozilla/dom/``,
          append to ``EXPORTS`` like so::
-@@ -1724,256 +1736,256 @@ VARIABLES = {
+@@ -1711,256 +1723,256 @@ VARIABLES = {
             EXPORTS += ['foo.h']
             EXPORTS.mozilla.dom += ['bar.h']
  
@@ -5814,7 +5816,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          files are elsehwere (jar.mn, for instance) and this state of affairs
          is OK.
          """),
-@@ -2034,196 +2046,196 @@ VARIABLES = {
+@@ -2021,196 +2033,196 @@ VARIABLES = {
              - non_unified_sources, a list containing sources files, relative to
                the current moz.build, that should be excluded from source file
                unification.
@@ -6034,7 +6036,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
      'HOST_PROGRAM',
      'HOST_LIBRARY_NAME',
      'HOST_SIMPLE_PROGRAMS',
-@@ -2232,34 +2244,34 @@ TEMPLATE_VARIABLES = {
+@@ -2219,34 +2231,34 @@ TEMPLATE_VARIABLES = {
      'PROGRAM',
      'SIMPLE_PROGRAMS',
  }
@@ -6071,7 +6073,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          If a relative path is given, it is evaluated as relative to the file
          currently being processed. If there is a chain of multiple include(),
          the relative path computation is from the most recent/active file.
-@@ -2276,17 +2288,17 @@ FUNCTIONS = {
+@@ -2263,17 +2275,17 @@ FUNCTIONS = {
             include('sibling.build')
  
          Include ``foo.build`` from a path within the top source directory::
@@ -6090,7 +6092,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
          The value used for the variable is the final value at the end of the
          moz.build file, so it is possible (but not recommended style) to place
-@@ -2303,31 +2315,31 @@ FUNCTIONS = {
+@@ -2290,31 +2302,31 @@ FUNCTIONS = {
  
          To make all children directories install as the given extension::
  
@@ -6125,7 +6127,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          Contrary to traditional python functions:
             - return values from template functions are ignored,
             - template functions don't have access to the global scope.
-@@ -2376,48 +2388,48 @@ TestDirsPlaceHolder = List()
+@@ -2363,48 +2375,48 @@ TestDirsPlaceHolder = List()
  # Special variables. These complement VARIABLES.
  #
  # Each entry is a tuple of:
@@ -6179,7 +6181,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
              lambda key: context.config.substs_unicode.get(key)), dict,
          """Dictionary containing the current configuration variables.
  
-@@ -2426,92 +2438,92 @@ SPECIAL_VARIABLES = {
+@@ -2413,92 +2425,92 @@ SPECIAL_VARIABLES = {
  
          Values in this container are read-only. Attempts at changing values
          will result in a run-time error.
@@ -10247,14 +10249,14 @@ diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/sh
 diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py
 --- a/python/mozbuild/mozbuild/sphinx.py
 +++ b/python/mozbuild/mozbuild/sphinx.py
-@@ -187,14 +187,14 @@ def setup(app):
- 
-     app.srcdir = os.path.join(app.outdir, '_staging')
+@@ -182,14 +182,14 @@ def setup(app):
+     # documentation.
+     manager.generate_docs(app)
+     app.srcdir = manager.staging_dir
  
      # We need to adjust sys.path in order for Python API docs to get generated
      # properly. We leverage the in-tree virtualenv for this.
-     from mozbuild.virtualenv import VirtualenvManager
- 
+     topsrcdir = manager.topsrcdir
      ve = VirtualenvManager(topsrcdir,
 -        os.path.join(topsrcdir, 'dummy-objdir'),
 -        os.path.join(app.outdir, '_venv'),
@@ -10302,9 +10304,6 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 -    rmtree(self.tmpdir)
 +    def setUp(self):
 +        self.tmpdir = mkdtemp()
-+
-+    def tearDown(self):
-+        rmtree(self.tmpdir)
  
 -  # utility methods for tests
 -  def touch(self, file, dir=None):
@@ -10313,13 +10312,8 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 -    f = os.path.join(dir, file)
 -    open(f, 'w').close()
 -    return f
-+    # utility methods for tests
-+    def touch(self, file, dir=None):
-+        if dir is None:
-+            dir = self.tmpdir
-+        f = os.path.join(dir, file)
-+        open(f, 'w').close()
-+        return f
++    def tearDown(self):
++        rmtree(self.tmpdir)
  
 -  def assertFileContains(self, filename, l):
 -    """Assert that the lines in the file |filename| are equal
@@ -10336,6 +10330,23 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 -    self.assert_(len(l) == 0, 
 -                 "not enough lines in file! (expected '{0}',"
 -                 " got '{1}'".format(l, lines))
++    # utility methods for tests
++    def touch(self, file, dir=None):
++        if dir is None:
++            dir = self.tmpdir
++        f = os.path.join(dir, file)
++        open(f, 'w').close()
++        return f
+ 
+-  def test_basic(self):
+-    "Test that addEntriesToListFile works when file doesn't exist."
+-    testfile = os.path.join(self.tmpdir, "test.list")
+-    l = ["a", "b", "c"]
+-    addEntriesToListFile(testfile, l)
+-    self.assertFileContains(testfile, l)
+-    # ensure that attempting to add the same entries again doesn't change it
+-    addEntriesToListFile(testfile, l)
+-    self.assertFileContains(testfile, l)
 +    def assertFileContains(self, filename, l):
 +        """Assert that the lines in the file |filename| are equal
 +        to the contents of the list |l|, in order."""
@@ -10352,14 +10363,15 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 +                     "not enough lines in file! (expected '{0}',"
 +                     " got '{1}'".format(l, lines))
  
--  def test_basic(self):
--    "Test that addEntriesToListFile works when file doesn't exist."
+-  def test_append(self):
+-    "Test adding new entries."
 -    testfile = os.path.join(self.tmpdir, "test.list")
 -    l = ["a", "b", "c"]
 -    addEntriesToListFile(testfile, l)
 -    self.assertFileContains(testfile, l)
--    # ensure that attempting to add the same entries again doesn't change it
--    addEntriesToListFile(testfile, l)
+-    l2 = ["x","y","z"]
+-    addEntriesToListFile(testfile, l2)
+-    l.extend(l2)
 -    self.assertFileContains(testfile, l)
 +    def test_basic(self):
 +        "Test that addEntriesToListFile works when file doesn't exist."
@@ -10371,16 +10383,14 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 +        addEntriesToListFile(testfile, l)
 +        self.assertFileContains(testfile, l)
  
--  def test_append(self):
--    "Test adding new entries."
+-  def test_append_some(self):
+-    "Test adding new entries mixed with existing entries."
 -    testfile = os.path.join(self.tmpdir, "test.list")
 -    l = ["a", "b", "c"]
 -    addEntriesToListFile(testfile, l)
 -    self.assertFileContains(testfile, l)
--    l2 = ["x","y","z"]
--    addEntriesToListFile(testfile, l2)
--    l.extend(l2)
--    self.assertFileContains(testfile, l)
+-    addEntriesToListFile(testfile, ["a", "x", "c", "z"])
+-    self.assertFileContains(testfile, ["a", "b", "c", "x", "z"])
 +    def test_append(self):
 +        "Test adding new entries."
 +        testfile = os.path.join(self.tmpdir, "test.list")
@@ -10392,14 +10402,14 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 +        l.extend(l2)
 +        self.assertFileContains(testfile, l)
  
--  def test_append_some(self):
--    "Test adding new entries mixed with existing entries."
+-  def test_add_multiple(self):
+-    """Test that attempting to add the same entry multiple times results in
+-    only one entry being added."""
 -    testfile = os.path.join(self.tmpdir, "test.list")
--    l = ["a", "b", "c"]
--    addEntriesToListFile(testfile, l)
--    self.assertFileContains(testfile, l)
--    addEntriesToListFile(testfile, ["a", "x", "c", "z"])
--    self.assertFileContains(testfile, ["a", "b", "c", "x", "z"])
+-    addEntriesToListFile(testfile, ["a","b","a","a","b"])
+-    self.assertFileContains(testfile, ["a","b"])
+-    addEntriesToListFile(testfile, ["c","a","c","b","c"])
+-    self.assertFileContains(testfile, ["a","b","c"])
 +    def test_append_some(self):
 +        "Test adding new entries mixed with existing entries."
 +        testfile = os.path.join(self.tmpdir, "test.list")
@@ -10408,15 +10418,7 @@ diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/moz
 +        self.assertFileContains(testfile, l)
 +        addEntriesToListFile(testfile, ["a", "x", "c", "z"])
 +        self.assertFileContains(testfile, ["a", "b", "c", "x", "z"])
- 
--  def test_add_multiple(self):
--    """Test that attempting to add the same entry multiple times results in
--    only one entry being added."""
--    testfile = os.path.join(self.tmpdir, "test.list")
--    addEntriesToListFile(testfile, ["a","b","a","a","b"])
--    self.assertFileContains(testfile, ["a","b"])
--    addEntriesToListFile(testfile, ["c","a","c","b","c"])
--    self.assertFileContains(testfile, ["a","b","c"])
++
 +    def test_add_multiple(self):
 +        """Test that attempting to add the same entry multiple times results in
 +        only one entry being added."""
@@ -12021,7 +12023,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          self.assertEqual(len(objs), 1)
          self.assertIsInstance(objs[0], VariablePassthru)
  
-@@ -397,17 +397,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -398,17 +398,17 @@ class TestEmitterBasic(unittest.TestCase
          self.assertEqual(flags.flags['LIBRARY_DEFINES'],
                           ['-DMOZ_LIBRARY_DEFINE=MOZ_TEST'])
          self.assertEqual(flags.flags['DEFINES'],
@@ -12040,7 +12042,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          self.assertIsInstance(flags, ComputedFlags)
          self.assertEqual(flags.flags['BASE_INCLUDES'],
                           ['-I%s' % reader.config.topsrcdir,
-@@ -430,17 +430,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -431,17 +431,17 @@ class TestEmitterBasic(unittest.TestCase
          })
          sources, ldflags, lib, flags = self.read_topsrcdir(reader)
          self.assertEqual(flags.flags['WARNINGS_CFLAGS'], [])
@@ -12059,7 +12061,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
                                   YASM='yasm',
                                   YASM_ASFLAGS='-foo',
                               ))
-@@ -528,27 +528,26 @@ class TestEmitterBasic(unittest.TestCase
+@@ -529,27 +529,26 @@ class TestEmitterBasic(unittest.TestCase
          self.assertIsInstance(objs[1], LocalizedFiles)
  
      def test_localized_files_not_localized_generated(self):
@@ -12089,7 +12091,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
  
          self.assertEqual(len(objs), 2)
          for o in objs:
-@@ -572,29 +571,29 @@ class TestEmitterBasic(unittest.TestCase
+@@ -573,29 +572,29 @@ class TestEmitterBasic(unittest.TestCase
          self.assertEqual(o.outputs, ('bar.c',))
          self.assertRegexpMatches(o.script, 'script.py$')
          self.assertEqual(o.method, 'make_bar')
@@ -12122,7 +12124,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
  
          self.assertEqual(len(objs), 1)
          self.assertIsInstance(objs[0], Exports)
-@@ -613,26 +612,26 @@ class TestEmitterBasic(unittest.TestCase
+@@ -614,26 +613,26 @@ class TestEmitterBasic(unittest.TestCase
              self.assertEqual(expect_headers, actual_headers)
  
      def test_exports_missing(self):
@@ -12151,7 +12153,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
  
          self.assertEqual(len(objs), 2)
          self.assertIsInstance(objs[0], GeneratedFile)
-@@ -659,17 +658,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -660,17 +659,17 @@ class TestEmitterBasic(unittest.TestCase
          for path, strings in objs[0].files.walk():
              self.assertTrue(path in expected)
              basenames = sorted(mozpath.basename(s) for s in strings)
@@ -12170,7 +12172,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
  
          self.assertEqual(len(objs), 6)
          self.assertIsInstance(objs[0], Sources)
-@@ -730,32 +729,31 @@ class TestEmitterBasic(unittest.TestCase
+@@ -731,32 +730,31 @@ class TestEmitterBasic(unittest.TestCase
  
      def test_empty_test_manifest_rejected(self):
          """A test manifest without any entries is rejected."""
@@ -12204,7 +12206,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
  
          objs = self.read_topsrcdir(reader)
          self.assertEqual(len(objs), 1)
-@@ -924,17 +922,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -925,17 +923,17 @@ class TestEmitterBasic(unittest.TestCase
          for o in objs:
              m = metadata[mozpath.basename(o.manifest_relpath)]
  
@@ -12223,7 +12225,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
                  relpath = path[len(o.directory)+1:]
  
                  self.assertIn(relpath, m['installs'])
-@@ -942,49 +940,49 @@ class TestEmitterBasic(unittest.TestCase
+@@ -943,49 +941,49 @@ class TestEmitterBasic(unittest.TestCase
  
              if 'pattern-installs' in m:
                  self.assertEqual(len(o.pattern_installs), m['pattern-installs'])
@@ -12277,7 +12279,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          objs = self.read_topsrcdir(reader)
          ipdl_collection = objs[0]
          self.assertIsInstance(ipdl_collection, IPDLCollection)
-@@ -1022,17 +1020,16 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1023,17 +1021,16 @@ class TestEmitterBasic(unittest.TestCase
              'fooParent.cpp',
              'foo1.cpp',
              'foo1Child.cpp',
@@ -12295,7 +12297,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          local_includes = [o.path for o in objs if isinstance(o, LocalInclude)]
          expected = [
              '/bar/baz',
-@@ -1130,33 +1127,33 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1131,33 +1128,33 @@ class TestEmitterBasic(unittest.TestCase
      def test_jar_manifests_multiple_files(self):
          with self.assertRaisesRegexp(SandboxValidationError, 'limited to one value'):
              reader = self.reader('jar-manifests-multiple-files')
@@ -12332,7 +12334,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
              'libc': '-DIN_LIBA -DIN_LIBB',
              'libd': ''
          }
-@@ -1321,17 +1318,16 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1322,17 +1319,16 @@ class TestEmitterBasic(unittest.TestCase
                  [mozpath.join(reader.config.topsrcdir, f) for f in files])
  
              for f in files:
@@ -12350,7 +12352,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          # The last object is a ComputedFlags, the second to last a Linkable,
          # followed by ldflags, ignore them.
          linkable = objs[-2]
-@@ -1356,17 +1352,16 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1357,17 +1353,16 @@ class TestEmitterBasic(unittest.TestCase
              self.assertTrue(sources.have_unified_mapping)
  
              for f in dict(sources.unified_source_mapping).keys():
@@ -12368,7 +12370,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          # The last object is a Linkable, the second to last ComputedFlags,
          # followed by ldflags, ignore them.
          objs = objs[:-3]
-@@ -1405,25 +1400,25 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1406,25 +1401,25 @@ class TestEmitterBasic(unittest.TestCase
  
              expected = {'install.rdf', 'main.js'}
              for f in files:
@@ -12396,7 +12398,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          objs = self.read_topsrcdir(reader)
  
          self.assertEqual(len(objs), 1)
-@@ -1437,17 +1432,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1438,17 +1433,17 @@ class TestEmitterBasic(unittest.TestCase
              for f in files:
                  self.assertTrue(unicode(f) in expected)
  
@@ -12415,7 +12417,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
          objs = self.read_topsrcdir(reader)
  
          self.assertEqual(len(objs), 1)
-@@ -1460,38 +1455,38 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1461,38 +1456,38 @@ class TestEmitterBasic(unittest.TestCase
              expected = {'en-US/bar.ini', 'en-US/foo.js'}
              for f in files:
                  self.assertTrue(unicode(f) in expected)
@@ -12458,7 +12460,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
                               extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc'))
          objs = self.read_topsrcdir(reader)
  
-@@ -1504,17 +1499,17 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1505,17 +1500,17 @@ class TestEmitterBasic(unittest.TestCase
          self.assertRegexpMatches(lib.import_name, "random_crate")
          self.assertRegexpMatches(lib.basename, "random-crate")
  
@@ -12477,7 +12479,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
                               extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc'))
          objs = self.read_topsrcdir(reader)
  
-@@ -1524,47 +1519,47 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1525,47 +1520,47 @@ class TestEmitterBasic(unittest.TestCase
          self.assertIsInstance(cflags, ComputedFlags)
          self.assertIsInstance(lib, RustLibrary)
          self.assertEqual(lib.features, ['musthave', 'cantlivewithout'])
@@ -12530,7 +12532,7 @@ diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/moz
                               extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc',
                                                 BIN_SUFFIX='.exe'))
          objs = self.read_topsrcdir(reader)
-@@ -1658,14 +1653,14 @@ class TestEmitterBasic(unittest.TestCase
+@@ -1659,14 +1654,14 @@ class TestEmitterBasic(unittest.TestCase
          self.assertEqual(shlib.symbols_file, 'foo.symbols')
  
      def test_symbols_file_objdir_missing_generated(self):
@@ -13215,10 +13217,12 @@ diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/
          self.assertEqual(test['foo'], 1)
  
 -        self.assertEqual(test.keys(), ['foo', 'bar' ])
-+        self.assertEqual(test.keys(), ['foo', 'bar'])
- 
-     def test_defaults(self):
+-
+-    def test_defaults(self):
 -        test = OrderedDefaultDict(bool, {'foo': 1 })
++        self.assertEqual(test.keys(), ['foo', 'bar'])
++
++    def test_defaults(self):
 +        test = OrderedDefaultDict(bool, {'foo': 1})
  
          self.assertEqual(test['foo'], 1)
@@ -13938,18 +13942,13 @@ diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuil
 +        for line, ending in zip(['a', '#literal b', 'c'], lineendings):
 +            f.write(line+ending)
 +        f.close()
-+
-+    def testMac(self):
-+        self.createFile(['\x0D']*3)
-+        self.pp.do_include(self.tempnam)
-+        self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
  
 -  def testMac(self):
 -    self.createFile(['\x0D']*3)
 -    self.pp.do_include(self.tempnam)
 -    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
-+    def testUnix(self):
-+        self.createFile(['\x0A']*3)
++    def testMac(self):
++        self.createFile(['\x0D']*3)
 +        self.pp.do_include(self.tempnam)
 +        self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
  
@@ -13957,8 +13956,8 @@ diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuil
 -    self.createFile(['\x0A']*3)
 -    self.pp.do_include(self.tempnam)
 -    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
-+    def testWindows(self):
-+        self.createFile(['\x0D\x0A']*3)
++    def testUnix(self):
++        self.createFile(['\x0A']*3)
 +        self.pp.do_include(self.tempnam)
 +        self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
  
@@ -13966,6 +13965,11 @@ diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuil
 -    self.createFile(['\x0D\x0A']*3)
 -    self.pp.do_include(self.tempnam)
 -    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
++    def testWindows(self):
++        self.createFile(['\x0D\x0A']*3)
++        self.pp.do_include(self.tempnam)
++        self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
++
  
  if __name__ == '__main__':
 -  mozunit.main()

mozilla-release/patches/1559975-29-70a1.patch

@@ -2,7 +2,7 @@
 # User Edwin Gao <egao@mozilla.com>
 # Date 1563547524 0
 # Node ID 18c1e6d076564a25c12e497a65babf18f9821650
-# Parent  5af9e88ccc0bdbc12f16beeebc82c3922a50581f
+# Parent  b34c92e6f15b863d0d0dcd16fb80f3e6b69002a0
 Bug 1559975 - fix python2 linter error for tools/docs r=ahal
 
 Differential Revision: https://phabricator.services.mozilla.com/D37777
@@ -38,11 +38,11 @@ diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init_
  
  import os
  
+ from mozbuild.base import MozbuildObject
  from mozbuild.frontend.reader import BuildReader
- from mozpack.archive import create_tar_gz_from_files
+ from mozbuild.util import memoize
  from mozpack.copier import FileCopier
  from mozpack.files import FileFinder
- from mozpack.manifests import InstallManifest
 diff --git a/tools/lint/py2.yml b/tools/lint/py2.yml
 --- a/tools/lint/py2.yml
 +++ b/tools/lint/py2.yml

mozilla-release/patches/1563797-3-70a1.patch

@@ -2,7 +2,7 @@
 # User Andrew Halberstadt <ahalberstadt@mozilla.com>
 # Date 1562853819 0
 # Node ID 7f92f148ea546a606b4e434049ea3f4d29450b1c
-# Parent  bad333922574c6e7261f31ef33ac7d0a5df1de90
+# Parent  8a9d67aa10276d5bc30cc6e7366cabc6536f3c1f
 Bug 1563797 - Use 'backports.shutil_which' instead of 'which' across the tree r=Callek
 
 Differential Revision: https://phabricator.services.mozilla.com/D37097
@@ -22,12 +22,13 @@ diff --git a/js/src/builtin/embedjs.py b/js/src/builtin/embedjs.py
 +
 +import errno
  import re, sys, os, subprocess
--import mozpack.path as mozpath
- import shlex
--import which
++import shlex
 +
- import buildconfig
-+import mozpack.path as mozpath
++import buildconfig
+ import mozpack.path as mozpath
+-import shlex
+-import which
+-import buildconfig
 +from mozfile import which
  
  def ToCAsciiArray(lines):
@@ -79,7 +80,7 @@ diff --git a/python/mozboot/mozboot/mach_commands.py b/python/mozboot/mozboot/ma
      Command,
  )
  
-@@ -57,33 +58,34 @@ class VersionControlCommands(object):
+@@ -62,33 +63,34 @@ class VersionControlCommands(object):
  
          User choice is respected: no changes are made without explicit
          confirmation from you.
@@ -144,7 +145,7 @@ diff --git a/testing/mach_commands.py b/testing/mach_commands.py
  import subprocess
  import shutil
  
-@@ -567,19 +568,18 @@ class TestInfoCommand(MachCommandBase):
+@@ -562,19 +563,18 @@ class TestInfoCommand(MachCommandBase):
                       help='Retrieve and display ActiveData test result summary.')
      @CommandArgument('--show-durations', action='store_true',
                       help='Retrieve and display ActiveData test duration summary.')
@@ -165,7 +166,7 @@ diff --git a/testing/mach_commands.py b/testing/mach_commands.py
          self.show_results = params['show_results']
          self.show_durations = params['show_durations']
          self.show_bugs = params['show_bugs']
-@@ -595,27 +595,25 @@ class TestInfoCommand(MachCommandBase):
+@@ -590,27 +590,25 @@ class TestInfoCommand(MachCommandBase):
              self.show_durations = True
              self.show_bugs = True
  
@@ -228,8 +229,8 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
  class Documentation(MachCommandBase):
      """Helps manage in-tree documentation."""
  
-     @Command('doc', category='devenv',
-@@ -39,19 +37,20 @@ class Documentation(MachCommandBase):
+     def __init__(self, context):
+@@ -46,19 +44,20 @@ class Documentation(MachCommandBase):
                       help="Don't automatically open HTML docs in a browser.")
      @CommandArgument('--no-serve', dest='serve', default=True, action='store_false',
                       help="Don't serve the generated docs after building.")
@@ -251,8 +252,8 @@ diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py
          self.virtualenv_manager.install_pip_requirements(
              os.path.join(here, 'requirements.txt'), quiet=True)
  
-         import moztreedocs
          import webbrowser
+         from livereload import Server
 diff --git a/tools/lint/docs/create.rst b/tools/lint/docs/create.rst
 --- a/tools/lint/docs/create.rst
 +++ b/tools/lint/docs/create.rst
@@ -428,7 +429,7 @@ diff --git a/tools/lint/spell/__init__.py b/tools/lint/spell/__init__.py
  CODESPELL_REQUIREMENTS_PATH = os.path.join(here, 'codespell_requirements.txt')
  
  CODESPELL_NOT_FOUND = """
-@@ -87,20 +87,17 @@ def get_codespell_binary():
+@@ -85,20 +85,17 @@ def get_codespell_binary():
      """
      Returns the path of the first codespell binary available
      if not found returns None

mozilla-release/patches/1583360-71a1.patch

@@ -2,7 +2,7 @@
 # User Andrew Halberstadt <ahalberstadt@mozilla.com>
 # Date 1571407549 0
 # Node ID 3cb0c1d7e71689d1f0198855b5b6064203aba8fc
-# Parent  e21512862f9607a801e8ebfe9a477af9e317ab54
+# Parent  66944fbd3ad89b69b21e4eab1d9f9b11e860b751
 Bug 1583360 - [mozbuild] Make reader.find_sphinx_variables a little more generic, r=nalexander
 
 The BuildReader's 'find_sphinx_variables' function is hardcoded to look for the
@@ -49,6 +49,13 @@ diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbu
  
 -    def find_sphinx_variables(self, path=None):
 -        """This function finds all assignments of Sphinx documentation variables.
+-
+-        This is a generator of tuples of (moz.build path, var, key, value). For
+-        variables that assign to keys in objects, key will be defined.
+-
+-        With a little work, this function could be made more generic. But if we
+-        end up writing a lot of ast code, it might be best to import a
+-        high-level AST manipulation library into the tree.
 +    def find_variables_from_ast(self, variables, path=None):
 +        """Finds all assignments to the specified variables by parsing
 +        moz.build abstract syntax trees.
@@ -66,9 +73,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbu
 +        2) A simple list. Values should be strings, e.g: The target of the
 +        assignment should be a Name node. Values should be a List node,
 +        whose elements are Str nodes. e.g:
- 
--        This is a generator of tuples of (moz.build path, var, key, value). For
--        variables that assign to keys in objects, key will be defined.
++
 +            VARIABLE += ['foo']
 +
 +        This is an `AugAssign` node with a `Name` target with id "VARIABLE".
@@ -78,10 +83,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbu
 +        With a little work, this function could support other types of
 +        assignment. But if we end up writing a lot of AST code, it might be
 +        best to import a high-level AST manipulation library into the tree.
- 
--        With a little work, this function could be made more generic. But if we
--        end up writing a lot of ast code, it might be best to import a
--        high-level AST manipulation library into the tree.
++
 +        Args:
 +            variables (list): A list of variable assignments to capture.
 +            path (str): A path relative to the source dir. If specified, only
@@ -149,32 +151,28 @@ diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbu
                  assert isinstance(target.slice.value, ast.Str)
                  key = target.slice.value.s
  
-diff --git a/tools/docs/moztreedocs/__init__.py.1583360.later b/tools/docs/moztreedocs/__init__.py.1583360.later
-new file mode 100644
---- /dev/null
-+++ b/tools/docs/moztreedocs/__init__.py.1583360.later
-@@ -0,0 +1,24 @@
-+--- __init__.py
-++++ __init__.py
-+@@ -36,18 +36,20 @@ def read_build_config(docdir):
-+     is_main = docdir == MAIN_DOC_PATH
-+     relevant_mozbuild_path = None if is_main else docdir
-+ 
-+     # Reading the Sphinx variables doesn't require a full build context.
-+     # Only define the parts we need.
-+     class fakeconfig(object):
-+         topsrcdir = build.topsrcdir
-+ 
-++    variables = ('SPHINX_TREES', 'SPHINX_PYTHON_PACKAGE_DIRS')
-+     reader = BuildReader(fakeconfig())
-+-    for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
-++    result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path)
-++    for path, name, key, value in result:
-+         reldir = os.path.dirname(path)
-+ 
-+         if name == 'SPHINX_TREES':
-+             # If we're building a subtree, only process that specific subtree.
-+             absdir = os.path.join(build.topsrcdir, reldir, value)
-+             if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
-+                 continue
-+ 
+diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
+--- a/tools/docs/moztreedocs/__init__.py
++++ b/tools/docs/moztreedocs/__init__.py
+@@ -34,18 +34,20 @@ def read_build_config(docdir):
+     is_main = docdir == MAIN_DOC_PATH
+     relevant_mozbuild_path = None if is_main else docdir
+ 
+     # Reading the Sphinx variables doesn't require a full build context.
+     # Only define the parts we need.
+     class fakeconfig(object):
+         topsrcdir = build.topsrcdir
+ 
++    variables = ('SPHINX_TREES', 'SPHINX_PYTHON_PACKAGE_DIRS')
+     reader = BuildReader(fakeconfig())
+-    for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
++    result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path)
++    for path, name, key, value in result
+         reldir = os.path.dirname(path)
+ 
+         if name == 'SPHINX_TREES':
+             # If we're building a subtree, only process that specific subtree.
+             absdir = os.path.join(build.topsrcdir, reldir, value)
+             if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
+                 continue
+ 

mozilla-release/patches/1600664-73a1.patch

@@ -0,0 +1,277 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1575405515 0
+# Node ID d2f8a8875708e694346e9ec7b5d5d0761e3662ca
+# Parent  13ddfb117ddb0b124b6eeb5e615faae69b7d1611
+Bug 1600664 - Download the wasi sysroot during bootstrap for Linux r=firefox-build-system-reviewers,chmanchester
+
+Differential Revision: https://phabricator.services.mozilla.com/D55481
+
+diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
+--- a/python/mozboot/mozboot/archlinux.py
++++ b/python/mozboot/mozboot/archlinux.py
+@@ -12,28 +12,29 @@ import glob
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
++    WasiSysrootInstall,
+ )
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class ArchlinuxBootstrapper(
+         NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
+-        LucetcInstall, BaseBootstrapper):
++        LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     '''Archlinux experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf2.13',
+         'base-devel',
+         'nodejs',
+         'python2',
+         'python2-setuptools',
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -296,16 +296,22 @@ class BaseBootstrapper(object):
+         pass
+ 
+     def ensure_lucetc_packages(self, state_dir, checkout_root):
+         '''
+         Install lucetc.
+         '''
+         pass
+ 
++    def ensure_wasi_sysroot_packages(self, state_dir, checkout_root):
++        '''
++        Install the wasi sysroot.
++        '''
++        pass
++
+     def ensure_node_packages(self, state_dir, checkout_root):
+         '''
+         Install any necessary packages needed to supply NodeJS'''
+         raise NotImplementedError(
+             '%s does not yet implement ensure_node_packages()'
+             % __name__)
+ 
+     def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -349,16 +349,17 @@ class Bootstrapper(object):
+ 
+         self.instance.state_dir = state_dir
+         self.instance.ensure_node_packages(state_dir, checkout_root)
+         self.instance.ensure_stylo_packages(state_dir, checkout_root)
+         self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+         self.instance.ensure_nasm_packages(state_dir, checkout_root)
+         self.instance.ensure_sccache_packages(state_dir, checkout_root)
+         self.instance.ensure_lucetc_packages(state_dir, checkout_root)
++        self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
+ 
+     def bootstrap(self):
+         if self.choice is None:
+             # Like ['1. Firefox for Desktop', '2. Firefox for Android Artifact Mode', ...].
+             labels = ['%s. %s' % (i + 1, name) for (i, (name, _)) in enumerate(APPLICATIONS_LIST)]
+             prompt = APPLICATION_CHOICE % '\n'.join('  {}'.format(label) for label in labels)
+             prompt_choice = self.instance.prompt_int(prompt=prompt, low=1, high=len(APPLICATIONS))
+             name, application = APPLICATIONS_LIST[prompt_choice-1]
+diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py
+--- a/python/mozboot/mozboot/centosfedora.py
++++ b/python/mozboot/mozboot/centosfedora.py
+@@ -9,22 +9,24 @@ import platform
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
++    WasiSysrootInstall,
+ )
+ 
+ 
+-class CentOSFedoraBootstrapper(NasmInstall, NodeInstall, StyloInstall,
+-                               SccacheInstall, ClangStaticAnalysisInstall,
+-                               LucetcInstall, BaseBootstrapper):
++class CentOSFedoraBootstrapper(
++        NasmInstall, NodeInstall, StyloInstall, SccacheInstall,
++        ClangStaticAnalysisInstall, LucetcInstall, WasiSysrootInstall,
++        BaseBootstrapper):
+     def __init__(self, distro, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.distro = distro
+         self.version = int(version.split('.')[0])
+         self.dist_id = dist_id
+ 
+         self.group_packages = []
+diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
+--- a/python/mozboot/mozboot/debian.py
++++ b/python/mozboot/mozboot/debian.py
+@@ -7,16 +7,17 @@ from __future__ import absolute_import, 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
++    WasiSysrootInstall,
+ )
+ 
+ 
+ MERCURIAL_INSTALL_PROMPT = '''
+ Mercurial releases a new version every 3 months and your distro's package
+ may become out of date. This may cause incompatibility with some
+ Mercurial extensions that rely on new Mercurial features. As a result,
+ you may not have an optimal version control experience.
+@@ -27,18 +28,19 @@ in files being placed in /usr/local/bin 
+ 
+ How would you like to continue?
+   1. Install a modern Mercurial via pip (recommended)
+   2. Install a legacy Mercurial via apt
+   3. Do not install Mercurial
+ Your choice: '''
+ 
+ 
+-class DebianBootstrapper(NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+-                         SccacheInstall, LucetcInstall, BaseBootstrapper):
++class DebianBootstrapper(
++        NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
++        SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     # These are common packages for all Debian-derived distros (such as
+     # Ubuntu).
+     COMMON_PACKAGES = [
+         'autoconf2.13',
+         'build-essential',
+         'nodejs',
+         'python-dev',
+         'python-pip',
+diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py
+--- a/python/mozboot/mozboot/gentoo.py
++++ b/python/mozboot/mozboot/gentoo.py
+@@ -7,30 +7,31 @@ from __future__ import absolute_import, 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
++    WasiSysrootInstall,
+ )
+ 
+ try:
+     from urllib2 import urlopen
+ except ImportError:
+     from urllib.request import urlopen
+ 
+ import re
+ import subprocess
+ 
+ 
+ class GentooBootstrapper(
+         NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+-        SccacheInstall, LucetcInstall, BaseBootstrapper):
++        SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+ 
+     def __init__(self, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.version = version
+         self.dist_id = dist_id
+ 
+     def install_system_packages(self):
+diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py
+--- a/python/mozboot/mozboot/linux_common.py
++++ b/python/mozboot/mozboot/linux_common.py
+@@ -31,16 +31,27 @@ class LucetcInstall(object):
+ 
+     def ensure_lucetc_packages(self, state_dir, checkout_root):
+         from mozboot import lucetc
+ 
+         self.install_toolchain_artifact(state_dir, checkout_root,
+                                         lucetc.LINUX_LUCETC)
+ 
+ 
++class WasiSysrootInstall(object):
++    def __init__(self, **kwargs):
++        pass
++
++    def ensure_wasi_sysroot_packages(self, state_dir, checkout_root):
++        from mozboot import wasi_sysroot
++
++        self.install_toolchain_artifact(state_dir, checkout_root,
++                                        wasi_sysroot.LINUX_WASI_SYSROOT)
++
++
+ class StyloInstall(object):
+     def __init__(self, **kwargs):
+         pass
+ 
+     def ensure_stylo_packages(self, state_dir, checkout_root):
+         from mozboot import stylo
+ 
+         if is_non_x86_64():
+diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py
+--- a/python/mozboot/mozboot/solus.py
++++ b/python/mozboot/mozboot/solus.py
+@@ -9,28 +9,29 @@ import subprocess
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
++    WasiSysrootInstall,
+ )
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class SolusBootstrapper(
+         NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
+-        LucetcInstall, BaseBootstrapper):
++        LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     '''Solus experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf213',
+         'nodejs',
+         'python',
+         'python3',
+         'unzip',
+diff --git a/python/mozboot/mozboot/wasi_sysroot.py b/python/mozboot/mozboot/wasi_sysroot.py
+new file mode 100644
+--- /dev/null
++++ b/python/mozboot/mozboot/wasi_sysroot.py
+@@ -0,0 +1,7 @@
++# This Source Code Form is subject to the terms of the Mozilla Public
++# License, v. 2.0. If a copy of the MPL was not distributed with this
++# file, You can obtain one at http://mozilla.org/MPL/2.0/.
++
++from __future__ import absolute_import, print_function, unicode_literals
++
++LINUX_WASI_SYSROOT = 'wasi-sysroot'

mozilla-release/patches/1601578-73a1.patch

@@ -2,7 +2,7 @@
 # User Sylvestre Ledru <sledru@mozilla.com>
 # Date 1576610263 0
 # Node ID 1b761079366e755b61eb6080f0210ad9c443c036
-# Parent  6b800395e5c8e42a23a29a1d99dde9a3850a2bd9
+# Parent  3413e17310ddb80ab63235ef388601bbe24054f0
 Bug 1601578 - mach - doc environment with python 3 r=ahal,firefox-build-system-reviewers,chmanchester
 
 Differential Revision: https://phabricator.services.mozilla.com/D55937
@@ -84,7 +84,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  class HostCompileFlags(BaseCompileFlags):
      def __init__(self, context):
          self._context = context
-@@ -540,27 +541,27 @@ class CompileFlags(BaseCompileFlags):
+@@ -529,27 +530,27 @@ class CompileFlags(BaseCompileFlags):
              raise ValueError('`%s` may not be set in COMPILE_FLAGS from moz.build, this '
                               'value is resolved from the emitter.' % key)
          if not (isinstance(value, list) and all(isinstance(v, basestring) for v in value)):
@@ -114,7 +114,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
      class EnumClass(object):
          def __new__(cls, value=None):
-@@ -599,17 +600,17 @@ class PathMeta(type):
+@@ -588,17 +589,17 @@ class PathMeta(type):
                  cls = ObjDirPath
              elif value.startswith('%'):
                  cls = AbsolutePath
@@ -133,7 +133,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
        - 'srcdir/relative/paths'
        - '!/topobjdir/relative/paths'
        - '!objdir/relative/paths'
-@@ -630,17 +631,17 @@ class Path(ContextDerivedValue, unicode)
+@@ -619,17 +620,17 @@ class Path(ContextDerivedValue, unicode)
          """ContextDerived equivalent of mozpath.join(self, *p), returning a
          new Path instance.
          """
@@ -152,7 +152,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
      def __ne__(self, other):
          return self.__cmp__(other) != 0
-@@ -938,28 +939,28 @@ def TypedListWithAction(typ, action):
+@@ -927,28 +928,28 @@ def TypedListWithAction(typ, action):
  
  
  ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest)
@@ -186,7 +186,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  class Files(SubContext):
      """Metadata attached to files.
  
-@@ -1271,18 +1272,18 @@ VARIABLES = {
+@@ -1260,18 +1261,18 @@ VARIABLES = {
          """Cargo features to activate for this library.
  
          This variable should not be used directly; you should be using the
@@ -207,7 +207,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          RustLibrary template instead.
          """
          ),
-@@ -1293,23 +1294,23 @@ VARIABLES = {
+@@ -1282,23 +1283,23 @@ VARIABLES = {
          """Cargo features to activate for this host library.
  
          This variable should not be used directly; you should be using the
@@ -233,7 +233,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
      'UNIFIED_SOURCES': (
          ContextDerivedTypedList(
              SourcePath,
-@@ -1556,17 +1557,17 @@ VARIABLES = {
+@@ -1545,17 +1546,17 @@ VARIABLES = {
          cases, not for general use. If you wish to add entries to OBJDIR_FILES,
          please consult a build peer.
          """),
@@ -252,7 +252,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          """),
  
      'CPP_UNIT_TESTS': (StrictOrderingOnAppendList, list,
-@@ -1597,66 +1598,66 @@ VARIABLES = {
+@@ -1586,66 +1587,66 @@ VARIABLES = {
  
      'HOST_SOURCES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list,
                       """Source code files to compile with the host compiler.
@@ -325,7 +325,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
  
      'USE_LIBS': (StrictOrderingOnAppendList, list,
                   """List of libraries to link to programs and libraries.
-@@ -1678,41 +1679,41 @@ VARIABLES = {
+@@ -1667,41 +1668,41 @@ VARIABLES = {
                 """Whether profile-guided optimization is disable in this directory.
          """),
  
@@ -372,7 +372,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          points to data instead of code, so that the Windows linker can treat
          them correctly.
          """),
-@@ -1723,17 +1724,17 @@ VARIABLES = {
+@@ -1712,17 +1713,17 @@ VARIABLES = {
          Each name in this variable corresponds to an executable built from the
          corresponding source file with the same base name.
  
@@ -391,7 +391,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          """),
  
      'HOST_SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list,
-@@ -1805,25 +1806,25 @@ VARIABLES = {
+@@ -1794,25 +1795,25 @@ VARIABLES = {
             EXPORTS += ['foo.h']
             EXPORTS.mozilla.dom += ['bar.h']
  
@@ -419,7 +419,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          """),
  
      'DIST_INSTALL': (Enum(None, False, True), bool,
-@@ -1851,17 +1852,17 @@ VARIABLES = {
+@@ -1840,17 +1841,17 @@ VARIABLES = {
      'XPIDL_SOURCES': (StrictOrderingOnAppendList, list,
                        """XPCOM Interface Definition Files (xpidl).
  
@@ -438,7 +438,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          """),
  
      'XPIDL_NO_MANIFEST': (bool, bool,
-@@ -2004,34 +2005,33 @@ VARIABLES = {
+@@ -1993,34 +1994,33 @@ VARIABLES = {
      'PYTHON_UNITTEST_MANIFESTS': (ManifestparserManifestList, list,
                                    """List of manifest files defining python unit tests.
          """),
@@ -476,7 +476,7 @@ diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozb
          result is dist/xpi-stage/$(XPI_NAME). If DIST_SUBDIR is present, then
          the $(DIST_SUBDIR) directory of the otherwise default value is used.
          """),
-@@ -2052,17 +2052,17 @@ VARIABLES = {
+@@ -2041,17 +2041,17 @@ VARIABLES = {
          .manifest file to go with those .js files.  Setting ``NO_JS_MANIFEST``
          indicates that the relevant .manifest file and entries for those .js
          files are elsehwere (jar.mn, for instance) and this state of affairs
@@ -707,42 +707,38 @@ new file mode 100644
 diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py
 --- a/tools/docs/moztreedocs/__init__.py
 +++ b/tools/docs/moztreedocs/__init__.py
-@@ -107,28 +107,28 @@ class SphinxManager(object):
+@@ -124,24 +124,24 @@ class _SphinxManager(object):
                      rel_source = source_path[len(source_dir) + 1:]
  
                      m.add_link(source_path, os.path.join(dest, rel_source))
  
          copier = FileCopier()
          m.populate_registry(copier)
-         copier.copy(self._docs_dir)
+         copier.copy(self.staging_dir)
  
--        with open(self._index_path, 'rb') as fh:
+-        with open(self.index_path, 'rb') as fh:
 +        with open(self.index_path, 'r') as fh:
              data = fh.read()
  
-         indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
+         indexes = ['%s/index' % p for p in sorted(self.trees.keys())]
          indexes = '\n   '.join(indexes)
  
-         packages = [os.path.basename(p) for p in self._python_package_dirs]
+         packages = [os.path.basename(p) for p in self.python_package_dirs]
          packages = ['python/%s' % p for p in packages]
          packages = '\n   '.join(sorted(packages))
          data = data.format(indexes=indexes, python_packages=packages)
  
--        with open(os.path.join(self._docs_dir, 'index.rst'), 'wb') as fh:
-+        with open(os.path.join(self._docs_dir, 'index.rst'), 'w') as fh:
+-        with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
++        with open(os.path.join(self.staging_dir, 'index.rst'), 'w') as fh:
              fh.write(data)
  
  
- def distribution_files(root):
-     """Find all files suitable for distributing.
- 
-     Given the path to generated Sphinx documentation, returns an iterable
-     of (path, BaseFile) for files that should be archived, uploaded, etc.
+ manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH)
 diff --git a/tools/docs/moztreedocs/__init__.py.1601578.later b/tools/docs/moztreedocs/__init__.py.1601578.later
 new file mode 100644
 --- /dev/null
 +++ b/tools/docs/moztreedocs/__init__.py.1601578.later
-@@ -0,0 +1,61 @@
+@@ -0,0 +1,27 @@
 +--- __init__.py
 ++++ __init__.py
 +@@ -117,22 +117,22 @@ class _SphinxManager(object):
@@ -770,37 +766,3 @@ new file mode 100644
 +                     if "summary" in post:
 +                         fh.write(post["summary"] + "\n")
 +                     # Write the content
-+@@ -164,17 +164,17 @@ class _SphinxManager(object):
-+                         self._process_markdown(m, source_path, os.path.join(".", target))
-+                     else:
-+                         m.add_link(source_path, target)
-+ 
-+         copier = FileCopier()
-+         m.populate_registry(copier)
-+         copier.copy(self.staging_dir, remove_empty_directories=False)
-+ 
-+-        with open(self.index_path, 'rb') as fh:
-++        with open(self.index_path, 'r') as fh:
-+             data = fh.read()
-+ 
-+         def is_toplevel(key):
-+             """Whether the tree is nested under the toplevel index, or is
-+             nested under another tree's index.
-+             """
-+             for k in self.trees:
-+                 if k == key:
-+@@ -202,13 +202,13 @@ class _SphinxManager(object):
-+         indexes = tuple(set(indexes) - set(cats))
-+         if indexes:
-+             # In case a new doc isn't categorized
-+             print(indexes)
-+             raise Exception("Uncategorized documentation. Please add it in tools/docs/config.yml")
-+ 
-+         data = data.format(**CATEGORIES)
-+ 
-+-        with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
-++        with open(os.path.join(self.staging_dir, 'index.rst'), 'w') as fh:
-+             fh.write(data)
-+ 
-+ 
-+ manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH)

mozilla-release/patches/1606825-73a1.patch

@@ -0,0 +1,152 @@
+# HG changeset patch
+# User James Graham <james@hoppipolla.co.uk>
+# Date 1578073023 0
+# Node ID 89045eb8ecafc07fb4e8f21036aae216d8fc2ce6
+# Parent  1fc419db70541d79aff7ccae14d44fa466488974
+Bug 1606825 - Prompt which VCS to use for gecko clone, r=firefox-build-system-reviewers,rstewart
+
+Make the choice of VCS work via an interactive prompt rather than by
+command line argument, just like we do for all other choices.
+
+Differential Revision: https://phabricator.services.mozilla.com/D58614
+
+diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
+--- a/python/mozboot/bin/bootstrap.py
++++ b/python/mozboot/bin/bootstrap.py
+@@ -141,35 +141,34 @@ def ensure_environment(repo_url=None, re
+             # This should always work.
+             sys.path.append(TEMPDIR)
+             from mozboot.bootstrap import Bootstrapper
+             return Bootstrapper
+ 
+ 
+ def main(args):
+     parser = OptionParser()
+-    parser.add_option('--vcs', dest='vcs',
+-                      default='hg',
+-                      help='VCS (hg or git) to use for downloading the source code. '
+-                      'Uses hg if omitted.')
+     parser.add_option('-r', '--repo-url', dest='repo_url',
+                       default='https://hg.mozilla.org/mozilla-central/',
+                       help='Base URL of source control repository where bootstrap files can '
+                       'be downloaded.')
+     parser.add_option('--repo-rev', dest='repo_rev',
+                       default='default',
+                       help='Revision of files in repository to fetch')
+     parser.add_option('--repo-type', dest='repo_type',
+                       default='hgweb',
+                       help='The type of the repository. This defines how we fetch file '
+                       'content. Like --repo, you should not need to set this.')
+ 
+     parser.add_option('--application-choice', dest='application_choice',
+                       help='Pass in an application choice (see mozboot.bootstrap.APPLICATIONS) '
+                       'instead of using the default interactive prompt.')
++    parser.add_option('--vcs', dest='vcs', default=None,
++                      help='VCS (hg or git) to use for downloading the source code, '
++                      'instead of using the default interactive prompt.')
+     parser.add_option('--no-interactive', dest='no_interactive', action='store_true',
+                       help='Answer yes to any (Y/n) interactive prompts.')
+     parser.add_option('--debug', dest='debug', action='store_true',
+                       help='Print extra runtime information useful for debugging and '
+                       'bug reports.')
+ 
+     options, leftover = parser.parse_args(args)
+ 
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -66,16 +66,24 @@ Your choice: '''
+ 
+ APPLICATIONS = OrderedDict([
+     ('Firefox for Desktop Artifact Mode', 'browser_artifact_mode'),
+     ('Firefox for Desktop', 'browser'),
+     ('Firefox for Android Artifact Mode', 'mobile_android_artifact_mode'),
+     ('Firefox for Android', 'mobile_android'),
+ ])
+ 
++VCS_CHOICE = '''
++Firefox can be cloned using either Git or Mercurial.
++
++Please choose the VCS you want to use:
++1. Mercurial
++2. Git
++Your choice: '''
++
+ STATE_DIR_INFO = '''
+ The Firefox build system and related tools store shared, persistent state
+ in a common directory on the filesystem. On this machine, that directory
+ is:
+ 
+   {statedir}
+ 
+ If you would like to use a different directory, hit CTRL+c and set the
+@@ -393,52 +401,65 @@ class Bootstrapper(object):
+ 
+         # We need to enable the loading of hgrc in case extensions are
+         # required to open the repo.
+         r = current_firefox_checkout(check_output=self.instance.check_output,
+                                      env=self.instance._hg_cleanenv(load_hgrc=True),
+                                      hg=self.instance.which('hg'))
+         (checkout_type, checkout_root) = r
+ 
++        # If we didn't specify a VCS, and we aren't in an exiting clone,
++        # offer a choice
++        if not self.vcs:
++            if checkout_type and False:
++                vcs = checkout_type
++            elif self.instance.no_interactive:
++                vcs = "hg"
++            else:
++                prompt_choice = self.instance.prompt_int(prompt=VCS_CHOICE, low=1, high=2)
++                vcs = ["hg", "git"][prompt_choice - 1]
++        else:
++            vcs = self.vcs
++
+         # Possibly configure Mercurial, but not if the current checkout or repo
+         # type is Git.
+-        if hg_installed and state_dir_available and (checkout_type == 'hg' or self.vcs == 'hg'):
++        if hg_installed and state_dir_available and (checkout_type == 'hg' or vcs == 'hg'):
+             configure_hg = False
+             if not self.instance.no_interactive:
+                 configure_hg = self.instance.prompt_yesno(prompt=CONFIGURE_MERCURIAL)
+             else:
+                 configure_hg = self.hg_configure
+ 
+             if configure_hg:
+                 configure_mercurial(self.instance.which('hg'), state_dir)
+ 
+         # Offer to configure Git, if the current checkout or repo type is Git.
+-        elif self.instance.which('git') and (checkout_type == 'git' or self.vcs == 'git'):
++        elif self.instance.which('git') and (checkout_type == 'git' or vcs == 'git'):
+             should_configure_git = False
+             if not self.instance.no_interactive:
+                 should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT)
+             else:
+                 # Assuming default configuration setting applies to all VCS.
+                 should_configure_git = self.hg_configure
+ 
+             if should_configure_git:
+                 configure_git(self.instance.which('git'), state_dir,
+                               checkout_root)
+ 
+         # Offer to clone if we're not inside a clone.
+         have_clone = False
+ 
+         if checkout_type:
+             have_clone = True
+-        elif hg_installed and not self.instance.no_interactive and self.vcs == 'hg':
++        elif hg_installed and not self.instance.no_interactive and vcs == 'hg':
+             dest = self.input_clone_dest()
+             if dest:
+                 have_clone = hg_clone_firefox(self.instance.which('hg'), dest)
+                 checkout_root = dest
+-        elif self.instance.which('git') and self.vcs == 'git':
++        elif self.instance.which('git') and vcs == 'git':
+             dest = self.input_clone_dest(False)
+             if dest:
+                 git = self.instance.which('git')
+                 watchman = self.instance.which('watchman')
+                 have_clone = git_clone_firefox(git, dest, watchman)
+                 checkout_root = dest
+ 
+         if not have_clone:

mozilla-release/patches/1614518-1-75a1.patch

@@ -2,7 +2,7 @@
 # User Steve Fink <sfink@mozilla.com>
 # Date 1581441405 0
 # Node ID e5f9b1159797eb830acdb6f78be62b69ead5cb01
-# Parent  e0d333acf9dcb7ea5bca9b31c30c77a9e9e89654
+# Parent  b1170ff86d6dfa1d20c04465e35facd4a7a46fba
 Bug 1614518 - Remove code only present for python 2.6 compatibility. r=ahal
 
 Differential Revision: https://phabricator.services.mozilla.com/D62378
@@ -62,11 +62,11 @@ diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootst
  
    {statedir}
  
-@@ -338,25 +330,25 @@ class Bootstrapper(object):
-         self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+@@ -339,25 +331,25 @@ class Bootstrapper(object):
          self.instance.ensure_nasm_packages(state_dir, checkout_root)
          self.instance.ensure_sccache_packages(state_dir, checkout_root)
          self.instance.ensure_lucetc_packages(state_dir, checkout_root)
+         self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
  
      def bootstrap(self):
          if self.choice is None:

mozilla-release/patches/1614518-2-75a1.patch

@@ -2,7 +2,7 @@
 # User Steve Fink <sfink@mozilla.com>
 # Date 1582223590 0
 # Node ID 788e4b59e885b237c83bae1c9a1da6c74e827b41
-# Parent  cfcb7c3237e5a255aa813c37265f553f00141c06
+# Parent  0a09523588fff5bfd25cece73e312ae3e01c090f
 Bug 1614518 - Allow using internal name for selecting application to bootstrap. r=nalexander
 
 Differential Revision: https://phabricator.services.mozilla.com/D62379
@@ -10,11 +10,11 @@ Differential Revision: https://phabricator.services.mozilla.com/D62379
 diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
 --- a/python/mozboot/mozboot/bootstrap.py
 +++ b/python/mozboot/mozboot/bootstrap.py
-@@ -330,25 +330,27 @@ class Bootstrapper(object):
-         self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+@@ -331,25 +331,27 @@ class Bootstrapper(object):
          self.instance.ensure_nasm_packages(state_dir, checkout_root)
          self.instance.ensure_sccache_packages(state_dir, checkout_root)
          self.instance.ensure_lucetc_packages(state_dir, checkout_root)
+         self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
  
      def bootstrap(self):
          if self.choice is None:

mozilla-release/patches/1617095-75a1.patch

@@ -2,7 +2,7 @@
 # User Emilio Cobos Alvarez <emilio@crisal.io>
 # Date 1582294784 0
 # Node ID 05f206719d98841d8b637c66dbdddbe2eadb791e
-# Parent  5931bf1f0e6f702c86e8ba88dddc7188d10aae57
+# Parent  d05b0e8e6687d2eb237172accffc3b806bb7ed14
 Bug 1617095 - Fix mach bootstrap interactive mode with python 3. r=froydnj
 
 Differential Revision: https://phabricator.services.mozilla.com/D63661
@@ -10,8 +10,8 @@ Differential Revision: https://phabricator.services.mozilla.com/D63661
 diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
 --- a/python/mozboot/mozboot/bootstrap.py
 +++ b/python/mozboot/mozboot/bootstrap.py
-@@ -333,17 +333,17 @@ class Bootstrapper(object):
-         self.instance.ensure_lucetc_packages(state_dir, checkout_root)
+@@ -334,17 +334,17 @@ class Bootstrapper(object):
+         self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
  
      def bootstrap(self):
          if self.choice is None:

mozilla-release/patches/1617984-75a1.patch

@@ -0,0 +1,39 @@
+# HG changeset patch
+# User Mohammad H Sekhavat <sekhavat17@gmail.com>
+# Date 1582734804 0
+# Node ID a50491a731c2a18ae6728baff5543a93cf9856aa
+# Parent  1a28609bb40541a14600790c6ed0231619debb9d
+Bug 1617984 - Fix mach bootstrap failure on Archlinux when PKGDEST is set; r=firefox-build-system-reviewers,rstewart
+
+Makepkg has a commented config in /etc/makepkg.conf to set the path for packages to be built:
+```
+#PKGDEST=/home/packages
+```
+When you uncommit this line, packages will be built in the specified path, instead of cwd. This causes the bootstrap script to fail.
+
+This commit will override the PKGDEST config so that package is built in cwd and could be located by the following lines of script.
+
+Differential Revision: https://phabricator.services.mozilla.com/D64195
+
+diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
+--- a/python/mozboot/mozboot/archlinux.py
++++ b/python/mozboot/mozboot/archlinux.py
+@@ -178,16 +178,17 @@ class ArchlinuxBootstrapper(
+ 
+         name = os.path.join(path, name) + '.tar.' + ext
+         command = ['tar', '-x', compression, '-f', name, '-C', path]
+         self.run(command)
+ 
+     def makepkg(self, name):
+         command = ['makepkg', '-s']
+         makepkg_env = os.environ.copy()
++        makepkg_env['PKGDEST'] = '.'
+         makepkg_env['PKGEXT'] = '.pkg.tar.xz'
+         self.run(command, env=makepkg_env)
+         pack = glob.glob(name + '*.pkg.tar.xz')[0]
+         command = ['pacman', '-U']
+         if self.no_interactive:
+             command.append('--noconfirm')
+         command.append(pack)
+         self.run_as_root(command)
+

mozilla-release/patches/1635834-78a1.patch

@@ -0,0 +1,332 @@
+# HG changeset patch
+# User Geoff Brown <gbrown@mozilla.com>
+# Date 1588964983 0
+# Node ID d97b600d3846804f429bc71154e968490f571168
+# Parent  8f58b22269ad1c3d627a57a91c82a9c049676cb0
+Bug 1635834 - Install minidump_stackwalk in 'mach bootstrap'; r=nalexander
+
+Install minidump_stackwalk as part of 'mach bootstrap' so that it is readily available
+for generating crash reports, if desired.
+
+Differential Revision: https://phabricator.services.mozilla.com/D74442
+
+diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
+--- a/python/mozboot/mozboot/archlinux.py
++++ b/python/mozboot/mozboot/archlinux.py
+@@ -9,31 +9,33 @@ import sys
+ import tempfile
+ import subprocess
+ import glob
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
++    MinidumpStackwalkInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
+     WasiSysrootInstall,
+ )
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class ArchlinuxBootstrapper(
+         NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
++        MinidumpStackwalkInstall,
+         LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     '''Archlinux experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf2.13',
+         'base-devel',
+         'nodejs',
+         'python2',
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -309,16 +309,22 @@ class BaseBootstrapper(object):
+ 
+     def ensure_node_packages(self, state_dir, checkout_root):
+         '''
+         Install any necessary packages needed to supply NodeJS'''
+         raise NotImplementedError(
+             '%s does not yet implement ensure_node_packages()'
+             % __name__)
+ 
++    def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root):
++        '''
++        Install minidump_stackwalk.
++        '''
++        pass
++
+     def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):
+         clang_tools_path = os.path.join(state_dir, 'clang-tools')
+         if not os.path.exists(clang_tools_path):
+             os.mkdir(clang_tools_path)
+         self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job)
+ 
+     def install_toolchain_artifact(self, state_dir, checkout_root, toolchain_job):
+         mach_binary = os.path.join(checkout_root, 'mach')
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -324,16 +324,17 @@ class Bootstrapper(object):
+             sys.exit(1)
+ 
+         if not have_clone:
+             print(STYLE_NODEJS_REQUIRES_CLONE)
+             sys.exit(1)
+ 
+         self.instance.state_dir = state_dir
+         self.instance.ensure_node_packages(state_dir, checkout_root)
++        self.instance.ensure_minidump_stackwalk_packages(state_dir, checkout_root)
+         self.instance.ensure_stylo_packages(state_dir, checkout_root)
+         self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+         self.instance.ensure_nasm_packages(state_dir, checkout_root)
+         self.instance.ensure_sccache_packages(state_dir, checkout_root)
+         self.instance.ensure_lucetc_packages(state_dir, checkout_root)
+         self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
+ 
+     def bootstrap(self):
+diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py
+--- a/python/mozboot/mozboot/centosfedora.py
++++ b/python/mozboot/mozboot/centosfedora.py
+@@ -5,26 +5,28 @@
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import platform
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
++    MinidumpStackwalkInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
+     WasiSysrootInstall,
+ )
+ 
+ 
+ class CentOSFedoraBootstrapper(
+         NasmInstall, NodeInstall, StyloInstall, SccacheInstall,
++        MinidumpStackwalkInstall,
+         ClangStaticAnalysisInstall, LucetcInstall, WasiSysrootInstall,
+         BaseBootstrapper):
+     def __init__(self, distro, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.distro = distro
+         self.version = int(version.split('.')[0])
+         self.dist_id = dist_id
+diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
+--- a/python/mozboot/mozboot/debian.py
++++ b/python/mozboot/mozboot/debian.py
+@@ -3,16 +3,17 @@
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
++    MinidumpStackwalkInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
+     WasiSysrootInstall,
+ )
+ 
+ 
+@@ -30,16 +31,17 @@ How would you like to continue?
+   1. Install a modern Mercurial via pip (recommended)
+   2. Install a legacy Mercurial via apt
+   3. Do not install Mercurial
+ Your choice: '''
+ 
+ 
+ class DebianBootstrapper(
+         NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
++        MinidumpStackwalkInstall,
+         SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     # These are common packages for all Debian-derived distros (such as
+     # Ubuntu).
+     COMMON_PACKAGES = [
+         'autoconf2.13',
+         'build-essential',
+         'nodejs',
+         'python-dev',
+diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py
+--- a/python/mozboot/mozboot/gentoo.py
++++ b/python/mozboot/mozboot/gentoo.py
+@@ -3,16 +3,17 @@
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
++    MinidumpStackwalkInstall,
+     NasmInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
+     WasiSysrootInstall,
+ )
+ 
+ try:
+@@ -21,16 +22,17 @@ except ImportError:
+     from urllib.request import urlopen
+ 
+ import re
+ import subprocess
+ 
+ 
+ class GentooBootstrapper(
+         NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
++        MinidumpStackwalkInstall,
+         SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+ 
+     def __init__(self, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.version = version
+         self.dist_id = dist_id
+ 
+diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py
+--- a/python/mozboot/mozboot/linux_common.py
++++ b/python/mozboot/mozboot/linux_common.py
+@@ -99,8 +99,19 @@ class ClangStaticAnalysisInstall(object)
+         if is_non_x86_64():
+             print('Cannot install static analysis tools from taskcluster.\n'
+                   'Please install these tools manually.')
+             return
+ 
+         from mozboot import static_analysis
+         self.install_toolchain_static_analysis(
+             state_dir, checkout_root, static_analysis.LINUX_CLANG_TIDY)
++
++
++class MinidumpStackwalkInstall(object):
++    def __init__(self, **kwargs):
++        pass
++
++    def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root):
++        from mozboot import minidump_stackwalk
++
++        self.install_toolchain_artifact(state_dir, checkout_root,
++                                        minidump_stackwalk.LINUX_MINIDUMP_STACKWALK)
+diff --git a/python/mozboot/mozboot/minidump_stackwalk.py b/python/mozboot/mozboot/minidump_stackwalk.py
+new file mode 100644
+--- /dev/null
++++ b/python/mozboot/mozboot/minidump_stackwalk.py
+@@ -0,0 +1,9 @@
++# This Source Code Form is subject to the terms of the Mozilla Public
++# License, v. 2.0. If a copy of the MPL was not distributed with this
++# file, You can obtain one at http://mozilla.org/MPL/2.0/.
++
++from __future__ import absolute_import, print_function, unicode_literals
++
++LINUX_MINIDUMP_STACKWALK = 'linux64-minidump-stackwalk'
++MACOS_MINIDUMP_STACKWALK = 'macosx64-minidump-stackwalk'
++WINDOWS_MINIDUMP_STACKWALK = 'win32-minidump-stackwalk'
+diff --git a/python/mozboot/mozboot/mozillabuild.py b/python/mozboot/mozboot/mozillabuild.py
+--- a/python/mozboot/mozboot/mozillabuild.py
++++ b/python/mozboot/mozboot/mozillabuild.py
+@@ -96,16 +96,22 @@ class MozillaBuildBootstrapper(BaseBoots
+     def ensure_node_packages(self, state_dir, checkout_root):
+         from mozboot import node
+         # We don't have native aarch64 node available, but aarch64 windows
+         # runs x86 binaries, so just use the x86 packages for such hosts.
+         node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64
+         self.install_toolchain_artifact(
+             state_dir, checkout_root, node_artifact)
+ 
++    def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root):
++        from mozboot import minidump_stackwalk
++
++        self.install_toolchain_artifact(state_dir, checkout_root,
++                                        minidump_stackwalk.WINDOWS_MINIDUMP_STACKWALK)
++
+     def _update_package_manager(self):
+         pass
+ 
+     def run(self, command):
+         subprocess.check_call(command, stdin=sys.stdin)
+ 
+     def pip_install(self, *packages):
+         pip_dir = os.path.join(os.environ['MOZILLABUILD'], 'python', 'Scripts', 'pip.exe')
+diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py
+--- a/python/mozboot/mozboot/osx.py
++++ b/python/mozboot/mozboot/osx.py
+@@ -527,16 +527,22 @@ class OSXBootstrapper(BaseBootstrapper):
+         # installed via ensure_browser_packages
+         pass
+ 
+     def ensure_node_packages(self, state_dir, checkout_root):
+         # XXX from necessary?
+         from mozboot import node
+         self.install_toolchain_artifact(state_dir, checkout_root, node.OSX)
+ 
++    def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root):
++        from mozboot import minidump_stackwalk
++
++        self.install_toolchain_artifact(state_dir, checkout_root,
++                                        minidump_stackwalk.MACOS_MINIDUMP_STACKWALK)
++
+     def install_homebrew(self):
+         print(PACKAGE_MANAGER_INSTALL % ('Homebrew', 'Homebrew', 'Homebrew', 'brew'))
+         bootstrap = urlopen(url=HOMEBREW_BOOTSTRAP, timeout=20).read()
+         with tempfile.NamedTemporaryFile() as tf:
+             tf.write(bootstrap)
+             tf.flush()
+ 
+             subprocess.check_call(['ruby', tf.name])
+diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py
+--- a/python/mozboot/mozboot/solus.py
++++ b/python/mozboot/mozboot/solus.py
+@@ -6,31 +6,33 @@ from __future__ import absolute_import, 
+ 
+ import sys
+ import subprocess
+ 
+ from mozboot.base import BaseBootstrapper
+ from mozboot.linux_common import (
+     ClangStaticAnalysisInstall,
+     LucetcInstall,
++    MinidumpStackwalkInstall,
+     NodeInstall,
+     SccacheInstall,
+     StyloInstall,
+     WasiSysrootInstall,
+ )
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class SolusBootstrapper(
+         NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
++        MinidumpStackwalkInstall,
+         LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
+     '''Solus experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf213',
+         'nodejs',
+         'python',
+         'python3',

mozilla-release/patches/1638012-78a1.patch

@@ -0,0 +1,298 @@
+# HG changeset patch
+# User Nathan Froyd <froydnj@mozilla.com>
+# Date 1589819229 0
+# Node ID 69e7bacf059bcef8befee258f8a0bb631a15c4f9
+# Parent  8647434b0d2a6a1e0b4fc549a846b90832de22df
+Bug 1638012 - commonize taskcluster-related bootstrappers for Linux; r=nalexander
+
+This change doesn't fix all of the boilerplate involved in declaring
+that certain packages should be fetched from taskcluster, but it's a
+start, at least.
+
+Differential Revision: https://phabricator.services.mozilla.com/D75330
+
+diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
+--- a/python/mozboot/mozboot/archlinux.py
++++ b/python/mozboot/mozboot/archlinux.py
+@@ -6,37 +6,28 @@ from __future__ import absolute_import, 
+ 
+ import os
+ import sys
+ import tempfile
+ import subprocess
+ import glob
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    MinidumpStackwalkInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class ArchlinuxBootstrapper(
+-        NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
+-        MinidumpStackwalkInstall,
+-        LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
++        LinuxBootstrapper,
++        BaseBootstrapper):
+     '''Archlinux experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf2.13',
+         'base-devel',
+         'nodejs',
+         'python2',
+         'python2-setuptools',
+diff --git a/python/mozboot/mozboot/centosfedora.py b/python/mozboot/mozboot/centosfedora.py
+--- a/python/mozboot/mozboot/centosfedora.py
++++ b/python/mozboot/mozboot/centosfedora.py
+@@ -2,32 +2,21 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ # You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import platform
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    MinidumpStackwalkInstall,
+-    NasmInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ 
+ class CentOSFedoraBootstrapper(
+-        NasmInstall, NodeInstall, StyloInstall, SccacheInstall,
+-        MinidumpStackwalkInstall,
+-        ClangStaticAnalysisInstall, LucetcInstall, WasiSysrootInstall,
++        LinuxBootstrapper,
+         BaseBootstrapper):
+     def __init__(self, distro, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.distro = distro
+         self.version = int(version.split('.')[0])
+         self.dist_id = dist_id
+ 
+diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
+--- a/python/mozboot/mozboot/debian.py
++++ b/python/mozboot/mozboot/debian.py
+@@ -1,25 +1,16 @@
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    MinidumpStackwalkInstall,
+-    NasmInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ 
+ MERCURIAL_INSTALL_PROMPT = '''
+ Mercurial releases a new version every 3 months and your distro's package
+ may become out of date. This may cause incompatibility with some
+ Mercurial extensions that rely on new Mercurial features. As a result,
+ you may not have an optimal version control experience.
+ 
+@@ -30,19 +21,18 @@ in files being placed in /usr/local/bin 
+ How would you like to continue?
+   1. Install a modern Mercurial via pip (recommended)
+   2. Install a legacy Mercurial via apt
+   3. Do not install Mercurial
+ Your choice: '''
+ 
+ 
+ class DebianBootstrapper(
+-        NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+-        MinidumpStackwalkInstall,
+-        SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
++        LinuxBootstrapper,
++        BaseBootstrapper):
+     # These are common packages for all Debian-derived distros (such as
+     # Ubuntu).
+     COMMON_PACKAGES = [
+         'autoconf2.13',
+         'build-essential',
+         'nodejs',
+         'python-dev',
+         'python-pip',
+diff --git a/python/mozboot/mozboot/gentoo.py b/python/mozboot/mozboot/gentoo.py
+--- a/python/mozboot/mozboot/gentoo.py
++++ b/python/mozboot/mozboot/gentoo.py
+@@ -1,39 +1,29 @@
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    MinidumpStackwalkInstall,
+-    NasmInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ try:
+     from urllib2 import urlopen
+ except ImportError:
+     from urllib.request import urlopen
+ 
+ import re
+ import subprocess
+ 
+ 
+ class GentooBootstrapper(
+-        NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+-        MinidumpStackwalkInstall,
+-        SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
++        LinuxBootstrapper,
++        BaseBootstrapper):
+ 
+     def __init__(self, version, dist_id, **kwargs):
+         BaseBootstrapper.__init__(self, **kwargs)
+ 
+         self.version = version
+         self.dist_id = dist_id
+ 
+     def install_system_packages(self):
+diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py
+--- a/python/mozboot/mozboot/linux_common.py
++++ b/python/mozboot/mozboot/linux_common.py
+@@ -110,8 +110,22 @@ class MinidumpStackwalkInstall(object):
+     def __init__(self, **kwargs):
+         pass
+ 
+     def ensure_minidump_stackwalk_packages(self, state_dir, checkout_root):
+         from mozboot import minidump_stackwalk
+ 
+         self.install_toolchain_artifact(state_dir, checkout_root,
+                                         minidump_stackwalk.LINUX_MINIDUMP_STACKWALK)
++
++
++class LinuxBootstrapper(
++        ClangStaticAnalysisInstall,
++        LucetcInstall,
++        MinidumpStackwalkInstall,
++        NasmInstall,
++        NodeInstall,
++        SccacheInstall,
++        StyloInstall,
++        WasiSysrootInstall):
++
++    def __init__(self, **kwargs):
++        pass
+diff --git a/python/mozboot/mozboot/opensuse.py b/python/mozboot/mozboot/opensuse.py
+--- a/python/mozboot/mozboot/opensuse.py
++++ b/python/mozboot/mozboot/opensuse.py
+@@ -1,29 +1,20 @@
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    NasmInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ 
+ class OpenSUSEBootstrapper(
+-        NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+-        SccacheInstall, LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
++        LinuxBootstrapper, BaseBootstrapper):
+     '''openSUSE experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf213',
+         'nodejs',
+         'npm',
+         'which',
+         'python3-devel',
+diff --git a/python/mozboot/mozboot/solus.py b/python/mozboot/mozboot/solus.py
+--- a/python/mozboot/mozboot/solus.py
++++ b/python/mozboot/mozboot/solus.py
+@@ -3,37 +3,28 @@
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import sys
+ import subprocess
+ 
+ from mozboot.base import BaseBootstrapper
+-from mozboot.linux_common import (
+-    ClangStaticAnalysisInstall,
+-    LucetcInstall,
+-    MinidumpStackwalkInstall,
+-    NodeInstall,
+-    SccacheInstall,
+-    StyloInstall,
+-    WasiSysrootInstall,
+-)
++from mozboot.linux_common import LinuxBootstrapper
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     input = raw_input  # noqa
+ 
+ 
+ class SolusBootstrapper(
+-        NodeInstall, StyloInstall, SccacheInstall, ClangStaticAnalysisInstall,
+-        MinidumpStackwalkInstall,
+-        LucetcInstall, WasiSysrootInstall, BaseBootstrapper):
++        LinuxBootstrapper,
++        BaseBootstrapper):
+     '''Solus experimental bootstrapper.'''
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf213',
+         'nodejs',
+         'python',
+         'python3',
+         'unzip',

mozilla-release/patches/1638799-78a1.patch

@@ -0,0 +1,32 @@
+# HG changeset patch
+# User Corentin Arnould <koalab1999@gmail.com>
+# Date 1589806254 0
+# Node ID aa52e6897c3f2fdab900c89d90e81d15f6462e37
+# Parent  e96739449e390d24ffc0d5c9a26e3aa9dc4921ee
+Bug 1638799 - Added `python-pip` to archlinux's bootstrap. r=froydnj
+
+pip3 is an optional dependency of python on archlinux. It needs to be installed explicitly.
+
+Differential Revision: https://phabricator.services.mozilla.com/D75785
+
+diff --git a/python/mozboot/mozboot/archlinux.py b/python/mozboot/mozboot/archlinux.py
+--- a/python/mozboot/mozboot/archlinux.py
++++ b/python/mozboot/mozboot/archlinux.py
+@@ -36,16 +36,17 @@ class ArchlinuxBootstrapper(
+ 
+     SYSTEM_PACKAGES = [
+         'autoconf2.13',
+         'base-devel',
+         'nodejs',
+         'python2',
+         'python2-setuptools',
+         'python',  # This is Python 3 on Arch.
++        'python-pip',
+         'unzip',
+         'zip',
+     ]
+ 
+     BROWSER_PACKAGES = [
+         'alsa-lib',
+         'dbus-glib',
+         'gtk2',

mozilla-release/patches/1643298-79a1.patch

@@ -0,0 +1,50 @@
+# HG changeset patch
+# User Mitchell Hentges <mhentges@mozilla.com>
+# Date 1591744788 0
+# Node ID 5595d6b0dedd72abe1d69f38d0c4e405002b8394
+# Parent  375e688261ca7851903a2322314c99afb360274d
+Bug 1643298: |mach bootstrap --no-system-changes| should still output mozconfig r=rstewart,geckoview-reviewers,agi
+
+Differential Revision: https://phabricator.services.mozilla.com/D78997
+
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -373,16 +373,17 @@ class Bootstrapper(object):
+                 hg=self.instance.which('hg'))
+             (checkout_type, checkout_root) = r
+             have_clone = bool(checkout_type)
+ 
+             self.maybe_install_private_packages_or_exit(state_dir,
+                                                         state_dir_available,
+                                                         have_clone,
+                                                         checkout_root)
++            self._output_mozconfig(application)
+             sys.exit(0)
+ 
+         self.instance.install_system_packages()
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+         hg_installed, hg_modern = self.instance.ensure_mercurial_modern()
+@@ -449,16 +450,19 @@ class Bootstrapper(object):
+                                                     have_clone,
+                                                     checkout_root)
+ 
+         print(self.finished % name)
+         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
+                 >= MODERN_RUST_VERSION):
+             print("To build %s, please restart the shell (Start a new terminal window)" % name)
+ 
++        self._output_mozconfig(application)
++
++    def _output_mozconfig(self, application):
+         # Like 'generate_browser_mozconfig' or 'generate_mobile_android_mozconfig'.
+         mozconfig = getattr(self.instance, 'generate_%s_mozconfig' % application)()
+ 
+         if mozconfig:
+             mozconfig_path = find_mozconfig(self.mach_context.topdir)
+             if not mozconfig_path:
+                 # No mozconfig file exists yet
+                 mozconfig_path = os.path.join(self.mach_context.topdir, 'mozconfig')

mozilla-release/patches/1643317-79a1.patch

@@ -0,0 +1,75 @@
+# HG changeset patch
+# User Mitchell Hentges <mhentges@mozilla.com>
+# Date 1591806444 0
+# Node ID f37526b7eb67cb4f39839f91b8184607c27cd51a
+# Parent  15f35bd2931db062c1083b767c80ecc272ca1082
+Bug 1643317: |mach bootstrap --help| uses file ref to explain --app-choice r=rstewart
+
+Previously, python import syntax was used to refer users to the definition of the --application-choice arguments.
+However, it isn't straightforward to the uninitiated, especially since it doesn't work in searchfox.
+
+Instead, by providing a file reference, it should be more accessible.
+
+Differential Revision: https://phabricator.services.mozilla.com/D79003
+
+diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
+--- a/python/mozboot/bin/bootstrap.py
++++ b/python/mozboot/bin/bootstrap.py
+@@ -154,18 +154,19 @@ def main(args):
+                       default='default',
+                       help='Revision of files in repository to fetch')
+     parser.add_option('--repo-type', dest='repo_type',
+                       default='hgweb',
+                       help='The type of the repository. This defines how we fetch file '
+                       'content. Like --repo, you should not need to set this.')
+ 
+     parser.add_option('--application-choice', dest='application_choice',
+-                      help='Pass in an application choice (see mozboot.bootstrap.APPLICATIONS) '
+-                      'instead of using the default interactive prompt.')
++                      help='Pass in an application choice (see "APPLICATIONS" in '
++                      'python/mozboot/mozboot/bootstrap.py) instead of using the '
++                      'default interactive prompt.')
+     parser.add_option('--vcs', dest='vcs', default=None,
+                       help='VCS (hg or git) to use for downloading the source code, '
+                       'instead of using the default interactive prompt.')
+     parser.add_option('--no-interactive', dest='no_interactive', action='store_true',
+                       help='Answer yes to any (Y/n) interactive prompts.')
+     parser.add_option('--debug', dest='debug', action='store_true',
+                       help='Print extra runtime information useful for debugging and '
+                       'bug reports.')
+diff --git a/python/mozboot/mozboot/mach_commands.py b/python/mozboot/mozboot/mach_commands.py
+--- a/python/mozboot/mozboot/mach_commands.py
++++ b/python/mozboot/mozboot/mach_commands.py
+@@ -8,28 +8,30 @@ import errno
+ import sys
+ 
+ from mach.decorators import (
+     CommandArgument,
+     CommandProvider,
+     Command,
+ )
+ from mozbuild.base import MachCommandBase
++from mozboot.bootstrap import APPLICATIONS
+ 
+ 
+ @CommandProvider
+ class Bootstrap(MachCommandBase):
+     """Bootstrap system and mach for optimal development experience."""
+ 
+     @Command('bootstrap', category='devenv',
+              description='Install required system packages for building.')
+     @CommandArgument('--application-choice',
++                     choices=list(APPLICATIONS.keys()) + list(APPLICATIONS.values()),
+                      default=None,
+-                     help='Pass in an application choice (see mozboot.bootstrap.APPLICATIONS) '
+-                     'instead of using the default interactive prompt.')
++                     help='Pass in an application choice instead of using the default '
++                     'interactive prompt.')
+     @CommandArgument('--no-interactive', dest='no_interactive', action='store_true',
+                      help='Answer yes to any (Y/n) interactive prompts.')
+     @CommandArgument('--no-system-changes', dest='no_system_changes',
+                      action='store_true',
+                      help='Only execute actions that leave the system '
+                           'configuration alone.')
+     def bootstrap(self, application_choice=None, no_interactive=False, no_system_changes=False):
+         from mozboot.bootstrap import Bootstrapper

mozilla-release/patches/1646406-81a1.patch

@@ -0,0 +1,107 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1596058457 0
+# Node ID 2ae4af0ee490209f1488546b45544e80671e1397
+# Parent  583760c1aa8765e978645afc736fa5e3d1f676eb
+Bug 1646406 - Distinguish between user errors and actual reportable exceptions in `mach` r=mhentges,froydnj
+
+Add a new `UserError` class which when thrown doesn't get reported to Sentry.
+
+Differential Revision: https://phabricator.services.mozilla.com/D85026
+
+diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py
+--- a/python/mach/mach/main.py
++++ b/python/mach/mach/main.py
+@@ -28,17 +28,17 @@ from .base import (
+     UnknownCommandError,
+     UnrecognizedArgumentError,
+     FailedCommandError,
+ )
+ from .config import ConfigSettings
+ from .dispatcher import CommandAction
+ from .logging import LoggingManager
+ from .registrar import Registrar
+-from .util import setenv
++from .util import setenv, UserError
+ 
+ MACH_ERROR = r'''
+ The error occurred in mach itself. This is likely a bug in mach itself or a
+ fundamental problem with a loaded module.
+ 
+ Please consider filing a bug against mach by going to the URL:
+ 
+     https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Mach%20Core
+@@ -47,16 +47,20 @@ Please consider filing a bug against mac
+ 
+ ERROR_FOOTER = r'''
+ If filing a bug, please include the full output of mach, including this error
+ message.
+ 
+ The details of the failure are as follows:
+ '''.lstrip()
+ 
++USER_ERROR = r'''
++This is a user error and does not appear to be a bug in mach.
++'''.lstrip()
++
+ COMMAND_ERROR = r'''
+ The error occurred in the implementation of the invoked mach command.
+ 
+ This should never occur and is likely a bug in the implementation of that
+ command. Consider filing a bug for this issue.
+ '''.lstrip()
+ 
+ MODULE_ERROR = r'''
+@@ -445,16 +449,25 @@ To see more help for a specific command,
+             return Registrar._run_command_handler(
+                 handler, context, debug_command=args.debug_command,
+                 **vars(args.command_args))
+         except KeyboardInterrupt as ki:
+             raise ki
+         except FailedCommandError as e:
+             print(e)
+             return e.exit_code
++        except UserError:
++            # We explicitly don't report UserErrors to Sentry.
++            exc_type, exc_value, exc_tb = sys.exc_info()
++            # The first two frames are us and are never used.
++            stack = traceback.extract_tb(exc_tb)[2:]
++            self._print_error_header(argv, sys.stdout)
++            print(USER_ERROR)
++            self._print_exception(sys.stdout, exc_type, exc_value, stack)
++            return 1
+         except Exception:
+             exc_type, exc_value, exc_tb = sys.exc_info()
+ 
+             # The first two frames are us and are never used.
+             stack = traceback.extract_tb(exc_tb)[2:]
+ 
+             # If we have nothing on the stack, the exception was raised as part
+             # of calling the @Command method itself. This likely means a
+diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py
+--- a/python/mach/mach/util.py
++++ b/python/mach/mach/util.py
+@@ -5,16 +5,23 @@
+ from __future__ import absolute_import, unicode_literals
+ 
+ import os
+ import sys
+ 
+ from six import text_type
+ 
+ 
++class UserError(Exception):
++    """Represents an error caused by something the user did wrong rather than
++    an internal `mach` failure. Exceptions that are subclasses of this class
++    will not be reported as failures to Sentry.
++    """
++
++
+ def setenv(key, value):
+     """Compatibility shim to ensure the proper string type is used with
+     os.environ for the version of Python being used.
+     """
+     encoding = "mbcs" if sys.platform == "win32" else "utf-8"
+ 
+     if sys.version_info[0] == 2:
+         if isinstance(key, text_type):

mozilla-release/patches/1647792-82a1.patch

@@ -0,0 +1,750 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1598985756 0
+# Node ID 8134800276e920f0ae7f83438d13fcea80d63776
+# Parent  6cf8892e0d257a108af8d1045e21a225ab40d23b
+Bug 1647792 - Standalone `bootstrap.py` script clones before running `mach bootstrap` r=mhentges,froydnj,mhoye
+
+Note that when I refer to "standalone `bootstrap.py`" here, I'm referring to the file `python/mozboot/bin/bootstrap.py` and no other similarly-named file in-tree.
+
+The current design, where standalone `bootstrap.py` downloads a small portion of the `mozilla-central` repo and then works through all the `bootstrap` logic, performing a clone in the middle of the `bootstrap` process, has some deficiencies, namely:
+
+1. Refactoring code that is shared between the `bootstrap` logic and the mainline `mach` logic is, if not impossible, more difficult than it needs to be, since standalone `bootstrap.py` needs to download a set of files that bootstraps a build environment perfectly with no other dependencies in `mozilla-central`. As a result we have some [duplicated or redundant code](https://searchfox.org/mozilla-central/rev/c6676771df58c6e0098574bc6b11517acbf264cf/python/mozboot/mozboot/base.py#349) and some stuff that has been [refactored into the `mozboot` directory](https://searchfox.org/mozilla-central/source/python/mozboot/mozboot/util.py) irrespective of whether it actually makes sense to go there (see bug 1649850).
+
+2. Since `mach bootstrap` has access to the entire `mozilla-central` environment, but the (much less frequently exercised) standalone `bootstrap.py` script does *not*, this can lead people to write patches that work fine in `mach bootstrap` but which regress standalone `bootstrap.py`. Furthermore, testing `bootstrap` patches with standalone `bootstrap.py` is difficult. So this is a not infrequent source of regressions; bugs 1652736 and 1643158 are recent examples. Furthermore, typically these regressions are "fixed" by adding more code duplication or by replacing battle-tested frequently-used libraries (either `m-c`-internal or third-party) with less robust bespoke code.
+
+3. Issue (2) is avoidable if people are sufficiently critical during code review, but at *best*, this is a weird extra level of mental overhead that we need to keep in mind only for `bootstrap` patches.
+
+This patch preserves back-compatibility and the validity of existing documentation by factoring out all the logic to clone `mozilla-central` into standalone `bootstrap.py` directly, and cloning *before* calling into `mach bootstrap` directly. This gives us only one official entry point into the bootstrapper, namely, `mach bootstrap`.
+
+There are a couple concrete implications of this change:
+
+1. Now, the clone happens before any other interesting work happens, so people may have to wait ~an hour before actually beginning to engage with the `bootstrap` wizard. While it's arguably slightly less convenient, I'm not sure it matters enough that we should block this patch or a similar one on it.
+
+2. The `hg`/`git` configuration step now happens *after* the clone rather than before it. Looking at what the `hg` and `git` configuration wizards actually do today, I don't think this matters either (all of the configurations can easily happen after cloning the repo).
+
+Another note: `bootstrap` installs `git-cinnabar` to the `.mozbuild` state directory. We could have duplicated all of that logic over to standalone `bootstrap.py`, but it's non-trivial and I didn't think that made any sense, so instead we have standalone `bootstrap.py` download a temporary version and use it for the initial clone if necessary.
+
+Differential Revision: https://phabricator.services.mozilla.com/D85058
+
+diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
+--- a/python/mozboot/bin/bootstrap.py
++++ b/python/mozboot/bin/bootstrap.py
+@@ -1,206 +1,296 @@
+ #!/usr/bin/env python
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ # You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ # This script provides one-line bootstrap support to configure systems to build
+-# the tree.
+-#
+-# The role of this script is to load the Python modules containing actual
+-# bootstrap support. It does this through various means, including fetching
+-# content from the upstream source repository.
++# the tree. It does so by cloning the repo before calling directly into `mach
++# bootstrap`.
++
++# Note that this script can't assume anything in particular about the host
++# Python environment (except that it's run with a sufficiently recent version of
++# Python 3), so we are restricted to stdlib modules.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+-WRONG_PYTHON_VERSION_MESSAGE = '''
+-Bootstrap currently only runs on Python 2.7 or Python 3.5+.
+-Please try re-running with python2.7 or python3.5+.
+-
+-If these aren't available on your system, you may need to install them.
+-Look for a "python2" or "python3.5" package in your package manager.
+-'''
+-
+ import sys
+ 
+ major, minor = sys.version_info[:2]
+-if (major == 2 and minor < 7) or (major == 3 and minor < 5):
+-    print(WRONG_PYTHON_VERSION_MESSAGE)
++if (major < 3) or (major == 3 and minor < 5):
++    print('Bootstrap currently only runs on Python 3.5+.'
++          'Please try re-running with python3.5+.')
+     sys.exit(1)
+ 
+ import os
+ import shutil
++import stat
++import subprocess
+ import tempfile
+ import zipfile
+ 
+-from io import BytesIO
+ from optparse import OptionParser
+-
+-# NOTE: This script is intended to be run with a vanilla Python install.  We
+-# have to rely on the standard library instead of Python 2+3 helpers like
+-# the six module.
+-try:
+-    from urllib2 import urlopen
+-except ImportError:
+-    from urllib.request import urlopen
+-
+-
+-# The next two variables define where in the repository the Python files
+-# reside. This is used to remotely download file content when it isn't
+-# available locally.
+-REPOSITORY_PATH_PREFIX = 'python/mozboot/'
+-
+-TEMPDIR = None
+-
+-
+-def setup_proxy():
+-    # Some Linux environments define ALL_PROXY, which is a SOCKS proxy
+-    # intended for all protocols. Python doesn't currently automatically
+-    # detect this like it does for http_proxy and https_proxy.
+-    if 'ALL_PROXY' in os.environ and 'https_proxy' not in os.environ:
+-        os.environ['https_proxy'] = os.environ['ALL_PROXY']
+-    if 'ALL_PROXY' in os.environ and 'http_proxy' not in os.environ:
+-        os.environ['http_proxy'] = os.environ['ALL_PROXY']
+-
+-
+-def fetch_files(repo_url, repo_rev, repo_type):
+-    setup_proxy()
+-    repo_url = repo_url.rstrip('/')
+-
+-    files = {}
+-
+-    if repo_type == 'hgweb':
+-        url = repo_url + '/archive/%s.zip/python/mozboot' % repo_rev
+-        req = urlopen(url=url, timeout=30)
+-        data = BytesIO(req.read())
+-        data.seek(0)
+-        zip = zipfile.ZipFile(data, 'r')
+-        for f in zip.infolist():
+-            # The paths are prefixed with the repo and revision name before the
+-            # directory name.
+-            offset = f.filename.find(REPOSITORY_PATH_PREFIX) + len(REPOSITORY_PATH_PREFIX)
+-            name = f.filename[offset:]
+-
+-            # We only care about the Python modules.
+-            if not name.startswith('mozboot/'):
+-                continue
+-
+-            files[name] = zip.read(f)
+-
+-        # Retrieve distro script
+-        url = repo_url + '/archive/%s.zip/third_party/python/distro/distro.py' % repo_rev
+-        req = urlopen(url=url, timeout=30)
+-        data = BytesIO(req.read())
+-        data.seek(0)
+-        zip = zipfile.ZipFile(data, 'r')
+-        files["distro.py"] = zip.read(zip.infolist()[0])
+-
++from urllib.request import urlopen
++
++CLONE_MERCURIAL_PULL_FAIL = '''
++Failed to pull from hg.mozilla.org.
++
++This is most likely because of unstable network connection.
++Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually,
++or download a mercurial bundle and use it:
++https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles'''
++
++WINDOWS = sys.platform.startswith('win32') or sys.platform.startswith('msys')
++VCS_HUMAN_READABLE = {
++    'hg': 'Mercurial',
++    'git': 'Git',
++}
++
++
++def which(name):
++    """Python implementation of which.
++
++    It returns the path of an executable or None if it couldn't be found.
++    """
++    # git-cinnabar.exe doesn't exist, but .exe versions of the other executables
++    # do.
++    if WINDOWS and name != 'git-cinnabar':
++        name += '.exe'
++    search_dirs = os.environ['PATH'].split(os.pathsep)
++
++    for path in search_dirs:
++        test = os.path.join(path, name)
++        if os.path.isfile(test) and os.access(test, os.X_OK):
++            return test
++
++    return None
++
++
++def validate_clone_dest(dest):
++    dest = os.path.abspath(dest)
++
++    if not os.path.exists(dest):
++        return dest
++
++    if not os.path.isdir(dest):
++        print('ERROR! Destination %s exists but is not a directory.' % dest)
++        return None
++
++    if not os.listdir(dest):
++        return dest
+     else:
+-        raise NotImplementedError('Not sure how to handle repo type.', repo_type)
+-
+-    return files
+-
+-
+-def ensure_environment(repo_url=None, repo_rev=None, repo_type=None):
+-    """Ensure we can load the Python modules necessary to perform bootstrap."""
+-
++        print('ERROR! Destination directory %s exists but is nonempty.' %
++              dest)
++        return None
++
++
++def input_clone_dest(vcs, no_interactive):
++    repo_name = 'mozilla-unified'
++    print('Cloning into %s using %s...' % (repo_name, VCS_HUMAN_READABLE[vcs]))
++    while True:
++        dest = None
++        if not no_interactive:
++            dest = input('Destination directory for clone (leave empty to use '
++                         'default destination of %s): ' % repo_name).strip()
++        if not dest:
++            dest = repo_name
++        dest = validate_clone_dest(os.path.expanduser(dest))
++        if dest:
++            return dest
++        if no_interactive:
++            return None
++
++
++def hg_clone_firefox(hg, dest):
++    # We create an empty repo then modify the config before adding data.
++    # This is necessary to ensure storage settings are optimally
++    # configured.
++    args = [
++        hg,
++        # The unified repo is generaldelta, so ensure the client is as
++        # well.
++        '--config', 'format.generaldelta=true',
++        'init',
++        dest
++    ]
++    res = subprocess.call(args)
++    if res:
++        print('unable to create destination repo; please try cloning manually')
++        return None
++
++    # Strictly speaking, this could overwrite a config based on a template
++    # the user has installed. Let's pretend this problem doesn't exist
++    # unless someone complains about it.
++    with open(os.path.join(dest, '.hg', 'hgrc'), 'a') as fh:
++        fh.write('[paths]\n')
++        fh.write('default = https://hg.mozilla.org/mozilla-unified\n')
++        fh.write('\n')
++
++        # The server uses aggressivemergedeltas which can blow up delta chain
++        # length. This can cause performance to tank due to delta chains being
++        # too long. Limit the delta chain length to something reasonable
++        # to bound revlog read time.
++        fh.write('[format]\n')
++        fh.write('# This is necessary to keep performance in check\n')
++        fh.write('maxchainlen = 10000\n')
++
++    res = subprocess.call(
++        [hg, 'pull', 'https://hg.mozilla.org/mozilla-unified'], cwd=dest)
++    print('')
++    if res:
++        print(CLONE_MERCURIAL_PULL_FAIL % dest)
++        return None
++
++    print('updating to "central" - the development head of Gecko and Firefox')
++    res = subprocess.call([hg, 'update', '-r', 'central'], cwd=dest)
++    if res:
++        print('error updating; you will need to `cd %s && hg update -r central` '
++              'manually' % dest)
++    return dest
++
++
++def git_clone_firefox(git, dest, watchman):
++    tempdir = None
++    cinnabar = None
++    env = dict(os.environ)
+     try:
+-        from mozboot.bootstrap import Bootstrapper
+-        return Bootstrapper
+-    except ImportError:
+-        # The first fallback is to assume we are running from a tree checkout
+-        # and have the files in a sibling directory.
+-        pardir = os.path.join(os.path.dirname(__file__), os.path.pardir)
+-        include = os.path.normpath(pardir)
+-
+-        sys.path.append(include)
+-        try:
+-            from mozboot.bootstrap import Bootstrapper
+-            return Bootstrapper
+-        except ImportError:
+-            sys.path.pop()
+-
+-            # The next fallback is to download the files from the source
+-            # repository.
+-            files = fetch_files(repo_url, repo_rev, repo_type)
+-
+-            # Install them into a temporary location. They will be deleted
+-            # after this script has finished executing.
+-            global TEMPDIR
+-            TEMPDIR = tempfile.mkdtemp()
+-
+-            for relpath in files.keys():
+-                destpath = os.path.join(TEMPDIR, relpath)
+-                destdir = os.path.dirname(destpath)
+-
+-                if not os.path.exists(destdir):
+-                    os.makedirs(destdir)
+-
+-                with open(destpath, 'wb') as fh:
+-                    fh.write(files[relpath])
+-
+-            # This should always work.
+-            sys.path.append(TEMPDIR)
+-            from mozboot.bootstrap import Bootstrapper
+-            return Bootstrapper
++        cinnabar = which('git-cinnabar')
++        if not cinnabar:
++            cinnabar_url = ('https://github.com/glandium/git-cinnabar/archive/'
++                            'master.zip')
++            # If git-cinnabar isn't installed already, that's fine; we can
++            # download a temporary copy. `mach bootstrap` will clone a full copy
++            # of the repo in the state dir; we don't want to copy all that logic
++            # to this tiny bootstrapping script.
++            tempdir = tempfile.mkdtemp()
++            with open(os.path.join(tempdir, 'git-cinnabar.zip'),
++                      mode='w+b') as archive:
++                with urlopen(cinnabar_url) as repo:
++                    shutil.copyfileobj(repo, archive)
++                archive.seek(0)
++                with zipfile.ZipFile(archive) as zipf:
++                    zipf.extractall(path=tempdir)
++            cinnabar_dir = os.path.join(tempdir, 'git-cinnabar-master')
++            cinnabar = os.path.join(cinnabar_dir, 'git-cinnabar')
++            # Make git-cinnabar and git-remote-hg executable.
++            st = os.stat(cinnabar)
++            os.chmod(cinnabar, st.st_mode | stat.S_IEXEC)
++            st = os.stat(os.path.join(cinnabar_dir, 'git-remote-hg'))
++            os.chmod(os.path.join(cinnabar_dir, 'git-remote-hg'),
++                     st.st_mode | stat.S_IEXEC)
++            env['PATH'] = cinnabar_dir + os.pathsep + env['PATH']
++            subprocess.check_call(['git', 'cinnabar', 'download'],
++                                  cwd=cinnabar_dir, env=env)
++            print('WARNING! git-cinnabar is required for Firefox development  '
++                  'with git. After the clone is complete, the bootstrapper '
++                  'will ask if you would like to configure git; answer yes, '
++                  'and be sure to add git-cinnabar to your PATH according to '
++                  'the bootstrapper output.')
++
++        # We're guaranteed to have `git-cinnabar` installed now.
++        # Configure git per the git-cinnabar requirements.
++        subprocess.check_call(
++            [git, 'clone', '-b', 'bookmarks/central',
++             'hg::https://hg.mozilla.org/mozilla-unified', dest], env=env)
++        subprocess.check_call([git, 'config', 'fetch.prune', 'true'], cwd=dest,
++                              env=env)
++        subprocess.check_call([git, 'config', 'pull.ff', 'only'], cwd=dest,
++                              env=env)
++
++        watchman_sample = os.path.join(
++            dest, '.git/hooks/fsmonitor-watchman.sample')
++        # Older versions of git didn't include fsmonitor-watchman.sample.
++        if watchman and os.path.exists(watchman_sample):
++            print('Configuring watchman')
++            watchman_config = os.path.join(dest, '.git/hooks/query-watchman')
++            if not os.path.exists(watchman_config):
++                print('Copying %s to %s' % (watchman_sample, watchman_config))
++                copy_args = ['cp', '.git/hooks/fsmonitor-watchman.sample',
++                             '.git/hooks/query-watchman']
++                subprocess.check_call(copy_args, cwd=dest)
++
++            config_args = [git, 'config', 'core.fsmonitor',
++                           '.git/hooks/query-watchman']
++            subprocess.check_call(config_args, cwd=dest, env=env)
++        return dest
++    finally:
++        if not cinnabar:
++            print('Failed to install git-cinnabar. Try performing a manual '
++                  'installation: https://github.com/glandium/git-cinnabar/wiki/'
++                  'Mozilla:-A-git-workflow-for-Gecko-development')
++        if tempdir:
++            shutil.rmtree(tempdir)
++
++
++def clone(vcs, no_interactive):
++    hg = which('hg')
++    if not hg:
++        print('Mercurial is not installed. Mercurial is required to clone '
++              'Firefox%s.' % (
++                  ', even when cloning with Git' if vcs == 'git' else ''))
++        print('Try installing hg with `pip3 install Mercurial`.')
++        return None
++    if vcs == 'hg':
++        binary = hg
++    else:
++        binary = which(vcs)
++        if not binary:
++            print('Git is not installed.')
++            print('Try installing git using your system package manager.')
++            return None
++
++    dest = input_clone_dest(vcs, no_interactive)
++    if not dest:
++        return None
++
++    print('Cloning Firefox %s repository to %s' % (VCS_HUMAN_READABLE[vcs],
++                                                   dest))
++    if vcs == 'hg':
++        return hg_clone_firefox(binary, dest)
++    else:
++        watchman = which('watchman')
++        return git_clone_firefox(binary, dest, watchman)
++
++
++def bootstrap(srcdir, application_choice, no_interactive, no_system_changes):
++    args = [sys.executable, os.path.join(srcdir, 'mach'), 'bootstrap']
++    if application_choice:
++        args += ['--application-choice', application_choice]
++    if no_interactive:
++        args += ['--no-interactive']
++    if no_system_changes:
++        args += ['--no-system-changes']
++    print('Running `%s`' % ' '.join(args))
++    return subprocess.call(args, cwd=srcdir)
+ 
+ 
+ def main(args):
+     parser = OptionParser()
+-    parser.add_option('-r', '--repo-url', dest='repo_url',
+-                      default='https://hg.mozilla.org/mozilla-central/',
+-                      help='Base URL of source control repository where bootstrap files can '
+-                      'be downloaded.')
+-    parser.add_option('--repo-rev', dest='repo_rev',
+-                      default='default',
+-                      help='Revision of files in repository to fetch')
+-    parser.add_option('--repo-type', dest='repo_type',
+-                      default='hgweb',
+-                      help='The type of the repository. This defines how we fetch file '
+-                      'content. Like --repo, you should not need to set this.')
+-
+     parser.add_option('--application-choice', dest='application_choice',
+                       help='Pass in an application choice (see "APPLICATIONS" in '
+                       'python/mozboot/mozboot/bootstrap.py) instead of using the '
+                       'default interactive prompt.')
+-    parser.add_option('--vcs', dest='vcs', default=None,
++    parser.add_option('--vcs', dest='vcs', default='hg', choices=['git', 'hg'],
+                       help='VCS (hg or git) to use for downloading the source code, '
+                       'instead of using the default interactive prompt.')
+     parser.add_option('--no-interactive', dest='no_interactive', action='store_true',
+                       help='Answer yes to any (Y/n) interactive prompts.')
+-    parser.add_option('--debug', dest='debug', action='store_true',
+-                      help='Print extra runtime information useful for debugging and '
+-                      'bug reports.')
+     parser.add_option('--no-system-changes', dest='no_system_changes', action='store_true',
+                       help='Only executes actions that leave the system '
+                       'configuration alone.')
+ 
+     options, leftover = parser.parse_args(args)
+ 
+     try:
+-        try:
+-            cls = ensure_environment(options.repo_url, options.repo_rev,
+-                                     options.repo_type)
+-        except Exception as e:
+-            print('Could not load the bootstrap Python environment.\n')
+-            print('This should never happen. Consider filing a bug.\n')
+-            print('\n')
+-
+-            if options.debug:
+-                # Raise full tracebacks during debugging and for bug reporting.
+-                raise
+-
+-            print(e)
++        srcdir = clone(options.vcs, options.no_interactive)
++        if not srcdir:
+             return 1
+-
+-        dasboot = cls(choice=options.application_choice, no_interactive=options.no_interactive,
+-                      vcs=options.vcs, no_system_changes=options.no_system_changes)
+-        dasboot.bootstrap()
+-
+-        return 0
+-    finally:
+-        if TEMPDIR is not None:
+-            shutil.rmtree(TEMPDIR)
++        print('Clone complete.')
++        return bootstrap(srcdir, options.application_choice,
++                         options.no_interactive, options.no_system_changes)
++    except Exception:
++        print('Could not bootstrap Firefox! Consider filing a bug.')
++        raise
+ 
+ 
+ if __name__ == '__main__':
+     sys.exit(main(sys.argv))
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -140,49 +140,16 @@ optimally configured?'''
+ 
+ CONFIGURE_GIT = '''
+ Mozilla recommends using git-cinnabar to work with mozilla-central (or
+ mozilla-unified).
+ 
+ Would you like to run a few configuration steps to ensure Git is
+ optimally configured?'''
+ 
+-CLONE_VCS = '''
+-If you would like to clone the {} {} repository, please
+-enter the destination path below.
+-'''
+-
+-CLONE_VCS_PROMPT = '''
+-Destination directory for {} clone (leave empty to not clone): '''.lstrip()
+-
+-CLONE_VCS_NOT_EMPTY = '''\
+-Destination directory '{}' is not empty.
+-
+-Would you like to clone to '{}' instead?
+-  1. Yes
+-  2. No, let me enter another path
+-  3. No, stop cloning
+-Your choice: '''
+-
+-CLONE_VCS_NOT_EMPTY_FALLBACK_FAILED = '''
+-ERROR! Destination directory '{}' is not empty and '{}' exists.
+-'''
+-
+-CLONE_VCS_NOT_DIR = '''
+-ERROR! Destination '{}' exists but is not a directory.
+-'''
+-
+-CLONE_MERCURIAL_PULL_FAIL = '''
+-Failed to pull from hg.mozilla.org.
+-
+-This is most likely because of unstable network connection.
+-Try running `hg pull https://hg.mozilla.org/mozilla-unified` manually, or
+-download mercurial bundle and use it:
+-https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles'''
+-
+ DEBIAN_DISTROS = (
+     'debian',
+     'ubuntu',
+     'linuxmint',
+     'elementary',
+     'neon',
+ )
+ 
+@@ -266,53 +233,16 @@ class Bootstrapper(object):
+                 cls = WindowsBootstrapper
+ 
+         if cls is None:
+             raise NotImplementedError('Bootstrap support is not yet available '
+                                       'for your OS.')
+ 
+         self.instance = cls(**args)
+ 
+-    def input_clone_dest(self, with_hg=True):
+-        repo_name = 'mozilla-unified'
+-        vcs = 'Mercurial'
+-        if not with_hg:
+-            vcs = 'Git'
+-        print(CLONE_VCS.format(repo_name, vcs))
+-
+-        while True:
+-            dest = input(CLONE_VCS_PROMPT.format(vcs))
+-            dest = dest.strip()
+-            if not dest:
+-                return ''
+-
+-            dest = os.path.expanduser(dest)
+-            if not os.path.exists(dest):
+-                return dest
+-
+-            if not os.path.isdir(dest):
+-                print(CLONE_VCS_NOT_DIR.format(dest))
+-                continue
+-
+-            if os.listdir(dest) == []:
+-                return dest
+-
+-            newdest = os.path.join(dest, repo_name)
+-            if os.path.exists(newdest):
+-                print(CLONE_VCS_NOT_EMPTY_FALLBACK_FAILED.format(dest, newdest))
+-                continue
+-
+-            choice = self.instance.prompt_int(prompt=CLONE_VCS_NOT_EMPTY.format(dest,
+-                                              newdest), low=1, high=3)
+-            if choice == 1:
+-                return newdest
+-            if choice == 2:
+-                continue
+-            return ''
+-
+     # The state directory code is largely duplicated from mach_bootstrap.py.
+     # We can't easily import mach_bootstrap.py because the bootstrapper may
+     # run in self-contained mode and only the files in this directory will
+     # be available. We /could/ refactor parts of mach_bootstrap.py to be
+     # part of this directory to avoid the code duplication.
+     def try_to_create_state_dir(self):
+         state_dir = get_state_dir()
+ 
+@@ -441,37 +371,16 @@ class Bootstrapper(object):
+             else:
+                 # Assuming default configuration setting applies to all VCS.
+                 should_configure_git = self.hg_configure
+ 
+             if should_configure_git:
+                 configure_git(self.instance.which('git'), state_dir,
+                               checkout_root)
+ 
+-        # Offer to clone if we're not inside a clone.
+-        have_clone = False
+-
+-        if checkout_type:
+-            have_clone = True
+-        elif hg_installed and not self.instance.no_interactive and vcs == 'hg':
+-            dest = self.input_clone_dest()
+-            if dest:
+-                have_clone = hg_clone_firefox(self.instance.which('hg'), dest)
+-                checkout_root = dest
+-        elif self.instance.which('git') and vcs == 'git':
+-            dest = self.input_clone_dest(False)
+-            if dest:
+-                git = self.instance.which('git')
+-                watchman = self.instance.which('watchman')
+-                have_clone = git_clone_firefox(git, dest, watchman)
+-                checkout_root = dest
+-
+-        if not have_clone:
+-            print(SOURCE_ADVERTISE)
+-
+         self.maybe_install_private_packages_or_exit(state_dir,
+                                                     state_dir_available,
+                                                     have_clone,
+                                                     checkout_root)
+         self.instance.ensure_mach_environment(checkout_root)
+ 
+         print(self.finished % name)
+         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
+@@ -561,67 +470,16 @@ def update_mercurial_repo(hg, url, dest,
+ 
+     try:
+         subprocess.check_call(pull_args, cwd=cwd)
+         subprocess.check_call(update_args, cwd=dest)
+     finally:
+         print('=' * 80)
+ 
+ 
+-def hg_clone_firefox(hg, dest):
+-    """Clone the Firefox repository to a specified destination."""
+-    print('Cloning Firefox Mercurial repository to %s' % dest)
+-
+-    # We create an empty repo then modify the config before adding data.
+-    # This is necessary to ensure storage settings are optimally
+-    # configured.
+-    args = [
+-        hg,
+-        # The unified repo is generaldelta, so ensure the client is as
+-        # well.
+-        '--config', 'format.generaldelta=true',
+-        'init',
+-        dest
+-    ]
+-    res = subprocess.call(args)
+-    if res:
+-        print('unable to create destination repo; please try cloning manually')
+-        return False
+-
+-    # Strictly speaking, this could overwrite a config based on a template
+-    # the user has installed. Let's pretend this problem doesn't exist
+-    # unless someone complains about it.
+-    with open(os.path.join(dest, '.hg', 'hgrc'), 'ab') as fh:
+-        fh.write('[paths]\n')
+-        fh.write('default = https://hg.mozilla.org/mozilla-unified\n')
+-        fh.write('\n')
+-
+-        # The server uses aggressivemergedeltas which can blow up delta chain
+-        # length. This can cause performance to tank due to delta chains being
+-        # too long. Limit the delta chain length to something reasonable
+-        # to bound revlog read time.
+-        fh.write('[format]\n')
+-        fh.write('# This is necessary to keep performance in check\n')
+-        fh.write('maxchainlen = 10000\n')
+-
+-    res = subprocess.call([hg, 'pull', 'https://hg.mozilla.org/mozilla-unified'], cwd=dest)
+-    print('')
+-    if res:
+-        print(CLONE_MERCURIAL_PULL_FAIL)
+-        return False
+-
+-    print('updating to "central" - the development head of Gecko and Firefox')
+-    res = subprocess.call([hg, 'update', '-r', 'central'], cwd=dest)
+-    if res:
+-        print('error updating; you will need to `hg update` manually')
+-
+-    print('Firefox source code available at %s' % dest)
+-    return True
+-
+-
+ def current_firefox_checkout(check_output, env, hg=None):
+     """Determine whether we're in a Firefox checkout.
+ 
+     Returns one of None, ``git``, or ``hg``.
+     """
+     HG_ROOT_REVISIONS = set([
+         # From mozilla-unified.
+         '8ba995b74e18334ab3707f27e9eb8f4e37ba3d29',
+@@ -702,47 +560,8 @@ def update_git_repo(git, url, dest):
+         print('=' * 80)
+ 
+ 
+ def configure_git(git, root_state_dir, top_src_dir):
+     """Run the Git configuration steps."""
+     cinnabar_dir = update_git_tools(git, root_state_dir, top_src_dir)
+ 
+     print(ADD_GIT_TOOLS_PATH.format(cinnabar_dir))
+-
+-
+-def git_clone_firefox(git, dest, watchman=None):
+-    """Clone the Firefox repository to a specified destination."""
+-    print('Cloning Firefox repository to %s' % dest)
+-
+-    try:
+-        # Configure git per the git-cinnabar requirements.
+-        subprocess.check_call([git, 'clone', '-b', 'bookmarks/central',
+-                               'hg::https://hg.mozilla.org/mozilla-unified', dest])
+-        subprocess.check_call([git, 'remote', 'add', 'inbound',
+-                               'hg::ssh://hg.mozilla.org/integration/mozilla-inbound'],
+-                              cwd=dest)
+-        subprocess.check_call([git, 'config', 'remote.inbound.skipDefaultUpdate',
+-                               'true'], cwd=dest)
+-        subprocess.check_call([git, 'config', 'remote.inbound.push',
+-                               '+HEAD:refs/heads/branches/default/tip'], cwd=dest)
+-        subprocess.check_call([git, 'config', 'fetch.prune', 'true'], cwd=dest)
+-        subprocess.check_call([git, 'config', 'pull.ff', 'only'], cwd=dest)
+-
+-        watchman_sample = os.path.join(dest, '.git/hooks/fsmonitor-watchman.sample')
+-        # Older versions of git didn't include fsmonitor-watchman.sample.
+-        if watchman and watchman_sample:
+-            print('Configuring watchman')
+-            watchman_config = os.path.join(dest, '.git/hooks/query-watchman')
+-            if not os.path.exists(watchman_config):
+-                print('Copying %s to %s' % (watchman_sample, watchman_config))
+-                copy_args = ['cp', '.git/hooks/fsmonitor-watchman.sample',
+-                             '.git/hooks/query-watchman']
+-                subprocess.check_call(copy_args, cwd=dest)
+-
+-            config_args = [git, 'config', 'core.fsmonitor', '.git/hooks/query-watchman']
+-            subprocess.check_call(config_args, cwd=dest)
+-    except Exception as e:
+-        print(e)
+-        return False
+-
+-    print('Firefox source code available at %s' % dest)
+-    return True

mozilla-release/patches/1654663-80a1.patch

@@ -0,0 +1,81 @@
+# HG changeset patch
+# User Mitchell Hentges <mhentges@mozilla.com>
+# Date 1595606749 0
+# Node ID aae7491870ddb91b333d2a90b3dce3c60d84c3e9
+# Parent  55a7d1ffde71dcf1068670ee035d2cf06daad7da
+Bug 1654663: Removes glean_parser from virtualenv_packages.txt r=firefox-build-system-reviewers,rstewart
+
+In preparation for Glean telemetry, we scope the availability of the out-of-date vendored
+"glean_parser" library to its one usage: "run_glean_parser.py".
+
+This allows Glean telemetry to load its modern "glean_parser" dependency from the
+"--user" package environment.
+
+Differential Revision: https://phabricator.services.mozilla.com/D84610
+
+diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt
+--- a/build/virtualenv_packages.txt
++++ b/build/virtualenv_packages.txt
+@@ -21,17 +21,16 @@ mozilla.pth:third_party/python/diskcache
+ mozilla.pth:third_party/python/distro
+ mozilla.pth:third_party/python/dlmanager
+ mozilla.pth:third_party/python/ecdsa/src
+ python2:mozilla.pth:third_party/python/enum34
+ mozilla.pth:third_party/python/fluent.migrate
+ mozilla.pth:third_party/python/fluent.syntax
+ mozilla.pth:third_party/python/funcsigs
+ python2:mozilla.pth:third_party/python/futures
+-python3:mozilla.pth:third_party/python/glean_parser
+ mozilla.pth:third_party/python/importlib_metadata
+ mozilla.pth:third_party/python/iso8601
+ mozilla.pth:third_party/python/Jinja2/src
+ mozilla.pth:third_party/python/jsonschema
+ mozilla.pth:third_party/python/MarkupSafe/src
+ mozilla.pth:third_party/python/more-itertools
+ mozilla.pth:third_party/python/packaging
+ mozilla.pth:third_party/python/pathlib2
+diff --git a/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1654663.later b/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1654663.later
+new file mode 100644
+--- /dev/null
++++ b/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1654663.later
+@@ -0,0 +1,40 @@
++--- run_glean_parser.py
+++++ run_glean_parser.py
++@@ -1,17 +1,35 @@
++ # This Source Code Form is subject to the terms of the Mozilla Public
++ # License, v. 2.0. If a copy of the MPL was not distributed with this
++ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
++ 
++ import sys
++-
++-from glean_parser import lint
++ from pathlib import Path
++ 
++ 
++ def main(output, *filenames):
+++    # Unlike most vendored packages, we don't want "glean_parser" in
+++    # "build/virtualenv_packages.txt" because it interferes with mach's usage of Glean.
+++    # This is because Glean ("glean_sdk"):
+++    # * Has native code (so, can't be vendored)
+++    # * Also depends on "glean_parser"
+++    # Since this file has different version constraints than mach on "glean_parser",
+++    # we want the two different consumers to own their own separate "glean_parser"
+++    # packages.
+++    #
+++    # This is solved by:
+++    # * Having mach import Glean (and transitively "glean_parser") via the "--user"
+++    #   package environment. To accomplish this, the vendored "glean_parser" is removed
+++    #   from "virtualenv_packages.txt".
+++    # * Having this script import "glean_parser" from the vendored location. This is
+++    #   done by manually adding it to the pythonpath.
+++
+++    srcdir = Path(__file__).joinpath('../../../../../')
+++    glean_parser_path = srcdir.joinpath('third_party/python/glean_parser')
+++    sys.path.insert(0, str(glean_parser_path.resolve()))
+++    from glean_parser import lint
++     if lint.glinter([Path(x) for x in filenames], {"allow_reserved": False}):
++         sys.exit(1)
++ 
++ 
++ if __name__ == '__main__':
++     main(sys.stdout, *sys.argv[1:])

mozilla-release/patches/1655781-81a1.patch

@@ -0,0 +1,140 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1595960995 0
+# Node ID 3cf9282868b948b34759eb4924cffd18ba1ec7e7
+# Parent  11d1b0cf2db06a56d821b7d35d196b37b66af384
+Bug 1655781- Allow specifying that certain packages should only be included in a `virtualenv` when parsing `virtualenv_packages.txt` r=chutten,nalexander
+
+This solves the same problem we attempted to solve in bug 1654663. That was a low-cost, sensible solution when there was only one in-build reference to `glean_parser`, but with project FOG we're about to drastically increase the in-build reliance on the library, so the ad-hoc `sys.path` manipulation is an increasingly insensible solution. Here we address this in a first-class way by specifying that `glean_parser` should be imported in `virtualenv`s, but NOT by top-level `mach` commands that run outside of an in-`objdir` `virtualenv`.
+
+Differential Revision: https://phabricator.services.mozilla.com/D85182
+
+diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py
+--- a/build/mach_bootstrap.py
++++ b/build/mach_bootstrap.py
+@@ -112,16 +112,19 @@ CATEGORIES = {
+ }
+ 
+ 
+ def search_path(mozilla_dir, packages_txt):
+     with open(os.path.join(mozilla_dir, packages_txt)) as f:
+         packages = [line.rstrip().split(':') for line in f]
+ 
+     def handle_package(package):
++        if package[0] == 'in-virtualenv':
++            return
++
+         if package[0] == 'optional':
+             try:
+                 for path in handle_package(package[1:]):
+                     yield path
+             except Exception:
+                 pass
+ 
+         if package[0] in ('windows', '!windows'):
+diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt
+--- a/build/virtualenv_packages.txt
++++ b/build/virtualenv_packages.txt
+@@ -21,16 +21,17 @@ mozilla.pth:third_party/python/diskcache
+ mozilla.pth:third_party/python/distro
+ mozilla.pth:third_party/python/dlmanager
+ mozilla.pth:third_party/python/ecdsa/src
+ python2:mozilla.pth:third_party/python/enum34
+ mozilla.pth:third_party/python/fluent.migrate
+ mozilla.pth:third_party/python/fluent.syntax
+ mozilla.pth:third_party/python/funcsigs
+ python2:mozilla.pth:third_party/python/futures
++in-virtualenv:python3:mozilla.pth:third_party/python/glean_parser
+ mozilla.pth:third_party/python/importlib_metadata
+ mozilla.pth:third_party/python/iso8601
+ mozilla.pth:third_party/python/Jinja2/src
+ mozilla.pth:third_party/python/jsonschema
+ mozilla.pth:third_party/python/MarkupSafe/src
+ mozilla.pth:third_party/python/more-itertools
+ mozilla.pth:third_party/python/packaging
+ mozilla.pth:third_party/python/pathlib2
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -272,27 +272,36 @@ class VirtualenvManager(object):
+             on non-Windows systems.
+ 
+         python3 -- This denotes that the action should only be taken when run
+             on Python 3.
+ 
+         python2 -- This denotes that the action should only be taken when run
+             on python 2.
+ 
++        in-virtualenv -- This denotes that the action should only be taken when
++            constructing a virtualenv (and not when bootstrapping a `mach`
++            action).
++
+         Note that the Python interpreter running this function should be the
+         one from the virtualenv. If it is the system Python or if the
+         environment is not configured properly, packages could be installed
+         into the wrong place. This is how virtualenv's work.
+         """
+         import distutils.sysconfig
+ 
+         packages = self.packages()
+         python_lib = distutils.sysconfig.get_python_lib()
+ 
+         def handle_package(package):
++            if package[0] == 'in-virtualenv':
++                assert len(package) >= 2
++                package = package[1:]
++                # Continue processing normally.
++
+             if package[0] == 'setup.py':
+                 assert len(package) >= 2
+ 
+                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
+                                 package[2:])
+ 
+                 return True
+ 
+diff --git a/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1655781.later b/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1655781.later
+new file mode 100644
+--- /dev/null
++++ b/toolkit/components/telemetry/build_scripts/run_glean_parser.py.1655781.later
+@@ -0,0 +1,40 @@
++--- run_glean_parser.py
+++++ run_glean_parser.py
++@@ -1,35 +1,17 @@
++ # This Source Code Form is subject to the terms of the Mozilla Public
++ # License, v. 2.0. If a copy of the MPL was not distributed with this
++ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
++ 
++ import sys
+++
+++from glean_parser import lint
++ from pathlib import Path
++ 
++ 
++ def main(output, *filenames):
++-    # Unlike most vendored packages, we don't want "glean_parser" in
++-    # "build/virtualenv_packages.txt" because it interferes with mach's usage of Glean.
++-    # This is because Glean ("glean_sdk"):
++-    # * Has native code (so, can't be vendored)
++-    # * Also depends on "glean_parser"
++-    # Since this file has different version constraints than mach on "glean_parser",
++-    # we want the two different consumers to own their own separate "glean_parser"
++-    # packages.
++-    #
++-    # This is solved by:
++-    # * Having mach import Glean (and transitively "glean_parser") via the "--user"
++-    #   package environment. To accomplish this, the vendored "glean_parser" is removed
++-    #   from "virtualenv_packages.txt".
++-    # * Having this script import "glean_parser" from the vendored location. This is
++-    #   done by manually adding it to the pythonpath.
++-
++-    srcdir = Path(__file__).joinpath('../../../../../')
++-    glean_parser_path = srcdir.joinpath('third_party/python/glean_parser')
++-    sys.path.insert(0, str(glean_parser_path.resolve()))
++-    from glean_parser import lint
++     if lint.glinter([Path(x) for x in filenames], {"allow_reserved": False}):
++         sys.exit(1)
++ 
++ 
++ if __name__ == '__main__':
++     main(sys.stdout, *sys.argv[1:])

mozilla-release/patches/1656611-81a1.patch

@@ -2,7 +2,7 @@
 # User Ricky Stewart <rstewart@mozilla.com>
 # Date 1596816216 0
 # Node ID 44bb58dd097180de8024107f4149e1e9c0cb5adc
-# Parent  581adf3ce88acdda658270e1a00e3df034e88041
+# Parent  45ddb4a356a0247bec19fa81fe0d757a0d5c295d
 Bug 1656611 - Remove `objdir` support from `virtualenv_packages.txt` handling r=mhentges,froydnj
 
 I noticed that the `objdir:build` entry in `build/virtualenv_packages.txt` entry was apparently unused. This originates from bug 841713, seven years ago, when the `objdir` handling was introduced. Today, this doesn't appear to be serving a purpose. There is no Python library in my `$objdir/build` directory; nor can I see anything in `build/moz.build` or any related files suggesting one could ever appear. I can only assume this feature has outlived its usefulness, so delete it and the relevant in-tree support.
@@ -137,14 +137,14 @@ diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
 diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py
 --- a/python/mozbuild/mozbuild/sphinx.py
 +++ b/python/mozbuild/mozbuild/sphinx.py
-@@ -187,14 +187,13 @@ def setup(app):
- 
-     app.srcdir = os.path.join(app.outdir, '_staging')
+@@ -182,14 +182,13 @@ def setup(app):
+     # documentation.
+     manager.generate_docs(app)
+     app.srcdir = manager.staging_dir
  
      # We need to adjust sys.path in order for Python API docs to get generated
      # properly. We leverage the in-tree virtualenv for this.
-     from mozbuild.virtualenv import VirtualenvManager
- 
+     topsrcdir = manager.topsrcdir
      ve = VirtualenvManager(topsrcdir,
 -                           os.path.join(topsrcdir, 'dummy-objdir'),
                             os.path.join(app.outdir, '_venv'),

mozilla-release/patches/1656993-81a1.patch

@@ -0,0 +1,1375 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1597684862 0
+# Node ID eff0a199fae6727caebd03b687824a398fe132ba
+# Parent  3cdb6781a7297a2bb9cdf3573854e09f233a36fe
+Bug 1656993: Create and require by default global `virtualenv`s in `~/.mozbuild` for `mach` r=mhentges,ahal
+
+In two different places we've been encountering issues regarding 1) how we configure the system Python environment and 2) how the system Python environment relates to the `virtualenv`s that we use for building, testing, and other dev tasks. Specifically:
+
+1. With the push to use `glean` for telemetry in `mach`, we are requiring (or rather, strongly encouraging) the `glean_sdk` Python package to be installed with bug 1651424. `mach bootstrap` upgrades the library using your system Python 3 in bug 1654607. We can't vendor it due to the package containing native code. Since we generally vendor all code required for `mach` to function, requiring that the system Python be configured with a certain version of `glean` is an unfortunate change.
+
+2. The build uses the vendored `glean_parser` for a number of build tasks. Since the vendored `glean_parser` conflicts with the globally-installed `glean_sdk` package, we had to add special ad-hoc handling to allow us to circumvent this conflict in bug 1655781.
+
+3. We begin to rely more and more on the `zstandard` package during build tasks, this package again being one that we can't vendor due to containing native code. Bug 1654994 contained more ad-hoc code which subprocesses out from the build system's `virtualenv` to the SYSTEM `python3` binary, assuming that the system `python3` has `zstandard` installed.
+
+As we rely more on `glean_sdk`, `zstandard`, and other packages that are not vendorable, we need to settle on a standard model for how `mach`, the build process, and other `mach` commands that may make their own `virtualenv`s work in the presence of unvendorable packages.
+
+With that in mind, this patch does all the following:
+
+1. Separate out the `mach` `virtualenv_packages` from the in-build `virtualenv_packages`. Refactor the common stuff into `common_virtualenv_packages.txt`. Add functionality to the `virtualenv_packages` manifest parsing to allow the build `virtualenv` to "inherit" from the parent by pointing to the parent's `site-packages`. The `in-virtualenv` feature from bug 1655781 is no longer necessary, so delete it.
+
+2. Add code to `bootstrap`, as well as a new `mach` command `create-mach-environment` to create `virtualenv`s in `~/.mozbuild`.
+
+3. Add code to `mach` to dispatch either to the in-`~/.mozbuild` `virtualenv`s (or to the system Python 3 for commands which cannot run in the `virtualenv`s, namely `bootstrap` and `create-mach-environment`).
+
+4. Remove the "add global argument" feature from `mach`. It isn't used and conflicts with (3).
+
+5. Remove the `--print-command` feature from `mach` which is obsoleted by these changes.
+
+This has the effect of allowing us to install packages that cannot be vendored into a "common" place (namely the global `~/.mozbuild` `virtualenv`s) and use those from the build without requiring us to hit the network. Miscellaneous implementation notes:
+
+1. We allow users to force running `mach` with the system Python if they like. For now it doesn't make any sense to require 100% of people to create these `virtualenv`s when they're allowed to continue on with the old behavior if they like. We also skip this in CI.
+
+2. We needed to duplicate the global-argument logic into the `mach` script to allow for the dispatch behavior. This is something we avoided with the Python 2 -> Python 3 migration with the `--print-command` feature, justifying its use by saying it was only temporarily required until all `mach` commands were running with Python 3. With this change, we'll need to be able to determine the `mach` command from the shell script for the forseeable future, and committing to this forever with the cost that `--print-command` incurs (namely `mach` startup time, an additional .4s on my machine) didn't seem worth it to me. It's not a ton of duplicated code.
+
+Differential Revision: https://phabricator.services.mozilla.com/D85916
+
+diff --git a/build/build_virtualenv_packages.txt b/build/build_virtualenv_packages.txt
+new file mode 100644
+--- /dev/null
++++ b/build/build_virtualenv_packages.txt
+@@ -0,0 +1,4 @@
++inherit-from-parent-environment
++packages.txt:build/common_virtualenv_packages.txt
++python3:mozilla.pth:third_party/python/glean_parser
++set-variable MOZBUILD_VIRTUALENV=1
+diff --git a/build/virtualenv_packages.txt b/build/common_virtualenv_packages.txt
+rename from build/virtualenv_packages.txt
+rename to build/common_virtualenv_packages.txt
+--- a/build/virtualenv_packages.txt
++++ b/build/common_virtualenv_packages.txt
+@@ -21,17 +21,16 @@ mozilla.pth:third_party/python/diskcache
+ mozilla.pth:third_party/python/distro
+ mozilla.pth:third_party/python/dlmanager
+ mozilla.pth:third_party/python/ecdsa/src
+ python2:mozilla.pth:third_party/python/enum34
+ mozilla.pth:third_party/python/fluent.migrate
+ mozilla.pth:third_party/python/fluent.syntax
+ mozilla.pth:third_party/python/funcsigs
+ python2:mozilla.pth:third_party/python/futures
+-in-virtualenv:python3:mozilla.pth:third_party/python/glean_parser
+ mozilla.pth:third_party/python/importlib_metadata
+ mozilla.pth:third_party/python/iso8601
+ mozilla.pth:third_party/python/Jinja2/src
+ mozilla.pth:third_party/python/jsonschema
+ mozilla.pth:third_party/python/MarkupSafe/src
+ mozilla.pth:third_party/python/more-itertools
+ mozilla.pth:third_party/python/packaging
+ mozilla.pth:third_party/python/pathlib2
+diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py
+--- a/build/mach_bootstrap.py
++++ b/build/mach_bootstrap.py
+@@ -112,19 +112,16 @@ CATEGORIES = {
+ }
+ 
+ 
+ def search_path(mozilla_dir, packages_txt):
+     with open(os.path.join(mozilla_dir, packages_txt)) as f:
+         packages = [line.rstrip().split(':') for line in f]
+ 
+     def handle_package(package):
+-        if package[0] == 'in-virtualenv':
+-            return
+-
+         if package[0] == 'optional':
+             try:
+                 for path in handle_package(package[1:]):
+                     yield path
+             except Exception:
+                 pass
+ 
+         if package[0] in ('windows', '!windows'):
+@@ -171,19 +168,20 @@ def bootstrap(topsrcdir, mozilla_dir=Non
+     # Global build system and mach state is stored in a central directory. By
+     # default, this is ~/.mozbuild. However, it can be defined via an
+     # environment variable. We detect first run (by lack of this directory
+     # existing) and notify the user that it will be created. The logic for
+     # creation is much simpler for the "advanced" environment variable use
+     # case. For default behavior, we educate users and give them an opportunity
+     # to react. We always exit after creating the directory because users don't
+     # like surprises.
+-    sys.path[0:0] = [os.path.join(mozilla_dir, path)
+-                     for path in search_path(mozilla_dir,
+-                                             'build/virtualenv_packages.txt')]
++    sys.path[0:0] = [
++        os.path.join(mozilla_dir, path)
++        for path in search_path(mozilla_dir,
++                                'build/mach_virtualenv_packages.txt')]
+     import mach.base
+     import mach.main
+     from mach.util import setenv
+     from mozboot.util import get_state_dir
+ 
+     # Set a reasonable limit to the number of open files.
+     #
+     # Some linux systems set `ulimit -n` to a very high number, which works
+diff --git a/build/mach_virtualenv_packages.txt b/build/mach_virtualenv_packages.txt
+new file mode 100644
+--- /dev/null
++++ b/build/mach_virtualenv_packages.txt
+@@ -0,0 +1,2 @@
++packages.txt:build/common_virtualenv_packages.txt
++set-variable MACH_VIRTUALENV=1
+diff --git a/build/moz.configure/init.configure b/build/moz.configure/init.configure
+--- a/build/moz.configure/init.configure
++++ b/build/moz.configure/init.configure
+@@ -298,27 +298,30 @@ def virtualenv_python3(env_python, build
+     if topobjdir.endswith('/js/src'):
+         topobjdir = topobjdir[:-7]
+ 
+     virtualenvs_root = os.path.join(topobjdir, '_virtualenvs')
+     with LineIO(lambda l: log.info(l), 'replace') as out:
+         manager = VirtualenvManager(
+             topsrcdir,
+             os.path.join(virtualenvs_root, 'init_py3'), out,
+-            os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
++            os.path.join(topsrcdir, 'build', 'build_virtualenv_packages.txt'))
+ 
+     # If we're not in the virtualenv, we need to update the path to include some
+     # necessary modules for find_program.
+-    if normsep(sys.executable) != normsep(manager.python_path):
++    try:
++        import mozfile
++    except ImportError:
+         sys.path.insert(
+             0, os.path.join(topsrcdir, 'testing', 'mozbase', 'mozfile'))
+         sys.path.insert(
+             0, os.path.join(topsrcdir, 'third_party', 'python', 'backports'))
+     else:
+-        python = sys.executable
++        if 'MOZBUILD_VIRTUALENV' in os.environ or not python:
++            python = sys.executable
+ 
+     # If we know the Python executable the caller is asking for then verify its
+     # version. If the caller did not ask for a specific executable then find
+     # a reasonable default.
+     if python:
+         found_python = find_program(python)
+         if not found_python:
+             die('The PYTHON3 environment variable does not contain '
+diff --git a/build/sparse-profiles/mach.1656993.later b/build/sparse-profiles/mach.1656993.later
+new file mode 100644
+--- /dev/null
++++ b/build/sparse-profiles/mach.1656993.later
+@@ -0,0 +1,23 @@
++--- mach
+++++ mach
++@@ -2,17 +2,19 @@
++ # Various mach commands call config.guess to resolve the default objdir name.
++ path:build/autoconf/config.guess
++ path:build/autoconf/config.sub
++ path:build/moz.configure/checks.configure
++ path:build/moz.configure/init.configure
++ path:build/moz.configure/util.configure
++ # Used for bootstrapping the mach driver.
++ path:build/mach_bootstrap.py
++-path:build/virtualenv_packages.txt
+++path:build/build_virtualenv_packages.txt
+++path:build/common_virtualenv_packages.txt
+++path:build/mach_virtualenv_packages.txt
++ path:mach
++ # Various dependencies. There is room to trim fat, especially in
++ # third_party/python.
++ path:python/
++ path:testing/mozbase/
++ path:third_party/python/
++ # certifi is needed for Sentry
++ path:testing/web-platform/tests/tools/third_party/certifi
+diff --git a/mach b/mach
+--- a/mach
++++ b/mach
+@@ -5,16 +5,17 @@
+ 
+ # The beginning of this script is both valid POSIX shell and valid Python,
+ # such that the script starts with the shell and is reexecuted with
+ # the right Python.
+ 
+ # Embeds a shell script inside a Python triple quote. This pattern is valid
+ # shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
+ ''':'
++# Commands that are to be run with Python 2.
+ py2commands="
+     android
+     awsy-test
+     browsertime
+     check-spidermonkey
+     cramtest
+     crashtest
+     devtools-css-db
+@@ -57,78 +58,122 @@ py2commands="
+     wpt-metadata-summary
+     wpt-serve
+     wpt-test-paths
+     wpt-unittest
+     wpt-update
+     xpcshell-test
+ "
+ 
++# Commands that are to be run with the system Python 3 instead of the
++# virtualenv.
++nativecmds="
++    bootstrap
++    create-mach-environment
++"
++
+ run_py() {
+-    # Try to run a specific Python interpreter. Fall back to the system
+-    # default Python if the specific interpreter couldn't be found.
++    # Try to run a specific Python interpreter.
+     py_executable="$1"
+     shift
+     if which "$py_executable" > /dev/null
+     then
+         exec "$py_executable" "$0" "$@"
+-    elif [ "$py_executable" = "python2.7" ]; then
+-        exec python "$0" "$@"
+     else
+         echo "This mach command requires $py_executable, which wasn't found on the system!"
++        case "$py_executable" in
++            python2.7|python3) ;;
++            *)
++                echo "Consider running 'mach bootstrap' or 'mach create-mach-environment' to create the mach virtualenvs, or set MACH_USE_SYSTEM_PYTHON to use the system Python installation over a virtualenv."
++                ;;
++        esac
+         exit 1
+     fi
+ }
+ 
+-first_arg=$1
+-if [ "$first_arg" = "help" ]; then
+-    # When running `./mach help <command>`, the correct Python for <command>
+-    # needs to be used.
+-    first_arg=$2
+-elif [ "$first_arg" = "mach-completion" ]; then
+-    # When running `./mach mach-completion /path/to/mach <command>`, the
+-    # correct Python for <command> needs to be used.
+-    first_arg=$3
++get_command() {
++    # Parse the name of the mach command out of the arguments. This is necessary
++    # in the presence of global mach arguments that come before the name of the
++    # command, e.g. `mach -v build`. We dispatch to the correct Python
++    # interpreter depending on the command.
++    while true; do
++    case $1 in
++        -v|--verbose) shift;;
++        -l|--log-file)
++            if [ "$#" -lt 2 ]
++            then
++                echo
++                break
++            else
++                shift 2
++            fi
++            ;;
++        --log-interval) shift;;
++        --log-no-times) shift;;
++        -h) shift;;
++        --debug-command) shift;;
++        --settings)
++            if [ "$#" -lt 2 ]
++            then
++                echo
++                break
++            else
++                shift 2
++            fi
++            ;;
++        # When running `./mach help <command>`, the correct Python for <command>
++        # needs to be used.
++        help) echo $2; break;;
++        # When running `./mach mach-completion /path/to/mach <command>`, the
++        # correct Python for <command> needs to be used.
++        mach-completion) echo $3; break;;
++        "") echo; break;;
++        *) echo $1; break;;
++    esac
++    done
++}
++
++state_dir=${MOZBUILD_STATE_PATH:-~/.mozbuild}
++command=$(get_command "$@")
++
++# If MACH_USE_SYSTEM_PYTHON or MOZ_AUTOMATION are set, always use the
++# python{2.7,3} executables and not the virtualenv locations.
++if [ -z ${MACH_USE_SYSTEM_PYTHON} ] && [ -z ${MOZ_AUTOMATION} ]
++then
++    case "$OSTYPE" in
++        cygwin|msys|win32) bin_path=Scripts;;
++        *) bin_path=bin;;
++    esac
++    py2executable=$state_dir/_virtualenvs/mach_py2/$bin_path/python
++    py3executable=$state_dir/_virtualenvs/mach/$bin_path/python
++else
++    py2executable=python2.7
++    py3executable=python3
+ fi
+ 
+-if [ -z "$first_arg" ]; then
+-    # User ran `./mach` or `./mach help`, use Python 3.
+-    run_py python3 "$@"
+-fi
+-
+-case "${first_arg}" in
+-    "-"*)
+-        # We have global arguments which are tricky to parse from this shell
+-        # script. So invoke `mach` with a special --print-command argument to
+-        # return the name of the command. This adds extra overhead when using
+-        # global arguments, but global arguments are an edge case and this hack
+-        # is only needed temporarily for the Python 3 migration. We use Python
+-        # 2.7 because using Python 3 hits this error in build tasks:
+-        # https://searchfox.org/mozilla-central/rev/c7e8bc4996f9/build/moz.configure/init.configure#319
+-        command=`run_py python2.7 --print-command "$@" | tail -n1`
+-        ;;
+-    *)
+-        # In the common case, the first argument is the command.
+-        command=${first_arg};
++# Check whether we need to run with the native Python 3 interpreter.
++case " $(echo $nativecmds) " in
++    *\ $command\ *)
++        run_py python3 "$@"
+         ;;
+ esac
+ 
+ # Check for the mach subcommand in the Python 2 commands list and run it
+ # with the correct interpreter.
+ case " $(echo $py2commands) " in
+     *\ $command\ *)
+-        run_py python2.7 "$@"
++        run_py "$py2executable" "$@"
+         ;;
+     *)
+-        run_py python3 "$@"
++        run_py "$py3executable" "$@"
+         ;;
+ esac
+ 
+ # Run Python 3 for everything else.
+-run_py python3 "$@"
++run_py "$py3executable" "$@"
+ '''
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import os
+ import sys
+ 
+ def ancestors(path):
+diff --git a/moz.configure b/moz.configure
+--- a/moz.configure
++++ b/moz.configure
+@@ -655,17 +655,19 @@ def config_status_deps(build_env, build_
+         os.path.join(topsrcdir, 'configure'),
+         os.path.join(topsrcdir, 'js', 'src', 'configure'),
+         os.path.join(topsrcdir, 'configure.in'),
+         os.path.join(topsrcdir, 'js', 'src', 'configure.in'),
+         os.path.join(topsrcdir, 'nsprpub', 'configure'),
+         os.path.join(topsrcdir, 'config', 'milestone.txt'),
+         os.path.join(topsrcdir, 'browser', 'config', 'version.txt'),
+         os.path.join(topsrcdir, 'browser', 'config', 'version_display.txt'),
+-        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'),
++        os.path.join(topsrcdir, 'build', 'build_virtualenv_packages.txt'),
++        os.path.join(topsrcdir, 'build', 'common_virtualenv_packages.txt'),
++        os.path.join(topsrcdir, 'build', 'mach_virtualenv_packages.txt'),
+         os.path.join(topsrcdir, 'python', 'mozbuild', 'mozbuild', 'virtualenv.py'),
+         os.path.join(topsrcdir, 'testing', 'mozbase', 'packages.txt'),
+         os.path.join(topsrcdir, 'aclocal.m4'),
+         os.path.join(topsrcdir, 'old-configure.in'),
+         os.path.join(topsrcdir, 'js', 'src', 'aclocal.m4'),
+         os.path.join(topsrcdir, 'js', 'src', 'old-configure.in'),
+     ] + glob.glob(os.path.join(topsrcdir, 'build', 'autoconf', '*.m4'))
+ 
+diff --git a/python/mach/docs/driver.rst b/python/mach/docs/driver.rst
+--- a/python/mach/docs/driver.rst
++++ b/python/mach/docs/driver.rst
+@@ -25,27 +25,8 @@ providers. e.g.:
+ See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
+ for more information on creating an entry point. To search for entry
+ point plugins, you can call
+ :py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.:
+ 
+ .. code-block:: python
+ 
+    mach.load_commands_from_entry_point("mach.external.providers")
+-
+-Adding Global Arguments
+-=======================
+-
+-Arguments to mach commands are usually command-specific. However,
+-mach ships with a handful of global arguments that apply to all
+-commands.
+-
+-It is possible to extend the list of global arguments. In your
+-*mach driver*, simply call
+-:py:meth:`mach.main.Mach.add_global_argument`. e.g.:
+-
+-.. code-block:: python
+-
+-   mach = mach.main.Mach(os.getcwd())
+-
+-   # Will allow --example to be specified on every mach command.
+-   mach.add_global_argument('--example', action='store_true',
+-       help='Demonstrate an example global argument.')
+diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py
+--- a/python/mach/mach/main.py
++++ b/python/mach/mach/main.py
+@@ -209,27 +209,18 @@ To see more help for a specific command,
+         self.logger = logging.getLogger(__name__)
+         self.settings = ConfigSettings()
+         self.settings_paths = []
+ 
+         if 'MACHRC' in os.environ:
+             self.settings_paths.append(os.environ['MACHRC'])
+ 
+         self.log_manager.register_structured_logger(self.logger)
+-        self.global_arguments = []
+         self.populate_context_handler = None
+ 
+-    def add_global_argument(self, *args, **kwargs):
+-        """Register a global argument with the argument parser.
+-
+-        Arguments are proxied to ArgumentParser.add_argument()
+-        """
+-
+-        self.global_arguments.append((args, kwargs))
+-
+     def load_commands_from_directory(self, path):
+         """Scan for mach commands from modules in a directory.
+ 
+         This takes a path to a directory, loads the .py files in it, and
+         registers and found mach command providers with this mach instance.
+         """
+         for f in sorted(os.listdir(path)):
+             if not f.endswith('.py') or f == '__init__.py':
+@@ -399,28 +390,17 @@ To see more help for a specific command,
+             # We don't register the usage until here because if it is globally
+             # registered, argparse always prints it. This is not desired when
+             # running with --help.
+             parser.usage = Mach.USAGE
+             parser.print_usage()
+             return 0
+ 
+         try:
+-            try:
+-                args = parser.parse_args(argv)
+-            except NoCommandError as e:
+-                if e.namespace.print_command:
+-                    context.get_command = True
+-                    args = parser.parse_args(e.namespace.print_command)
+-                    if args.command == 'mach-completion':
+-                        args = parser.parse_args(e.namespace.print_command[2:])
+-                    print(args.command)
+-                    return 0
+-                else:
+-                    raise
++            args = parser.parse_args(argv)
+         except NoCommandError:
+             print(NO_COMMAND_ERROR)
+             return 1
+         except UnknownCommandError as e:
+             suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (
+                 e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else ''
+             print(UNKNOWN_COMMAND_ERROR %
+                   (e.verb, e.command, suggestion_message))
+@@ -567,16 +547,18 @@ To see more help for a specific command,
+ 
+     def get_argument_parser(self, context):
+         """Returns an argument parser for the command-line interface."""
+ 
+         parser = ArgumentParser(add_help=False,
+                                 usage='%(prog)s [global arguments] '
+                                 'command [command arguments]')
+ 
++        # WARNING!!! If you add a global argument here, also add it to the
++        # global argument handling in the top-level `mach` script.
+         # Order is important here as it dictates the order the auto-generated
+         # help messages are printed.
+         global_group = parser.add_argument_group('Global Arguments')
+ 
+         global_group.add_argument('-v', '--verbose', dest='verbose',
+                                   action='store_true', default=False,
+                                   help='Print verbose output.')
+         global_group.add_argument('-l', '--log-file', dest='logfile',
+@@ -598,20 +580,15 @@ To see more help for a specific command,
+         global_group.add_argument('-h', '--help', dest='help',
+                                   action='store_true', default=False,
+                                   help='Show this help message.')
+         global_group.add_argument('--debug-command', action='store_true',
+                                   help='Start a Python debugger when command is dispatched.')
+         global_group.add_argument('--settings', dest='settings_file',
+                                   metavar='FILENAME', default=None,
+                                   help='Path to settings file.')
+-        global_group.add_argument('--print-command', nargs=argparse.REMAINDER,
+-                                  help=argparse.SUPPRESS)
+-
+-        for args, kwargs in self.global_arguments:
+-            global_group.add_argument(*args, **kwargs)
+ 
+         # We need to be last because CommandAction swallows all remaining
+         # arguments and argparse parses arguments in the order they were added.
+         parser.add_argument('command', action=CommandAction,
+                             registrar=Registrar, context=context)
+ 
+         return parser
+diff --git a/python/mach/mach/test/test_commands.py b/python/mach/mach/test/test_commands.py
+--- a/python/mach/mach/test/test_commands.py
++++ b/python/mach/mach/test/test_commands.py
+@@ -51,30 +51,11 @@ class TestCommands(TestBase):
+         # 'cmd_f' as a prefix, the completion script will handle this case
+         # properly.
+         assert stdout == self.format(self.all_commands)
+ 
+         result, stdout, stderr = self._run_mach(['mach-completion', 'cmd_foo'])
+         assert result == 0
+         assert stdout == self.format(['help', '--arg'])
+ 
+-    def test_print_command(self):
+-        result, stdout, stderr = self._run_mach(['--print-command', 'cmd_foo', '-flag'])
+-        assert result == 0
+-        assert stdout == 'cmd_foo\n'
+-
+-        result, stdout, stderr = self._run_mach(['--print-command', '-v', 'cmd_foo', '-flag'])
+-        assert result == 0
+-        assert stdout == 'cmd_foo\n'
+-
+-        result, stdout, stderr = self._run_mach(
+-            ['--print-command', 'mach-completion', 'mach', 'cmd_foo', '-flag'])
+-        assert result == 0
+-        assert stdout == 'cmd_foo\n'
+-
+-        result, stdout, stderr = self._run_mach(
+-            ['--print-command', 'mach-completion', 'mach', '-v', 'cmd_foo', '-flag'])
+-        assert result == 0
+-        assert stdout == 'cmd_foo\n'
+-
+ 
+ if __name__ == '__main__':
+     main()
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -256,16 +256,24 @@ class BaseBootstrapper(object):
+ 
+         Firefox for Android Artifact Mode needs an application and an ABI set,
+         and it needs paths to the Android SDK.
+         '''
+         raise NotImplementedError(
+             '%s does not yet implement generate_mobile_android_artifact_mode_mozconfig()'
+             % __name__)
+ 
++    def ensure_mach_environment(self, checkout_root):
++        if checkout_root:
++            mach_binary = os.path.abspath(os.path.join(checkout_root, 'mach'))
++            if not os.path.exists(mach_binary):
++                raise ValueError('mach not found at %s' % mach_binary)
++            cmd = [sys.executable, mach_binary, 'create-mach-environment']
++            subprocess.check_call(cmd, cwd=checkout_root)
++
+     def ensure_clang_static_analysis_package(self, state_dir, checkout_root):
+         '''
+         Install the clang static analysis package
+         '''
+         raise NotImplementedError(
+             '%s does not yet implement ensure_clang_static_analysis_package()'
+             % __name__)
+ 
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -373,16 +373,17 @@ class Bootstrapper(object):
+                 hg=self.instance.which('hg'))
+             (checkout_type, checkout_root) = r
+             have_clone = bool(checkout_type)
+ 
+             self.maybe_install_private_packages_or_exit(state_dir,
+                                                         state_dir_available,
+                                                         have_clone,
+                                                         checkout_root)
++            self.instance.ensure_mach_environment(checkout_root)
+             self._output_mozconfig(application)
+             sys.exit(0)
+ 
+         self.instance.install_system_packages()
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+@@ -444,16 +445,17 @@ class Bootstrapper(object):
+ 
+         if not have_clone:
+             print(SOURCE_ADVERTISE)
+ 
+         self.maybe_install_private_packages_or_exit(state_dir,
+                                                     state_dir_available,
+                                                     have_clone,
+                                                     checkout_root)
++        self.instance.ensure_mach_environment(checkout_root)
+ 
+         print(self.finished % name)
+         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
+                 >= MODERN_RUST_VERSION):
+             print("To build %s, please restart the shell (Start a new terminal window)" % name)
+ 
+         self._output_mozconfig(application)
+ 
+diff --git a/python/mozboot/mozboot/bootstrap.py.1656933.later b/python/mozboot/mozboot/bootstrap.py.1656933.later
+new file mode 100644
+--- /dev/null
++++ b/python/mozboot/mozboot/bootstrap.py.1656933.later
+@@ -0,0 +1,82 @@
++--- bootstrap.py
+++++ bootstrap.py
++@@ -12,18 +12,16 @@ import re
++ import sys
++ import subprocess
++ import time
++ from distutils.version import LooseVersion
++ 
++ # NOTE: This script is intended to be run with a vanilla Python install.  We
++ # have to rely on the standard library instead of Python 2+3 helpers like
++ # the six module.
++-from subprocess import CalledProcessError
++-
++ if sys.version_info < (3,):
++     from ConfigParser import (
++         Error as ConfigParserError,
++         RawConfigParser,
++     )
++     input = raw_input  # noqa
++ else:
++     from configparser import (
++@@ -568,21 +567,16 @@ class Bootstrapper(object):
++                 git = self.instance.which('git')
++                 watchman = self.instance.which('watchman')
++                 have_clone = git_clone_firefox(git, dest, watchman)
++                 checkout_root = dest
++ 
++         if not have_clone:
++             print(SOURCE_ADVERTISE)
++ 
++-        if state_dir_available:
++-            is_telemetry_enabled = self.check_telemetry_opt_in(state_dir)
++-            if is_telemetry_enabled:
++-                _install_glean()
++-
++         self.maybe_install_private_packages_or_exit(state_dir,
++                                                     state_dir_available,
++                                                     have_clone,
++                                                     checkout_root)
++         self.instance.ensure_mach_environment(checkout_root)
++ 
++         print(self.finished % name)
++         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
++@@ -877,38 +872,16 @@ def git_clone_firefox(git, dest, watchma
++     except Exception as e:
++         print(e)
++         return False
++ 
++     print('Firefox source code available at %s' % dest)
++     return True
++ 
++ 
++-def _install_glean():
++-    """Installs glean to the current python environment.
++-
++-    If the current python instance is a virtualenv, then glean is installed
++-    directly.
++-    If not, then glean is installed to the Python user install directory.
++-    Upgrades glean if it's out-of-date.
++-    """
++-    pip_call = [sys.executable, '-m', 'pip', 'install', 'glean_sdk~=31.5.0']
++-    if not os.environ.get('VIRTUAL_ENV'):
++-        # If the user is already using a virtual environment before they invoked
++-        # `mach bootstrap`, then we shouldn't add the "--user" flag. This is because
++-        # virtual environments don't support the flags since they don't have a
++-        # separate "user install directory".
++-        pip_call.append('--user')
++-
++-    try:
++-        subprocess.check_output(pip_call)
++-    except CalledProcessError:
++-        print("Failed to install glean, telemetry will not be gathered")
++-
++-
++ def _warn_if_risky_revision(path):
++     # Warn the user if they're trying to bootstrap from an obviously old
++     # version of tree as reported by the version control system (a month in
++     # this case). This is an approximate calculation but is probably good
++     # enough for our purposes.
++     NUM_SECONDS_IN_MONTH = 60 * 60 * 24 * 30
++     from mozversioncontrol import get_repository_object
++     repo = get_repository_object(path)
+diff --git a/python/mozbuild/mozbuild/action/test_archive.py.1656993.later b/python/mozbuild/mozbuild/action/test_archive.py.1656993.later
+new file mode 100644
+--- /dev/null
++++ b/python/mozbuild/mozbuild/action/test_archive.py.1656993.later
+@@ -0,0 +1,29 @@
++--- test_archive.py
+++++ test_archive.py
++@@ -501,17 +501,25 @@ ARCHIVE_FILES = {
++             'pattern': 'python/**'
++         },
++         {
++             'source': buildconfig.topsrcdir,
++             'pattern': 'build/mach_bootstrap.py'
++         },
++         {
++             'source': buildconfig.topsrcdir,
++-            'pattern': 'build/virtualenv_packages.txt'
+++            'pattern': 'build/build_virtualenv_packages.txt'
+++        },
+++        {
+++            'source': buildconfig.topsrcdir,
+++            'pattern': 'build/common_virtualenv_packages.txt'
+++        },
+++        {
+++            'source': buildconfig.topsrcdir,
+++            'pattern': 'build/mach_virtualenv_packages.txt'
++         },
++         {
++             'source': buildconfig.topsrcdir,
++             'pattern': 'mach/**'
++         },
++         {
++             'source': buildconfig.topsrcdir,
++             'pattern': 'testing/web-platform/tests/tools/third_party/certifi/**'
+diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
+--- a/python/mozbuild/mozbuild/base.py
++++ b/python/mozbuild/mozbuild/base.py
+@@ -264,17 +264,18 @@ class MozbuildObject(ProcessExecutionMix
+         from .virtualenv import VirtualenvManager
+ 
+         if self._virtualenv_manager is None:
+             self._virtualenv_manager = VirtualenvManager(
+                 self.topsrcdir,
+                 os.path.join(self.topobjdir, '_virtualenvs',
+                              self._virtualenv_name),
+                 sys.stdout,
+-                os.path.join(self.topsrcdir, 'build', 'virtualenv_packages.txt')
++                os.path.join(self.topsrcdir, 'build',
++                             'build_virtualenv_packages.txt')
+                 )
+ 
+         return self._virtualenv_manager
+ 
+     @staticmethod
+     @memoize
+     def get_mozconfig_and_target(topsrcdir, path, env_mozconfig):
+         # env_mozconfig is only useful for unittests, which change the value of
+diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py
+--- a/python/mozbuild/mozbuild/configure/__init__.py
++++ b/python/mozbuild/mozbuild/configure/__init__.py
+@@ -290,17 +290,17 @@ class ConfigureSandbox(dict):
+ 
+     # The default set of builtins. We expose unicode as str to make sandboxed
+     # files more python3-ready.
+     BUILTINS = ReadOnlyDict({
+         b: getattr(__builtin__, b, None)
+         for b in ('None', 'False', 'True', 'int', 'bool', 'any', 'all', 'len',
+                   'list', 'tuple', 'set', 'dict', 'isinstance', 'getattr',
+                   'hasattr', 'enumerate', 'range', 'zip', 'AssertionError',
+-                  '__build_class__',  # will be None on py2
++                  'ImportError', '__build_class__',  # will be None on py2
+                   )
+     }, __import__=forbidden_import, str=six.text_type)
+ 
+     # Expose a limited set of functions from os.path
+     OS = ReadOnlyNamespace(path=ReadOnlyNamespace(**{
+         k: getattr(mozpath, k, getattr(os.path, k))
+         for k in ('abspath', 'basename', 'dirname', 'isabs', 'join',
+                   'normcase', 'normpath', 'realpath', 'relpath')
+diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py
+--- a/python/mozbuild/mozbuild/controller/building.py
++++ b/python/mozbuild/mozbuild/controller/building.py
+@@ -1580,16 +1580,22 @@ class BuildDriver(MozbuildObject):
+ 
+         if mozconfig_make_lines:
+             self.log(logging.WARNING, 'mozconfig_content', {
+                 'path': mozconfig['path'],
+                 'content': '\n    '.join(mozconfig_make_lines),
+             }, 'Adding make options from {path}\n    {content}')
+ 
+         append_env['OBJDIR'] = mozpath.normsep(self.topobjdir)
++        if (mozpath.normpath(os.path.dirname(sys.executable)) not in
++            [mozpath.normpath(s) for s in
++             os.environ['PATH'].split(os.pathsep)]):
++            append_env['PATH'] = (
++                os.path.dirname(sys.executable) + os.pathsep +
++                os.environ['PATH'])
+ 
+         return self._run_make(srcdir=True,
+                               filename='client.mk',
+                               allow_parallel=False,
+                               ensure_exit_code=False,
+                               print_directory=False,
+                               target=target,
+                               line_handler=line_handler,
+diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
+--- a/python/mozbuild/mozbuild/mach_commands.py
++++ b/python/mozbuild/mozbuild/mach_commands.py
+@@ -1167,8 +1167,60 @@ class Repackage(MachCommandBase):
+     @CommandArgument('--format', type=str, default='lzma',
+                      choices=('lzma', 'bz2'),
+                      help='Mar format')
+     @CommandArgument('--arch', type=str, required=True,
+                      help='The archtecture you are building.')
+     def repackage_mar(self, input, mar, output, format, arch):
+         from mozbuild.repackaging.mar import repackage_mar
+         repackage_mar(self.topsrcdir, input, mar, output, format, arch=arch)
++
++
++@CommandProvider
++class CreateMachEnvironment(MachCommandBase):
++    """Create the mach virtualenvs."""
++
++    @Command('create-mach-environment', category='devenv',
++             description=(
++                 'Create the `mach` virtualenvs. If executed with python3 (the '
++                 'default when entering from `mach`), create both a python3 '
++                 'and python2.7 virtualenv. If executed with python2, only '
++                 'create the python2.7 virtualenv.'))
++    def create_mach_environment(self):
++        from mozboot.util import get_state_dir
++        from mozbuild.pythonutil import find_python2_executable
++        from mozbuild.virtualenv import VirtualenvManager
++        from six import PY3
++
++        state_dir = get_state_dir()
++        virtualenv_path = os.path.join(state_dir, '_virtualenvs',
++                                       'mach' if PY3 else 'mach_py2')
++        if sys.executable.startswith(virtualenv_path):
++            print('You can only create a mach environment with the system '
++                  'Python. Re-run this `mach` command with the system Python.',
++                  file=sys.stderr)
++            return 1
++
++        manager = VirtualenvManager(
++            self.topsrcdir, virtualenv_path, sys.stdout,
++            os.path.join(self.topsrcdir, 'build',
++                         'mach_virtualenv_packages.txt'),
++            populate_local_paths=False)
++        manager.build(sys.executable)
++
++        manager.install_pip_package('zstandard>=0.9.0,<=0.13.0')
++
++        if PY3:
++            manager.install_pip_package('glean_sdk~=31.5.0')
++            print('Python 3 mach environment created.')
++            python2, _ = find_python2_executable()
++            if not python2:
++                print('WARNING! Could not find a Python 2 executable to create '
++                      'a Python 2 virtualenv', file=sys.stderr)
++                return 0
++            ret = subprocess.call([
++                python2, os.path.join(self.topsrcdir, 'mach'),
++                'create-mach-environment'])
++            if ret:
++                print('WARNING! Failed to create a Python 2 mach environment.',
++                      file=sys.stderr)
++        else:
++            print('Python 2 mach environment created.')
+diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py
+--- a/python/mozbuild/mozbuild/sphinx.py
++++ b/python/mozbuild/mozbuild/sphinx.py
+@@ -181,14 +181,13 @@ def setup(app):
+     # Here, we invoke our custom code for staging/generating all our
+     # documentation.
+     manager.generate_docs(app)
+     app.srcdir = manager.staging_dir
+ 
+     # We need to adjust sys.path in order for Python API docs to get generated
+     # properly. We leverage the in-tree virtualenv for this.
+     topsrcdir = manager.topsrcdir
+-    ve = VirtualenvManager(topsrcdir,
+-                           os.path.join(app.outdir, '_venv'),
+-                           sys.stderr,
+-                           os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
++    ve = VirtualenvManager(
++        topsrcdir, os.path.join(app.outdir, '_venv'), sys.stderr,
++        os.path.join(topsrcdir, 'build', 'build_virtualenv_packages.txt'))
+     ve.ensure()
+     ve.activate()
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -2,16 +2,17 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ # This file contains code for populating the virtualenv environment for
+ # Mozilla's build system. It is typically called as part of configure.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
++import argparse
+ import os
+ import platform
+ import shutil
+ import subprocess
+ import sys
+ 
+ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
+ IS_CYGWIN = (sys.platform == 'cygwin')
+@@ -30,20 +31,51 @@ Run |mach bootstrap| to ensure your syst
+ If you still receive this error, your shell environment is likely detecting
+ another Python version. Ensure a modern Python can be found in the paths
+ defined by the $PATH environment variable and try again.
+ '''.lstrip()
+ 
+ here = os.path.abspath(os.path.dirname(__file__))
+ 
+ 
++# We can't import six.ensure_binary() or six.ensure_text() because this module
++# has to run stand-alone.  Instead we'll implement an abbreviated version of the
++# checks it does.
++if PY3:
++    text_type = str
++    binary_type = bytes
++else:
++    text_type = unicode
++    binary_type = str
++
++
++def ensure_binary(s, encoding='utf-8'):
++    if isinstance(s, text_type):
++        return s.encode(encoding, errors='strict')
++    elif isinstance(s, binary_type):
++        return s
++    else:
++        raise TypeError("not expecting type '%s'" % type(s))
++
++
++def ensure_text(s, encoding='utf-8'):
++    if isinstance(s, binary_type):
++        return s.decode(encoding, errors='strict')
++    elif isinstance(s, text_type):
++        return s
++    else:
++        raise TypeError("not expecting type '%s'" % type(s))
++
++
+ class VirtualenvManager(object):
+     """Contains logic for managing virtualenvs for building the tree."""
+ 
+-    def __init__(self, topsrcdir, virtualenv_path, log_handle, manifest_path):
++    def __init__(
++            self, topsrcdir, virtualenv_path, log_handle, manifest_path,
++            parent_site_dir=None, populate_local_paths=True):
+         """Create a new manager.
+ 
+         Each manager is associated with a source directory, a path where you
+         want the virtualenv to be created, and a handle to write output to.
+         """
+         assert os.path.isabs(
+             manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
+         self.topsrcdir = topsrcdir
+@@ -52,16 +84,21 @@ class VirtualenvManager(object):
+         # Record the Python executable that was used to create the Virtualenv
+         # so we can check this against sys.executable when verifying the
+         # integrity of the virtualenv.
+         self.exe_info_path = os.path.join(self.virtualenv_root,
+                                           'python_exe.txt')
+ 
+         self.log_handle = log_handle
+         self.manifest_path = manifest_path
++        self.parent_site_dir = parent_site_dir
++        if not self.parent_site_dir:
++            import distutils.sysconfig
++            self.parent_site_dir = distutils.sysconfig.get_python_lib()
++        self.populate_local_paths = populate_local_paths
+ 
+     @property
+     def virtualenv_script_path(self):
+         """Path to virtualenv's own populator script."""
+         return os.path.join(self.topsrcdir, 'third_party', 'python',
+                             'virtualenv', 'virtualenv.py')
+ 
+     @property
+@@ -272,35 +309,57 @@ class VirtualenvManager(object):
+             on non-Windows systems.
+ 
+         python3 -- This denotes that the action should only be taken when run
+             on Python 3.
+ 
+         python2 -- This denotes that the action should only be taken when run
+             on python 2.
+ 
+-        in-virtualenv -- This denotes that the action should only be taken when
+-            constructing a virtualenv (and not when bootstrapping a `mach`
+-            action).
++        inherit-from-parent-environment -- This denotes that we should add the
++            configured site directory of the "parent" to the virtualenv's list
++            of site directories. This can be specified on the command line as
++            --parent-site-dir or passed in the constructor of this class. This
++            defaults to the site-packages directory of the current Python
++            interpreter if not provided.
++
++        set-variable -- Set the given environment variable; e.g.
++            `set-variable FOO=1`.
+ 
+         Note that the Python interpreter running this function should be the
+         one from the virtualenv. If it is the system Python or if the
+         environment is not configured properly, packages could be installed
+         into the wrong place. This is how virtualenv's work.
+         """
+         import distutils.sysconfig
+ 
+         packages = self.packages()
+         python_lib = distutils.sysconfig.get_python_lib()
++        sitecustomize = open(
++            os.path.join(os.path.dirname(python_lib), 'sitecustomize.py'),
++            mode='w')
+ 
+         def handle_package(package):
+-            if package[0] == 'in-virtualenv':
+-                assert len(package) >= 2
+-                package = package[1:]
+-                # Continue processing normally.
++            if package[0] == 'inherit-from-parent-environment':
++                assert len(package) == 1
++                sitecustomize.write(
++                    'import site\n'
++                    "site.addsitedir(%s)\n" % repr(self.parent_site_dir))
++                return True
++
++            if package[0].startswith('set-variable '):
++                assert len(package) == 1
++                assignment = package[0][len('set-variable '):].strip()
++                var, val = assignment.split('=', 1)
++                var = var if PY3 else ensure_binary(var)
++                val = val if PY3 else ensure_binary(val)
++                sitecustomize.write(
++                    'import os\n'
++                    "os.environ[%s] = %s\n" % (repr(var), repr(val)))
++                return True
+ 
+             if package[0] == 'setup.py':
+                 assert len(package) >= 2
+ 
+                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
+                                 package[2:])
+ 
+                 return True
+@@ -315,27 +374,30 @@ class VirtualenvManager(object):
+ 
+                 return True
+ 
+             if package[0] == 'packages.txt':
+                 assert len(package) == 2
+ 
+                 src = os.path.join(self.topsrcdir, package[1])
+                 assert os.path.isfile(src), "'%s' does not exist" % src
+-                submanager = VirtualenvManager(self.topsrcdir,
+-                                               self.virtualenv_root,
+-                                               self.log_handle,
+-                                               src)
++                submanager = VirtualenvManager(
++                    self.topsrcdir, self.virtualenv_root, self.log_handle, src,
++                    parent_site_dir=self.parent_site_dir,
++                    populate_local_paths=self.populate_local_paths)
+                 submanager.populate()
+ 
+                 return True
+ 
+             if package[0].endswith('.pth'):
+                 assert len(package) == 2
+ 
++                if not self.populate_local_paths:
++                    return True
++
+                 path = os.path.join(self.topsrcdir, package[1])
+ 
+                 with open(os.path.join(python_lib, package[0]), 'a') as f:
+                     # This path is relative to the .pth file.  Using a
+                     # relative path allows the srcdir/objdir combination
+                     # to be moved around (as long as the paths relative to
+                     # each other remain the same).
+                     f.write("%s\n" % os.path.relpath(path, python_lib))
+@@ -397,26 +459,25 @@ class VirtualenvManager(object):
+                     continue
+ 
+                 old_env_variables[k] = os.environ[k]
+                 del os.environ[k]
+ 
+             for package in packages:
+                 handle_package(package)
+ 
+-            sitecustomize = os.path.join(
+-                os.path.dirname(python_lib), 'sitecustomize.py')
+-            with open(sitecustomize, 'w') as f:
+-                f.write(
+-                    '# Importing mach_bootstrap has the side effect of\n'
+-                    '# installing an import hook\n'
+-                    'import mach_bootstrap\n'
+-                )
++            sitecustomize.write(
++                '# Importing mach_bootstrap has the side effect of\n'
++                '# installing an import hook\n'
++                'import mach_bootstrap\n'
++            )
+ 
+         finally:
++            sitecustomize.close()
++
+             os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
+ 
+             if old_target is not None:
+                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
+ 
+             os.environ.update(old_env_variables)
+ 
+     def call_setup(self, directory, arguments):
+@@ -442,16 +503,17 @@ class VirtualenvManager(object):
+ 
+             raise Exception('Error installing package: %s' % directory)
+ 
+     def build(self, python):
+         """Build a virtualenv per tree conventions.
+ 
+         This returns the path of the created virtualenv.
+         """
++        import distutils
+ 
+         self.create(python)
+ 
+         # We need to populate the virtualenv using the Python executable in
+         # the virtualenv for paths to be proper.
+ 
+         # If this module was run from Python 2 then the __file__ attribute may
+         # point to a Python 2 .pyc file. If we are generating a Python 3
+@@ -461,17 +523,20 @@ class VirtualenvManager(object):
+             thismodule = __file__[:-1]
+         else:
+             thismodule = __file__
+ 
+         # __PYVENV_LAUNCHER__ confuses pip about the python interpreter
+         # See https://bugzilla.mozilla.org/show_bug.cgi?id=1635481
+         os.environ.pop('__PYVENV_LAUNCHER__', None)
+         args = [self.python_path, thismodule, 'populate', self.topsrcdir,
+-                self.virtualenv_root, self.manifest_path]
++                self.virtualenv_root, self.manifest_path, '--parent-site-dir',
++                distutils.sysconfig.get_python_lib()]
++        if self.populate_local_paths:
++            args.append('--populate-local-paths')
+ 
+         result = self._log_process_output(args, cwd=self.topsrcdir)
+ 
+         if result != 0:
+             raise Exception('Error populating virtualenv.')
+ 
+         os.utime(self.activate_path, None)
+ 
+@@ -481,36 +546,42 @@ class VirtualenvManager(object):
+         """Activate the virtualenv in this Python context.
+ 
+         If you run a random Python script and wish to "activate" the
+         virtualenv, you can simply instantiate an instance of this class
+         and call .ensure() and .activate() to make the virtualenv active.
+         """
+ 
+         exec(open(self.activate_path).read(), dict(__file__=self.activate_path))
+-        if PY2 and isinstance(os.environ['PATH'], unicode):
+-            os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
++        # Activating the virtualenv can make `os.environ` a little janky under
++        # Python 2.
++        env = ensure_subprocess_env(os.environ)
++        os.environ.clear()
++        os.environ.update(env)
+ 
+     def install_pip_package(self, package, vendored=False):
+         """Install a package via pip.
+ 
+         The supplied package is specified using a pip requirement specifier.
+         e.g. 'foo' or 'foo==1.0'.
+ 
+         If the package is already installed, this is a no-op.
+ 
+         If vendored is True, no package index will be used and no dependencies
+         will be installed.
+         """
+-        from pip._internal.req.constructors import install_req_from_line
+-
+-        req = install_req_from_line(package)
+-        req.check_if_exists(use_user_site=False)
+-        if req.satisfied_by is not None:
+-            return
++        if sys.executable.startswith(self.bin_path):
++            # If we're already running in this interpreter, we can optimize in
++            # the case that the package requirement is already satisfied.
++            from pip._internal.req.constructors import install_req_from_line
++
++            req = install_req_from_line(package)
++            req.check_if_exists(use_user_site=False)
++            if req.satisfied_by is not None:
++                return
+ 
+         args = [
+             'install',
+             package,
+         ]
+ 
+         if vendored:
+             args.extend([
+@@ -581,16 +652,18 @@ class VirtualenvManager(object):
+         """Activate a virtual environment managed by pipenv
+ 
+         If ``pipfile`` is not ``None`` then the Pipfile located at the path
+         provided will be used to create the virtual environment. If
+         ``populate`` is ``True`` then the virtual environment will be
+         populated from the manifest file. The optional ``python`` argument
+         indicates the version of Python for pipenv to use.
+         """
++
++        import distutils.sysconfig
+         from distutils.version import LooseVersion
+ 
+         pipenv = os.path.join(self.bin_path, 'pipenv')
+         env = ensure_subprocess_env(os.environ.copy())
+         env.update(ensure_subprocess_env({
+             'PIPENV_IGNORE_VIRTUALENVS': '1',
+             'PIP_NO_INDEX': '1',
+             'WORKON_HOME': str(os.path.normpath(workon_home)),
+@@ -666,20 +739,23 @@ class VirtualenvManager(object):
+                 'PIPENV_PIPFILE': str(pipfile)
+             }))
+             subprocess.check_call([pipenv, 'install'], stderr=subprocess.STDOUT, env=env_)
+ 
+         self.virtualenv_root = ensure_venv()
+ 
+         if populate:
+             # Populate from the manifest
+-            subprocess.check_call([
++            args = [
+                 pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+-                self.topsrcdir, self.virtualenv_root, self.manifest_path],
+-                stderr=subprocess.STDOUT, env=env)
++                self.topsrcdir, self.virtualenv_root, self.manifest_path,
++                '--parent-site-dir', distutils.sysconfig.get_python_lib()]
++            if self.populate_local_paths:
++                args.append('--populate-local-paths')
++            subprocess.check_call(args, stderr=subprocess.STDOUT, env=env)
+ 
+         self.activate()
+ 
+ 
+ def verify_python_version(log_handle):
+     """Ensure the current version of Python is sufficient."""
+     from distutils.version import LooseVersion
+ 
+@@ -715,67 +791,53 @@ def ensure_subprocess_env(env, encoding=
+     This will convert all keys and values to bytes on Python 2, and text on
+     Python 3.
+ 
+     Args:
+         env (dict): Environment to ensure.
+         encoding (str): Encoding to use when converting to/from bytes/text
+                         (default: utf-8).
+     """
+-    # We can't import six.ensure_binary() or six.ensure_text() because this module
+-    # has to run stand-alone.  Instead we'll implement an abbreviated version of the
+-    # checks it does.
+-
+-    if PY3:
+-        text_type = str
+-        binary_type = bytes
++    ensure = ensure_binary if PY2 else ensure_text
++
++    try:
++        return {
++            ensure(k, encoding=encoding): ensure(v, encoding=encoding)
++            for k, v in env.iteritems()
++        }
++    except AttributeError:
++        return {
++            ensure(k, encoding=encoding): ensure(v, encoding=encoding)
++            for k, v in env.items()
++        }
++
++
++if __name__ == '__main__':
++    verify_python_version(sys.stdout)
++
++    if len(sys.argv) < 2:
++        print('Too few arguments', file=sys.stderr)
++        sys.exit(1)
++
++    parser = argparse.ArgumentParser()
++    parser.add_argument('topsrcdir')
++    parser.add_argument('virtualenv_path')
++    parser.add_argument('manifest_path')
++    parser.add_argument('--parent-site-dir', default=None)
++    parser.add_argument('--populate-local-paths', action='store_true')
++
++    if sys.argv[1] == 'populate':
++        # This should only be called internally.
++        populate = True
++        opts = parser.parse_args(sys.argv[2:])
+     else:
+-        text_type = unicode
+-        binary_type = str
+-
+-    def ensure_binary(s):
+-        if isinstance(s, text_type):
+-            return s.encode(encoding, errors='strict')
+-        elif isinstance(s, binary_type):
+-            return s
+-        else:
+-            raise TypeError("not expecting type '%s'" % type(s))
+-
+-    def ensure_text(s):
+-        if isinstance(s, binary_type):
+-            return s.decode(encoding, errors='strict')
+-        elif isinstance(s, text_type):
+-            return s
+-        else:
+-            raise TypeError("not expecting type '%s'" % type(s))
+-
+-    ensure = ensure_binary if PY2 else ensure_text
+-
+-    try:
+-        return {ensure(k): ensure(v) for k, v in env.iteritems()}
+-    except AttributeError:
+-        return {ensure(k): ensure(v) for k, v in env.items()}
+-
+-
+-if __name__ == '__main__':
+-    if len(sys.argv) < 4:
+-        print(
+-            'Usage: virtualenv.py /path/to/topsrcdir '
+-            '/path/to/virtualenv /path/to/virtualenv_manifest')
+-        sys.exit(1)
+-
+-    verify_python_version(sys.stdout)
+-
+-    topsrcdir, virtualenv_path, manifest_path = sys.argv[1:4]
+-    populate = False
+-
+-    # This should only be called internally.
+-    if sys.argv[1] == 'populate':
+-        populate = True
+-        topsrcdir, virtualenv_path, manifest_path = sys.argv[2:]
+-
+-    manager = VirtualenvManager(topsrcdir, virtualenv_path,
+-                                sys.stdout, manifest_path)
++        populate = False
++        opts = parser.parse_args(sys.argv[1:])
++
++    manager = VirtualenvManager(
++        opts.topsrcdir, opts.virtualenv_path, sys.stdout, opts.manifest_path,
++        parent_site_dir=opts.parent_site_dir,
++        populate_local_paths=opts.populate_local_paths)
+ 
+     if populate:
+         manager.populate()
+     else:
+         manager.ensure()

mozilla-release/patches/1659542-82a1.patch

@@ -2,7 +2,7 @@
 # User Ricky Stewart <rstewart@mozilla.com>
 # Date 1599076557 0
 # Node ID a6be919846bd27373128ecc24e6c99bc124f214c
-# Parent  2dad38173190f3aaaf64f81fe4f58d64be9b03e2
+# Parent  e9ad873cfe5cc10a801718f35b200172be106018
 Bug 1659542 - Remove support for `pipenv` in `mozilla-central` r=mhentges,dmajor
 
 Differential Revision: https://phabricator.services.mozilla.com/D89192
@@ -10,7 +10,7 @@ Differential Revision: https://phabricator.services.mozilla.com/D89192
 diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
 --- a/python/mozbuild/mozbuild/base.py
 +++ b/python/mozbuild/mozbuild/base.py
-@@ -824,34 +824,16 @@ class MozbuildObject(ProcessExecutionMix
+@@ -825,34 +825,16 @@ class MozbuildObject(ProcessExecutionMix
  
      def activate_virtualenv(self):
          self.virtualenv_manager.ensure()
@@ -48,14 +48,14 @@ diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
 diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
 --- a/python/mozbuild/mozbuild/virtualenv.py
 +++ b/python/mozbuild/mozbuild/virtualenv.py
-@@ -3,17 +3,16 @@
- # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+@@ -4,17 +4,16 @@
  
  # This file contains code for populating the virtualenv environment for
  # Mozilla's build system. It is typically called as part of configure.
  
  from __future__ import absolute_import, print_function, unicode_literals
  
+ import argparse
  import os
 -import platform
  import shutil
@@ -66,7 +66,7 @@ diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/v
  IS_CYGWIN = (sys.platform == 'cygwin')
  
  PY2 = sys.version_info[0] == 2
-@@ -562,118 +561,16 @@ class VirtualenvManager(object):
+@@ -651,123 +650,16 @@ class VirtualenvManager(VirtualenvHelper
          # against the executing interpreter. By creating a new process, we
          # force the virtualenv's interpreter to be used and all is well.
          # It /might/ be possible to cheat and set sys.executable to
@@ -85,6 +85,8 @@ diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/v
 -        populated from the manifest file. The optional ``python`` argument
 -        indicates the version of Python for pipenv to use.
 -        """
+-
+-        import distutils.sysconfig
 -        from distutils.version import LooseVersion
 -
 -        pipenv = os.path.join(self.bin_path, 'pipenv')
@@ -170,10 +172,13 @@ diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/v
 -
 -        if populate:
 -            # Populate from the manifest
--            subprocess.check_call([
+-            args = [
 -                pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
--                self.topsrcdir, self.virtualenv_root, self.manifest_path],
--                stderr=subprocess.STDOUT, env=env)
+-                self.topsrcdir, self.virtualenv_root, self.manifest_path,
+-                '--parent-site-dir', distutils.sysconfig.get_python_lib()]
+-            if self.populate_local_paths:
+-                args.append('--populate-local-paths')
+-            subprocess.check_call(args, stderr=subprocess.STDOUT, env=env)
 -
 -        self.activate()
 -

mozilla-release/patches/1660105-81a1.patch

@@ -0,0 +1,39 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1597942019 0
+# Node ID 59edb9c938c148891a1c8c5905897ff4e40962fc
+# Parent  de33002bfafc6ae3ab3e7521b7e1a92a1d69d271
+Bug 1660105 - Allow installing `glean_sdk` to fail in `create-mach-environment`. r=mhentges,froydnj
+
+`glean_sdk` can't currently be installed on Apple Silicon or OpenBSD since Glean can't be built from source. While that issue is being resolved (see bug 1660120), allow this installation to fail.
+
+Differential Revision: https://phabricator.services.mozilla.com/D87667
+
+diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
+--- a/python/mozbuild/mozbuild/mach_commands.py
++++ b/python/mozbuild/mozbuild/mach_commands.py
+@@ -1204,17 +1204,23 @@ class CreateMachEnvironment(MachCommandB
+             os.path.join(self.topsrcdir, 'build',
+                          'mach_virtualenv_packages.txt'),
+             populate_local_paths=False)
+         manager.build(sys.executable)
+ 
+         manager.install_pip_package('zstandard>=0.9.0,<=0.13.0')
+ 
+         if PY3:
+-            manager.install_pip_package('glean_sdk~=31.5.0')
++            # This can fail on some platforms. See
++            # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
++            try:
++                manager.install_pip_package('glean_sdk~=31.5.0')
++            except subprocess.CalledProcessError:
++                print('Could not install glean_sdk, so telemetry will not be '
++                      'collected. Continuing.')
+             print('Python 3 mach environment created.')
+             python2, _ = find_python2_executable()
+             if not python2:
+                 print('WARNING! Could not find a Python 2 executable to create '
+                       'a Python 2 virtualenv', file=sys.stderr)
+                 return 0
+             ret = subprocess.call([
+                 python2, os.path.join(self.topsrcdir, 'mach'),

mozilla-release/patches/1660351-82a1.patch

@@ -0,0 +1,339 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1598564886 0
+# Node ID f50e4bee2efb3afbd6016fa7286078500599775b
+# Parent  b77a72ae89b0f9ebc58c87c06157245b215923b7
+Bug 1660351 - Tweak `inherit-from-parent-environment` implementation in `virtualenv.py` r=ahal
+
+The intended behavior of `inherit-from-parent-environment` is that the packages from the parent Python environment are available to the sub-`virtualenv`. The implementation of that behavior thus far has been around "site directories", the idea being that custom (non-stdlib) packages are likely to be installed in the "site directory". The limitation of this approach is that there's no one location, in practice, where packages are installed, and it's hard to enumerate a static list of all those possible locations across all platforms.
+
+This patch circumvents the issue by ignoring the "site directory" question entirely and just looking at `sys.path`. If we're inheriting from the parent environment when creating a `virtualenv`, we just ask the parent Python what its `sys.path` is and configure the `virtualenv`'s `sys.path` on startup.
+
+Differential Revision: https://phabricator.services.mozilla.com/D87808
+
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -3,16 +3,17 @@
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ # This file contains code for populating the virtualenv environment for
+ # Mozilla's build system. It is typically called as part of configure.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import argparse
++import json
+ import os
+ import platform
+ import shutil
+ import subprocess
+ import sys
+ 
+ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
+ IS_CYGWIN = (sys.platform == 'cygwin')
+@@ -65,17 +66,17 @@ def ensure_text(s, encoding='utf-8'):
+         raise TypeError("not expecting type '%s'" % type(s))
+ 
+ 
+ class VirtualenvManager(object):
+     """Contains logic for managing virtualenvs for building the tree."""
+ 
+     def __init__(
+             self, topsrcdir, virtualenv_path, log_handle, manifest_path,
+-            parent_site_dir=None, populate_local_paths=True):
++            populate_local_paths=True):
+         """Create a new manager.
+ 
+         Each manager is associated with a source directory, a path where you
+         want the virtualenv to be created, and a handle to write output to.
+         """
+         assert os.path.isabs(
+             manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
+         self.topsrcdir = topsrcdir
+@@ -84,20 +85,16 @@ class VirtualenvManager(object):
+         # Record the Python executable that was used to create the Virtualenv
+         # so we can check this against sys.executable when verifying the
+         # integrity of the virtualenv.
+         self.exe_info_path = os.path.join(self.virtualenv_root,
+                                           'python_exe.txt')
+ 
+         self.log_handle = log_handle
+         self.manifest_path = manifest_path
+-        self.parent_site_dir = parent_site_dir
+-        if not self.parent_site_dir:
+-            import distutils.sysconfig
+-            self.parent_site_dir = distutils.sysconfig.get_python_lib()
+         self.populate_local_paths = populate_local_paths
+ 
+     @property
+     def virtualenv_script_path(self):
+         """Path to virtualenv's own populator script."""
+         return os.path.join(self.topsrcdir, 'third_party', 'python',
+                             'virtualenv', 'virtualenv.py')
+ 
+@@ -148,16 +145,21 @@ class VirtualenvManager(object):
+             fh.write("%s\n" % ver)
+ 
+     def python_executable_hexversion(self, python):
+         """Run a Python executable and return its sys.hexversion value."""
+         program = 'import sys; print(sys.hexversion)'
+         out = subprocess.check_output([python, '-c', program]).rstrip()
+         return int(out)
+ 
++    def _python_sys_path(self, python):
++        program = 'import json, sys; print(json.dumps(sys.path))'
++        return json.loads(ensure_text(subprocess.check_output(
++            [python, '-c', program])).strip())
++
+     def up_to_date(self, python):
+         """Returns whether the virtualenv is present and up to date.
+ 
+         Args:
+             python: Full path string to the Python executable that this virtualenv
+                 should be running.  If the Python executable passed in to this
+                 argument is not the same version as the Python the virtualenv was
+                 built with then this method will return False.
+@@ -268,18 +270,18 @@ class VirtualenvManager(object):
+ 
+     def packages(self):
+         mode = 'rU' if PY2 else 'r'
+         with open(self.manifest_path, mode) as fh:
+             packages = [line.rstrip().split(':')
+                         for line in fh]
+         return packages
+ 
+-    def populate(self):
+-        """Populate the virtualenv.
++    def populate(self, python, sitecustomize=None):
++        """Populate the virtualenv from the given python.
+ 
+         The manifest file consists of colon-delimited fields. The first field
+         specifies the action. The remaining fields are arguments to that
+         action. The following actions are supported:
+ 
+         setup.py -- Invoke setup.py for a package. Expects the arguments:
+             1. relative path directory containing setup.py.
+             2. argument(s) to setup.py. e.g. "develop". Each program argument
+@@ -309,45 +311,44 @@ class VirtualenvManager(object):
+             on non-Windows systems.
+ 
+         python3 -- This denotes that the action should only be taken when run
+             on Python 3.
+ 
+         python2 -- This denotes that the action should only be taken when run
+             on python 2.
+ 
+-        inherit-from-parent-environment -- This denotes that we should add the
+-            configured site directory of the "parent" to the virtualenv's list
+-            of site directories. This can be specified on the command line as
+-            --parent-site-dir or passed in the constructor of this class. This
+-            defaults to the site-packages directory of the current Python
+-            interpreter if not provided.
++        inherit-from-parent-environment -- This denotes that we should append
++            the path from the parent python to the sys.path of the virtualenv.
+ 
+         set-variable -- Set the given environment variable; e.g.
+             `set-variable FOO=1`.
+ 
+         Note that the Python interpreter running this function should be the
+         one from the virtualenv. If it is the system Python or if the
+         environment is not configured properly, packages could be installed
+         into the wrong place. This is how virtualenv's work.
+         """
+         import distutils.sysconfig
+ 
+         packages = self.packages()
+         python_lib = distutils.sysconfig.get_python_lib()
+-        sitecustomize = open(
++        do_close = not bool(sitecustomize)
++        sitecustomize = sitecustomize or open(
+             os.path.join(os.path.dirname(python_lib), 'sitecustomize.py'),
+             mode='w')
+ 
+         def handle_package(package):
+             if package[0] == 'inherit-from-parent-environment':
+                 assert len(package) == 1
+-                sitecustomize.write(
+-                    'import site\n'
+-                    "site.addsitedir(%s)\n" % repr(self.parent_site_dir))
++                sitecustomize.write('import sys\n')
++                paths = self._python_sys_path(python)
++                for d in paths:
++                    sitecustomize.write('if %s not in sys.path:\n' % repr(d))
++                    sitecustomize.write('    sys.path.append(%s)\n' % repr(d))
+                 return True
+ 
+             if package[0].startswith('set-variable '):
+                 assert len(package) == 1
+                 assignment = package[0][len('set-variable '):].strip()
+                 var, val = assignment.split('=', 1)
+                 var = var if PY3 else ensure_binary(var)
+                 val = val if PY3 else ensure_binary(val)
+@@ -376,19 +377,18 @@ class VirtualenvManager(object):
+ 
+             if package[0] == 'packages.txt':
+                 assert len(package) == 2
+ 
+                 src = os.path.join(self.topsrcdir, package[1])
+                 assert os.path.isfile(src), "'%s' does not exist" % src
+                 submanager = VirtualenvManager(
+                     self.topsrcdir, self.virtualenv_root, self.log_handle, src,
+-                    parent_site_dir=self.parent_site_dir,
+                     populate_local_paths=self.populate_local_paths)
+-                submanager.populate()
++                submanager.populate(python, sitecustomize=sitecustomize)
+ 
+                 return True
+ 
+             if package[0].endswith('.pth'):
+                 assert len(package) == 2
+ 
+                 if not self.populate_local_paths:
+                     return True
+@@ -466,18 +466,18 @@ class VirtualenvManager(object):
+ 
+             sitecustomize.write(
+                 '# Importing mach_bootstrap has the side effect of\n'
+                 '# installing an import hook\n'
+                 'import mach_bootstrap\n'
+             )
+ 
+         finally:
+-            sitecustomize.close()
+-
++            if do_close:
++                sitecustomize.close()
+             os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
+ 
+             if old_target is not None:
+                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
+ 
+             os.environ.update(old_env_variables)
+ 
+     def call_setup(self, directory, arguments):
+@@ -504,18 +504,16 @@ class VirtualenvManager(object):
+ 
+             raise Exception('Error installing package: %s' % directory)
+ 
+     def build(self, python):
+         """Build a virtualenv per tree conventions.
+ 
+         This returns the path of the created virtualenv.
+         """
+-        import distutils
+-
+         self.create(python)
+ 
+         # We need to populate the virtualenv using the Python executable in
+         # the virtualenv for paths to be proper.
+ 
+         # If this module was run from Python 2 then the __file__ attribute may
+         # point to a Python 2 .pyc file. If we are generating a Python 3
+         # virtualenv from Python 2 make sure we call Python 3 with the path to
+@@ -524,18 +522,17 @@ class VirtualenvManager(object):
+             thismodule = __file__[:-1]
+         else:
+             thismodule = __file__
+ 
+         # __PYVENV_LAUNCHER__ confuses pip about the python interpreter
+         # See https://bugzilla.mozilla.org/show_bug.cgi?id=1635481
+         os.environ.pop('__PYVENV_LAUNCHER__', None)
+         args = [self.python_path, thismodule, 'populate', self.topsrcdir,
+-                self.virtualenv_root, self.manifest_path, '--parent-site-dir',
+-                distutils.sysconfig.get_python_lib()]
++                self.virtualenv_root, self.manifest_path, '--python', python]
+         if self.populate_local_paths:
+             args.append('--populate-local-paths')
+ 
+         result = self._log_process_output(args, cwd=self.topsrcdir)
+ 
+         if result != 0:
+             raise Exception('Error populating virtualenv.')
+ 
+@@ -654,17 +651,16 @@ class VirtualenvManager(object):
+ 
+         If ``pipfile`` is not ``None`` then the Pipfile located at the path
+         provided will be used to create the virtual environment. If
+         ``populate`` is ``True`` then the virtual environment will be
+         populated from the manifest file. The optional ``python`` argument
+         indicates the version of Python for pipenv to use.
+         """
+ 
+-        import distutils.sysconfig
+         from distutils.version import LooseVersion
+ 
+         pipenv = os.path.join(self.bin_path, 'pipenv')
+         env = ensure_subprocess_env(os.environ.copy())
+         env.update(ensure_subprocess_env({
+             'PIPENV_IGNORE_VIRTUALENVS': '1',
+             'PIP_NO_INDEX': '1',
+             'WORKON_HOME': str(os.path.normpath(workon_home)),
+@@ -738,22 +734,34 @@ class VirtualenvManager(object):
+             del env_['PIP_NO_INDEX']
+             env_.update(ensure_subprocess_env({
+                 'PIPENV_PIPFILE': str(pipfile)
+             }))
+             subprocess.check_call([pipenv, 'install'], stderr=subprocess.STDOUT, env=env_)
+ 
+         self.virtualenv_root = ensure_venv()
+ 
++        # Awful hack, but only necessary until bug 1659542 is closed.
++        windows = IS_CYGWIN or IS_NATIVE_WIN
++        populate_python = sys.executable
++        if not python:
++            pass
++        elif os.path.exists(python):
++            populate_python = python
++        elif python.startswith('2'):
++            populate_python = 'python2.7' + ('.exe' if windows else '')
++        elif python.startswith('3'):
++            populate_python = 'python3' + ('.exe' if windows else '')
++
+         if populate:
+             # Populate from the manifest
+             args = [
+                 pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+                 self.topsrcdir, self.virtualenv_root, self.manifest_path,
+-                '--parent-site-dir', distutils.sysconfig.get_python_lib()]
++                '--python', populate_python]
+             if self.populate_local_paths:
+                 args.append('--populate-local-paths')
+             subprocess.check_call(args, stderr=subprocess.STDOUT, env=env)
+ 
+         self.activate()
+ 
+ 
+ def verify_python_version(log_handle):
+@@ -817,28 +825,28 @@ if __name__ == '__main__':
+     if len(sys.argv) < 2:
+         print('Too few arguments', file=sys.stderr)
+         sys.exit(1)
+ 
+     parser = argparse.ArgumentParser()
+     parser.add_argument('topsrcdir')
+     parser.add_argument('virtualenv_path')
+     parser.add_argument('manifest_path')
+-    parser.add_argument('--parent-site-dir', default=None)
++    parser.add_argument('--python', default=None)
+     parser.add_argument('--populate-local-paths', action='store_true')
+ 
+     if sys.argv[1] == 'populate':
+         # This should only be called internally.
+         populate = True
+         opts = parser.parse_args(sys.argv[2:])
+     else:
+         populate = False
+         opts = parser.parse_args(sys.argv[1:])
+ 
+     manager = VirtualenvManager(
+         opts.topsrcdir, opts.virtualenv_path, sys.stdout, opts.manifest_path,
+-        parent_site_dir=opts.parent_site_dir,
+         populate_local_paths=opts.populate_local_paths)
+ 
+     if populate:
+-        manager.populate()
++        assert opts.python
++        manager.populate(opts.python)
+     else:
+         manager.ensure()

mozilla-release/patches/1660559-82a1.patch

@@ -0,0 +1,117 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1599066132 0
+# Node ID 637126b84cee525c4bb40589bfbbe789102bb62c
+# Parent  5a0023f28369d2ed19b61ca57e7015851224abde
+Bug 1660559 - In `mach bootstrap`, initialize the `mach` `virtualenv`s early and run `mach artifact` with them r=mhentges,ahal
+
+`mach artifact` has a dependency on `zstandard`, which is installed in the `mach` `virtualenv`s, so we have to run `mach artifact` with the correct `virtualenv`. Also create the `virtualenv`s earlier in the process to account for this.
+
+This all has a dependency on the existence of a checkout (which has the `mach` script with all its dependencies on everything else), but after bug 1647792 that's not a concern.
+
+Differential Revision: https://phabricator.services.mozilla.com/D87920
+
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -8,16 +8,17 @@ import hashlib
+ import os
+ import re
+ import subprocess
+ import sys
+ 
+ from distutils.version import LooseVersion
+ from mozboot import rust
+ from mozboot.util import MINIMUM_RUST_VERSION
++from mozbuild.virtualenv import VirtualenvHelper
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     from urllib2 import urlopen
+     input = raw_input  # noqa
+ else:
+@@ -319,23 +320,28 @@ class BaseBootstrapper(object):
+         self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job)
+ 
+     def install_toolchain_artifact(self, state_dir, checkout_root, toolchain_job):
+         mach_binary = os.path.join(checkout_root, 'mach')
+         mach_binary = os.path.abspath(mach_binary)
+         if not os.path.exists(mach_binary):
+             raise ValueError("mach not found at %s" % mach_binary)
+ 
+-        # If Python can't figure out what its own executable is, there's little
+-        # chance we're going to be able to execute mach on its own, particularly
+-        # on Windows.
+-        if not sys.executable:
+-            raise ValueError("cannot determine path to Python executable")
+-
+-        cmd = [sys.executable, mach_binary, 'artifact', 'toolchain',
++        # NOTE: Use self.state_dir over the passed-in state_dir, which might be
++        # a subdirectory of the actual state directory.
++        if not self.state_dir:
++            raise ValueError(
++                'Need a state directory (e.g. ~/.mozbuild) to download '
++                'artifacts')
++        python_location = VirtualenvHelper(os.path.join(
++            self.state_dir, '_virtualenvs', 'mach')).python_path
++        if not os.path.exists(python_location):
++            raise ValueError('python not found at %s' % python_location)
++
++        cmd = [python_location, mach_binary, 'artifact', 'toolchain',
+                '--bootstrap', '--from-build', toolchain_job]
+ 
+         subprocess.check_call(cmd, cwd=state_dir)
+ 
+     def which(self, name, *extra_search_dirs):
+         """Python implementation of which.
+ 
+         It returns the path of an executable or None if it couldn't be found.
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -305,24 +305,24 @@ class Bootstrapper(object):
+         if state_dir_available:
+             self.instance.state_dir = state_dir
+ 
+         # We need to enable the loading of hgrc in case extensions are
+         # required to open the repo.
+         (checkout_type, checkout_root) = current_firefox_checkout(
+             env=self.instance._hg_cleanenv(load_hgrc=True),
+             hg=self.instance.which('hg'))
++        self.instance.ensure_mach_environment(checkout_root)
+         have_clone = bool(checkout_type)
+ 
+         if self.instance.no_system_changes:
+             self.maybe_install_private_packages_or_exit(state_dir,
+                                                         state_dir_available,
+                                                         have_clone,
+                                                         checkout_root)
+-            self.instance.ensure_mach_environment(checkout_root)
+             self._output_mozconfig(application)
+             sys.exit(0)
+ 
+         self.instance.install_system_packages()
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+@@ -367,17 +367,16 @@ class Bootstrapper(object):
+             if should_configure_git:
+                 configure_git(self.instance.which('git'), state_dir,
+                               checkout_root)
+ 
+         self.maybe_install_private_packages_or_exit(state_dir,
+                                                     state_dir_available,
+                                                     have_clone,
+                                                     checkout_root)
+-        self.instance.ensure_mach_environment(checkout_root)
+ 
+         print(self.finished % name)
+         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
+                 >= MODERN_RUST_VERSION):
+             print("To build %s, please restart the shell (Start a new terminal window)" % name)
+ 
+         self._output_mozconfig(application)
+ 

mozilla-release/patches/1661391-82a1.patch

@@ -0,0 +1,39 @@
+# HG changeset patch
+# User Mike Hommey <mh+mozilla@glandium.org>
+# Date 1598531274 0
+# Node ID b62abe7770477e7d8471b1bbf27b696107977733
+# Parent  40d9baa8432fc1e3b66e4e8704b16384997e47d6
+Bug 1661391 - Read the output from setup.py as unicode. r=froydnj
+
+There are various problems happening when dealing with the output from
+setup.py during virtualenv setup, all of which step from the process
+command output not being a unicode string in python.
+
+As this code is still used to setup python2 virtualenv, we need to use
+the backwards-compatible universal_newlines=True trick.
+
+Differential Revision: https://phabricator.services.mozilla.com/D88372
+
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -488,17 +488,18 @@ class VirtualenvManager(object):
+         program.extend(arguments)
+ 
+         # We probably could call the contents of this file inside the context
+         # of this interpreter using execfile() or similar. However, if global
+         # variables like sys.path are adjusted, this could cause all kinds of
+         # havoc. While this may work, invoking a new process is safer.
+ 
+         try:
+-            output = subprocess.check_output(program, cwd=directory, stderr=subprocess.STDOUT)
++            output = subprocess.check_output(program, cwd=directory, stderr=subprocess.STDOUT,
++                                             universal_newlines=True)
+             print(output)
+         except subprocess.CalledProcessError as e:
+             if 'Python.h: No such file or directory' in e.output:
+                 print('WARNING: Python.h not found. Install Python development headers.')
+             else:
+                 print(e.output)
+ 
+             raise Exception('Error installing package: %s' % directory)

mozilla-release/patches/1661624-85a1.patch

@@ -0,0 +1,68 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1606511100 0
+# Node ID 4058e147401f15cfca916f32156381725b4d2137
+# Parent  c99b23794af45a5011b0051700757f9e5869d40c
+Bug 1661624 - Include `psutil` in the `mach` `virtualenv`s r=firefox-build-system-reviewers,rstewart
+
+Install `psutil` when setting up the `mach` `virtualenv`s and stop importing the in-tree version in the build.
+
+Nothing in-tree currently assumes or mandates the installation of `psutil` (all uses of `psutil` are guarded with imports of the form `try : import psutil; except ImportError: psutil = None`), so there's no back-incompatibility concerns here. There will be an awkward period where telemetry will be lacking CPU/disk data for everyone until they re-run `mach bootstrap` or `mach create-mach-environment`, but that will come back as people gradually update their `virtualenv`s.
+
+An alternative to circumvent that issue is REQUIRING that `psutil` be installed by adding an assertion in `mach` that `psutil` can be found (allowing us to remove all the conditional logic in-tree around whether `psutil` is installed), but I wouldn't claim that we're ready to do that and deal with whatever fallout might occur.
+
+Differential Revision: https://phabricator.services.mozilla.com/D90914
+
+diff --git a/build/common_virtualenv_packages.txt b/build/common_virtualenv_packages.txt
+--- a/build/common_virtualenv_packages.txt
++++ b/build/common_virtualenv_packages.txt
+@@ -36,19 +36,16 @@ mozilla.pth:third_party/python/packaging
+ mozilla.pth:third_party/python/pathlib2
+ mozilla.pth:third_party/python/pathspec
+ mozilla.pth:third_party/python/pep487/lib
+ mozilla.pth:third_party/python/gyp/pylib
+ mozilla.pth:third_party/python/pyrsistent
+ mozilla.pth:third_party/python/python-hglib
+ mozilla.pth:third_party/python/pluggy
+ mozilla.pth:third_party/python/jsmin
+-!windows:optional:setup.py:third_party/python/psutil:build_ext:--inplace
+-!windows:mozilla.pth:third_party/python/psutil
+-windows:mozilla.pth:third_party/python/psutil-cp27-none-win_amd64
+ mozilla.pth:third_party/python/pylru
+ mozilla.pth:third_party/python/pyparsing
+ mozilla.pth:third_party/python/pystache
+ python2:mozilla.pth:third_party/python/PyYAML/lib
+ python3:mozilla.pth:third_party/python/PyYAML/lib3/
+ mozilla.pth:third_party/python/requests
+ mozilla.pth:third_party/python/requests-unixsocket
+ python2:mozilla.pth:third_party/python/scandir
+diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
+--- a/python/mozbuild/mozbuild/mach_commands.py
++++ b/python/mozbuild/mozbuild/mach_commands.py
+@@ -1201,16 +1201,26 @@ class CreateMachEnvironment(MachCommandB
+             self.topsrcdir, virtualenv_path, sys.stdout,
+             os.path.join(self.topsrcdir, 'build',
+                          'mach_virtualenv_packages.txt'),
+             populate_local_paths=False)
+         manager.build(sys.executable)
+ 
+         manager.install_pip_package('zstandard>=0.9.0,<=0.13.0')
+ 
++        try:
++            # `mach` can handle it perfectly fine if `psutil` is missing, so
++            # there's no reason to freak out in this case.
++            manager.install_pip_package("psutil==5.7.0")
++        except subprocess.CalledProcessError:
++            print(
++                "Could not install psutil, so telemetry will be missing some "
++                "data. Continuing."
++            )
++
+         if not PY2:
+             # This can fail on some platforms. See
+             # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
+             try:
+                 manager.install_pip_package('glean_sdk~=31.5.0')
+             except subprocess.CalledProcessError:
+                 print('Could not install glean_sdk, so telemetry will not be '
+                       'collected. Continuing.')

mozilla-release/patches/1661635-82a1.patch

@@ -0,0 +1,40 @@
+# HG changeset patch
+# User Mike Hommey <mh+mozilla@glandium.org>
+# Date 1598626140 0
+# Node ID 6e90e5d68be7256d291bdf66a63e305011896ec7
+# Parent  8472cd3c2fed5c9720690031afe04481259549f0
+Bug 1661635 - Make windows/!windows and python2/python3 entries in virtualenv.txt non-optional. r=firefox-build-system-reviewers,rstewart
+
+Except when optional is actually specified.
+
+Differential Revision: https://phabricator.services.mozilla.com/D88527
+
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -412,23 +412,23 @@ class VirtualenvManager(object):
+                           'because optional. (%s)' % ':'.join(package),
+                           file=self.log_handle)
+                     return False
+ 
+             if package[0] in ('windows', '!windows'):
+                 for_win = not package[0].startswith('!')
+                 is_win = sys.platform == 'win32'
+                 if is_win == for_win:
+-                    handle_package(package[1:])
++                    return handle_package(package[1:])
+                 return True
+ 
+             if package[0] in ('python2', 'python3'):
+                 for_python3 = package[0].endswith('3')
+                 if PY3 == for_python3:
+-                    handle_package(package[1:])
++                    return handle_package(package[1:])
+                 return True
+ 
+             raise Exception('Unknown action: %s' % package[0])
+ 
+         # We always target the OS X deployment target that Python itself was
+         # built with, regardless of what's in the current environment. If we
+         # don't do # this, we may run into a Python bug. See
+         # http://bugs.python.org/issue9516 and bug 659881.

mozilla-release/patches/1662130-82a1.patch

@@ -0,0 +1,413 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1598979267 0
+# Node ID 3ac7571a94983003726c19f3f6d68bd61a4232aa
+# Parent  7d54b28cff6f2b821296805fb1721e65b052f882
+Bug 1662130 - Walk back new `inherit-from-parent-environment` logic in `virtualenv` handling r=ahal
+
+This logic is meant to expose packages from a globally-installed Python to be used by the in-`objdir` `virtualenv`s, so for example we don't have to figure out how to install `zstandard` (or other Python packages with native code that may or may not have prebuilt wheels for any given platform) in those `virtualenv`s. Bug 1660351 augmented that logic to work within the requirements of bug 1660353. This worked mostly, but is causing builds to unconditionally break on Arch Linux, caused a couple test failures, and in general is just introducing other weird behaviors downstream, and issues with the resultant `PYTHONPATH`s are hard to diagnose and fix.
+
+In the long-term we'll have to permanently solve the `zstandard` problem and pave the way for other Python packages with native code as well, but that's not an urgent need.
+
+The ultimate goal is to completely remove `inherit-from-parent-environment`, but we can't do that until bug 1659539 is solved.
+
+Partially reverts bugs 1660351. Entirely reverts bug 1660353, restoring that file to as it was before that patch.
+
+Differential Revision: https://phabricator.services.mozilla.com/D89001
+
+diff --git a/python/mozbuild/mozbuild/action/symbols_archive.py.1662130.later b/python/mozbuild/mozbuild/action/symbols_archive.py.1662130.later
+new file mode 100644
+--- /dev/null
++++ b/python/mozbuild/mozbuild/action/symbols_archive.py.1662130.later
+@@ -0,0 +1,60 @@
++--- symbols_archive.py
+++++ symbols_archive.py
++@@ -30,26 +30,48 @@ def make_archive(archive_name, base, exc
++             from mozpack.mozjar import JarWriter
++             with JarWriter(fileobj=fh, compress_level=5) as writer:
++                 def add_file(p, f):
++                     should_compress = any(mozpath.match(p, pat) for pat in compress)
++                     writer.add(p.encode('utf-8'), f, mode=f.mode,
++                                compress=should_compress, skip_duplicates=True)
++                 fill_archive(add_file)
++         elif archive_basename.endswith('.tar.zst'):
+++            import mozfile
+++            import subprocess
++             import tarfile
++-            import zstandard
++-            ctx = zstandard.ZstdCompressor(threads=-1)
++-            with ctx.stream_writer(fh) as zstdwriter:
++-                with tarfile.open(mode='w|', fileobj=zstdwriter,
++-                                  bufsize=1024*1024) as tar:
++-                    def add_file(p, f):
++-                        info = tar.gettarinfo(os.path.join(base, p), p)
++-                        tar.addfile(info, f.open())
++-                    fill_archive(add_file)
+++            from buildconfig import topsrcdir
+++            # Ideally, we'd do this:
+++            #   import zstandard
+++            #   ctx = zstandard.ZstdCompressor(threads=-1)
+++            #   zstdwriter = ctx.stream_writer(fh)
+++            # and use `zstdwriter` as the fileobj input for `tarfile.open`.
+++            # But this script is invoked with `PYTHON3` in a Makefile, which
+++            # uses the virtualenv python where zstandard is not installed.
+++            # Both `sys.executable` and `buildconfig.substs['PYTHON3']` would
+++            # be the same. Instead, search within PATH to find another python3,
+++            # which hopefully has zstandard available (and that's the case on
+++            # automation, where this code path is expected to be followed).
+++            python_path = mozpath.normpath(os.path.dirname(sys.executable))
+++            path = [p for p in os.environ['PATH'].split(os.pathsep)
+++                    if mozpath.normpath(p) != python_path]
+++            python3 = mozfile.which('python3', path=path)
+++            proc = subprocess.Popen([
+++                python3,
+++                os.path.join(topsrcdir, 'taskcluster', 'scripts', 'misc', 'zstdpy'),
+++                '-T0',
+++            ], stdin=subprocess.PIPE, stdout=fh)
+++
+++            with tarfile.open(mode='w|', fileobj=proc.stdin, bufsize=1024*1024) as tar:
+++                def add_file(p, f):
+++                    info = tar.gettarinfo(os.path.join(base, p), p)
+++                    tar.addfile(info, f.open())
+++                fill_archive(add_file)
+++            proc.stdin.close()
+++            proc.wait()
++         else:
++             raise Exception('Unsupported archive format for {}'.format(archive_basename))
++ 
++ 
++ def main(argv):
++     parser = argparse.ArgumentParser(description='Produce a symbols archive')
++     parser.add_argument('archive', help='Which archive to generate')
++     parser.add_argument('base', help='Base directory to package')
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -3,17 +3,16 @@
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ # This file contains code for populating the virtualenv environment for
+ # Mozilla's build system. It is typically called as part of configure.
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import argparse
+-import json
+ import os
+ import platform
+ import shutil
+ import subprocess
+ import sys
+ 
+ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
+ IS_CYGWIN = (sys.platform == 'cygwin')
+@@ -66,17 +65,17 @@ def ensure_text(s, encoding='utf-8'):
+         raise TypeError("not expecting type '%s'" % type(s))
+ 
+ 
+ class VirtualenvManager(object):
+     """Contains logic for managing virtualenvs for building the tree."""
+ 
+     def __init__(
+             self, topsrcdir, virtualenv_path, log_handle, manifest_path,
+-            populate_local_paths=True):
++            parent_site_dir=None, populate_local_paths=True):
+         """Create a new manager.
+ 
+         Each manager is associated with a source directory, a path where you
+         want the virtualenv to be created, and a handle to write output to.
+         """
+         assert os.path.isabs(
+             manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
+         self.topsrcdir = topsrcdir
+@@ -85,16 +84,20 @@ class VirtualenvManager(object):
+         # Record the Python executable that was used to create the Virtualenv
+         # so we can check this against sys.executable when verifying the
+         # integrity of the virtualenv.
+         self.exe_info_path = os.path.join(self.virtualenv_root,
+                                           'python_exe.txt')
+ 
+         self.log_handle = log_handle
+         self.manifest_path = manifest_path
++        self.parent_site_dir = parent_site_dir
++        if not self.parent_site_dir:
++            import distutils.sysconfig
++            self.parent_site_dir = distutils.sysconfig.get_python_lib()
+         self.populate_local_paths = populate_local_paths
+ 
+     @property
+     def virtualenv_script_path(self):
+         """Path to virtualenv's own populator script."""
+         return os.path.join(self.topsrcdir, 'third_party', 'python',
+                             'virtualenv', 'virtualenv.py')
+ 
+@@ -145,21 +148,16 @@ class VirtualenvManager(object):
+             fh.write("%s\n" % ver)
+ 
+     def python_executable_hexversion(self, python):
+         """Run a Python executable and return its sys.hexversion value."""
+         program = 'import sys; print(sys.hexversion)'
+         out = subprocess.check_output([python, '-c', program]).rstrip()
+         return int(out)
+ 
+-    def _python_sys_path(self, python):
+-        program = 'import json, sys; print(json.dumps(sys.path))'
+-        return json.loads(ensure_text(subprocess.check_output(
+-            [python, '-c', program])).strip())
+-
+     def up_to_date(self, python):
+         """Returns whether the virtualenv is present and up to date.
+ 
+         Args:
+             python: Full path string to the Python executable that this virtualenv
+                 should be running.  If the Python executable passed in to this
+                 argument is not the same version as the Python the virtualenv was
+                 built with then this method will return False.
+@@ -270,18 +268,18 @@ class VirtualenvManager(object):
+ 
+     def packages(self):
+         mode = 'rU' if PY2 else 'r'
+         with open(self.manifest_path, mode) as fh:
+             packages = [line.rstrip().split(':')
+                         for line in fh]
+         return packages
+ 
+-    def populate(self, python, sitecustomize=None):
+-        """Populate the virtualenv from the given python.
++    def populate(self, sitecustomize=None):
++        """Populate the virtualenv.
+ 
+         The manifest file consists of colon-delimited fields. The first field
+         specifies the action. The remaining fields are arguments to that
+         action. The following actions are supported:
+ 
+         setup.py -- Invoke setup.py for a package. Expects the arguments:
+             1. relative path directory containing setup.py.
+             2. argument(s) to setup.py. e.g. "develop". Each program argument
+@@ -311,18 +309,22 @@ class VirtualenvManager(object):
+             on non-Windows systems.
+ 
+         python3 -- This denotes that the action should only be taken when run
+             on Python 3.
+ 
+         python2 -- This denotes that the action should only be taken when run
+             on python 2.
+ 
+-        inherit-from-parent-environment -- This denotes that we should append
+-            the path from the parent python to the sys.path of the virtualenv.
++        inherit-from-parent-environment -- This denotes that we should add the
++            configured site directory of the "parent" to the virtualenv's list
++            of site directories. This can be specified on the command line as
++            --parent-site-dir or passed in the constructor of this class. This
++            defaults to the site-packages directory of the current Python
++            interpreter if not provided.
+ 
+         set-variable -- Set the given environment variable; e.g.
+             `set-variable FOO=1`.
+ 
+         Note that the Python interpreter running this function should be the
+         one from the virtualenv. If it is the system Python or if the
+         environment is not configured properly, packages could be installed
+         into the wrong place. This is how virtualenv's work.
+@@ -334,21 +336,19 @@ class VirtualenvManager(object):
+         do_close = not bool(sitecustomize)
+         sitecustomize = sitecustomize or open(
+             os.path.join(os.path.dirname(python_lib), 'sitecustomize.py'),
+             mode='w')
+ 
+         def handle_package(package):
+             if package[0] == 'inherit-from-parent-environment':
+                 assert len(package) == 1
+-                sitecustomize.write('import sys\n')
+-                paths = self._python_sys_path(python)
+-                for d in paths:
+-                    sitecustomize.write('if %s not in sys.path:\n' % repr(d))
+-                    sitecustomize.write('    sys.path.append(%s)\n' % repr(d))
++                sitecustomize.write(
++                    'import site\n'
++                    "site.addsitedir(%s)\n" % repr(self.parent_site_dir))
+                 return True
+ 
+             if package[0].startswith('set-variable '):
+                 assert len(package) == 1
+                 assignment = package[0][len('set-variable '):].strip()
+                 var, val = assignment.split('=', 1)
+                 var = var if PY3 else ensure_binary(var)
+                 val = val if PY3 else ensure_binary(val)
+@@ -377,18 +377,19 @@ class VirtualenvManager(object):
+ 
+             if package[0] == 'packages.txt':
+                 assert len(package) == 2
+ 
+                 src = os.path.join(self.topsrcdir, package[1])
+                 assert os.path.isfile(src), "'%s' does not exist" % src
+                 submanager = VirtualenvManager(
+                     self.topsrcdir, self.virtualenv_root, self.log_handle, src,
++                    parent_site_dir=self.parent_site_dir,
+                     populate_local_paths=self.populate_local_paths)
+-                submanager.populate(python, sitecustomize=sitecustomize)
++                submanager.populate(sitecustomize=sitecustomize)
+ 
+                 return True
+ 
+             if package[0].endswith('.pth'):
+                 assert len(package) == 2
+ 
+                 if not self.populate_local_paths:
+                     return True
+@@ -459,25 +460,24 @@ class VirtualenvManager(object):
+                     continue
+ 
+                 old_env_variables[k] = os.environ[k]
+                 del os.environ[k]
+ 
+             for package in packages:
+                 handle_package(package)
+ 
+-            sitecustomize.write(
+-                '# Importing mach_bootstrap has the side effect of\n'
+-                '# installing an import hook\n'
+-                'import mach_bootstrap\n'
+-            )
+-
+         finally:
+             if do_close:
++                sitecustomize.write(
++                    '# Importing mach_bootstrap has the side effect of\n'
++                    '# installing an import hook\n'
++                    'import mach_bootstrap\n')
+                 sitecustomize.close()
++
+             os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
+ 
+             if old_target is not None:
+                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
+ 
+             os.environ.update(old_env_variables)
+ 
+     def call_setup(self, directory, arguments):
+@@ -504,16 +504,18 @@ class VirtualenvManager(object):
+ 
+             raise Exception('Error installing package: %s' % directory)
+ 
+     def build(self, python):
+         """Build a virtualenv per tree conventions.
+ 
+         This returns the path of the created virtualenv.
+         """
++        import distutils
++
+         self.create(python)
+ 
+         # We need to populate the virtualenv using the Python executable in
+         # the virtualenv for paths to be proper.
+ 
+         # If this module was run from Python 2 then the __file__ attribute may
+         # point to a Python 2 .pyc file. If we are generating a Python 3
+         # virtualenv from Python 2 make sure we call Python 3 with the path to
+@@ -522,17 +524,18 @@ class VirtualenvManager(object):
+             thismodule = __file__[:-1]
+         else:
+             thismodule = __file__
+ 
+         # __PYVENV_LAUNCHER__ confuses pip about the python interpreter
+         # See https://bugzilla.mozilla.org/show_bug.cgi?id=1635481
+         os.environ.pop('__PYVENV_LAUNCHER__', None)
+         args = [self.python_path, thismodule, 'populate', self.topsrcdir,
+-                self.virtualenv_root, self.manifest_path, '--python', python]
++                self.virtualenv_root, self.manifest_path, '--parent-site-dir',
++                distutils.sysconfig.get_python_lib()]
+         if self.populate_local_paths:
+             args.append('--populate-local-paths')
+ 
+         result = self._log_process_output(args, cwd=self.topsrcdir)
+ 
+         if result != 0:
+             raise Exception('Error populating virtualenv.')
+ 
+@@ -651,16 +654,17 @@ class VirtualenvManager(object):
+ 
+         If ``pipfile`` is not ``None`` then the Pipfile located at the path
+         provided will be used to create the virtual environment. If
+         ``populate`` is ``True`` then the virtual environment will be
+         populated from the manifest file. The optional ``python`` argument
+         indicates the version of Python for pipenv to use.
+         """
+ 
++        import distutils.sysconfig
+         from distutils.version import LooseVersion
+ 
+         pipenv = os.path.join(self.bin_path, 'pipenv')
+         env = ensure_subprocess_env(os.environ.copy())
+         env.update(ensure_subprocess_env({
+             'PIPENV_IGNORE_VIRTUALENVS': '1',
+             'PIP_NO_INDEX': '1',
+             'WORKON_HOME': str(os.path.normpath(workon_home)),
+@@ -734,34 +738,22 @@ class VirtualenvManager(object):
+             del env_['PIP_NO_INDEX']
+             env_.update(ensure_subprocess_env({
+                 'PIPENV_PIPFILE': str(pipfile)
+             }))
+             subprocess.check_call([pipenv, 'install'], stderr=subprocess.STDOUT, env=env_)
+ 
+         self.virtualenv_root = ensure_venv()
+ 
+-        # Awful hack, but only necessary until bug 1659542 is closed.
+-        windows = IS_CYGWIN or IS_NATIVE_WIN
+-        populate_python = sys.executable
+-        if not python:
+-            pass
+-        elif os.path.exists(python):
+-            populate_python = python
+-        elif python.startswith('2'):
+-            populate_python = 'python2.7' + ('.exe' if windows else '')
+-        elif python.startswith('3'):
+-            populate_python = 'python3' + ('.exe' if windows else '')
+-
+         if populate:
+             # Populate from the manifest
+             args = [
+                 pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+                 self.topsrcdir, self.virtualenv_root, self.manifest_path,
+-                '--python', populate_python]
++                '--parent-site-dir', distutils.sysconfig.get_python_lib()]
+             if self.populate_local_paths:
+                 args.append('--populate-local-paths')
+             subprocess.check_call(args, stderr=subprocess.STDOUT, env=env)
+ 
+         self.activate()
+ 
+ 
+ def verify_python_version(log_handle):
+@@ -825,28 +817,28 @@ if __name__ == '__main__':
+     if len(sys.argv) < 2:
+         print('Too few arguments', file=sys.stderr)
+         sys.exit(1)
+ 
+     parser = argparse.ArgumentParser()
+     parser.add_argument('topsrcdir')
+     parser.add_argument('virtualenv_path')
+     parser.add_argument('manifest_path')
+-    parser.add_argument('--python', default=None)
++    parser.add_argument('--parent-site-dir', default=None)
+     parser.add_argument('--populate-local-paths', action='store_true')
+ 
+     if sys.argv[1] == 'populate':
+         # This should only be called internally.
+         populate = True
+         opts = parser.parse_args(sys.argv[2:])
+     else:
+         populate = False
+         opts = parser.parse_args(sys.argv[1:])
+ 
+     manager = VirtualenvManager(
+         opts.topsrcdir, opts.virtualenv_path, sys.stdout, opts.manifest_path,
++        parent_site_dir=opts.parent_site_dir,
+         populate_local_paths=opts.populate_local_paths)
+ 
+     if populate:
+-        assert opts.python
+-        manager.populate(opts.python)
++        manager.populate()
+     else:
+         manager.ensure()

mozilla-release/patches/1662632-82a1.patch

@@ -0,0 +1,81 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1599058642 0
+# Node ID fc60813b36e9b19e4f746601b5bf6b326a8c7e36
+# Parent  f427208a0f6ca892495a1871bb86321a52d2054e
+Bug 1662632 - Fix `UnboundLocalError` due to undefined variable `have_clone` in `mach bootstrap` r=froydnj
+
+This consolidates the `have_clone` logic in one place unconditionally. After bug 1647792 we're deprecating the use case where `bootstrap` is run without a clone, so that's not a problem.
+
+In reality the whole `have_clone` thing isn't necessary any more (`have_clone` is always going to be `True` in practice), but I'll save that for a bigger refactoring.
+
+Differential Revision: https://phabricator.services.mozilla.com/D89152
+
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -296,27 +296,28 @@ class Bootstrapper(object):
+         elif self.choice in APPLICATIONS.keys():
+             name, application = self.choice, APPLICATIONS[self.choice]
+         elif self.choice in APPLICATIONS.values():
+             name, application = next((k, v) for k, v in APPLICATIONS.items() if v == self.choice)
+         else:
+             raise Exception('Please pick a valid application choice: (%s)' %
+                             '/'.join(APPLICATIONS.keys()))
+ 
++        state_dir_available, state_dir = self.try_to_create_state_dir()
++        if state_dir_available:
++            self.instance.state_dir = state_dir
++
++        # We need to enable the loading of hgrc in case extensions are
++        # required to open the repo.
++        (checkout_type, checkout_root) = current_firefox_checkout(
++            env=self.instance._hg_cleanenv(load_hgrc=True),
++            hg=self.instance.which('hg'))
++        have_clone = bool(checkout_type)
++
+         if self.instance.no_system_changes:
+-            state_dir_available, state_dir = self.try_to_create_state_dir()
+-            # We need to enable the loading of hgrc in case extensions are
+-            # required to open the repo.
+-            r = current_firefox_checkout(
+-                check_output=self.instance.check_output,
+-                env=self.instance._hg_cleanenv(load_hgrc=True),
+-                hg=self.instance.which('hg'))
+-            (checkout_type, checkout_root) = r
+-            have_clone = bool(checkout_type)
+-
+             self.maybe_install_private_packages_or_exit(state_dir,
+                                                         state_dir_available,
+                                                         have_clone,
+                                                         checkout_root)
+             self.instance.ensure_mach_environment(checkout_root)
+             self._output_mozconfig(application)
+             sys.exit(0)
+ 
+@@ -324,25 +325,16 @@ class Bootstrapper(object):
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+         hg_installed, hg_modern = self.instance.ensure_mercurial_modern()
+         self.instance.ensure_python_modern()
+         self.instance.ensure_rust_modern()
+ 
+-        state_dir_available, state_dir = self.try_to_create_state_dir()
+-
+-        # We need to enable the loading of hgrc in case extensions are
+-        # required to open the repo.
+-        r = current_firefox_checkout(check_output=self.instance.check_output,
+-                                     env=self.instance._hg_cleanenv(load_hgrc=True),
+-                                     hg=self.instance.which('hg'))
+-        (checkout_type, checkout_root) = r
+-
+         # If we didn't specify a VCS, and we aren't in an exiting clone,
+         # offer a choice
+         if not self.vcs:
+             if checkout_type and False:
+                 vcs = checkout_type
+             elif self.instance.no_interactive:
+                 vcs = "hg"
+             else:

mozilla-release/patches/1662775-82a1.patch

@@ -0,0 +1,149 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1599554463 0
+# Node ID 1575ac14206f1c9527dc50545d9e2a2825fdf822
+# Parent  fc803ff178a5b81486d2d72f91b4b1dae92bd7d3
+Bug 1662775 - Refactor logic to locate `mach` `virtualenv`s r=glandium
+
+This will be found in a couple places, so we might as well make a helper function. For symmetry put it in the same file where we keep the helper function to locate the `state_dir`.
+
+Differential Revision: https://phabricator.services.mozilla.com/D89156
+
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -7,18 +7,20 @@ from __future__ import absolute_import, 
+ import hashlib
+ import os
+ import re
+ import subprocess
+ import sys
+ 
+ from distutils.version import LooseVersion
+ from mozboot import rust
+-from mozboot.util import MINIMUM_RUST_VERSION
+-from mozbuild.virtualenv import VirtualenvHelper
++from mozboot.util import (
++    get_mach_virtualenv_binary,
++    MINIMUM_RUST_VERSION,
++)
+ 
+ # NOTE: This script is intended to be run with a vanilla Python install.  We
+ # have to rely on the standard library instead of Python 2+3 helpers like
+ # the six module.
+ if sys.version_info < (3,):
+     from urllib2 import urlopen
+     input = raw_input  # noqa
+ else:
+@@ -326,18 +328,17 @@ class BaseBootstrapper(object):
+             raise ValueError("mach not found at %s" % mach_binary)
+ 
+         # NOTE: Use self.state_dir over the passed-in state_dir, which might be
+         # a subdirectory of the actual state directory.
+         if not self.state_dir:
+             raise ValueError(
+                 'Need a state directory (e.g. ~/.mozbuild) to download '
+                 'artifacts')
+-        python_location = VirtualenvHelper(os.path.join(
+-            self.state_dir, '_virtualenvs', 'mach')).python_path
++        python_location = get_mach_virtualenv_binary(state_dir=self.state_dir)
+         if not os.path.exists(python_location):
+             raise ValueError('python not found at %s' % python_location)
+ 
+         cmd = [python_location, mach_binary, 'artifact', 'toolchain',
+                '--bootstrap', '--from-build', toolchain_job]
+ 
+         subprocess.check_call(cmd, cwd=state_dir)
+ 
+diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py
+--- a/python/mozboot/mozboot/util.py
++++ b/python/mozboot/mozboot/util.py
+@@ -4,16 +4,19 @@
+ 
+ from __future__ import absolute_import, print_function, unicode_literals
+ 
+ import hashlib
+ import os
+ 
+ import six
+ 
++from mozbuild.virtualenv import VirtualenvHelper
++
++
+ here = os.path.join(os.path.dirname(__file__))
+ 
+ 
+ MINIMUM_RUST_VERSION = '1.61.0'
+ 
+ 
+ def get_state_dir(srcdir=False):
+     """Obtain path to a directory to hold state.
+@@ -48,8 +51,19 @@ def get_state_dir(srcdir=False):
+         print('Creating local state directory: %s' % state_dir)
+         os.makedirs(state_dir, mode=0o770)
+         # Save the topsrcdir that this state dir corresponds to so we can clean
+         # it up in the event its srcdir was deleted.
+         with open(os.path.join(state_dir, 'topsrcdir.txt'), 'w') as fh:
+             fh.write(srcdir)
+ 
+     return state_dir
++
++
++def get_mach_virtualenv_root(state_dir=None, py2=False):
++    return os.path.join(
++        state_dir or get_state_dir(), '_virtualenvs',
++        'mach_py2' if py2 else 'mach')
++
++
++def get_mach_virtualenv_binary(state_dir=None, py2=False):
++    root = get_mach_virtualenv_root(state_dir=state_dir, py2=py2)
++    return VirtualenvHelper(root).python_path
+diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
+--- a/python/mozbuild/mozbuild/mach_commands.py
++++ b/python/mozbuild/mozbuild/mach_commands.py
+@@ -1180,40 +1180,38 @@ class CreateMachEnvironment(MachCommandB
+ 
+     @Command('create-mach-environment', category='devenv',
+              description=(
+                  'Create the `mach` virtualenvs. If executed with python3 (the '
+                  'default when entering from `mach`), create both a python3 '
+                  'and python2.7 virtualenv. If executed with python2, only '
+                  'create the python2.7 virtualenv.'))
+     def create_mach_environment(self):
+-        from mozboot.util import get_state_dir
++        from mozboot.util import get_mach_virtualenv_root
+         from mozbuild.pythonutil import find_python2_executable
+         from mozbuild.virtualenv import VirtualenvManager
+-        from six import PY3
+-
+-        state_dir = get_state_dir()
+-        virtualenv_path = os.path.join(state_dir, '_virtualenvs',
+-                                       'mach' if PY3 else 'mach_py2')
++        from six import PY2
++
++        virtualenv_path = get_mach_virtualenv_root(py2=PY2)
+         if sys.executable.startswith(virtualenv_path):
+             print('You can only create a mach environment with the system '
+                   'Python. Re-run this `mach` command with the system Python.',
+                   file=sys.stderr)
+             return 1
+ 
+         manager = VirtualenvManager(
+             self.topsrcdir, virtualenv_path, sys.stdout,
+             os.path.join(self.topsrcdir, 'build',
+                          'mach_virtualenv_packages.txt'),
+             populate_local_paths=False)
+         manager.build(sys.executable)
+ 
+         manager.install_pip_package('zstandard>=0.9.0,<=0.13.0')
+ 
+-        if PY3:
++        if not PY2:
+             # This can fail on some platforms. See
+             # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
+             try:
+                 manager.install_pip_package('glean_sdk~=31.5.0')
+             except subprocess.CalledProcessError:
+                 print('Could not install glean_sdk, so telemetry will not be '
+                       'collected. Continuing.')
+             print('Python 3 mach environment created.')

mozilla-release/patches/1662819-82a1.patch

@@ -0,0 +1,111 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1599072778 0
+# Node ID 952e5212e142b31ddf20a13e17c4ffb8b18c8cf0
+# Parent  0d65a95b2c1c56eadfd82c8428ce9059c59b0075
+Bug 1662819 - Refactor `virtualenv` script path locating logic r=mhentges,froydnj
+
+The ability to get the path to the Python executable from a given `virtualenv` location is generally useful outside the context of all the extra stuff a `VirtualenvManager` provides, so refactor it out into a lighter-weight helper class.
+
+Differential Revision: https://phabricator.services.mozilla.com/D89175
+
+diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
+--- a/python/mozbuild/mozbuild/virtualenv.py
++++ b/python/mozbuild/mozbuild/virtualenv.py
+@@ -60,31 +60,58 @@ def ensure_text(s, encoding='utf-8'):
+     if isinstance(s, binary_type):
+         return s.decode(encoding, errors='strict')
+     elif isinstance(s, text_type):
+         return s
+     else:
+         raise TypeError("not expecting type '%s'" % type(s))
+ 
+ 
+-class VirtualenvManager(object):
++class VirtualenvHelper(object):
++    """Contains basic logic for getting information about virtualenvs."""
++
++    def __init__(self, virtualenv_path):
++        self.virtualenv_root = virtualenv_path
++
++    @property
++    def bin_path(self):
++        # virtualenv.py provides a similar API via path_locations(). However,
++        # we have a bit of a chicken-and-egg problem and can't reliably
++        # import virtualenv. The functionality is trivial, so just implement
++        # it here.
++        if IS_CYGWIN or IS_NATIVE_WIN:
++            return os.path.join(self.virtualenv_root, 'Scripts')
++
++        return os.path.join(self.virtualenv_root, 'bin')
++
++    @property
++    def python_path(self):
++        binary = 'python'
++        if sys.platform in ('win32', 'cygwin'):
++            binary += '.exe'
++
++        return os.path.join(self.bin_path, binary)
++
++
++class VirtualenvManager(VirtualenvHelper):
+     """Contains logic for managing virtualenvs for building the tree."""
+ 
+     def __init__(
+             self, topsrcdir, virtualenv_path, log_handle, manifest_path,
+             parent_site_dir=None, populate_local_paths=True):
+         """Create a new manager.
+ 
+         Each manager is associated with a source directory, a path where you
+         want the virtualenv to be created, and a handle to write output to.
+         """
++        super(VirtualenvManager, self).__init__(virtualenv_path)
++
+         assert os.path.isabs(
+             manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
+         self.topsrcdir = topsrcdir
+-        self.virtualenv_root = virtualenv_path
+ 
+         # Record the Python executable that was used to create the Virtualenv
+         # so we can check this against sys.executable when verifying the
+         # integrity of the virtualenv.
+         self.exe_info_path = os.path.join(self.virtualenv_root,
+                                           'python_exe.txt')
+ 
+         self.log_handle = log_handle
+@@ -97,35 +124,16 @@ class VirtualenvManager(object):
+ 
+     @property
+     def virtualenv_script_path(self):
+         """Path to virtualenv's own populator script."""
+         return os.path.join(self.topsrcdir, 'third_party', 'python',
+                             'virtualenv', 'virtualenv.py')
+ 
+     @property
+-    def bin_path(self):
+-        # virtualenv.py provides a similar API via path_locations(). However,
+-        # we have a bit of a chicken-and-egg problem and can't reliably
+-        # import virtualenv. The functionality is trivial, so just implement
+-        # it here.
+-        if IS_CYGWIN or IS_NATIVE_WIN:
+-            return os.path.join(self.virtualenv_root, 'Scripts')
+-
+-        return os.path.join(self.virtualenv_root, 'bin')
+-
+-    @property
+-    def python_path(self):
+-        binary = 'python'
+-        if sys.platform in ('win32', 'cygwin'):
+-            binary += '.exe'
+-
+-        return os.path.join(self.bin_path, binary)
+-
+-    @property
+     def version_info(self):
+         return eval(subprocess.check_output([
+             self.python_path, '-c', 'import sys; print(sys.version_info[:])']))
+ 
+     @property
+     def activate_path(self):
+         return os.path.join(self.bin_path, 'activate_this.py')
+ 

mozilla-release/patches/1662893-82a1.patch

@@ -0,0 +1,388 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1599582786 0
+# Node ID a4f631604a0cc37c67b2d694fb913a0226505088
+# Parent  e87f3b15df0880c568ae60e60f858999fce1fdca
+Bug 1662893 - Refactor `bootstrap` to reflect that the `~/.mozbuild` state directory and a local checkout are both mandatory r=glandium
+
+`bootstrap` is written in such a way that we don't necessarily assume the existence of either the global state directory (`~/.mozbuild`) or a local checkout of `mozilla-central`. The independence from `~/.mozbuild` is a design decision that may have been appropriate at some point, but today it's arguably useless to continue executing `bootstrap` without a global state directory (we install a bunch of build dependencies there, as well as the `mach` `virtualenv`s that are needed for running non-`bootstrap` `mach` commands after bug 1656993). The independence from a local checkout is an artifact of the old design of `python/mozboot/bin/bootstrap.py`, which is no longer necessary as of bug 1647792.
+
+With that in mind, 1) throw errors if we can't create the global state directory or find the topsrcdir, and 2) remove all existing conditionals of the form "if the global state directory exists" or "if the root of the checkout exists" since these conditions are always going to be true in practice.
+
+Differential Revision: https://phabricator.services.mozilla.com/D89220
+
+diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py
+--- a/python/mozboot/mozboot/base.py
++++ b/python/mozboot/mozboot/base.py
+@@ -260,22 +260,21 @@ class BaseBootstrapper(object):
+         Firefox for Android Artifact Mode needs an application and an ABI set,
+         and it needs paths to the Android SDK.
+         '''
+         raise NotImplementedError(
+             '%s does not yet implement generate_mobile_android_artifact_mode_mozconfig()'
+             % __name__)
+ 
+     def ensure_mach_environment(self, checkout_root):
+-        if checkout_root:
+-            mach_binary = os.path.abspath(os.path.join(checkout_root, 'mach'))
+-            if not os.path.exists(mach_binary):
+-                raise ValueError('mach not found at %s' % mach_binary)
+-            cmd = [sys.executable, mach_binary, 'create-mach-environment']
+-            subprocess.check_call(cmd, cwd=checkout_root)
++        mach_binary = os.path.abspath(os.path.join(checkout_root, 'mach'))
++        if not os.path.exists(mach_binary):
++            raise ValueError('mach not found at %s' % mach_binary)
++        cmd = [sys.executable, mach_binary, 'create-mach-environment']
++        subprocess.check_call(cmd, cwd=checkout_root)
+ 
+     def ensure_clang_static_analysis_package(self, state_dir, checkout_root):
+         '''
+         Install the clang static analysis package
+         '''
+         raise NotImplementedError(
+             '%s does not yet implement ensure_clang_static_analysis_package()'
+             % __name__)
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -21,18 +21,18 @@ if sys.version_info < (3,):
+     )
+     input = raw_input  # noqa
+ else:
+     from configparser import (
+         Error as ConfigParserError,
+         RawConfigParser,
+     )
+ 
+-# Don't forgot to add new mozboot modules to the bootstrap download
+-# list in bin/bootstrap.py!
++from mach.util import UserError
++
+ from mozboot.base import MODERN_RUST_VERSION
+ from mozboot.centosfedora import CentOSFedoraBootstrapper
+ from mozboot.opensuse import OpenSUSEBootstrapper
+ from mozboot.debian import DebianBootstrapper
+ from mozboot.freebsd import FreeBSDBootstrapper
+ from mozboot.gentoo import GentooBootstrapper
+ from mozboot.osx import OSXBootstrapper
+ from mozboot.openbsd import OpenBSDBootstrapper
+@@ -66,76 +66,42 @@ Your choice: '''
+ 
+ APPLICATIONS = OrderedDict([
+     ('Firefox for Desktop Artifact Mode', 'browser_artifact_mode'),
+     ('Firefox for Desktop', 'browser'),
+     ('Firefox for Android Artifact Mode', 'mobile_android_artifact_mode'),
+     ('Firefox for Android', 'mobile_android'),
+ ])
+ 
+-VCS_CHOICE = '''
+-Firefox can be cloned using either Git or Mercurial.
+-
+-Please choose the VCS you want to use:
+-1. Mercurial
+-2. Git
+-Your choice: '''
+-
+ STATE_DIR_INFO = '''
+ The Firefox build system and related tools store shared, persistent state
+ in a common directory on the filesystem. On this machine, that directory
+ is:
+ 
+   {statedir}
+ 
+ If you would like to use a different directory, hit CTRL+c and set the
+ MOZBUILD_STATE_PATH environment variable to the directory you'd like to
+ use and re-run the bootstrapper.
+ 
+ Would you like to create this directory?'''
+ 
+-STYLO_NODEJS_DIRECTORY_MESSAGE = '''
+-Stylo and NodeJS packages require a directory to store shared, persistent
+-state.  On this machine, that directory is:
+-
+-  {statedir}
+-
+-Please restart bootstrap and create that directory when prompted.
+-'''
+-
+-STYLE_NODEJS_REQUIRES_CLONE = '''
+-Installing Stylo and NodeJS packages requires a checkout of mozilla-central
+-(or mozilla-unified). Once you have such a checkout, please re-run
+-`./mach bootstrap` from the checkout directory.
+-'''
+-
+ FINISHED = '''
+ Your system should be ready to build %s!
+ '''
+ 
+ MOZCONFIG_SUGGESTION_TEMPLATE = '''
+ Paste the lines between the chevrons (>>> and <<<) into
+ %s:
+ 
+ >>>
+ %s
+ <<<
+ '''.strip()
+ 
+-SOURCE_ADVERTISE = '''
+-Source code can be obtained by running
+-
+-    hg clone https://hg.mozilla.org/mozilla-unified
+-
+-Or, if you prefer Git, by following the instruction here to clone from the
+-Mercurial repository:
+-
+-    https://github.com/glandium/git-cinnabar/wiki/Mozilla:-A-git-workflow-for-Gecko-development
+-'''
+-
+ CONFIGURE_MERCURIAL = '''
+ Mozilla recommends a number of changes to Mercurial to enhance your
+ experience with it.
+ 
+ Would you like to run a configuration wizard to ensure Mercurial is
+ optimally configured?'''
+ 
+ CONFIGURE_GIT = '''
+@@ -162,26 +128,23 @@ lines:
+ 
+ Then restart your shell.
+ '''
+ 
+ 
+ class Bootstrapper(object):
+     """Main class that performs system bootstrap."""
+ 
+-    def __init__(self, finished=FINISHED, choice=None, no_interactive=False,
+-                 hg_configure=False, no_system_changes=False, mach_context=None,
+-                 vcs=None):
++    def __init__(self, choice=None, no_interactive=False, hg_configure=False,
++                 no_system_changes=False, mach_context=None):
+         self.instance = None
+-        self.finished = finished
+         self.choice = choice
+         self.hg_configure = hg_configure
+         self.no_system_changes = no_system_changes
+         self.mach_context = mach_context
+-        self.vcs = vcs
+         cls = None
+         args = {'no_interactive': no_interactive,
+                 'no_system_changes': no_system_changes}
+ 
+         if sys.platform.startswith('linux'):
+             # distro package provides reliable ids for popular distributions so
+             # we use those instead of the full distribution name
+             dist_id, version, codename = distro.linux_distribution(full_distribution_name=False)
+@@ -233,57 +196,43 @@ class Bootstrapper(object):
+                 cls = WindowsBootstrapper
+ 
+         if cls is None:
+             raise NotImplementedError('Bootstrap support is not yet available '
+                                       'for your OS.')
+ 
+         self.instance = cls(**args)
+ 
+-    # The state directory code is largely duplicated from mach_bootstrap.py.
+-    # We can't easily import mach_bootstrap.py because the bootstrapper may
+-    # run in self-contained mode and only the files in this directory will
+-    # be available. We /could/ refactor parts of mach_bootstrap.py to be
+-    # part of this directory to avoid the code duplication.
+-    def try_to_create_state_dir(self):
++    def create_state_dir(self):
+         state_dir = get_state_dir()
+ 
+         if not os.path.exists(state_dir):
+             should_create_state_dir = True
+             if not self.instance.no_interactive:
+                 should_create_state_dir = self.instance.prompt_yesno(
+                     prompt=STATE_DIR_INFO.format(statedir=state_dir))
+ 
+             # This directory is by default in $HOME, or overridden via an env
+             # var, so we probably shouldn't gate it on --no-system-changes.
+             if should_create_state_dir:
+                 print('Creating global state directory: %s' % state_dir)
+                 os.makedirs(state_dir, mode=0o770)
+-
+-        state_dir_available = os.path.exists(state_dir)
+-        return state_dir_available, state_dir
++            else:
++                raise UserError('Need permission to create global state '
++                                'directory at %s' % state_dir)
++
++        return state_dir
+ 
+     def maybe_install_private_packages_or_exit(self, state_dir,
+-                                               state_dir_available,
+-                                               have_clone,
+                                                checkout_root):
+         # Install the clang packages needed for building the style system, as
+         # well as the version of NodeJS that we currently support.
+         # Also install the clang static-analysis package by default
+         # The best place to install our packages is in the state directory
+         # we have.  We should have created one above in non-interactive mode.
+-        if not state_dir_available:
+-            print(STYLO_NODEJS_DIRECTORY_MESSAGE.format(statedir=state_dir))
+-            sys.exit(1)
+-
+-        if not have_clone:
+-            print(STYLE_NODEJS_REQUIRES_CLONE)
+-            sys.exit(1)
+-
+-        self.instance.state_dir = state_dir
+         self.instance.ensure_node_packages(state_dir, checkout_root)
+         self.instance.ensure_minidump_stackwalk_packages(state_dir, checkout_root)
+         self.instance.ensure_stylo_packages(state_dir, checkout_root)
+         self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+         self.instance.ensure_nasm_packages(state_dir, checkout_root)
+         self.instance.ensure_sccache_packages(state_dir, checkout_root)
+         self.instance.ensure_lucetc_packages(state_dir, checkout_root)
+         self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
+@@ -298,89 +247,69 @@ class Bootstrapper(object):
+         elif self.choice in APPLICATIONS.keys():
+             name, application = self.choice, APPLICATIONS[self.choice]
+         elif self.choice in APPLICATIONS.values():
+             name, application = next((k, v) for k, v in APPLICATIONS.items() if v == self.choice)
+         else:
+             raise Exception('Please pick a valid application choice: (%s)' %
+                             '/'.join(APPLICATIONS.keys()))
+ 
+-        state_dir_available, state_dir = self.try_to_create_state_dir()
+-        if state_dir_available:
+-            self.instance.state_dir = state_dir
++        state_dir = self.create_state_dir()
++        self.instance.state_dir = state_dir
+ 
+         # We need to enable the loading of hgrc in case extensions are
+         # required to open the repo.
+         (checkout_type, checkout_root) = current_firefox_checkout(
+             env=self.instance._hg_cleanenv(load_hgrc=True),
+             hg=self.instance.which('hg'))
+         self.instance.ensure_mach_environment(checkout_root)
+-        have_clone = bool(checkout_type)
+ 
+         if self.instance.no_system_changes:
+             self.maybe_install_private_packages_or_exit(state_dir,
+-                                                        state_dir_available,
+-                                                        have_clone,
+                                                         checkout_root)
+             self._output_mozconfig(application)
+             sys.exit(0)
+ 
+         self.instance.install_system_packages()
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+         hg_installed, hg_modern = self.instance.ensure_mercurial_modern()
+         self.instance.ensure_python_modern()
+         self.instance.ensure_rust_modern()
+ 
+-        # If we didn't specify a VCS, and we aren't in an exiting clone,
+-        # offer a choice
+-        if not self.vcs:
+-            if checkout_type and False:
+-                vcs = checkout_type
+-            elif self.instance.no_interactive:
+-                vcs = "hg"
+-            else:
+-                prompt_choice = self.instance.prompt_int(prompt=VCS_CHOICE, low=1, high=2)
+-                vcs = ["hg", "git"][prompt_choice - 1]
+-        else:
+-            vcs = self.vcs
+-
+         # Possibly configure Mercurial, but not if the current checkout or repo
+         # type is Git.
+-        if hg_installed and state_dir_available and (checkout_type == 'hg' or vcs == 'hg'):
++        if hg_installed and checkout_type == 'hg':
+             configure_hg = False
+             if not self.instance.no_interactive:
+                 configure_hg = self.instance.prompt_yesno(prompt=CONFIGURE_MERCURIAL)
+             else:
+                 configure_hg = self.hg_configure
+ 
+             if configure_hg:
+                 configure_mercurial(self.instance.which('hg'), state_dir)
+ 
+         # Offer to configure Git, if the current checkout or repo type is Git.
+-        elif self.instance.which('git') and (checkout_type == 'git' or vcs == 'git'):
++        elif self.instance.which('git') and checkout_type == 'git':
+             should_configure_git = False
+             if not self.instance.no_interactive:
+                 should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT)
+             else:
+                 # Assuming default configuration setting applies to all VCS.
+                 should_configure_git = self.hg_configure
+ 
+             if should_configure_git:
+                 configure_git(self.instance.which('git'), state_dir,
+                               checkout_root)
+ 
+-        self.maybe_install_private_packages_or_exit(state_dir,
+-                                                    state_dir_available,
+-                                                    have_clone,
+-                                                    checkout_root)
+-
+-        print(self.finished % name)
++        self.maybe_install_private_packages_or_exit(state_dir, checkout_root)
++
++        print(FINISHED % name)
+         if not (self.instance.which('rustc') and self.instance._parse_version('rustc')
+                 >= MODERN_RUST_VERSION):
+             print("To build %s, please restart the shell (Start a new terminal window)" % name)
+ 
+         self._output_mozconfig(application)
+ 
+     def _output_mozconfig(self, application):
+         # Like 'generate_browser_mozconfig' or 'generate_mobile_android_mozconfig'.
+@@ -503,17 +432,18 @@ def current_firefox_checkout(check_outpu
+             moz_configure = os.path.join(path, 'moz.configure')
+             if os.path.exists(moz_configure):
+                 return ('git', path)
+ 
+         path, child = os.path.split(path)
+         if child == '':
+             break
+ 
+-    return (None, None)
++    raise UserError('Could not identify the root directory of your checkout! '
++                    'Are you running `mach bootstrap` in an hg or git clone?')
+ 
+ 
+ def update_git_tools(git, root_state_dir, top_src_dir):
+     """Update git tools, hooks and extensions"""
+     # Ensure git-cinnabar is up to date.
+     cinnabar_dir = os.path.join(root_state_dir, 'git-cinnabar')
+ 
+     # Ensure the latest revision of git-cinnabar is present.
+diff --git a/python/mozboot/mozboot/bootstrap.py.1662893.later b/python/mozboot/mozboot/bootstrap.py.1662893.later
+new file mode 100644
+--- /dev/null
++++ b/python/mozboot/mozboot/bootstrap.py.1662893.later
+@@ -0,0 +1,28 @@
++--- bootstrap.py
+++++ bootstrap.py
++@@ -670,21 +599,20 @@ def configure_git(git, cinnabar, root_st
++         raise Exception('Could not find git version')
++     git_version = LooseVersion(match.group(1))
++ 
++     if git_version < MINIMUM_RECOMMENDED_GIT_VERSION:
++         print(OLD_GIT_WARNING.format(
++             old_version=git_version,
++             minimum_recommended_version=MINIMUM_RECOMMENDED_GIT_VERSION))
++ 
++-    if top_src_dir and os.path.exists(top_src_dir):
++-        if git_version >= LooseVersion('2.17'):
++-            # "core.untrackedCache" has a bug before 2.17
++-            subprocess.check_call(
++-                [git, 'config', 'core.untrackedCache', 'true'], cwd=top_src_dir)
+++    if git_version >= LooseVersion('2.17'):
+++        # "core.untrackedCache" has a bug before 2.17
+++        subprocess.check_call(
+++            [git, 'config', 'core.untrackedCache', 'true'], cwd=top_src_dir)
++ 
++     cinnabar_dir = update_git_tools(git, root_state_dir, top_src_dir)
++ 
++     if not cinnabar:
++         print(ADD_GIT_CINNABAR_PATH.format(cinnabar_dir))
++ 
++ 
++ def _warn_if_risky_revision(path):

mozilla-release/patches/1664581-1-82a1.patch

@@ -0,0 +1,41 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1600118307 0
+# Node ID d2fcca85cb163aad52263c420c9d9317c40a2af5
+# Parent  4d8c015a30148b92ae826141873762c283a6a8de
+Bug 1664581 - Explicitly recommend putting `hg` in your `PATH` if `bootstrap` can't find it r=nalexander
+
+Differential Revision: https://phabricator.services.mozilla.com/D90171
+
+diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
+--- a/python/mozboot/bin/bootstrap.py
++++ b/python/mozboot/bin/bootstrap.py
+@@ -222,17 +222,27 @@ def git_clone_firefox(git, dest, watchma
+ 
+ 
+ def clone(vcs, no_interactive):
+     hg = which('hg')
+     if not hg:
+         print('Mercurial is not installed. Mercurial is required to clone '
+               'Firefox%s.' % (
+                   ', even when cloning with Git' if vcs == 'git' else ''))
+-        print('Try installing hg with `pip3 install Mercurial`.')
++        try:
++            # We're going to recommend people install the Mercurial package with
++            # pip3. That will work if `pip3` installs binaries to a location
++            # that's in the PATH, but it might not be. To help out, if we CAN
++            # import "mercurial" (in which case it's already been installed),
++            # offer that as a solution.
++            import mercurial  # noqa: F401
++            print('Hint: have you made sure that Mercurial is installed to a '
++                  'location in your PATH?')
++        except ImportError:
++            print('Try installing hg with `pip3 install Mercurial`.')
+         return None
+     if vcs == 'hg':
+         binary = hg
+     else:
+         binary = which(vcs)
+         if not binary:
+             print('Git is not installed.')
+             print('Try installing git using your system package manager.')

mozilla-release/patches/1680162-85a1.patch

@@ -0,0 +1,77 @@
+# HG changeset patch
+# User Mitchell Hentges <mhentges@mozilla.com>
+# Date 1607441705 0
+# Node ID 1a5dcdffe6c181fd18e38bdf9de96f77a99ca400
+# Parent  894391018841bf8611089aa7cb962bbb41f78e5a
+Bug 1680162: Install python packages after system packages r=firefox-build-system-reviewers,glandium
+
+Native python packages will probably require python headers. For some
+systems, these are available in a separate system package (such as
+python[3]-dev).
+
+If the user is bootstrapping with "--no-system-changes", we just
+try to install these native packages and fail gracefully if required
+headers aren't installed.
+
+Differential Revision: https://phabricator.services.mozilla.com/D98530
+
+diff --git a/python/mozboot/mozboot/base.py.1680162.later b/python/mozboot/mozboot/base.py.1680162.later
+new file mode 100644
+--- /dev/null
++++ b/python/mozboot/mozboot/base.py.1680162.later
+@@ -0,0 +1,21 @@
++--- base.py
+++++ base.py
++@@ -170,17 +170,17 @@ class BaseBootstrapper(object):
++         Called once the current firefox checkout has been detected.
++         Platform-specific implementations should check the environment and offer advice/warnings
++         to the user, if necessary.
++         """
++ 
++     def suggest_install_distutils(self):
++         """Called if distutils.{sysconfig,spawn} can't be imported."""
++         print(
++-            "Does your distro require installing another package for " "distutils?",
+++            "Does your distro require installing another package for distutils?",
++             file=sys.stderr,
++         )
++ 
++     def suggest_install_pip3(self):
++         """Called if pip3 can't be found."""
++         print(
++             "Try installing pip3 with your system's package manager.", file=sys.stderr
++         )
+diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py
+--- a/python/mozboot/mozboot/bootstrap.py
++++ b/python/mozboot/mozboot/bootstrap.py
+@@ -255,25 +255,29 @@ class Bootstrapper(object):
+         state_dir = self.create_state_dir()
+         self.instance.state_dir = state_dir
+ 
+         # We need to enable the loading of hgrc in case extensions are
+         # required to open the repo.
+         (checkout_type, checkout_root) = current_firefox_checkout(
+             env=self.instance._hg_cleanenv(load_hgrc=True),
+             hg=self.instance.which('hg'))
+-        self.instance.ensure_mach_environment(checkout_root)
+ 
+         if self.instance.no_system_changes:
++            self.instance.ensure_mach_environment(checkout_root)
+             self.maybe_install_private_packages_or_exit(state_dir,
+                                                         checkout_root)
+             self._output_mozconfig(application)
+             sys.exit(0)
+ 
+         self.instance.install_system_packages()
++        # Install mach environment python packages after system packages.
++        # Some mach packages require building native modules, which require
++        # tools which are installed to the system.
++        self.instance.ensure_mach_environment(checkout_root)
+ 
+         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
+         getattr(self.instance, 'install_%s_packages' % application)()
+ 
+         hg_installed, hg_modern = self.instance.ensure_mercurial_modern()
+         self.instance.ensure_python_modern()
+         self.instance.ensure_rust_modern()
+ 

mozilla-release/patches/1712819-1-90a1.patch

@@ -2,7 +2,7 @@
 # User Mitchell Hentges <mhentges@mozilla.com>
 # Date 1622156646 0
 # Node ID 9c25bbd071297bb44a0d36f02b40fda07411959e
-# Parent  07abde1fb1c5827cc617f22694373d602ad92dfc
+# Parent  a41506266ebe6e571e7c5dfe11e66afa94d6f927
 Bug 1712819: Fix VirtualenvManager not being expandable in debugger r=ahal
 
 At least in PyCharm, expanding a `VirtualenvManager` instance means
@@ -38,14 +38,14 @@ diff --git a/python/mach_commands.py b/python/mach_commands.py
 diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
 --- a/python/mozbuild/mozbuild/virtualenv.py
 +++ b/python/mozbuild/mozbuild/virtualenv.py
-@@ -77,17 +77,16 @@ class VirtualenvManager(object):
-     @property
-     def python_path(self):
-         binary = 'python'
-         if sys.platform in ('win32', 'cygwin'):
-             binary += '.exe'
+@@ -122,17 +122,16 @@ class VirtualenvManager(VirtualenvHelper
+         self.populate_local_paths = populate_local_paths
  
-         return os.path.join(self.bin_path, binary)
+     @property
+     def virtualenv_script_path(self):
+         """Path to virtualenv's own populator script."""
+         return os.path.join(self.topsrcdir, 'third_party', 'python',
+                             'virtualenv', 'virtualenv.py')
  
 -    @property
      def version_info(self):

mozilla-release/patches/1712819-2-90a1.patch

@@ -2,7 +2,7 @@
 # User Mitchell Hentges <mhentges@mozilla.com>
 # Date 1622156646 0
 # Node ID 3a5b335bdc8c52a384f3f3a03a87ea03d12d2935
-# Parent  5dc310ecb395e9a5eb92ffcdcc3559ed8b0d1970
+# Parent  f6667092787e0d622f35afc4ac52851ac0992fdf
 Bug 1712819: Avoid pip's "outdated" warning in virtualenvs r=ahal
 
 Now, when running mach commands that invoke `pip`, it will no longer
@@ -16,7 +16,7 @@ Differential Revision: https://phabricator.services.mozilla.com/D115940
 diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py
 --- a/python/mozbuild/mozbuild/virtualenv.py
 +++ b/python/mozbuild/mozbuild/virtualenv.py
-@@ -219,16 +219,17 @@ class VirtualenvManager(object):
+@@ -264,16 +264,17 @@ class VirtualenvManager(VirtualenvHelper
          result = self._log_process_output(args)
  
          if result:
@@ -34,7 +34,7 @@ diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/v
          with open(self.manifest_path, mode) as fh:
              packages = [line.rstrip().split(':')
                          for line in fh]
-@@ -548,16 +549,65 @@ class VirtualenvManager(object):
+@@ -637,16 +638,65 @@ class VirtualenvManager(VirtualenvHelper
          if vendored:
              args.extend([
                  '--no-deps',

mozilla-release/patches/series

@@ -43,6 +43,7 @@ NOBUG-20170803-promisehelper-57a1.patch
 1380617-10-57a1.patch
 1380512-57a1.patch
 1390675-57a1.patch
+1390693-8-57a1.patch
 1387088-57a1.patch
 1372592-57a1.patch
 1393900-57a1.patch
@@ -441,6 +442,7 @@ NOBUG-20170803-promisehelper-57a1.patch
 1403871-2-59a1.patch
 1403871-3-59a1.patch
 1418274-59a1.patch
+1402154-59a1.patch
 1419447-59a1.patch
 1408401-59a1.patch
 1419665-59a1.patch
@@ -3896,7 +3898,12 @@ NOBUG-20180505-lint-61a1.patch
 1346291-2-62a1.patch
 1346291-3-62a1.patch
 1457359-62a1.patch
+1454640-1-62a1.patch
+1454640-2-62a1.patch
+1454640-3-62a1.patch
 1454640-4-62a1.patch
+1454640-5-62a1.patch
+1454640-6-62a1.patch
 1441914-1-62a1.patch
 1441914-2-62a1.patch
 1346291-4-62a1.patch
@@ -4247,6 +4254,8 @@ NOBUG-20180505-lint-61a1.patch
 1472490-63a1.patch
 1475649-1-63a1.patch
 1475642-63a1.patch
+1474746-1-63a1.patch
+1474746-2-63a1.patch
 1448976-63a1.patch
 1475650-63a1.patch
 1448980-0-63a1.patch
@@ -5872,6 +5881,7 @@ NOBUG-20190207-crashreporter-67a1.patch
 1280561-73a1.patch
 1596660-PARTIAL-removeonly-73a1.patch
 1602317-2-PARTIAL-X11Undefonly-73a1.patch
+1600664-73a1.patch
 1606728-74a1.patch
 1605659-fix-74a1.patch
 1606124-74a1.patch
@@ -5977,6 +5987,7 @@ NOBUG-20190207-crashreporter-67a1.patch
 1620449-75a1.patch
 1618879-75a1.patch
 1583854-75a1.patch
+1617984-75a1.patch
 1620433-76a1.patch
 1620744-01-76a1.patch
 1620744-02-76a1.patch
@@ -6198,6 +6209,9 @@ NOBUG-20190207-crashreporter-67a1.patch
 1638954-78a1.patch
 1638978-78a1.patch
 1127565-78a1.patch
+1635834-78a1.patch
+1638799-78a1.patch
+1638012-78a1.patch
 1621960-781.patch
 1642032-1-79a1.patch
 1642032-2-79a1.patch
@@ -6922,6 +6936,7 @@ TOP-NOBUG-seamonkey-credits.patch
 1606009-2-73a1.patch
 1606009-3-73a1.patch
 1606009-4-73a1.patch
+1606825-73a1.patch
 1605879-74a1.patch
 1605850-74a1.patch
 1602773-1-75a1.patch
@@ -6946,6 +6961,7 @@ TOP-NOBUG-seamonkey-credits.patch
 1640878-78a1.patch
 1636251-1-PARTIAL-78a1.patch
 1634391-2-78a1.patch
+1643298-79a1.patch
 1639406-79a1.patch
 1638985-79a1.patch
 1436251-3-PARTIAL-79a1.patch
@@ -6984,10 +7000,14 @@ TOP-NOBUG-seamonkey-credits.patch
 1646427-3-79a1.patch
 1646427-4-79a1.patch
 1645097-79a1.patch
+1643317-79a1.patch
 1645948-80a1.patch
 1650057-80a1.patch
 1606475-80a1.patch
 1654795-80a1.patch
+1654663-80a1.patch
+1482675-80a1.patch
+1655781-81a1.patch
 1655701-81a1.patch
 1653560-81a1.patch
 985141-1-81a1.patch
@@ -7002,12 +7022,28 @@ TOP-NOBUG-seamonkey-credits.patch
 1659154-81a1.patch
 1659411-1-81a1.patch
 1659411-2-81a1.patch
+1656993-81a1.patch
+1660105-81a1.patch
+1646406-81a1.patch
+1661391-82a1.patch
+1660351-82a1.patch
+1661635-82a1.patch
+1662130-82a1.patch
+1662819-82a1.patch
 1659542-82a1.patch
+1647792-82a1.patch
+1664581-1-82a1.patch
+1662632-82a1.patch
+1660559-82a1.patch
+1662775-82a1.patch
 1662851-82a1.patch
+1662893-82a1.patch
 1666232-83a1.patch
+1661624-85a1.patch
 1666345-85a1.patch
 1666347-85a1.patch
 1680051-85a1.patch
+1680162-85a1.patch
 1712819-1-90a1.patch
 1712819-2-90a1.patch
 1712819-3-91a1.patch