ensure_subprocess_env

frg-buildfixes/mozilla/TOP-1566826-1-PLASTER-define-ensure_subprocess_env.patch

@@ -0,0 +1,55 @@
+# HG changeset patch
+# User Andrew Halberstadt <ahalberstadt@mozilla.com>
+# Date 1567640973 0
+# Node ID d64a120a589b9197889592531692b86f5922c609
+# Parent  efdbcf5c4addc6f6ef8be5c4df916c9b3a8ba1b2
+Bug 1577826 - [mozbuild] Create an 'ensure_subprocess_env' utility function, r=glandium a=lizzard
+
+Differential Revision: https://phabricator.services.mozilla.com/D44667
+
+diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py
+--- a/python/mozbuild/mozbuild/util.py
++++ b/python/mozbuild/mozbuild/util.py
+@@ -1359,8 +1360,42 @@ def patch_main():
+             fork_string = ("main_file_name = '%s'\n" % main_file_name +
+                            "main_module_name = '%s'\n" % main_module_name +
+                            ''.join(x[12:] for x in fork_code[1:]))
+             cmdline = orig_command_line()
+             cmdline[2] = fork_string
+             return cmdline
+         orig_command_line = forking.get_command_line
+         forking.get_command_line = my_get_command_line
++
++
++def ensure_subprocess_env(env, encoding='utf-8'):
++    """Ensure the environment is in the correct format for the `subprocess`
++    module.
++
++    This method uses the method with same name from virtualenv.py in
++    mozilla-central.
++
++    This will convert all keys and values to bytes on Python 2, and text on
++    Python 3.
++
++    Args:
++        env (dict): Environment to ensure.
++        encoding (str): Encoding to use when converting to/from bytes/text
++                        (default: utf-8).
++    """
++    def ensure_bytes(value, encoding='utf-8'):
++        try:
++            return value.decode('latin_1').encode(encoding)
++        except AttributeError:
++            return value
++
++    def ensure_unicode(value, encoding='utf-8'):
++        try:
++            return value.decode(encoding)
++        except AttributeError:
++            return value
++
++    ensure = ensure_bytes if sys.version_info[0] < 3 else ensure_unicode
++    try:
++        return {ensure(k, encoding): ensure(v, encoding) for k, v in env.iteritems()}
++    except AttributeError:
++        return {ensure(k, encoding): ensure(v, encoding) for k, v in env.items()}

frg-buildfixes/mozilla/TOP-1606160-PLASTER-use-ensure_subprocess_env.patch

@@ -0,0 +1,106 @@
+# HG changeset patch
+# User Ricky Stewart <rstewart@mozilla.com>
+# Date 1577806210 0
+# Node ID 6e02a9956f57b0c32e8bad6b023a39d56343f7cb
+# Parent  84a9f1e261d3ac46e4675473755af9212e67e89c
+Bug 1606160 - mozbuild/base.py and mozbuild/mozconfig.py support Python 3 r=firefox-build-system-reviewers,mshal
+
+Differential Revision: https://phabricator.services.mozilla.com/D58323
+
+diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
+--- a/python/mozbuild/mozbuild/base.py
++++ b/python/mozbuild/mozbuild/base.py
+@@ -44,17 +44,20 @@ def ancestors(path):
+     while path:
+         yield path
+         newpath = os.path.dirname(path)
+         if newpath == path:
+             break
+         path = newpath
+ 
+ def samepath(path1, path2):
+-    if hasattr(os.path, 'samefile'):
++    # Under Python 3 (but NOT Python 2), MozillaBuild exposes the
++    # os.path.samefile function despite it not working, so only use it if we're
++    # not running under Windows.
++    if hasattr(os.path, 'samefile') and os.name != 'nt':
+         return os.path.samefile(path1, path2)
+     return os.path.normcase(os.path.realpath(path1)) == \
+         os.path.normcase(os.path.realpath(path2))
+ 
+ class BadEnvironmentException(Exception):
+     """Base class for errors raised when the build environment is not sane."""
+ 
+ 
+diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py
+--- a/python/mozbuild/mozbuild/mozconfig.py
++++ b/python/mozbuild/mozbuild/mozconfig.py
+@@ -2,22 +2,23 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ 
+ from __future__ import absolute_import, unicode_literals
+ 
+ import filecmp
+ import os
+ import re
+ import sys
+ import subprocess
+ import traceback
+ 
+ from collections import defaultdict
+ from mozpack import path as mozpath
++from mozbuild.util import ensure_subprocess_env
+ 
+ 
+ MOZ_MYCONFIG_ERROR = '''
+ The MOZ_MYCONFIG environment variable to define the location of mozconfigs
+ is deprecated. If you wish to define the mozconfig path via an environment
+ variable, use MOZCONFIG instead.
+ '''.strip()
+ 
+@@ -237,18 +239,19 @@ class MozconfigLoader(object):
+         command = [shell, mozpath.normsep(self._loader_script),
+                    mozpath.normsep(self.topsrcdir), path, sys.executable,
+                    mozpath.join(mozpath.dirname(self._loader_script),
+                                 'action', 'dump_env.py')]
+ 
+         try:
+             # We need to capture stderr because that's where the shell sends
+             # errors if execution fails.
+-            output = subprocess.check_output(command, stderr=subprocess.STDOUT,
+-                cwd=self.topsrcdir, env=env)
++            output = subprocess.check_output(
++                command, stderr=subprocess.STDOUT, cwd=self.topsrcdir,
++                env=ensure_subprocess_env(os.environ), universal_newlines=True)
+         except subprocess.CalledProcessError as e:
+             lines = e.output.splitlines()
+ 
+             # Output before actual execution shouldn't be relevant.
+             try:
+                 index = lines.index('------END_BEFORE_SOURCE')
+                 lines = lines[index + 1:]
+             except ValueError:
+@@ -349,22 +352,16 @@ class MozconfigLoader(object):
+         env_after_source = {}
+ 
+         current = None
+         current_type = None
+         in_variable = None
+ 
+         for line in output.splitlines():
+ 
+-            # XXX This is an ugly hack. Data may be lost from things
+-            # like environment variable values.
+-            # See https://bugzilla.mozilla.org/show_bug.cgi?id=831381
+-            line = line.decode('mbcs' if sys.platform == 'win32' else 'utf-8',
+-                               'ignore')
+-
+             if not line:
+                 continue
+ 
+             if line.startswith('------BEGIN_'):
+                 assert current_type is None
+                 assert current is None
+                 assert not in_variable
+                 current_type = line[len('------BEGIN_'):]

frg-buildfixes/mozilla/series

@@ -2425,6 +2425,8 @@ TOP-1580697-rust137_servo_webrender-56.patch
 TOP-1493400-6-PLASTER-dav1d-avoid-mColorDepth-257.patch
 TOP-1445683-14-PLASTER-aom-fix-win32-bustage-257.patch
 TOP-1294490-7-PLASTER-webp-257.patch
+TOP-1566826-1-PLASTER-define-ensure_subprocess_env.patch
+TOP-1606160-PLASTER-use-ensure_subprocess_env.patch
 TOP-9999999-fixlangpack-257.patch
 1395504-61a1.patch
 1457545-1-62a1.patch

frg-buildfixes/mozilla/series_wg9s

@@ -2425,6 +2425,8 @@ TOP-1580697-rust137_servo_webrender-56.patch
 TOP-1493400-6-PLASTER-dav1d-avoid-mColorDepth-257.patch
 TOP-1445683-14-PLASTER-aom-fix-win32-bustage-257.patch
 TOP-1294490-7-PLASTER-webp-257.patch
+TOP-1566826-1-PLASTER-define-ensure_subprocess_env.patch
+TOP-1606160-PLASTER-use-ensure_subprocess_env.patch
 TOP-9999999-fixlangpack-257.patch
 1395504-61a1.patch
 1457545-1-62a1.patch