# HG changeset patch
# User Ted Mielczarek <ted@mielczarek.org>
# Date 1515612372 18000
# Node ID ad09b92fb875944ba775ef452a245669b567619d
# Parent  200b8243af2a9ea3ea79a4a1a12c9e1e26d262ec
bug 1255485 - build PROGRAMs directly in dist/bin instead of copying them. r=nalexander

Historically we built all our binaries in directories in the objdir, then
symlinked them into dist/bin. Some binaries needed to be copied instead
so that certain relative path lookups work properly, so we resorted to
sprinkling `NSDISTMODE=copy` around Makefiles.

This change makes it so we build PROGRAMs (not any other sort of targets)
directly in dist/bin instead. We could do the same for our other targets
with a little more work.

There were several places in the tree that were copying built binaries to
some other place and needed fixup to match the new location of binaries.

On Windows pdb files are left in the objdir where the program was
originally linked. symbolstore.py needs to locate the pdb file both to
determine whether it should dump symbols for a binary and also to copy
the pdb file into the symbol package. We fix this by simply looking for
the pdb file in the current working directory if it isn't present next
to the binary, which matches how we invoke symbolstore.py.

MozReview-Commit-ID: 8TOD1uTXD5e

diff --git a/browser/app/Makefile.in b/browser/app/Makefile.in
--- a/browser/app/Makefile.in
+++ b/browser/app/Makefile.in
@@ -41,17 +41,17 @@ endif
 PROGRAMS_DEST = $(DIST)/bin
 
 include $(topsrcdir)/config/rules.mk
 
 ifneq (,$(filter-out WINNT,$(OS_ARCH)))
 
 ifdef COMPILE_ENVIRONMENT
 libs:: 
-	cp -p $(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
+	cp -p $(DIST)/bin/$(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
 endif
 
 GARBAGE += $(addprefix $(FINAL_TARGET)/defaults/pref/, firefox.js)
 
 endif
 
 # channel-prefs.js is handled separate from other prefs due to bug 756325
 libs:: $(srcdir)/profile/channel-prefs.js
diff --git a/config/makefiles/target_binaries.mk b/config/makefiles/target_binaries.mk
--- a/config/makefiles/target_binaries.mk
+++ b/config/makefiles/target_binaries.mk
@@ -2,18 +2,18 @@
 # vim:set ts=8 sw=8 sts=8 noet:
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 ifndef NO_DIST_INSTALL
 
-ifneq (,$(strip $(PROGRAM)$(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
-PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(PROGRAM) $(RUST_PROGRAMS)
+ifneq (,$(strip $(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
+PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(RUST_PROGRAMS)
 PROGRAMS_DEST ?= $(FINAL_TARGET)
 PROGRAMS_TARGET := target
 INSTALL_TARGETS += PROGRAMS
 endif
 
 
 ifdef SHARED_LIBRARY
 SHARED_LIBRARY_FILES = $(SHARED_LIBRARY)
diff --git a/config/rules.mk b/config/rules.mk
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -554,17 +554,17 @@ LINKER_OUT=$(subst /,\,$1)
 else
 LINKER_OUT=$1
 endif
 
 #
 # PROGRAM = Foo
 # creates OBJS, links with LIBS to create Foo
 #
-$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(EXE_DEF_FILE) $(RESFILE) $(GLOBAL_DEPS)
+$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
 	$(REPORT_BUILD)
 	@$(RM) $@.manifest
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 	$(EXPAND_LINK) -NOLOGO -OUT:$(call LINKER_OUT,$@) -PDB:$(LINK_PDBFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(PROGOBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(EXTRA_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$@.manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
diff --git a/js/src/shell/moz.build b/js/src/shell/moz.build
--- a/js/src/shell/moz.build
+++ b/js/src/shell/moz.build
@@ -54,9 +54,9 @@ if CONFIG['CC_TYPE'] in ('msvc', 'clang-
 
 # Place a GDB Python auto-load file next to the shell executable, both in
 # the build directory and in the dist/bin directory.
 DEFINES['topsrcdir'] = '%s/js/src' % TOPSRCDIR
 FINAL_TARGET_PP_FILES += ['js-gdb.py.in']
 OBJDIR_FILES.js.src.shell += ['!/dist/bin/js-gdb.py']
 
 # People expect the js shell to wind up in the top-level JS dir.
-OBJDIR_FILES.js.src += ['!js%s' % CONFIG['BIN_SUFFIX']]
+OBJDIR_FILES.js.src += ['!/dist/bin/js%s' % CONFIG['BIN_SUFFIX']]
diff --git a/modules/libmar/tests/moz.build b/modules/libmar/tests/moz.build
--- a/modules/libmar/tests/moz.build
+++ b/modules/libmar/tests/moz.build
@@ -3,10 +3,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
 
 if CONFIG['OS_TARGET'] != 'Android':
     TEST_HARNESS_FILES.xpcshell.modules.libmar.tests.unit += [
-        '!../tool/signmar%s' % CONFIG['BIN_SUFFIX'],
+        '!/dist/bin/signmar%s' % CONFIG['BIN_SUFFIX'],
     ]
diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1108,17 +1108,17 @@ class RecursiveMakeBackend(CommonBackend
             interfaces_manifests = ' '.join(interfaces_manifests),
             xpidl_rules=rules.getvalue(),
             xpidl_modules=' '.join(xpt_modules),
             xpt_files=' '.join(sorted(xpt_files - registered_xpt_files)),
             registered_xpt_files=' '.join(sorted(registered_xpt_files)),
         ))
 
     def _process_program(self, obj, backend_file):
-        backend_file.write('PROGRAM = %s\n' % obj.program)
+        backend_file.write('PROGRAM = %s\n' % self._pretty_path(obj.output_path, backend_file))
         if not obj.cxx_link and not self.environment.bin_suffix:
             backend_file.write('PROG_IS_C_ONLY_%s := 1\n' % obj.program)
 
     def _process_host_program(self, program, backend_file):
         backend_file.write('HOST_PROGRAM = %s\n' % program)
 
     def _process_rust_program_base(self, obj, backend_file,
                                    target_variable,
diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -12,16 +12,17 @@ All data structures of interest are chil
 Logic for populating these data structures is not defined in this class.
 Instead, what we have here are dumb container classes. The emitter module
 contains the code for converting executed mozbuild files into these data
 structures.
 """
 
 from __future__ import absolute_import, unicode_literals
 
+from mozbuild.frontend.context import ObjDirPath
 from mozbuild.util import StrictOrderingOnAppendList
 from mozpack.chrome.manifest import ManifestEntry
 
 import mozpack.path as mozpath
 from .context import FinalTargetValue
 
 from collections import defaultdict, OrderedDict
 
@@ -80,16 +81,20 @@ class ContextDerived(TreeMetadata):
 
         self._context = context
 
     @property
     def install_target(self):
         return self._context['FINAL_TARGET']
 
     @property
+    def installed(self):
+        return self._context['DIST_INSTALL'] is not False
+
+    @property
     def defines(self):
         defines = self._context['DEFINES']
         return Defines(self._context, defines) if defines else None
 
     @property
     def relobjdir(self):
         return mozpath.relpath(self.objdir, self.topobjdir)
 
@@ -409,16 +414,23 @@ class BaseProgram(Linkable):
         Linkable.__init__(self, context)
 
         bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '')
         if not program.endswith(bin_suffix):
             program += bin_suffix
         self.program = program
         self.is_unit_test = is_unit_test
 
+    @property
+    def output_path(self):
+        if self.installed:
+            return ObjDirPath(self._context, '!/' + mozpath.join(self.install_target, self.program))
+        else:
+            return ObjDirPath(self._context, '!' + self.program)
+
     def __repr__(self):
         return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.program)
 
 
 class Program(BaseProgram):
     """Context derived container object for PROGRAM"""
     SUFFIX_VAR = 'BIN_SUFFIX'
     KIND = 'target'
diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -759,17 +759,18 @@ class TreeMetadataEmitter(LoggingMixin):
                              'GENERATED_FILES: %s') % (symbols_file,), context)
                     shared_args['symbols_file'] = symbols_file.target_basename
 
             if shared_lib:
                 lib = SharedLibrary(context, libname, **shared_args)
                 self._libs[libname].append(lib)
                 self._linkage.append((context, lib, 'USE_LIBS'))
                 linkables.append(lib)
-                generated_files.add(lib.lib_name)
+                if not lib.installed:
+                    generated_files.add(lib.lib_name)
                 if symbols_file and isinstance(symbols_file, SourcePath):
                     script = mozpath.join(
                         mozpath.dirname(mozpath.dirname(__file__)),
                         'action', 'generate_symbols_file.py')
                     defines = ()
                     if lib.defines:
                         defines = lib.defines.get_defines()
                     yield GeneratedFile(context, script,
diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -610,31 +610,50 @@ class Dumper:
 
         elapsed = time.time() - t_start
         print('Finished processing %s in %.2fs' % (file, elapsed),
               file=sys.stderr)
 
 # Platform-specific subclasses.  For the most part, these just have
 # logic to determine what files to extract symbols from.
 
+def locate_pdb(path):
+    '''Given a path to a binary, attempt to locate the matching pdb file with simple heuristics:
+    * Look for a pdb file with the same base name next to the binary
+    * Look for a pdb file with the same base name in the cwd
+
+    Returns the path to the pdb file if it exists, or None if it could not be located.
+    '''
+    path, ext = os.path.splitext(path)
+    pdb = path + '.pdb'
+    if os.path.isfile(pdb):
+        return pdb
+    # If there's no pdb next to the file, see if there's a pdb with the same root name
+    # in the cwd. We build some binaries directly into dist/bin, but put the pdb files
+    # in the relative objdir, which is the cwd when running this script.
+    base = os.path.basename(pdb)
+    pdb = os.path.join(os.getcwd(), base)
+    if os.path.isfile(pdb):
+        return pdb
+    return None
+
 class Dumper_Win32(Dumper):
     fixedFilenameCaseCache = {}
 
     def ShouldProcess(self, file):
         """This function will allow processing of exe or dll files that have pdb
         files with the same base name next to them."""
         if file.endswith(".exe") or file.endswith(".dll"):
-            path, ext = os.path.splitext(file)
-            if os.path.isfile(path + ".pdb"):
+            if locate_pdb(file) is not None:
                 return True
         return False
 
 
     def CopyDebug(self, file, debug_file, guid, code_file, code_id):
-        file = "%s.pdb" % os.path.splitext(file)[0]
+        file = locate_pdb(file)
         def compress(path):
             compressed_file = path[:-1] + '_'
             # ignore makecab's output
             makecab = buildconfig.substs['MAKECAB']
             success = subprocess.call([makecab, "-D",
                                        "CompressionType=MSZIP",
                                        path, compressed_file],
                                       stdout=open(os.devnull, 'w'),
diff --git a/toolkit/crashreporter/tools/unit-symbolstore.py b/toolkit/crashreporter/tools/unit-symbolstore.py
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -464,18 +464,17 @@ class TestFunctional(HelperMixin, unitte
             else:
                 self.dump_syms = os.path.join(self.topsrcdir,
                                               'toolkit',
                                               'crashreporter',
                                               'tools',
                                               'win32',
                                               'dump_syms_vc{_MSC_VER}.exe'.format(**buildconfig.substs))
             self.target_bin = os.path.join(buildconfig.topobjdir,
-                                           'browser',
-                                           'app',
+                                           'dist', 'bin',
                                            'firefox.exe')
         else:
             self.dump_syms = os.path.join(buildconfig.topobjdir,
                                           'dist', 'host', 'bin',
                                           'dump_syms')
             self.target_bin = os.path.join(buildconfig.topobjdir,
                                            'dist', 'bin', 'firefox')
 
@@ -484,26 +483,28 @@ class TestFunctional(HelperMixin, unitte
         HelperMixin.tearDown(self)
 
     def testSymbolstore(self):
         if self.skip_test:
             raise unittest.SkipTest('Skipping test in non-Firefox product')
         dist_include_manifest = os.path.join(buildconfig.topobjdir,
                                              '_build_manifests/install/dist_include')
         dist_include = os.path.join(buildconfig.topobjdir, 'dist/include')
+        browser_app = os.path.join(buildconfig.topobjdir, 'browser/app')
         output = subprocess.check_output([sys.executable,
                                           self.script_path,
                                           '--vcs-info',
                                           '-s', self.topsrcdir,
                                           '--install-manifest=%s,%s' % (dist_include_manifest,
                                                                         dist_include),
                                           self.dump_syms,
                                           self.test_dir,
                                           self.target_bin],
-                                         stderr=open(os.devnull, 'w'))
+                                         stderr=open(os.devnull, 'w'),
+                                         cwd=browser_app)
         lines = filter(lambda x: x.strip(), output.splitlines())
         self.assertEqual(1, len(lines),
                          'should have one filename in the output')
         symbol_file = os.path.join(self.test_dir, lines[0])
         self.assertTrue(os.path.isfile(symbol_file))
         symlines = open(symbol_file, 'r').readlines()
         file_lines = [l for l in symlines if l.startswith('FILE')]
         def check_hg_path(lines, match):
