# HG changeset patch
# User Nick Alexander <nalexander@mozilla.com>
# Date 1520553846 28800
# Node ID fe70510127c52b9ac8c1c9287ffc9c7148c414aa
# Parent  0db6e65892e98f6fb98579b75d87b44beb65c936
Bug 1443208 - Pre: Add force flag to GENERATED_FILES. r=ted.mielczarek

The forces on the system are such that I really need to be able to
FORCE a few RecursiveMake targets in order to make Android and Gradle
use GENERATED_FILES and LOCALIZED_GENERATED_FILES.  Over time, I hope
to avoid FORCE, but that time is not now.

MozReview-Commit-ID: 453FpnihSRK

diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -573,31 +573,38 @@ class RecursiveMakeBackend(CommonBackend
                     # Android localized resources have special Makefile
                     # handling.
                     backend_file.write('%s:: %s\n' % (tier, first_output))
             for output in outputs:
                 if output != first_output:
                     backend_file.write('%s: %s ;\n' % (output, first_output))
                 backend_file.write('GARBAGE += %s\n' % output)
             backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file)
+
+            force = ''
+            if obj.force:
+                force = ' FORCE'
+            elif obj.localized:
+                force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)'
+
             if obj.script:
-                backend_file.write("""{output}: {script}{inputs}{backend}{repack_force}
+                backend_file.write("""{output}: {script}{inputs}{backend}{force}
 \t$(REPORT_BUILD)
 \t$(call py_action,file_generate,{locale}{script} {method} {output} $(MDDEPDIR)/{dep_file}{inputs}{flags})
 
 """.format(output=first_output,
            dep_file=dep_file,
            inputs=' ' + ' '.join(inputs) if inputs else '',
            flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '',
            backend=' backend.mk' if obj.flags else '',
            # Locale repacks repack multiple locales from a single configured objdir,
            # so standard mtime dependencies won't work properly when the build is re-run
            # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set
            # in this situation, so simply force the generation to run in that case.
-           repack_force=' $(if $(IS_LANGUAGE_REPACK),FORCE)' if obj.localized else '',
+           force=force,
            locale='--locale=$(AB_CD) ' if obj.localized else '',
            script=obj.script,
            method=obj.method))
 
         elif isinstance(obj, JARManifest):
             self._no_skip['libs'].add(backend_file.relobjdir)
             backend_file.write('JAR_MANIFEST := %s\n' % obj.path.full_path)
 
diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -926,16 +926,17 @@ BugzillaComponent = TypedNamedTuple('Bug
                         [('product', unicode), ('component', unicode)])
 SchedulingComponents = ContextDerivedTypedRecord(
         ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)),
         ('exclusive', TypedList(unicode, StrictOrderingOnAppendList)))
 
 GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({
     'script': unicode,
     'inputs': list,
+    'force': bool,
     'flags': list, })
 
 
 class Files(SubContext):
     """Metadata attached to files.
 
     It is common to want to annotate files with metadata, such as which
     Bugzilla component tracks issues with certain files. This sub-context is
@@ -1271,18 +1272,18 @@ VARIABLES = {
         size.
         """),
 
     'GENERATED_FILES': (GeneratedFilesList, list,
         """Generic generated files.
 
         This variable contains a list of files for the build system to
         generate at export time. The generation method may be declared
-        with optional ``script``, ``inputs`` and ``flags`` attributes on
-        individual entries.
+        with optional ``script``, ``inputs``, ``flags``, and ``force``
+        attributes on individual entries.
         If the optional ``script`` attribute is not present on an entry, it
         is assumed that rules for generating the file are present in
         the associated Makefile.in.
 
         Example::
 
            GENERATED_FILES += ['bar.c', 'baz.c', 'foo.c']
            bar = GENERATED_FILES['bar.c']
@@ -1310,16 +1311,21 @@ VARIABLES = {
           bar = GENERATED_FILES['bar.c']
           bar.script = 'generate.py:make_bar'
 
         The chosen script entry point may optionally return a set of strings,
         indicating extra files the output depends on.
 
         When the ``flags`` attribute is present, the given list of flags is
         passed as extra arguments following the inputs.
+
+        When the ``force`` attribute is present, the file is generated every
+        build, regardless of whether it is stale.  This is special to the
+        RecursiveMake backend and intended for special situations only (e.g.,
+        localization).  Please consult a build peer before using ``force``.
         """),
 
     'DEFINES': (InitializedDefines, dict,
         """Dictionary of compiler defines to declare.
 
         These are passed in to the compiler as ``-Dkey='value'`` for string
         values, ``-Dkey=value`` for numeric values, or ``-Dkey`` if the
         value is True. Note that for string values, the outer-level of
diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -1117,26 +1117,29 @@ class GeneratedFile(ContextDerived):
     __slots__ = (
         'script',
         'method',
         'outputs',
         'inputs',
         'flags',
         'required_for_compile',
         'localized',
+        'force',
     )
 
-    def __init__(self, context, script, method, outputs, inputs, flags=(), localized=False):
+    def __init__(self, context, script, method, outputs, inputs,
+                 flags=(), localized=False, force=False):
         ContextDerived.__init__(self, context)
         self.script = script
         self.method = method
         self.outputs = outputs if isinstance(outputs, tuple) else (outputs,)
         self.inputs = inputs
         self.flags = flags
         self.localized = localized
+        self.force = force
 
         suffixes = (
             '.c',
             '.cpp',
             '.h',
             '.inc',
             '.py',
             '.rs',
diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1386,17 +1386,17 @@ class TreeMetadataEmitter(LoggingMixin):
                     if (isinstance(p, SourcePath) and
                             not os.path.exists(p.full_path)):
                         raise SandboxValidationError(
                             'Input for generating %s does not exist: %s'
                             % (f, p.full_path), context)
                     inputs.append(p)
 
                 yield GeneratedFile(context, script, method, outputs, inputs,
-                                    flags.flags, localized=localized)
+                                    flags.flags, localized=localized, force=flags.force)
 
     def _process_test_manifests(self, context):
         for prefix, info in TEST_MANIFESTS.items():
             for path, manifest in context.get('%s_MANIFESTS' % prefix, []):
                 for obj in self._process_test_manifest(context, info, path, manifest):
                     yield obj
 
         for flavor in REFTEST_FLAVORS:
diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data
copy from python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data
copy to python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data
diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py
copy from python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py
copy to python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py
diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py
copy from python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py
copy to python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py
diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build
copy from python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build
copy to python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build
--- a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build
@@ -1,12 +1,14 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ]
 
 bar = GENERATED_FILES['bar.c']
 bar.script = 'generate-bar.py:baz'
+bar.force = True
 
 foo = GENERATED_FILES['foo.c']
 foo.script = 'generate-foo.py'
 foo.inputs = ['foo-data']
+foo.force = False
diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/en-US/localized-input b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input
copy from python/mozbuild/mozbuild/test/backend/data/localized-generated-files/en-US/localized-input
copy to python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input
diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/foo-data b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data
copy from python/mozbuild/mozbuild/test/backend/data/localized-generated-files/foo-data
copy to python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data
diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/generate-foo.py b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py
copy from python/mozbuild/mozbuild/test/backend/data/localized-generated-files/generate-foo.py
copy to python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py
diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build
copy from python/mozbuild/mozbuild/test/backend/data/localized-generated-files/moz.build
copy to python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build
--- a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build
@@ -6,10 +6,17 @@ LOCALIZED_GENERATED_FILES += [ 'foo.xyz'
 
 foo = LOCALIZED_GENERATED_FILES['foo.xyz']
 foo.script = 'generate-foo.py'
 foo.inputs = [
     'en-US/localized-input',
     'non-localized-input',
 ]
 
-# Also check that using it in LOCALIZED_FILES does the right thing.
-LOCALIZED_FILES += [ '!foo.xyz' ]
+LOCALIZED_GENERATED_FILES += [ 'abc.xyz' ]
+
+abc = LOCALIZED_GENERATED_FILES['abc.xyz']
+abc.script = 'generate-foo.py'
+abc.inputs = [
+    'en-US/localized-input',
+    'non-localized-input',
+]
+abc.force = True
diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files/non-localized-input b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input
copy from python/mozbuild/mozbuild/test/backend/data/localized-generated-files/non-localized-input
copy to python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input
diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -413,16 +413,46 @@ class TestRecursiveMakeBackend(BackendTe
             'export:: quux.c',
             'GARBAGE += quux.c',
             'EXTRA_MDDEPEND_FILES += quux.c.pp',
         ]
 
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
+    def test_generated_files_force(self):
+        """Ensure GENERATED_FILES with .force is handled properly."""
+        env = self._consume('generated-files-force', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'export:: bar.c',
+            'GARBAGE += bar.c',
+            'EXTRA_MDDEPEND_FILES += bar.c.pp',
+            'bar.c: %s/generate-bar.py FORCE' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp)' % env.topsrcdir,
+            '',
+            'export:: foo.c',
+            'GARBAGE += foo.c',
+            'EXTRA_MDDEPEND_FILES += foo.c.pp',
+            'foo.c: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(srcdir)/foo-data)' % (env.topsrcdir),
+            '',
+            'export:: quux.c',
+            'GARBAGE += quux.c',
+            'EXTRA_MDDEPEND_FILES += quux.c.pp',
+        ]
+
+        self.maxDiff = None
+        self.assertEqual(lines, expected)
+
     def test_localized_generated_files(self):
         """Ensure LOCALIZED_GENERATED_FILES is handled properly."""
         env = self._consume('localized-generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
@@ -437,16 +467,43 @@ class TestRecursiveMakeBackend(BackendTe
             'LOCALIZED_FILES_0_DEST = $(FINAL_TARGET)/',
             'LOCALIZED_FILES_0_TARGET := libs',
             'INSTALL_TARGETS += LOCALIZED_FILES_0',
         ]
 
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
+    def test_localized_generated_files_force(self):
+        """Ensure LOCALIZED_GENERATED_FILES with .force is handled properly."""
+        env = self._consume('localized-generated-files-force', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'libs:: foo.xyz',
+            'GARBAGE += foo.xyz',
+            'EXTRA_MDDEPEND_FILES += foo.xyz.pp',
+            'foo.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,
+            '',
+            'libs:: abc.xyz',
+            'GARBAGE += abc.xyz',
+            'EXTRA_MDDEPEND_FILES += abc.xyz.pp',
+            'abc.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,
+            '',
+        ]
+
+        self.maxDiff = None
+        self.assertEqual(lines, expected)
+
     def test_localized_generated_files_AB_CD(self):
         """Ensure LOCALIZED_GENERATED_FILES is handled properly
         when {AB_CD} and {AB_rCD} are used."""
         env = self._consume('localized-generated-files-AB_CD', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build
copy from python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build
@@ -1,5 +1,7 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 GENERATED_FILES += [ 'bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
+GENERATED_FILES['bar.c'].force = True
+GENERATED_FILES['foo.c'].force = False
diff --git a/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build
copy from python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build
@@ -1,5 +1,6 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ]
+LOCALIZED_GENERATED_FILES['abc.ini'].force = True
diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -446,34 +446,43 @@ class TestEmitterBasic(unittest.TestCase
         maxDiff = self.maxDiff
         self.maxDiff = None
         self.assertEqual(passthru.variables,
                          {'AS': 'yasm',
                           'AS_DASH_C_FLAG': '',
                           'ASOUTOPTION': '-o '})
         self.maxDiff = maxDiff
 
-
     def test_generated_files(self):
         reader = self.reader('generated-files')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 3)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)
             self.assertFalse(o.localized)
+            self.assertFalse(o.force)
 
         expected = ['bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
         for o, f in zip(objs, expected):
             expected_filename = f if isinstance(f, tuple) else (f,)
             self.assertEqual(o.outputs, expected_filename)
             self.assertEqual(o.script, None)
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_generated_files_force(self):
+        reader = self.reader('generated-files-force')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 3)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+            self.assertEqual(o.force, 'bar.c' in o.outputs)
+
     def test_localized_generated_files(self):
         reader = self.reader('localized-generated-files')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 2)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)
             self.assertTrue(o.localized)
@@ -482,16 +491,26 @@ class TestEmitterBasic(unittest.TestCase
         expected = ['abc.ini', ('bar', 'baz'), ]
         for o, f in zip(objs, expected):
             expected_filename = f if isinstance(f, tuple) else (f,)
             self.assertEqual(o.outputs, expected_filename)
             self.assertEqual(o.script, None)
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_localized_generated_files_force(self):
+        reader = self.reader('localized-generated-files-force')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+            self.assertTrue(o.localized)
+            self.assertEqual(o.force, 'abc.ini' in o.outputs)
+
     def test_localized_files_from_generated(self):
         """Test that using LOCALIZED_GENERATED_FILES and then putting the output in
         LOCALIZED_FILES as an objdir path works.
         """
         reader = self.reader('localized-files-from-generated')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 2)

