|
@@ -0,0 +1,349 @@
|
|
|
|
+# HG changeset patch
|
|
|
|
+# User Andi-Bogdan Postelnicu <bpostelnicu@mozilla.com>
|
|
|
|
+# Date 1569237023 0
|
|
|
|
+# Node ID de2c00f37a4ab8327776285ac53b6a554adbf4ec
|
|
|
|
+# Parent 62768df130c387a74a604153f23795038d7f431e
|
|
|
|
+Bug 1527818 - Add Coverity analysis for full build on try. r=marco,bastien
|
|
|
|
+
|
|
|
|
+Differential Revision: https://phabricator.services.mozilla.com/D46477
|
|
|
|
+
|
|
|
|
+diff --git a/python/mozbuild/mozbuild/code-analysis/mach_commands.py b/python/mozbuild/mozbuild/code-analysis/mach_commands.py
|
|
|
|
+--- a/python/mozbuild/mozbuild/code-analysis/mach_commands.py
|
|
|
|
++++ b/python/mozbuild/mozbuild/code-analysis/mach_commands.py
|
|
|
|
+@@ -286,109 +286,191 @@ class StaticAnalysis(MachCommandBase):
|
|
|
|
+ @CommandArgument('--output', '-o', default=None,
|
|
|
|
+ help='Write coverity output translated to json output in a file')
|
|
|
|
+ @CommandArgument('--coverity_output_path', '-co', default=None,
|
|
|
|
+ help='Path where to write coverity results as cov-results.json. '
|
|
|
|
+ 'If no path is specified the default path from the coverity working '
|
|
|
|
+ 'directory, ~./mozbuild/coverity is used.')
|
|
|
|
+ @CommandArgument('--outgoing', default=False, action='store_true',
|
|
|
|
+ help='Run coverity on outgoing files from mercurial or git repository')
|
|
|
|
++ @CommandArgument('--full-build', default=False, action='store_true',
|
|
|
|
++ help='Run a full build for coverity analisys.')
|
|
|
|
+ def check_coverity(self, source=[], output=None, coverity_output_path=None,
|
|
|
|
+- outgoing=False, verbose=False):
|
|
|
|
++ outgoing=False, full_build=False, verbose=False):
|
|
|
|
+ self._set_log_level(verbose)
|
|
|
|
+ self.log_manager.enable_all_structured_loggers()
|
|
|
|
+
|
|
|
|
+ if 'MOZ_AUTOMATION' not in os.environ:
|
|
|
|
+ self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+ 'Coverity based static-analysis cannot be ran outside automation.')
|
|
|
|
+ return
|
|
|
|
+
|
|
|
|
++ if full_build and outgoing:
|
|
|
|
++ self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
++ 'Coverity full build cannot be associated with outgoing.')
|
|
|
|
++ return
|
|
|
|
++
|
|
|
|
+ # Use outgoing files instead of source files
|
|
|
|
+ if outgoing:
|
|
|
|
+ repo = get_repository_object(self.topsrcdir)
|
|
|
|
+ files = repo.get_outgoing_files()
|
|
|
|
+ source = map(os.path.abspath, files)
|
|
|
|
+
|
|
|
|
+- if len(source) == 0:
|
|
|
|
++ # Verify that we have source files or we are dealing with a full-build
|
|
|
|
++ if len(source) == 0 and not full_build:
|
|
|
|
+ self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+ 'There are no files that coverity can use to scan.')
|
|
|
|
+ return 0
|
|
|
|
+
|
|
|
|
++ # Load the configuration file for coverity static-analysis
|
|
|
|
++ # For the moment we store only the reliability index for each checker
|
|
|
|
++ # as the rest is managed on the https://github.com/mozilla/release-services side.
|
|
|
|
++ self._cov_config = self._get_cov_config()
|
|
|
|
++
|
|
|
|
++ rc = self.setup_coverity()
|
|
|
|
++ if rc != 0:
|
|
|
|
++ return rc
|
|
|
|
++
|
|
|
|
++ # First run cov-run-desktop --setup in order to setup the analysis env
|
|
|
|
++ # We need this in both cases, per patch analysis or full tree build
|
|
|
|
++ cmd = [self.cov_run_desktop, '--setup']
|
|
|
|
++ if self.run_cov_command(cmd, self.cov_path):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ # Run cov-configure for clang, javascript and python
|
|
|
|
++ langs = ["clang", "javascript", "python"]
|
|
|
|
++ for lang in langs:
|
|
|
|
++ cmd = [self.cov_configure, '--{}'.format(lang)]
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ if full_build:
|
|
|
|
++ # 1. Build the model file that is going to be used for analysis
|
|
|
|
++ model_path = mozpath.join("tools", "coverity", "model.cpp")
|
|
|
|
++ cmd = [self.cov_make_library, "-sf", self.cov_lic_path, model_path]
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ # 2. Run cov-build
|
|
|
|
++
|
|
|
|
++ # Add cov_build command
|
|
|
|
++ cmd = [
|
|
|
|
++ self.cov_build,
|
|
|
|
++ '--dir',
|
|
|
|
++ 'cov-int'
|
|
|
|
++ ]
|
|
|
|
++ # Add fs capture search paths for languages that are not nuilt
|
|
|
|
++ cmd += [
|
|
|
|
++ "--fs-capture-search={}".format(path)
|
|
|
|
++ for path in self.cov_capture_search
|
|
|
|
++ ]
|
|
|
|
++
|
|
|
|
++ # Add the exclude criteria for test cases
|
|
|
|
++ cmd += [
|
|
|
|
++ '--fs-capture-search-exclude-regex',
|
|
|
|
++ '.*/test',
|
|
|
|
++ './mach', '--log-no-times', 'build'
|
|
|
|
++ ]
|
|
|
|
++ if self.run_cov_command(cmd):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ # 3. Run cov-analyze and exclude disabled checkers
|
|
|
|
++ cmd = [
|
|
|
|
++ self.cov_analyze,
|
|
|
|
++ '--dir',
|
|
|
|
++ 'cov-int',
|
|
|
|
++ '--all',
|
|
|
|
++ '--enable-virtual',
|
|
|
|
++ '--strip-path={}'.format(self.topsrcdir),
|
|
|
|
++ '-sf',
|
|
|
|
++ self.cov_lic_path
|
|
|
|
++ ]
|
|
|
|
++
|
|
|
|
++ cmd += [
|
|
|
|
++ "--disable={}".format(key)
|
|
|
|
++ for key, checker in self._cov_config['coverity_checkers'].items()
|
|
|
|
++ if checker.get("publish", True) is False
|
|
|
|
++ ]
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ # 4. Run cov-commit-defects
|
|
|
|
++ protocol = "https" if self.cov_server_ssl else "http"
|
|
|
|
++ server_url = "{0}://{1}:{2}".format(protocol, self.cov_url, self.cov_port)
|
|
|
|
++ cmd = [
|
|
|
|
++ self.cov_commit_defects,
|
|
|
|
++ "--auth-key-file",
|
|
|
|
++ self.cov_auth_path,
|
|
|
|
++ "--stream",
|
|
|
|
++ self.cov_stream,
|
|
|
|
++ "--dir",
|
|
|
|
++ "cov-int",
|
|
|
|
++ "--url",
|
|
|
|
++ server_url,
|
|
|
|
++ "-sf",
|
|
|
|
++ self.cov_lic_path
|
|
|
|
++ ]
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd):
|
|
|
|
++ return 1
|
|
|
|
++
|
|
|
|
++ return 0
|
|
|
|
++
|
|
|
|
+ rc = self._build_compile_db(verbose=verbose)
|
|
|
|
+ rc = rc or self._build_export(jobs=2, verbose=verbose)
|
|
|
|
+
|
|
|
|
+ if rc != 0:
|
|
|
|
+ return rc
|
|
|
|
+
|
|
|
|
+ commands_list = self.get_files_with_commands(source)
|
|
|
|
+ if len(commands_list) == 0:
|
|
|
|
+ self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+ 'There are no files that need to be analyzed.')
|
|
|
|
+ return 0
|
|
|
|
+
|
|
|
|
+- # Load the configuration file for coverity static-analysis
|
|
|
|
+- # For the moment we store only the reliability index for each checker
|
|
|
|
+- # as the rest is managed on the https://github.com/mozilla/release-services side.
|
|
|
|
+- self._cov_config = self._get_cov_config()
|
|
|
|
+-
|
|
|
|
+- rc = self.setup_coverity()
|
|
|
|
+- if rc != 0:
|
|
|
|
+- return rc
|
|
|
|
+-
|
|
|
|
+- # First run cov-run-desktop --setup in order to setup the analysis env
|
|
|
|
+- cmd = [self.cov_run_desktop, '--setup']
|
|
|
|
+- self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+- 'Running {} --setup'.format(self.cov_run_desktop))
|
|
|
|
+-
|
|
|
|
+- rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True)
|
|
|
|
+-
|
|
|
|
+- if rc != 0:
|
|
|
|
+- self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+- 'Running {} --setup failed!'.format(self.cov_run_desktop))
|
|
|
|
+- return rc
|
|
|
|
+-
|
|
|
|
+- # Run cov-configure for clang
|
|
|
|
+- cmd = [self.cov_configure, '--clang']
|
|
|
|
+- self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+- 'Running {} --clang'.format(self.cov_configure))
|
|
|
|
+-
|
|
|
|
+- rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True)
|
|
|
|
+-
|
|
|
|
+- if rc != 0:
|
|
|
|
+- self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+- 'Running {} --clang failed!'.format(self.cov_configure))
|
|
|
|
+- return rc
|
|
|
|
+-
|
|
|
|
+ # For each element in commands_list run `cov-translate`
|
|
|
|
+ for element in commands_list:
|
|
|
|
+- cmd = [self.cov_translate, '--dir', self.cov_idir_path] + element['command'].split(' ')
|
|
|
|
+- self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+- 'Running Coverity Tranlate for {}'.format(cmd))
|
|
|
|
+- rc = self.run_process(args=cmd, cwd=element['directory'], pass_thru=True)
|
|
|
|
+- if rc != 0:
|
|
|
|
+- self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+- 'Running Coverity Tranlate failed for {}'.format(cmd))
|
|
|
|
+- return cmd
|
|
|
|
++ cmd = [self.cov_translate, '--dir', self.cov_idir_path] + \
|
|
|
|
++ element['command'].split(' ')
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd, element['directory']):
|
|
|
|
++ return 1
|
|
|
|
+
|
|
|
|
+ if coverity_output_path is None:
|
|
|
|
+ cov_result = mozpath.join(self.cov_state_path, 'cov-results.json')
|
|
|
|
+ else:
|
|
|
|
+ cov_result = mozpath.join(coverity_output_path, 'cov-results.json')
|
|
|
|
+
|
|
|
|
+ # Once the capture is performed we need to do the actual Coverity Desktop analysis
|
|
|
|
+- cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--analyze-captured-source']
|
|
|
|
+- self.log(logging.INFO, 'static-analysis', {},
|
|
|
|
+- 'Running Coverity Analysis for {}'.format(cmd))
|
|
|
|
+- rc = self.run_process(cmd, cwd=self.cov_state_path, pass_thru=True)
|
|
|
|
+- if rc != 0:
|
|
|
|
+- self.log(logging.ERROR, 'static-analysis', {}, 'Coverity Analysis failed!')
|
|
|
|
++ cmd = [self.cov_run_desktop, '--json-output-v6',
|
|
|
|
++ cov_result, '--analyze-captured-source']
|
|
|
|
++
|
|
|
|
++ if self.run_cov_command(cmd, self.cov_state_path):
|
|
|
|
++ return 1
|
|
|
|
+
|
|
|
|
+ if output is not None:
|
|
|
|
+ self.dump_cov_artifact(cov_result, source, output)
|
|
|
|
+
|
|
|
|
++ def run_cov_command(self, cmd, path=None):
|
|
|
|
++ if path is None:
|
|
|
|
++ # We want to run it in topsrcdir
|
|
|
|
++ path = self.topsrcdir
|
|
|
|
++
|
|
|
|
++ self.log(logging.INFO, 'static-analysis', {}, 'Running '+' '.join(cmd))
|
|
|
|
++
|
|
|
|
++ rc = self.run_process(args=cmd, cwd=path, pass_thru=True)
|
|
|
|
++
|
|
|
|
++ if rc != 0:
|
|
|
|
++ self.log(logging.ERROR, 'static-analysis', {}, 'Running ' + ' '.join(cmd) + ' failed!')
|
|
|
|
++ return rc
|
|
|
|
++ return 0
|
|
|
|
++
|
|
|
|
+ def get_reliability_index_for_cov_checker(self, checker_name):
|
|
|
|
+ if self._cov_config is None:
|
|
|
|
+ self.log(logging.INFO, 'static-analysis', {}, 'Coverity config file not found, '
|
|
|
|
+ 'using default-value \'reliablity\' = medium. for checker {}'.format(
|
|
|
|
+ checker_name))
|
|
|
|
+ return 'medium'
|
|
|
|
+
|
|
|
|
+ checkers = self._cov_config['coverity_checkers']
|
|
|
|
+@@ -495,22 +577,26 @@ class StaticAnalysis(MachCommandBase):
|
|
|
|
+ if cov_config is None:
|
|
|
|
+ self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+ 'Ill formatted secret for Coverity. Aborting analysis.')
|
|
|
|
+ return 1
|
|
|
|
+
|
|
|
|
+ self.cov_analysis_url = cov_config.get('package_url')
|
|
|
|
+ self.cov_package_name = cov_config.get('package_name')
|
|
|
|
+ self.cov_url = cov_config.get('server_url')
|
|
|
|
++ self.cov_server_ssl = cov_config.get('server_ssl', True)
|
|
|
|
+ # In case we don't have a port in the secret we use the default one,
|
|
|
|
+ # for a default coverity deployment.
|
|
|
|
+ self.cov_port = cov_config.get('server_port', 8443)
|
|
|
|
+ self.cov_auth = cov_config.get('auth_key')
|
|
|
|
+ self.cov_package_ver = cov_config.get('package_ver')
|
|
|
|
++ self.cov_lic_name = cov_config.get('lic_name')
|
|
|
|
++ self.cov_capture_search = cov_config.get('fs_capture_search', None)
|
|
|
|
+ self.cov_full_stack = cov_config.get('full_stack', False)
|
|
|
|
++ self.cov_stream = cov_config.get('stream', False)
|
|
|
|
+
|
|
|
|
+ return 0
|
|
|
|
+
|
|
|
|
+ def download_coverity(self):
|
|
|
|
+ if self.cov_url is None or self.cov_port is None or \
|
|
|
|
+ self.cov_analysis_url is None or \
|
|
|
|
+ self.cov_auth is None:
|
|
|
|
+ self.log(logging.ERROR, 'static-analysis', {}, 'Missing Coverity secret on try job!')
|
|
|
|
+@@ -532,36 +618,36 @@ class StaticAnalysis(MachCommandBase):
|
|
|
|
+ "cov_run_desktop": {
|
|
|
|
+ "build_cmd": [],
|
|
|
|
+ "clean_cmd": []
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ '''
|
|
|
|
+ # Generate the coverity.conf and auth files
|
|
|
|
+- cov_auth_path = mozpath.join(self.cov_state_path, 'auth')
|
|
|
|
++ self.cov_auth_path = mozpath.join(self.cov_state_path, 'auth')
|
|
|
|
+ cov_setup_path = mozpath.join(self.cov_state_path, 'coverity.conf')
|
|
|
|
+- cov_conf = COVERITY_CONFIG % (self.cov_url, self.cov_port, cov_auth_path)
|
|
|
|
++ cov_conf = COVERITY_CONFIG % (self.cov_url, self.cov_port, self.cov_auth_path)
|
|
|
|
+
|
|
|
|
+ def download(artifact_url, target):
|
|
|
|
+ import requests
|
|
|
|
+ resp = requests.get(artifact_url, verify=False, stream=True)
|
|
|
|
+ resp.raise_for_status()
|
|
|
|
+
|
|
|
|
+ # Extract archive into destination
|
|
|
|
+ with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar:
|
|
|
|
+ tar.extractall(target)
|
|
|
|
+
|
|
|
|
+ download(self.cov_analysis_url, self.cov_state_path)
|
|
|
|
+
|
|
|
|
+- with open(cov_auth_path, 'w') as f:
|
|
|
|
++ with open(self.cov_auth_path, 'w') as f:
|
|
|
|
+ f.write(self.cov_auth)
|
|
|
|
+
|
|
|
|
+ # Modify it's permission to 600
|
|
|
|
+- os.chmod(cov_auth_path, 0o600)
|
|
|
|
++ os.chmod(self.cov_auth_path, 0o600)
|
|
|
|
+
|
|
|
|
+ with open(cov_setup_path, 'a') as f:
|
|
|
|
+ f.write(cov_conf)
|
|
|
|
+
|
|
|
|
+ def setup_coverity(self, force_download=True):
|
|
|
|
+ rc, config, _ = self._get_config_environment()
|
|
|
|
+ rc = rc or self.get_coverity_secrets()
|
|
|
|
+
|
|
|
|
+@@ -576,20 +662,27 @@ class StaticAnalysis(MachCommandBase):
|
|
|
|
+
|
|
|
|
+ os.mkdir(self.cov_state_path)
|
|
|
|
+
|
|
|
|
+ # Download everything that we need for Coverity from out private instance
|
|
|
|
+ self.download_coverity()
|
|
|
|
+
|
|
|
|
+ self.cov_path = mozpath.join(self.cov_state_path, self.cov_package_name)
|
|
|
|
+ self.cov_run_desktop = mozpath.join(self.cov_path, 'bin', 'cov-run-desktop')
|
|
|
|
++ self.cov_configure = mozpath.join(self.cov_path, 'bin', 'cov-configure')
|
|
|
|
++ self.cov_make_library = mozpath.join(self.cov_path, 'bin', 'cov-make-library')
|
|
|
|
++ self.cov_build = mozpath.join(self.cov_path, 'bin', 'cov-build')
|
|
|
|
++ self.cov_analyze = mozpath.join(self.cov_path, 'bin', 'cov-analyze')
|
|
|
|
++ self.cov_commit_defects = mozpath.join(self.cov_path, 'bin', 'cov-commit-defects')
|
|
|
|
+ self.cov_translate = mozpath.join(self.cov_path, 'bin', 'cov-translate')
|
|
|
|
+ self.cov_configure = mozpath.join(self.cov_path, 'bin', 'cov-configure')
|
|
|
|
+ self.cov_work_path = mozpath.join(self.cov_state_path, 'data-coverity')
|
|
|
|
+ self.cov_idir_path = mozpath.join(self.cov_work_path, self.cov_package_ver, 'idir')
|
|
|
|
++ self.cov_lic_path = mozpath.join(
|
|
|
|
++ self.cov_work_path, self.cov_package_ver, 'lic', self.cov_lic_name)
|
|
|
|
+
|
|
|
|
+ if not os.path.exists(self.cov_path):
|
|
|
|
+ self.log(logging.ERROR, 'static-analysis', {},
|
|
|
|
+ 'Missing Coverity in {}'.format(self.cov_path))
|
|
|
|
+ return 1
|
|
|
|
+
|
|
|
|
+ return 0
|
|
|
|
+
|