[5/9] support/scripts: Add a per configuration CVE checker

This scripts takes as entry on stdin a JSON description of the package
used for a given configuration. This description is the one generated
by "make show-info".

The script generates the list of all the package used and if they are
affected by a CVE. The output is either a JSON or an HTML file similar
to the one generated by pkg-stats.

Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
---
 support/scripts/cve-checker | 291 ++++++++++++++++++++++++++++++++++++
 1 file changed, 291 insertions(+)
 create mode 100755 support/scripts/cve-checker

Gregory,

On Wed, Jul 8, 2020 at 11:41 AM Gregory CLEMENT
<gregory.clement@bootlin.com> wrote:
>
> This scripts takes as entry on stdin a JSON description of the package
> used for a given configuration. This description is the one generated
> by "make show-info".
>
> The script generates the list of all the package used and if they are
> affected by a CVE. The output is either a JSON or an HTML file similar
> to the one generated by pkg-stats.
>
> Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
> ---
>  support/scripts/cve-checker | 291 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 291 insertions(+)
>  create mode 100755 support/scripts/cve-checker
>
> diff --git a/support/scripts/cve-checker b/support/scripts/cve-checker
> new file mode 100755
> index 0000000000..db8497d7aa
> --- /dev/null
> +++ b/support/scripts/cve-checker
> @@ -0,0 +1,291 @@
> +#!/usr/bin/env python
> +
> +# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
> +# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
> +
> +import argparse
> +import datetime
> +import fnmatch
> +import os
> +from collections import defaultdict
> +import re
> +import subprocess
> +import requests  # URL checking
> +import json
> +import ijson
> +import certifi
> +import distutils.version
> +import time
> +import gzip
> +import sys
> +from urllib3 import HTTPSConnectionPool
> +from urllib3.exceptions import HTTPError
> +from multiprocessing import Pool
> +
> +sys.path.append('utils/')
> +
> +import cve as cvecheck
> +
> +
> +INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
> +URL_RE = re.compile(r"\s*https?://\S*\s*$")
> +
> +RM_API_STATUS_ERROR = 1
> +RM_API_STATUS_FOUND_BY_DISTRO = 2
> +RM_API_STATUS_FOUND_BY_PATTERN = 3
> +RM_API_STATUS_NOT_FOUND = 4
> +
> +# Used to make multiple requests to the same host. It is global
> +# because it's used by sub-processes.
> +http_pool = None
> +
> +
> +class Package:
> +    def __init__(self, name, version, ignored_cves):
> +        self.name = name
> +        self.version = version
> +        self.cves = list()
> +        self.ignored_cves = ignored_cves
> +
> +def check_package_cves(nvd_path, packages):
> +    if not os.path.isdir(nvd_path):
> +        os.makedirs(nvd_path)
> +
> +    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):

This read_nvd_dir call that does the dictionary download has a whole
bunch of parsing package output  "Cannot parse package 'openssl'
version ''".  I assume some of that output will get cleaned up when we
add the additional CPE matching

> +        for pkg_name in cve.pkg_names:
> +            pkg = packages.get(pkg_name, '')
> +            if pkg and cve.affects(pkg.name, pkg.version, pkg.ignored_cves):
> +                pkg.cves.append(cve.identifier)
> +
> +html_header = """
> +<head>
> +<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
> +<style type=\"text/css\">
> +table {
> +  width: 100%;
> +}
> +td {
> +  border: 1px solid black;
> +}
> +td.centered {
> +  text-align: center;
> +}
> +td.wrong {
> +  background: #ff9a69;
> +}
> +td.correct {
> +  background: #d2ffc4;
> +}
> +td.nopatches {
> +  background: #d2ffc4;
> +}
> +td.somepatches {
> +  background: #ffd870;
> +}
> +td.lotsofpatches {
> +  background: #ff9a69;
> +}
> +
> +td.good_url {
> +  background: #d2ffc4;
> +}
> +td.missing_url {
> +  background: #ffd870;
> +}
> +td.invalid_url {
> +  background: #ff9a69;
> +}
> +
> +td.version-good {
> +  background: #d2ffc4;
> +}
> +td.version-needs-update {
> +  background: #ff9a69;
> +}
> +td.version-unknown {
> + background: #ffd870;
> +}
> +td.version-error {
> + background: #ccc;
> +}
> +
> +</style>
> +<title>CVE status for Buildroot packages</title>
> +</head>
> +
> +<a href=\"#results\">CVE Status</a><br/>
> +
> +<p id=\"sortable_hint\"></p>
> +"""
> +
> +
> +html_footer = """
> +</body>
> +<script>
> +if (typeof sorttable === \"object\") {
> +  document.getElementById(\"sortable_hint\").innerHTML =
> +  \"hint: the table can be sorted by clicking the column headers\"
> +}
> +</script>
> +</html>
> +"""
> +
> +
> +def infra_str(infra_list):
> +    if not infra_list:
> +        return "Unknown"
> +    elif len(infra_list) == 1:
> +        return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
> +    elif infra_list[0][1] == infra_list[1][1]:
> +        return "<b>%s</b><br/>%s + %s" % \
> +            (infra_list[0][1], infra_list[0][0], infra_list[1][0])
> +    else:
> +        return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
> +            (infra_list[0][1], infra_list[0][0],
> +             infra_list[1][1], infra_list[1][0])
> +
> +
> +def boolean_str(b):
> +    if b:
> +        return "Yes"
> +    else:
> +        return "No"
> +
> +
> +def dump_html_pkg(f, pkg):
> +    f.write(" <tr>\n")
> +    f.write("  <td>%s</td>\n" % pkg.name)
> +
> +    # Current version
> +    if len(pkg.version) > 20:
> +        version = pkg.version[:20] + "..."
> +    else:
> +        version = pkg.version
> +    f.write("  <td class=\"centered\">%s</td>\n" % version)
> +
> +    # CVEs
> +    td_class = ["centered"]
> +    if len(pkg.cves) == 0:
> +        td_class.append("correct")
> +    else:
> +        td_class.append("wrong")
> +    f.write("  <td class=\"%s\">\n" % " ".join(td_class))
> +    for cve in pkg.cves:
> +        f.write("   <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
> +    f.write("  </td>\n")
> +
> +    f.write(" </tr>\n")
> +
> +
> +def dump_html_all_pkgs(f, packages):
> +    f.write("""
> +<table class=\"sortable\">
> +<tr>
> +<td>Package</td>
> +<td class=\"centered\">Version</td>
> +<td class=\"centered\">CVEs</td>
> +</tr>
> +""")
> +    for pkg in packages:
> +        dump_html_pkg(f, pkg)
> +    f.write("</table>")
> +
> +
> +def dump_html_gen_info(f, date, commit):
> +    # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
> +    f.write("<p><i>Updated on %s, git commit %s</i></p>\n" % (str(date), commit))
> +
> +
> +def dump_html(packages, date, commit, output):
> +    with open(output, 'w') as f:
> +        f.write(html_header)
> +        dump_html_all_pkgs(f, packages)
> +        dump_html_gen_info(f, date, commit)
> +        f.write(html_footer)
> +
> +
> +def dump_json(packages, date, commit, output):
> +    # Format packages as a dictionnary instead of a list
> +    # Exclude local field that does not contains real date
> +    excluded_fields = ['url_worker', 'name']
> +    pkgs = {
> +        pkg.name: {
> +            k: v
> +            for k, v in pkg.__dict__.items()
> +            if k not in excluded_fields
> +        } for pkg in packages
> +    }
> +     # The actual structure to dump, add commit and date to it
> +    final = {'packages': pkgs,
> +             'commit': commit,
> +             'date': str(date)}
> +    with open(output, 'w') as f:
> +        json.dump(final, f, indent=2, separators=(',', ': '))
> +        f.write('\n')
> +
> +
> +def resolvepath(path):
> +        return os.path.abspath(os.path.expanduser(path))
> +
> +
> +def parse_args():
> +    parser = argparse.ArgumentParser()
> +    output = parser.add_argument_group('output', 'Output file(s)')
> +    output.add_argument('--html', dest='html', type=resolvepath,
> +                        help='HTML output file')
> +    output.add_argument('--json', dest='json', type=resolvepath,
> +                        help='JSON output file')
> +    packages = parser.add_mutually_exclusive_group()
> +    packages.add_argument('-n', dest='npackages', type=int, action='store',
> +                          help='Number of packages')
> +    packages.add_argument('-p', dest='packages', action='store',
> +                          help='List of packages (comma separated)')

Are the -n and -p options left over from pulling this tool out of
pkg-stats?  Since this report is based on a specific defconfig,

> +    parser.add_argument('--nvd-path', dest='nvd_path',
> +                        help='Path to the local NVD database', type=resolvepath)

I noticed this was a required item, maybe default to a folder name in
the current folder when one isn't provided?

> +    args = parser.parse_args()
> +    if not args.html and not args.json:
> +        parser.error('at least one of --html or --json (or both) is required')

plus nvd path unless you add a default

> +    return args
> +
> +
> +def __main__():
> +    packages = list()
> +    exclude_pacakges = ["linux", "gcc"]
> +    content = json.load(sys.stdin)
> +    for item in content:
> +        if item in exclude_pacakges:
> +            continue
> +        pkg = content[item]
> +        p = Package(item, pkg.get('version', ''), pkg.get('ignored_cves', ''))
> +        packages.append(p)
> +
> +    args = parse_args()
> +    date = datetime.datetime.utcnow()
> +    commit = subprocess.check_output(['git', 'rev-parse',
> +                                      'HEAD']).splitlines()[0].decode()

This git commit check doesn't work when the tools are used with out of
tree buildroot builds.

To reproduce from within Buildroot clone:
make O=../foobar  qemu_x86_64_defconfig
cd ../foobar
make show-info | support/scripts/cve-checker --html report.html --nvd-path nvd


> +
> +    if args.nvd_path:
> +        print("Checking packages CVEs")
> +        check_package_cves(args.nvd_path, {p.name: p for p in packages})
> +    if args.html:
> +        print("Write HTML")
> +        dump_html(packages, date, commit, args.html)
> +    if args.json:
> +        print("Write JSON")
> +        dump_json(packages, date, commit, args.json)
> +
> +__main__()
> --
> 2.27.0
>

Hi Matt,

> Gregory,
>> +# Used to make multiple requests to the same host. It is global
>> +# because it's used by sub-processes.
>> +http_pool = None
>> +
>> +
>> +class Package:
>> +    def __init__(self, name, version, ignored_cves):
>> +        self.name = name
>> +        self.version = version
>> +        self.cves = list()
>> +        self.ignored_cves = ignored_cves
>> +
>> +def check_package_cves(nvd_path, packages):
>> +    if not os.path.isdir(nvd_path):
>> +        os.makedirs(nvd_path)
>> +
>> +    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
>
> This read_nvd_dir call that does the dictionary download has a whole
> bunch of parsing package output  "Cannot parse package 'openssl'
> version ''".  I assume some of that output will get cleaned up when we
> add the additional CPE matching

Yes. Before this series, pkg-stat considered a pacakge as non- affected
by a CVE if it didn't manage to get the version of the package. Since
patch "package/pkg-utils/cve.py: Manage case when package version
doesn't exist" it is no more the case and the CVEs related to the package
are output in the colunm 'CVEs to check'.

However we might be less verbose during the parsing.

>
>> +def parse_args():
>> +    parser = argparse.ArgumentParser()
>> +    output = parser.add_argument_group('output', 'Output file(s)')
>> +    output.add_argument('--html', dest='html', type=resolvepath,
>> +                        help='HTML output file')
>> +    output.add_argument('--json', dest='json', type=resolvepath,
>> +                        help='JSON output file')
>> +    packages = parser.add_mutually_exclusive_group()
>> +    packages.add_argument('-n', dest='npackages', type=int, action='store',
>> +                          help='Number of packages')
>> +    packages.add_argument('-p', dest='packages', action='store',
>> +                          help='List of packages (comma separated)')
>
> Are the -n and -p options left over from pulling this tool out of
> pkg-stats?  Since this report is based on a specific defconfig,

Indeed they are left over and I will remove them

>
>> +    parser.add_argument('--nvd-path', dest='nvd_path',
>> +                        help='Path to the local NVD database', type=resolvepath)
>
> I noticed this was a required item, maybe default to a folder name in
> the current folder when one isn't provided?

I will also add a default value.

>
>> +    args = parser.parse_args()
>> +    if not args.html and not args.json:
>> +        parser.error('at least one of --html or --json (or both) is required')
>
> plus nvd path unless you add a default
>
>> +    return args
>> +
>> +
>> +def __main__():
>> +    packages = list()
>> +    exclude_pacakges = ["linux", "gcc"]
>> +    content = json.load(sys.stdin)
>> +    for item in content:
>> +        if item in exclude_pacakges:
>> +            continue
>> +        pkg = content[item]
>> +        p = Package(item, pkg.get('version', ''), pkg.get('ignored_cves', ''))
>> +        packages.append(p)
>> +
>> +    args = parse_args()
>> +    date = datetime.datetime.utcnow()
>> +    commit = subprocess.check_output(['git', 'rev-parse',
>> +                                      'HEAD']).splitlines()[0].decode()
>
> This git commit check doesn't work when the tools are used with out of
> tree buildroot builds.
>
> To reproduce from within Buildroot clone:
> make O=../foobar  qemu_x86_64_defconfig
> cd ../foobar
> make show-info | support/scripts/cve-checker --html report.html
> --nvd-path nvd


Thanks for the report I will check it.

Gregory

Hi,
>>> +
>>> +    args = parse_args()
>>> +    date = datetime.datetime.utcnow()
>>> +    commit = subprocess.check_output(['git', 'rev-parse',
>>> +                                      'HEAD']).splitlines()[0].decode()
>>
>> This git commit check doesn't work when the tools are used with out of
>> tree buildroot builds.
>>
>> To reproduce from within Buildroot clone:
>> make O=../foobar  qemu_x86_64_defconfig
>> cd ../foobar
>> make show-info | support/scripts/cve-checker --html report.html
>> --nvd-path nvd
>
>
> Thanks for the report I will check it.

I had a closer look on it, the way I tested was the following:
make O=../foobar  qemu_x86_64_defconfig
make O=../foobar show-info | support/scripts/cve-checker --html report.html --nvd-path nvd

That's why I didn't see this issue. However, I kept this information from
pkg-stat, but actually I don't think it has big value for a given
configuration.

I will just remove it.

Gregory

>
> -- 
> Gregory Clement, Bootlin
> Embedded Linux and Kernel engineering
> http://bootlin.com

Gregory,

On Wed, Jul 8, 2020 at 11:41 AM Gregory CLEMENT
<gregory.clement@bootlin.com> wrote:
>
> This scripts takes as entry on stdin a JSON description of the package
> used for a given configuration. This description is the one generated
> by "make show-info".
>
> The script generates the list of all the package used and if they are
> affected by a CVE. The output is either a JSON or an HTML file similar
> to the one generated by pkg-stats.
>
> Signed-off-by: Gregory CLEMENT <gregory.clement@bootlin.com>
> ---
>  support/scripts/cve-checker | 291 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 291 insertions(+)
>  create mode 100755 support/scripts/cve-checker
>
> diff --git a/support/scripts/cve-checker b/support/scripts/cve-checker
> new file mode 100755
> index 0000000000..db8497d7aa
> --- /dev/null
> +++ b/support/scripts/cve-checker
> @@ -0,0 +1,291 @@
> +#!/usr/bin/env python
> +
> +# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
> +# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
> +
> +import argparse
> +import datetime
> +import fnmatch
> +import os
> +from collections import defaultdict
> +import re
> +import subprocess
> +import requests  # URL checking
> +import json
> +import ijson

I noticed ijson wasn't something I had already installed on my host
system.  I'd suggest adding the dependency check similar to the
graph-size make target (support/scripts/size-stats), since this is a
user facing tool vs pkg-stats was for maintenance.  An easy test to
see what dependencies a user might run into on a basic system could be
to use the base buildroot Docker
image(https://hub.docker.com/r/buildroot/base) or create one using
(support/docker/Dockerfile).

> +import certifi
> +import distutils.version
> +import time
> +import gzip
> +import sys
> +from urllib3 import HTTPSConnectionPool
> +from urllib3.exceptions import HTTPError
> +from multiprocessing import Pool
> +

Regards,
Matt