[v7,08/10] support/scripts/cpe-report: add NIST xml generation

Message ID 1539812522-7171-8-git-send-email-matthew.weber@rockwellcollins.com
State New
Headers show
Series
  • [v7,01/10] cpe-info: new make target
Related show

Commit Message

Matthew Weber Oct. 17, 2018, 9:42 p.m.
From: Bryce Ferguson <bryce.ferguson@rockwellcollins.com>

Hooks to generate the NIST xml files required to propose an
update to the database. The generation uses existing Buildroot
information to create the submission information.

This design uses the generated cpe-info csv and a path to the relevant
buildroot clone for access to the packages/dir structure.

Signed-off-by: Paresh Chaudhary <paresh.chaudhary@rockwellcollins.com>
Signed-off-by: Bryce Ferguson <bryce.ferguson@rockwellcollins.com>
Signed-off-by: Matt Weber <matthew.weber@rockwellcollins.com>
---
Changes
v7
 - New
---
 support/scripts/cpe-report |  17 ++++--
 support/scripts/cpedb.py   | 128 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 132 insertions(+), 13 deletions(-)

Patch

diff --git a/support/scripts/cpe-report b/support/scripts/cpe-report
index 036eab2..da4ad96 100755
--- a/support/scripts/cpe-report
+++ b/support/scripts/cpe-report
@@ -6,7 +6,7 @@  import csv
 from cpedb import CPEDB
 
 
-def get_target_cpe_report(cpe_report_file, cpedb):
+def get_target_cpe_report(cpe_report_file, pkg_dir, cpedb):
     report_cpe_exact_match = ""
     report_cpe_needing_update = ""
     report_cpe_missing = ""
@@ -21,9 +21,9 @@  def get_target_cpe_report(cpe_report_file, cpedb):
                 if not result:
                     result = cpedb.find_partial(cpedb.get_cpe_no_version(cpe[0]))
                     if not result:
-                        report_cpe_missing += cpe[0] + "\n"
+                        report_cpe_missing += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
                     else:
-                        report_cpe_needing_update += cpe[0] + "\n"
+                        report_cpe_needing_update += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
                 else:
                     report_cpe_exact_match += cpe[0] + "\n"
     except (OSError, IOError) as e:
@@ -34,11 +34,20 @@  def get_target_cpe_report(cpe_report_file, cpedb):
     print("CPE: Found but REQUIRES UPDATE:\n" + report_cpe_needing_update)
     print("CPE: Not found (proposing the following to be added):\n" + report_cpe_missing)
 
+    for cpe in report_cpe_needing_update.splitlines():
+        cpedb.update(cpe, pkg_dir, 1)
+    for cpe in report_cpe_missing.splitlines():
+        cpedb.update(cpe, pkg_dir, 1)
+    print("XML Generation Complete of NIST update files, see ./cpe/*")
+
 
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('-c', dest='cpe_report', action='store', required=True,
                         help='CPE Report generated by make cpe-info (csv format)')
+    parser.add_argument('-d', dest='pkg_dir', action='store', required=True,
+                        help='Path to dir(s) of Buildroot and BR2_EXTERNAL(s) seperated with ":". ' +
+                             'By default the script looks for a ./package/<pkgname> and ./<pkgname> in each path)')
     return parser.parse_args()
 
 
@@ -47,7 +56,7 @@  def __main__():
     cpedb = CPEDB()
     cpedb.get_xml_dict()
     print("Performing Target CPE Report Analysis...")
-    get_target_cpe_report(args.cpe_report, cpedb)
+    get_target_cpe_report(args.cpe_report, args.pkg_dir, cpedb)
 
 
 __main__()
diff --git a/support/scripts/cpedb.py b/support/scripts/cpedb.py
index 77d1d17..9e2ea91 100644
--- a/support/scripts/cpedb.py
+++ b/support/scripts/cpedb.py
@@ -1,16 +1,78 @@ 
 import sys
 import urllib2
+from collections import OrderedDict
 import xmltodict
 import gzip
 from StringIO import StringIO
+import re
+import os
 
 CPE_XML_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
 
 
+class CPE:
+    cpe_str = None
+    cpe_str_short = None
+    cpe_desc = None
+    references = {}
+
+    def __init__(self, cpe_str, product_ref=None, version_ref=None):
+        self.cpe_str = cpe_str
+        self.cpe_str_short = ":".join(self.cpe_str.split(":")[2:6])
+        pkg_name = "".join(self.cpe_str.split(":")[4:5]).replace("_", " ")
+        pkg_ver = "".join(self.cpe_str.split(":")[5:6])
+        self.cpe_desc = pkg_name.title() + " " + pkg_ver
+        if product_ref and version_ref:
+            self.references = {
+                'references': {
+                    'reference': [{
+                        '@href': product_ref,
+                        '#text': 'PRODUCT'
+                    }, {
+                        '@href': version_ref,
+                        '#text': 'VERSION'
+                    }]
+                }
+            }
+
+    def update_ref(self, new_ref):
+        self.references = new_ref
+
+    def to_dict(self):
+        cpe_dict = OrderedDict([
+            ('cpe-item', OrderedDict([
+                ('@name', 'cpe:/' + self.cpe_str_short),
+                ('title', OrderedDict([
+                        ('@xml:lang', 'en-US'),
+                        ('#text', self.cpe_desc)
+                ])),
+                ('references', self.references),
+                ('cpe-23:cpe23-item', OrderedDict([
+                        ('@name', self.cpe_str)
+                ]))
+            ]))
+        ])
+        return cpe_dict
+
+
+def find_url_in_config_in(pkg_config_in):
+    product_url = "Not Found"
+    in_help_section = False
+    if os.path.exists(pkg_config_in):
+        fp = open(pkg_config_in, "r")
+        for config_line in fp:
+            if config_line.strip() == "help":
+                in_help_section = True
+            if in_help_section and re.match("(.*)https?://", config_line):
+                product_url = ''.join(config_line.split())
+                break
+        fp.close()
+    return product_url
+
+
 class CPEDB:
     all_cpedb = dict()
-    all_cpes = set()
-    all_cpes_no_version = set()
+    all_cpes = dict()
 
     def get_xml_dict(self):
         print("CPE: Fetching xml manifest...")
@@ -23,9 +85,9 @@  class CPEDB:
 
             for cpe in self.all_cpedb['cpe-list']['cpe-item']:
                 cpe_str = cpe['cpe-23:cpe23-item']['@name']
+                item = CPE(cpe_str)
                 cpe_str_no_version = self.get_cpe_no_version(cpe_str)
-                self.all_cpes.add(cpe_str)
-                self.all_cpes_no_version.add(cpe_str_no_version)
+                self.all_cpes.update({cpe_str_no_version: {cpe_str: item}})
 
         except urllib2.HTTPError:
             print("CPE: HTTP Error: %s" % CPE_XML_URL)
@@ -36,17 +98,65 @@  class CPEDB:
 
     def find_partial(self, cpe_str):
         cpe_str_no_version = self.get_cpe_no_version(cpe_str)
-        if cpe_str_no_version in self.all_cpes_no_version:
+        if cpe_str_no_version in self.all_cpes:
             return cpe_str_no_version
 
     def find(self, cpe_str):
-        if cpe_str in self.all_cpes:
-            return cpe_str
+        if self.find_partial(cpe_str):
+            cpe_str_no_version = self.get_cpe_no_version(cpe_str)
+            if cpe_str in self.all_cpes[cpe_str_no_version]:
+                return cpe_str
 
-    def get_cpe_no_version(self, cpe):
-        return ":".join(cpe.split(":")[:5])
+    def check_package(self, cpe_str, pkg_dir):
+        product_site = "Not Found"
+        # For each dir, check the boot/<pkgname>/, package/<pkgname>/ and <pkgname>/ folders for Config.in
+        # This should cover regular packages, linux and boot related packages.
+        for pkg_dir_path in pkg_dir.split(":"):
+            file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'package', cpe_str, 'Config.in')
+            product_site = find_url_in_config_in(file_to_open)
+            if product_site != "Not Found":
+                break
+            file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'boot', cpe_str, 'Config.in')
+            product_site = find_url_in_config_in(file_to_open)
+            if product_site != "Not Found":
+                break
+            file_to_open = os.path.join(os.path.abspath(pkg_dir_path), cpe_str, 'Config.in')
+            product_site = find_url_in_config_in(file_to_open)
+            if product_site != "Not Found":
+                break
+        return product_site
+
+    def update(self, cpe_str, pkg_dir=None, cpe_full_str=None):
+        if cpe_full_str:
+            cpe_str = cpe_str.split(",")
+        to_update = CPE(cpe_str[0], self.check_package(cpe_str[1], pkg_dir), cpe_str[2])
+        xml = self.__gen_xml__(to_update.to_dict())
+        if not os.path.exists("cpe"):
+                os.makedirs("cpe")
+        fp = open(os.path.join('cpe', cpe_str[1] + ".xml"), "w+")
+        fp.write(xmltodict.unparse(xml, pretty=True))
+        fp.close()
 
     def get_nvd_url(self, cpe_str):
         return "https://nvd.nist.gov/products/cpe/search/results?keyword=" + \
                 urllib2.quote(cpe_str) + \
                 "&status=FINAL&orderBy=CPEURI&namingFormat=2.3"
+
+    def get_cpe_no_version(self, cpe):
+        return ":".join(cpe.split(":")[:5])
+
+    def __gen_xml__(self, cpe_list):
+        list_header = {
+            "cpe-list": {
+                "@xmlns:config": "http://scap.nist.gov/schema/configuration/0.1",
+                "@xmlns": "http://cpe.mitre.org/dictionary/2.0",
+                "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+                "@xmlnsscap-core": "http://scap.nist.gov/schema/scap-core/0.3",
+                "@xmlns:cpe-23": "http://scap.nist.gov/schema/cpe-extension/2.3",
+                "@xmlns:ns6": "http://scap.nist.gov/schema/scap-core/0.1",
+                "@xmlns:meta": "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
+                "@xsi:schemaLocation": "http://scap.nist.gov/schema/cpe-extension/2.3 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd http://cpe.mitre.org/dictionary/2.0 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2 https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd http://scap.nist.gov/schema/scap-core/0.3 https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd http://scap.nist.gov/schema/configuration/0.1 https://scap.nist.gov/schema/nvd/configuration_0.1.xsd http://scap.nist.gov/schema/scap-core/0.1 https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"
+             }
+        }
+        list_header['cpe-list'].update(cpe_list)
+        return list_header