@@ -6,7 +6,7 @@ import csv
from cpedb import CPEDB
-def get_target_cpe_report(cpe_report_file, cpedb):
+def get_target_cpe_report(cpe_report_file, pkg_dir, cpedb):
report_cpe_exact_match = ""
report_cpe_needing_update = ""
report_cpe_missing = ""
@@ -21,9 +21,9 @@ def get_target_cpe_report(cpe_report_file, cpedb):
if not result:
result = cpedb.find_partial(cpedb.get_cpe_no_version(cpe[0]))
if not result:
- report_cpe_missing += cpe[0] + "\n"
+ report_cpe_missing += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
else:
- report_cpe_needing_update += cpe[0] + "\n"
+ report_cpe_needing_update += cpe[0] + "," + cpe[2] + "," + cpe[4] + "\n"
else:
report_cpe_exact_match += cpe[0] + "\n"
except (OSError, IOError) as e:
@@ -34,11 +34,20 @@ def get_target_cpe_report(cpe_report_file, cpedb):
print("CPE: Found but REQUIRES UPDATE:\n" + report_cpe_needing_update)
print("CPE: Not found (proposing the following to be added):\n" + report_cpe_missing)
+ for cpe in report_cpe_needing_update.splitlines():
+ cpedb.update(cpe, pkg_dir, 1)
+ for cpe in report_cpe_missing.splitlines():
+ cpedb.update(cpe, pkg_dir, 1)
+ print("XML Generation Complete of NIST update files, see ./cpe/*")
+
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('-c', dest='cpe_report', action='store', required=True,
help='CPE Report generated by make cpe-info (csv format)')
+ parser.add_argument('-d', dest='pkg_dir', action='store', required=True,
+ help='Path to dir(s) of Buildroot and BR2_EXTERNAL(s) seperated with ":". ' +
+ 'By default the script looks for a ./package/<pkgname> and ./<pkgname> in each path)')
return parser.parse_args()
@@ -47,7 +56,7 @@ def __main__():
cpedb = CPEDB()
cpedb.get_xml_dict()
print("Performing Target CPE Report Analysis...")
- get_target_cpe_report(args.cpe_report, cpedb)
+ get_target_cpe_report(args.cpe_report, args.pkg_dir, cpedb)
__main__()
@@ -1,16 +1,78 @@
import sys
import urllib2
+from collections import OrderedDict
import xmltodict
import gzip
from StringIO import StringIO
+import re
+import os
CPE_XML_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
+class CPE:
+ cpe_str = None
+ cpe_str_short = None
+ cpe_desc = None
+ references = {}
+
+ def __init__(self, cpe_str, product_ref=None, version_ref=None):
+ self.cpe_str = cpe_str
+ self.cpe_str_short = ":".join(self.cpe_str.split(":")[2:6])
+ pkg_name = "".join(self.cpe_str.split(":")[4:5]).replace("_", " ")
+ pkg_ver = "".join(self.cpe_str.split(":")[5:6])
+ self.cpe_desc = pkg_name.title() + " " + pkg_ver
+ if product_ref and version_ref:
+ self.references = {
+ 'references': {
+ 'reference': [{
+ '@href': product_ref,
+ '#text': 'PRODUCT'
+ }, {
+ '@href': version_ref,
+ '#text': 'VERSION'
+ }]
+ }
+ }
+
+ def update_ref(self, new_ref):
+ self.references = new_ref
+
+ def to_dict(self):
+ cpe_dict = OrderedDict([
+ ('cpe-item', OrderedDict([
+ ('@name', 'cpe:/' + self.cpe_str_short),
+ ('title', OrderedDict([
+ ('@xml:lang', 'en-US'),
+ ('#text', self.cpe_desc)
+ ])),
+ ('references', self.references),
+ ('cpe-23:cpe23-item', OrderedDict([
+ ('@name', self.cpe_str)
+ ]))
+ ]))
+ ])
+ return cpe_dict
+
+
+def find_url_in_config_in(pkg_config_in):
+ product_url = "Not Found"
+ in_help_section = False
+ if os.path.exists(pkg_config_in):
+ fp = open(pkg_config_in, "r")
+ for config_line in fp:
+ if config_line.strip() == "help":
+ in_help_section = True
+ if in_help_section and re.match("(.*)https?://", config_line):
+ product_url = ''.join(config_line.split())
+ break
+ fp.close()
+ return product_url
+
+
class CPEDB:
all_cpedb = dict()
- all_cpes = set()
- all_cpes_no_version = set()
+ all_cpes = dict()
def get_xml_dict(self):
print("CPE: Fetching xml manifest...")
@@ -23,9 +85,9 @@ class CPEDB:
for cpe in self.all_cpedb['cpe-list']['cpe-item']:
cpe_str = cpe['cpe-23:cpe23-item']['@name']
+ item = CPE(cpe_str)
cpe_str_no_version = self.get_cpe_no_version(cpe_str)
- self.all_cpes.add(cpe_str)
- self.all_cpes_no_version.add(cpe_str_no_version)
+ self.all_cpes.update({cpe_str_no_version: {cpe_str: item}})
except urllib2.HTTPError:
print("CPE: HTTP Error: %s" % CPE_XML_URL)
@@ -36,17 +98,65 @@ class CPEDB:
def find_partial(self, cpe_str):
cpe_str_no_version = self.get_cpe_no_version(cpe_str)
- if cpe_str_no_version in self.all_cpes_no_version:
+ if cpe_str_no_version in self.all_cpes:
return cpe_str_no_version
def find(self, cpe_str):
- if cpe_str in self.all_cpes:
- return cpe_str
+ if self.find_partial(cpe_str):
+ cpe_str_no_version = self.get_cpe_no_version(cpe_str)
+ if cpe_str in self.all_cpes[cpe_str_no_version]:
+ return cpe_str
- def get_cpe_no_version(self, cpe):
- return ":".join(cpe.split(":")[:5])
+ def check_package(self, cpe_str, pkg_dir):
+ product_site = "Not Found"
+ # For each dir, check the boot/<pkgname>/, package/<pkgname>/ and <pkgname>/ folders for Config.in
+ # This should cover regular packages, linux and boot related packages.
+ for pkg_dir_path in pkg_dir.split(":"):
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'package', cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), 'boot', cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ file_to_open = os.path.join(os.path.abspath(pkg_dir_path), cpe_str, 'Config.in')
+ product_site = find_url_in_config_in(file_to_open)
+ if product_site != "Not Found":
+ break
+ return product_site
+
+ def update(self, cpe_str, pkg_dir=None, cpe_full_str=None):
+ if cpe_full_str:
+ cpe_str = cpe_str.split(",")
+ to_update = CPE(cpe_str[0], self.check_package(cpe_str[1], pkg_dir), cpe_str[2])
+ xml = self.__gen_xml__(to_update.to_dict())
+ if not os.path.exists("cpe"):
+ os.makedirs("cpe")
+ fp = open(os.path.join('cpe', cpe_str[1] + ".xml"), "w+")
+ fp.write(xmltodict.unparse(xml, pretty=True))
+ fp.close()
def get_nvd_url(self, cpe_str):
return "https://nvd.nist.gov/products/cpe/search/results?keyword=" + \
urllib2.quote(cpe_str) + \
"&status=FINAL&orderBy=CPEURI&namingFormat=2.3"
+
+ def get_cpe_no_version(self, cpe):
+ return ":".join(cpe.split(":")[:5])
+
+ def __gen_xml__(self, cpe_list):
+ list_header = {
+ "cpe-list": {
+ "@xmlns:config": "http://scap.nist.gov/schema/configuration/0.1",
+ "@xmlns": "http://cpe.mitre.org/dictionary/2.0",
+ "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+ "@xmlnsscap-core": "http://scap.nist.gov/schema/scap-core/0.3",
+ "@xmlns:cpe-23": "http://scap.nist.gov/schema/cpe-extension/2.3",
+ "@xmlns:ns6": "http://scap.nist.gov/schema/scap-core/0.1",
+ "@xmlns:meta": "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
+ "@xsi:schemaLocation": "http://scap.nist.gov/schema/cpe-extension/2.3 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd http://cpe.mitre.org/dictionary/2.0 https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2 https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd http://scap.nist.gov/schema/scap-core/0.3 https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd http://scap.nist.gov/schema/configuration/0.1 https://scap.nist.gov/schema/nvd/configuration_0.1.xsd http://scap.nist.gov/schema/scap-core/0.1 https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"
+ }
+ }
+ list_header['cpe-list'].update(cpe_list)
+ return list_header