blob: 19875a49b1c92296c8b52054df66bc74199c9f5c [file] [log] [blame]
SUMMARY = "Updates the NVD CVE database"
LICENSE = "MIT"
INHIBIT_DEFAULT_DEPS = "1"
inherit native
deltask do_unpack
deltask do_patch
deltask do_configure
deltask do_compile
deltask do_install
deltask do_populate_sysroot
python () {
if not d.getVar("CVE_CHECK_DB_FILE"):
raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
}
python do_populate_cve_db() {
"""
Update NVD database with json data feed
"""
import sqlite3, urllib, urllib.parse, shutil, gzip
from datetime import date
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
YEAR_START = 2002
db_dir = os.path.join(d.getVar("DL_DIR"), 'CVE_CHECK')
db_file = os.path.join(db_dir, 'nvdcve_1.0.db')
json_tmpfile = os.path.join(db_dir, 'nvd.json.gz')
# Don't refresh the database more than once an hour
try:
import time
if time.time() - os.path.getmtime(db_file) < (60*60):
return
except OSError:
pass
proxy = d.getVar("https_proxy")
if proxy:
# instantiate an opener but do not install it as the global
# opener unless if we're really sure it's applicable for all
# urllib requests
proxy_handler = urllib.request.ProxyHandler({'https': proxy})
proxy_opener = urllib.request.build_opener(proxy_handler)
else:
proxy_opener = None
cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
# Connect to database
conn = sqlite3.connect(db_file)
c = conn.cursor()
initialize_db(c)
for year in range(YEAR_START, date.today().year + 1):
year_url = BASE_URL + str(year)
meta_url = year_url + ".meta"
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
response = None
if proxy_opener:
response = proxy_opener.open(meta_url)
else:
req = urllib.request.Request(meta_url)
response = urllib.request.urlopen(req)
if response:
for l in response.read().decode("utf-8").splitlines():
key, value = l.split(":", 1)
if key == "lastModifiedDate":
last_modified = value
break
else:
bb.warn("Cannot parse CVE metadata, update failed")
return
# Compare with current db last modified date
c.execute("select DATE from META where YEAR = ?", (year,))
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Clear products table entries corresponding to current year
c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,))
# Update db with current year json file
try:
if proxy_opener:
response = proxy_opener.open(json_url)
else:
req = urllib.request.Request(json_url)
response = urllib.request.urlopen(req)
if response:
update_db(c, gzip.decompress(response.read()).decode('utf-8'))
c.execute("insert or replace into META values (?, ?)", [year, last_modified])
except urllib.error.URLError as e:
cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
return
# Update success, set the date to cve_check file.
if year == date.today().year:
cve_f.write('CVE database update : %s\n\n' % date.today())
cve_f.close()
conn.commit()
conn.close()
}
def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
VERSION_END TEXT, OPERATOR_END TEXT)")
def parse_node_and_insert(c, node, cveId):
# Parse children node if needed
for child in node.get('children', ()):
parse_node_and_insert(c, child, cveId)
def cpe_generator():
for cpe in node.get('cpe_match', ()):
if not cpe['vulnerable']:
return
cpe23 = cpe['cpe23Uri'].split(':')
vendor = cpe23[3]
product = cpe23[4]
version = cpe23[5]
if version != '*':
# Version is defined, this is a '=' match
yield [cveId, vendor, product, version, '=', '', '']
else:
# Parse start version, end version and operators
op_start = ''
op_end = ''
v_start = ''
v_end = ''
if 'versionStartIncluding' in cpe:
op_start = '>='
v_start = cpe['versionStartIncluding']
if 'versionStartExcluding' in cpe:
op_start = '>'
v_start = cpe['versionStartExcluding']
if 'versionEndIncluding' in cpe:
op_end = '<='
v_end = cpe['versionEndIncluding']
if 'versionEndExcluding' in cpe:
op_end = '<'
v_end = cpe['versionEndExcluding']
yield [cveId, vendor, product, v_start, op_start, v_end, op_end]
c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator())
def update_db(c, jsondata):
import json
root = json.loads(jsondata)
for elt in root['CVE_Items']:
if not elt['impact']:
continue
cveId = elt['cve']['CVE_data_meta']['ID']
cveDesc = elt['cve']['description']['description_data'][0]['value']
date = elt['lastModifiedDate']
accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
try:
cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cvssv3 = 0.0
c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
configurations = elt['configurations']['nodes']
for config in configurations:
parse_node_and_insert(c, config, cveId)
addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"