[Fuego] [PATCH 01/11] vuls: make it a benchmark test
Daniel Sangorrin
daniel.sangorrin at toshiba.co.jp
Wed Dec 27 06:32:45 UTC 2017
The reason is that we are going to use thresholds against
the number of vulnerabilities (a number) so it is easier
to represent as a benchmark.
In fact the variables inside fuego_test were already assuming
to be a benchmark by mistake.
Signed-off-by: Daniel Sangorrin <daniel.sangorrin at toshiba.co.jp>
---
engine/tests/Benchmark.vuls/chart_config.json | 3 +
engine/tests/Benchmark.vuls/criteria.json | 21 ++++++
engine/tests/Benchmark.vuls/fuego_test.sh | 93 ++++++++++++++++++++++++++
engine/tests/Benchmark.vuls/parser.py | 17 +++++
engine/tests/Benchmark.vuls/spec.json | 17 +++++
engine/tests/Functional.vuls/chart_config.json | 3 -
engine/tests/Functional.vuls/criteria.json | 21 ------
engine/tests/Functional.vuls/fuego_test.sh | 93 --------------------------
engine/tests/Functional.vuls/parser.py | 17 -----
engine/tests/Functional.vuls/spec.json | 17 -----
10 files changed, 151 insertions(+), 151 deletions(-)
create mode 100644 engine/tests/Benchmark.vuls/chart_config.json
create mode 100644 engine/tests/Benchmark.vuls/criteria.json
create mode 100644 engine/tests/Benchmark.vuls/fuego_test.sh
create mode 100644 engine/tests/Benchmark.vuls/parser.py
create mode 100644 engine/tests/Benchmark.vuls/spec.json
delete mode 100644 engine/tests/Functional.vuls/chart_config.json
delete mode 100644 engine/tests/Functional.vuls/criteria.json
delete mode 100644 engine/tests/Functional.vuls/fuego_test.sh
delete mode 100644 engine/tests/Functional.vuls/parser.py
delete mode 100644 engine/tests/Functional.vuls/spec.json
diff --git a/engine/tests/Benchmark.vuls/chart_config.json b/engine/tests/Benchmark.vuls/chart_config.json
new file mode 100644
index 0000000..1849992
--- /dev/null
+++ b/engine/tests/Benchmark.vuls/chart_config.json
@@ -0,0 +1,3 @@
+{
+ "chart_type": "testset_summary_table"
+}
diff --git a/engine/tests/Benchmark.vuls/criteria.json b/engine/tests/Benchmark.vuls/criteria.json
new file mode 100644
index 0000000..91c0f8a
--- /dev/null
+++ b/engine/tests/Benchmark.vuls/criteria.json
@@ -0,0 +1,21 @@
+{
+ "schema_version":"1.0",
+ "criteria":[
+ {
+ "tguid":"HIGH",
+ "max_fail": 0
+ },
+ {
+ "tguid":"MEDIUM",
+ "max_fail": 100
+ },
+ {
+ "tguid":"LOW",
+ "max_fail": 100
+ },
+ {
+ "tguid":"(unknown)",
+ "max_fail": 200
+ }
+ ]
+}
diff --git a/engine/tests/Benchmark.vuls/fuego_test.sh b/engine/tests/Benchmark.vuls/fuego_test.sh
new file mode 100644
index 0000000..068acc4
--- /dev/null
+++ b/engine/tests/Benchmark.vuls/fuego_test.sh
@@ -0,0 +1,93 @@
+export GOPATH=${WORKSPACE}/vuls-shared/go/
+export GOROOT=/usr/local/go
+export PATH=$PATH:${GOROOT}/bin:${GOPATH}/bin
+
+function test_pre_check {
+ # VULS required an ssh connection without password
+ assert_define SSH_KEY "Missing file definition with ssh private key"
+ if [ "${TRANSPORT}" != "ssh" ]; then
+ abort_job "Vuls requires your board to define TRANSPORT to be ssh"
+ fi
+
+ # FIXTHIS: support other distributions (e.g. CentOS requires yum-utils)
+ if [ "${BENCHMARK_VULS_DISTRO}" == "debian" ]; then
+ is_on_target reboot-notifier SERVICE_REBOOT_NOTIFIER /etc/cron.daily/:/etc/default
+ assert_define SERVICE_REBOOT_NOTIFIER "Missing reboot-notifier package on target"
+ if [ "${BENCHMARK_VULS_ISDEEP}" == "true" ] ; then
+ is_on_target aptitude-curses PROGRAM_APTITUDE /usr/bin
+ assert_define PROGRAM_APTITUDE "Missing aptitude package on target"
+ fi
+ fi
+
+ # vuls uses strict checking and jenkins has no home directory so use /etc
+ if [ -e /etc/ssh/ssh_known_hosts ]; then
+ grep ${IPADDR} /etc/ssh/ssh_known_hosts || \
+ abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts"
+ else
+ abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts"
+ fi
+
+ # Make sure that go was installed in the system
+ is_on_target go PROGRAM_GO $PATH
+ assert_define PROGRAM_GO "run /fuego-ro/scripts/fuego-vuls-docker-preparation"
+}
+
+function test_build {
+ if [ ! -d ${GOPATH}/src/github.com/kotakanbe/ ]; then
+ # Install go-cve-dictionary into ${GOPATH}/bin
+ mkdir -p ${GOPATH}/src/github.com/kotakanbe
+ cd ${GOPATH}/src/github.com/kotakanbe
+ git clone https://github.com/kotakanbe/go-cve-dictionary.git
+ cd go-cve-dictionary && make install
+
+ # Install goval-dictionary into ${GOPATH}/bin
+ cd ${GOPATH}/src/github.com/kotakanbe
+ git clone https://github.com/kotakanbe/goval-dictionary.git
+ cd goval-dictionary && make install
+ fi
+
+ # Install Vuls into ${GOPATH}/bin
+ if [ ! -d $GOPATH/src/github.com/future-architect/ ]; then
+ mkdir -p $GOPATH/src/github.com/future-architect/
+ cd $GOPATH/src/github.com/future-architect/
+ git clone https://github.com/future-architect/vuls.git
+ cd $GOPATH/src/github.com/future-architect/vuls
+ make install
+ fi
+
+ # Fetch vulnerability data from NVD and OVAL data
+ if [ -n "${http_proxy}" ]; then
+ for i in `seq 2002 $(date +"%Y")`; do
+ go-cve-dictionary fetchnvd -http-proxy=$http_proxy -years $i
+ done
+ goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION} -http-proxy ${http_proxy}
+ else
+ for i in `seq 2002 $(date +"%Y")`; do
+ go-cve-dictionary fetchnvd -years $i
+ done
+ goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION}
+ fi
+
+ # Create config.toml file
+ echo "[servers]" > config.toml
+ echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> config.toml
+ echo "host = \"${IPADDR}\"" >> config.toml
+ echo "port = \"${SSH_PORT}\"" >> config.toml
+ echo "user = \"${LOGIN}\"" >> config.toml
+ echo "keyPath = \"${SSH_KEY}\"" >> config.toml
+}
+
+function test_run {
+ cd $GOPATH/src/github.com/future-architect/vuls
+ # Check config.toml and settings on the server before scanning
+ if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then
+ vuls configtest -deep ${BENCHMARK_VULS_DISTRO}
+ else
+ vuls configtest ${BENCHMARK_VULS_DISTRO}
+ fi
+ # Start Scanning
+ vuls scan ${BENCHMARK_VULS_DISTRO}
+ # Reporting
+ vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3
+ vuls report -format-short-text | tee ${LOGDIR}/testlog.txt
+}
diff --git a/engine/tests/Benchmark.vuls/parser.py b/engine/tests/Benchmark.vuls/parser.py
new file mode 100644
index 0000000..3bfa196
--- /dev/null
+++ b/engine/tests/Benchmark.vuls/parser.py
@@ -0,0 +1,17 @@
+#!/bin/python
+
+import os, re, sys
+
+sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
+import common as plib
+
+results = {}
+with open(plib.TEST_LOG) as f:
+ for line in f:
+ if line.startswith("CVE-"):
+ fields = line.split()
+ test_set = fields[2]
+ test_case = fields[0]
+ results[test_set+'.'+test_case] = "FAIL"
+
+sys.exit(plib.process(results))
diff --git a/engine/tests/Benchmark.vuls/spec.json b/engine/tests/Benchmark.vuls/spec.json
new file mode 100644
index 0000000..60e0d8a
--- /dev/null
+++ b/engine/tests/Benchmark.vuls/spec.json
@@ -0,0 +1,17 @@
+{
+ "testName": "Benchmark.vuls",
+ "specs": {
+ "default": {
+ "gitrepo": "https://github.com/future-architect/vuls.git",
+ "distro": "debian",
+ "all_version": "7 8 9 10",
+ "isdeep": "false"
+ },
+ "ubuntu": {
+ "gitrepo": "https://github.com/future-architect/vuls.git",
+ "distro": "ubuntu",
+ "all_version": "12 14 16",
+ "isdeep": "false"
+ }
+ }
+}
diff --git a/engine/tests/Functional.vuls/chart_config.json b/engine/tests/Functional.vuls/chart_config.json
deleted file mode 100644
index 1849992..0000000
--- a/engine/tests/Functional.vuls/chart_config.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "chart_type": "testset_summary_table"
-}
diff --git a/engine/tests/Functional.vuls/criteria.json b/engine/tests/Functional.vuls/criteria.json
deleted file mode 100644
index 91c0f8a..0000000
--- a/engine/tests/Functional.vuls/criteria.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "schema_version":"1.0",
- "criteria":[
- {
- "tguid":"HIGH",
- "max_fail": 0
- },
- {
- "tguid":"MEDIUM",
- "max_fail": 100
- },
- {
- "tguid":"LOW",
- "max_fail": 100
- },
- {
- "tguid":"(unknown)",
- "max_fail": 200
- }
- ]
-}
diff --git a/engine/tests/Functional.vuls/fuego_test.sh b/engine/tests/Functional.vuls/fuego_test.sh
deleted file mode 100644
index 068acc4..0000000
--- a/engine/tests/Functional.vuls/fuego_test.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-export GOPATH=${WORKSPACE}/vuls-shared/go/
-export GOROOT=/usr/local/go
-export PATH=$PATH:${GOROOT}/bin:${GOPATH}/bin
-
-function test_pre_check {
- # VULS required an ssh connection without password
- assert_define SSH_KEY "Missing file definition with ssh private key"
- if [ "${TRANSPORT}" != "ssh" ]; then
- abort_job "Vuls requires your board to define TRANSPORT to be ssh"
- fi
-
- # FIXTHIS: support other distributions (e.g. CentOS requires yum-utils)
- if [ "${BENCHMARK_VULS_DISTRO}" == "debian" ]; then
- is_on_target reboot-notifier SERVICE_REBOOT_NOTIFIER /etc/cron.daily/:/etc/default
- assert_define SERVICE_REBOOT_NOTIFIER "Missing reboot-notifier package on target"
- if [ "${BENCHMARK_VULS_ISDEEP}" == "true" ] ; then
- is_on_target aptitude-curses PROGRAM_APTITUDE /usr/bin
- assert_define PROGRAM_APTITUDE "Missing aptitude package on target"
- fi
- fi
-
- # vuls uses strict checking and jenkins has no home directory so use /etc
- if [ -e /etc/ssh/ssh_known_hosts ]; then
- grep ${IPADDR} /etc/ssh/ssh_known_hosts || \
- abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts"
- else
- abort_job "Run on docker as root: ssh-keyscan -t ecdsa ${IPADDR} >> /etc/ssh/ssh_known_hosts"
- fi
-
- # Make sure that go was installed in the system
- is_on_target go PROGRAM_GO $PATH
- assert_define PROGRAM_GO "run /fuego-ro/scripts/fuego-vuls-docker-preparation"
-}
-
-function test_build {
- if [ ! -d ${GOPATH}/src/github.com/kotakanbe/ ]; then
- # Install go-cve-dictionary into ${GOPATH}/bin
- mkdir -p ${GOPATH}/src/github.com/kotakanbe
- cd ${GOPATH}/src/github.com/kotakanbe
- git clone https://github.com/kotakanbe/go-cve-dictionary.git
- cd go-cve-dictionary && make install
-
- # Install goval-dictionary into ${GOPATH}/bin
- cd ${GOPATH}/src/github.com/kotakanbe
- git clone https://github.com/kotakanbe/goval-dictionary.git
- cd goval-dictionary && make install
- fi
-
- # Install Vuls into ${GOPATH}/bin
- if [ ! -d $GOPATH/src/github.com/future-architect/ ]; then
- mkdir -p $GOPATH/src/github.com/future-architect/
- cd $GOPATH/src/github.com/future-architect/
- git clone https://github.com/future-architect/vuls.git
- cd $GOPATH/src/github.com/future-architect/vuls
- make install
- fi
-
- # Fetch vulnerability data from NVD and OVAL data
- if [ -n "${http_proxy}" ]; then
- for i in `seq 2002 $(date +"%Y")`; do
- go-cve-dictionary fetchnvd -http-proxy=$http_proxy -years $i
- done
- goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION} -http-proxy ${http_proxy}
- else
- for i in `seq 2002 $(date +"%Y")`; do
- go-cve-dictionary fetchnvd -years $i
- done
- goval-dictionary fetch-${BENCHMARK_VULS_DISTRO} ${BENCHMARK_VULS_ALL_VERSION}
- fi
-
- # Create config.toml file
- echo "[servers]" > config.toml
- echo "[servers.${BENCHMARK_VULS_DISTRO}]" >> config.toml
- echo "host = \"${IPADDR}\"" >> config.toml
- echo "port = \"${SSH_PORT}\"" >> config.toml
- echo "user = \"${LOGIN}\"" >> config.toml
- echo "keyPath = \"${SSH_KEY}\"" >> config.toml
-}
-
-function test_run {
- cd $GOPATH/src/github.com/future-architect/vuls
- # Check config.toml and settings on the server before scanning
- if [ "${BENCHMARK_VULS_ISDEEP}" = "true" ]; then
- vuls configtest -deep ${BENCHMARK_VULS_DISTRO}
- else
- vuls configtest ${BENCHMARK_VULS_DISTRO}
- fi
- # Start Scanning
- vuls scan ${BENCHMARK_VULS_DISTRO}
- # Reporting
- vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3
- vuls report -format-short-text | tee ${LOGDIR}/testlog.txt
-}
diff --git a/engine/tests/Functional.vuls/parser.py b/engine/tests/Functional.vuls/parser.py
deleted file mode 100644
index 3bfa196..0000000
--- a/engine/tests/Functional.vuls/parser.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/python
-
-import os, re, sys
-
-sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
-import common as plib
-
-results = {}
-with open(plib.TEST_LOG) as f:
- for line in f:
- if line.startswith("CVE-"):
- fields = line.split()
- test_set = fields[2]
- test_case = fields[0]
- results[test_set+'.'+test_case] = "FAIL"
-
-sys.exit(plib.process(results))
diff --git a/engine/tests/Functional.vuls/spec.json b/engine/tests/Functional.vuls/spec.json
deleted file mode 100644
index 60e0d8a..0000000
--- a/engine/tests/Functional.vuls/spec.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "testName": "Benchmark.vuls",
- "specs": {
- "default": {
- "gitrepo": "https://github.com/future-architect/vuls.git",
- "distro": "debian",
- "all_version": "7 8 9 10",
- "isdeep": "false"
- },
- "ubuntu": {
- "gitrepo": "https://github.com/future-architect/vuls.git",
- "distro": "ubuntu",
- "all_version": "12 14 16",
- "isdeep": "false"
- }
- }
-}
--
2.7.4
More information about the Fuego
mailing list