[Fuego] [PATCH 08/16] testplans: use testspecs instead of testplans internally
Daniel Sangorrin
daniel.sangorrin at toshiba.co.jp
Thu Mar 30 01:04:36 UTC 2017
This is quite a big change but necessary. Testplans are useful
to batch tests and define which spec or timeout we want for
each test. However, we also want to be able to run tests
individually even if they are not inside a testplan.
Until now one prolog.sh was generated for a testplan. This is not
a good idea for several reasons: collisions between different
test plans or the inability to have the same test with two
different specs in the same testplan.
This patch puts the "prolog.sh" generated each time a job is run
on the corresponding job's log folder. It also modifies the names
of the jobs which are now <board>.<testspec>.<testdir>
Signed-off-by: Daniel Sangorrin <daniel.sangorrin at toshiba.co.jp>
---
engine/scripts/functions.sh | 7 +-
engine/scripts/overlays.sh | 36 +++------
engine/scripts/ovgen.py | 191 +++++++++++---------------------------------
3 files changed, 61 insertions(+), 173 deletions(-)
diff --git a/engine/scripts/functions.sh b/engine/scripts/functions.sh
index b7f0f72..2483f61 100755
--- a/engine/scripts/functions.sh
+++ b/engine/scripts/functions.sh
@@ -261,9 +261,7 @@ function pre_test {
cmd "true" || abort_job "Cannot connect to $DEVICE via $TRANSPORT"
-# Create the log directory for this test run
export LOGDIR="$FUEGO_RW/logs/$TESTDIR/${NODE_NAME}.${BUILD_TIMESTAMP}.${BUILD_NUMBER}"
- mkdir -p $LOGDIR
# make a link to the log dir that Jenkins can get to
# (Jenkins might not have the BUILD_TIMESTAMP)
@@ -400,10 +398,11 @@ function post_test {
# reset the signal handler to avoid an infinite loop
trap - SIGTERM SIGHUP SIGALRM SIGINT
+ export LOGDIR="$FUEGO_RW/logs/$TESTDIR/${NODE_NAME}.${BUILD_TIMESTAMP}.${BUILD_NUMBER}"
+
# source generated prolog.sh file since post_test is called separately
- source $FUEGO_RW/work/${NODE_NAME}_prolog.sh
+ source $LOGDIR/prolog.sh
export SSHPASS=$PASSWORD
- export LOGDIR="$FUEGO_RW/logs/$TESTDIR/${NODE_NAME}.${BUILD_TIMESTAMP}.${BUILD_NUMBER}"
# re-source params to set correct DEVICE, LOGIN, SSH vars
source $FUEGO_CORE/engine/scripts/params.sh
diff --git a/engine/scripts/overlays.sh b/engine/scripts/overlays.sh
index e1d2233..cfecc73 100644
--- a/engine/scripts/overlays.sh
+++ b/engine/scripts/overlays.sh
@@ -25,15 +25,13 @@
. $FUEGO_CORE/engine/scripts/common.sh
assert_define "NODE_NAME"
+assert_define "TESTSPEC"
+assert_define "BUILD_NUMBER"
+assert_define "TESTDIR"
-OF_CLASSDIR_ARGS="--classdir $FUEGO_CORE/engine/overlays/base"
-OF_OUTPUT_FILE="$FUEGO_RW/work/${NODE_NAME}_prolog.sh"
-OF_OUTPUT_FILE_ARGS="--output $OF_OUTPUT_FILE"
+OF_CLASSDIRS="$FUEGO_CORE/engine/overlays/base"
OF_OVGEN="$FUEGO_CORE/engine/scripts/ovgen.py"
-OF_DISTRIB_FILE=""
-OF_OVFILES_ARGS=""
-OF_TESTPLAN_ARGS=""
#OF_DEBUG_ARGS="--debug 3"
OF_DEBUG_ARGS=""
@@ -55,29 +53,19 @@ function set_overlay_vars() {
abort_job "DISTRIB is not defined"
fi
- if [ "$TESTPLAN" ] ; then
- echo "BATCH_TESTPLAN is not set, using $TESTPLAN.json testplan"
- OF_TESTPLAN="$FUEGO_CORE/engine/overlays/testplans/$TESTPLAN.json"
+ # Create the log directory for this test run here so we can place the prolog.sh
+ export LOGDIR="$FUEGO_RW/logs/$TESTDIR/${NODE_NAME}.${BUILD_TIMESTAMP}.${BUILD_NUMBER}"
+ mkdir -p $LOGDIR
- if [ ! -f $OF_TESTPLAN ] ; then
- if [ ! "$(basename $OF_TESTPLAN)" == "testplan_default.json" ] ; then
- abort_job "$OF_TESTPLAN does not exist"
- fi
- fi
- OF_TESTPLAN_ARGS="--testplan $OF_TESTPLAN"
- fi
-
- OF_OVFILES_ARGS="--ovfiles $OF_DISTRIB_FILE $OF_BOARD_FILE"
-
- rm -f $OF_OUTPUT_FILE
+ rm -f $LOGDIR/prolog.sh
- run_python $OF_OVGEN $OF_DEBUG_ARGS $OF_CLASSDIR_ARGS $OF_OVFILES_ARGS $OF_TESTPLAN_ARGS $OF_OUTPUT_FILE_ARGS || abort_job "Error while prolog.sh file generation"
+ run_python $OF_OVGEN $OF_DEBUG_ARGS --classdir $OF_CLASSDIRS --ovfiles $OF_DISTRIB_FILE $OF_BOARD_FILE --testdir $TESTDIR --testspec $TESTSPEC --output $LOGDIR/prolog.sh || abort_job "Error while prolog.sh file generation"
- if [ ! -f "$OF_OUTPUT_FILE" ]
+ if [ ! -f "$LOGDIR/prolog.sh" ]
then
- abort_job "$OF_OUTPUT_FILE not found"
+ abort_job "$LOGDIR/prolog.sh not found"
fi
- source $OF_OUTPUT_FILE
+ source $LOGDIR/prolog.sh
}
diff --git a/engine/scripts/ovgen.py b/engine/scripts/ovgen.py
index 6012453..10d1cf4 100755
--- a/engine/scripts/ovgen.py
+++ b/engine/scripts/ovgen.py
@@ -68,17 +68,15 @@ class OFLayer:
class TestSpecs:
def __init__(self):
- self.testName=""
- self.specList={}
- self.method=None
- self.tarball=None
+ self.name=""
+ self.variables={}
self.fail_case=None
def __repr__(self):
- return "<TestSpecs name:\"%s\" specList:\"%s\" method:%s>\n" % (self.testName, self.specList, self.method)
+ return "<TestSpecs name:\"%s\" variables:\"%s\" fail_case:%s>\n" % (self.name, self.variables, self.fail_case)
def __str__(self):
- return "<TestSpecs name:\"%s\" specList:\"%s\" method:%s>\n" % (self.testName, self.specList, self.method)
+ return "<TestSpecs name:\"%s\" variables:\"%s\" fail_case:%s>\n" % (self.name, self.variables, self.fail_case)
class OFParseException(Exception):
def __init__(self, value):
@@ -377,7 +375,7 @@ def parseOverrideFile(overrideFile, layer, ofcls):
return classes
-def generateProlog(outFilePath, ofcls, classes, tpFiles):
+def generateProlog(outFilePath, ofcls, classes, testdir, testspec):
outfile = open(outFilePath, "w")
for ofc in classes:
@@ -407,43 +405,34 @@ def generateProlog(outFilePath, ofcls, classes, tpFiles):
file.write(outfile, "\n")
- if tpFiles != None:
- for tpf in tpFiles:
- parseGenTestPlan(tpf, outfile, outFilePath)
-
-# generateSpec - generate shell output for all specs of specific test
-# Arguments:
-# curTestSpecs:Dict - selected spec from testplan for current test
-# testName: String - name of test
-# specName: String - name of test spec
-# curSpecs: TestSpecs - all specs for current test
-# fout - file output descriptor opened -> prolog.sh
-
-def generateSpec(curTestSpecs, testName, specName, curSpec, fout):
- del curTestSpecs["name"]
- for par in curTestSpecs:
- if (par == "method" or par == "tarball" or par == "fail_case"):
- continue
+ ts = parseSpec(testdir, testspec)
+ generateSpec(ts, outfile)
+
+# generateSpec - generate shell output for the spec
+# ts:TestSpecs - parsed specs for our testdir
+# testspec: String - name of test spec (e.g. 'default')
+# outFilePath - file output descriptor opened -> prolog.sh
+def generateSpec(ts, fout):
+ debug_print("generating spec '%s' for '%s'" % (ts.variables['name'], ts.name))
- varname = "%s_%s" % (testName, par)
+ for var in ts.variables:
+ if var == 'name':
+ continue
+ varname = "%s_%s" % (ts.name, var)
varname = string.replace(varname, ".", "_").upper()
- value = "%s" % (curTestSpecs[par])
+ value = "%s" % (ts.variables[var])
outStr = '%s="%s"' % (varname, value)
-
- debug_print (outStr, 3)
+ debug_print(outStr, 3)
fout.write(outStr + "\n")
- tNameUp = string.replace(testName, ".", "_").upper()
- tNameUp = string.replace(tNameUp, "-", "_").upper()
-
- #print "cts: %s\n" % (curTestSpecs)
-
- if curSpec.fail_case:
- fc_num = len(curSpec.fail_case)
+ if ts.fail_case:
+ tNameUp = string.replace(ts.name, ".", "_").upper()
+ tNameUp = string.replace(tNameUp, "-", "_").upper()
+ fc_num = len(ts.fail_case)
outNum = "%s_FAIL_CASE_COUNT=%s" % (tNameUp, fc_num)
fout.write(outNum + "\n")
- for fmsg, num in zip(curSpec.fail_case, range(fc_num)):
+ for fmsg, num in zip(ts.fail_case, range(fc_num)):
outPattern = "%s_FAIL_PATTERN_%s=\"%s\"" % (tNameUp, num, fmsg["fail_regexp"])
outMessage = "%s_FAIL_MESSAGE_%s=\"%s\"" % (tNameUp, num, fmsg["fail_message"])
@@ -453,127 +442,46 @@ def generateSpec(curTestSpecs, testName, specName, curSpec, fout):
fout.write(outPattern + "\n")
fout.write(outMessage + "\n")
- if "method" in curTestSpecs:
- if (curTestSpecs["method"] == "tarball"):
- outStr = "%s_METHOD=tarball\n%s_TARBALL_NAME=%s" % (tNameUp, tNameUp, curTestSpecs["tarball"])
- fout.write(outStr + "\n")
- else:
- raise SpecException ("unkown method: %s" % (curTestSpecs["method"]))
- elif curSpec.method:
- if (curSpec.method == "tarball"):
- outStr = ("%s_METHOD=tarball\n%s_TARBALL_NAME=%s" % (tNameUp, tNameUp, curSpec.tarball))
- fout.write(outStr + "\n")
- else:
- raise SpecException ("unkown method: %s" % (curSpec.method))
- else:
- pass
- # TRB 2016-09-09 - ignore missing test spec methods - this feature doesn't work or isn't used
- #print("Can not find method for %s[%s]" % (testName,specName))
- #outStr = ("%s_METHOD=\"none\"" % (tNameUp))
- #fout.write(outStr + "\n")
- # raise SpecException ("Can not find method for %s[%s]" % (testName,specName))
-
-
-def parseGenTestPlan(tpFilePath, fout, fname):
- with open(tpFilePath) as f:
- debug_print("parsing `%s' TP" % (tpFilePath), 1)
- try:
- jd = json.load(f)
- except:
- print "Error parsing testplan file %s" % tpFilePath
- f.seek(0)
- js = json.load(f)
-
- name = jd["testPlanName"]
- fout.write("#testplan: %s\n" % (name))
- for t in jd["tests"]:
- testName = t["testName"]
- specName = t["spec"]
-
- specs = {}
- specs = parseSpecDir("/fuego-core/engine/tests/" + testName)
-
- if testName not in specs:
- raise SpecException("Cannot find test %s in spec list" % (testName))
-
- curSpecs = specs[testName]
-
- if specName not in curSpecs.specList:
- raise SpecException("Cannot find spec %s in %s" % (specName, testName))
-
- curSpecList = curSpecs.specList[specName]
-
- debug_print("generating spec %s for `%s'" % (specName, testName))
- generateSpec(curSpecList, testName, specName, curSpecs, fout)
-
- fout.write("\n")
-
-def parseSpec(specFileName):
+def parseSpec(testdir, testspec):
+ # TODO: get fuego-core from env
+ specpath = '/fuego-core/engine/tests/%s/%s.spec' % (testdir, testdir)
ts = TestSpecs()
- ts.specList={}
- debug_print("Parsing %s spec file" % (specFileName))
+ debug_print("Parsing %s spec file" % (specpath))
- with open(specFileName) as f:
+ with open(specpath) as f:
try:
jd = json.load(f)
except:
- print "Error parsing spec file %s" % specFileName
- f.seek(0)
- jd = json.load(f)
+ raise Exception("Error parsing spec file %s" % specpath)
- name = jd["testName"]
- debug_print("parsing `%s' spec" % (name), 1)
-
- if "method" in jd:
- if jd["method"] == "tarball":
- ts.method = "tarball"
- ts.tarball = jd["tarball"]
- else:
- raise SpecException("%s: Unknown method: %s", (name, ts.tarball))
+ ts.name = jd["testName"]
if "fail_case" in jd:
ts.fail_case = jd["fail_case"]
- debug_print ("Found fail_case msgs for `%s' specs" % (name)
-)
- for spec in jd["specs"]:
- sn = spec["name"]
- ts.testName = name
- ts.specList[sn] = spec
-
- return ts
+ debug_print ("Found fail_case msgs in '%s'" % specpath)
+ for spec in jd["specs"]:
+ if spec["name"] == testspec:
+ ts.variables = spec
-def parseSpecDir(specDir):
- specFiles = glob.glob(specDir + "/*.spec")
- tspList = {}
-
- debug_print ("Found following %s spec files in %s" % (specFiles, specDir))
-
- for sf in specFiles:
- sp = parseSpec(sf)
- tspList[sp.testName] = sp
-
- debug_print ("parsed specs: %s" % (tspList))
-
- return tspList
+ if not ts.variables:
+ raise Exception("Could not find %s in %s" % (testspec, specpath))
+ return ts
def run(test_args=None):
parser = argparse.ArgumentParser(description='Read OF class files, override files and generate prolog defining all variables and functions')
parser.add_argument('--classdir', help='OF base class directory', required=True)
parser.add_argument('--ovfiles', nargs='+', metavar='OVFILE', help='list of directories containing .override files', required=True)
- parser.add_argument('--testplans', nargs='+', metavar='TESTPLAN', help='list of test plan files', required=False)
+ parser.add_argument('--testdir', help='e.g.: Benchmark.Dhrystone', required=True)
+ parser.add_argument('--testspec', nargs='+', metavar='TESTSPEC', help='testspec', required=True)
parser.add_argument('--output', help='output file name', required=True)
parser.add_argument('--debug', help='{1,2,3} debug level (default is no debugging)', type=int, required=False)
args = parser.parse_args(args=test_args)
- classdir = args.classdir
- ovfiles = args.ovfiles
- output = args.output
-
if args.debug:
if args.debug < 1 or args.debug > 3:
@@ -583,29 +491,22 @@ def run(test_args=None):
global log_lvl
log_lvl = args.debug
- testPlans = {}
- if args.testplans != None:
- testPlans = args.testplans
-
- debug_print ("Using =%s, ovfiles=%s" % (classdir, ovfiles), 1)
+ debug_print("Using classdir=%s, ovfiles=%s testdir=%s testspec=%s" % (args.classdir, args.ovfiles, args.testdir, args.testspec), 1)
ofcls = {}
- parseBaseDir(classdir, ofcls)
+ parseBaseDir(args.classdir, ofcls)
layers = {}
classes = []
-
- for ovf in ovfiles:
+ for ovf in args.ovfiles:
classes = classes + parseOverrideFile(ovf, layers, ofcls)
debug_print ("parsed %s override\n------------\n" % (ovf))
- generateProlog(output, ofcls, classes, testPlans)
-
+ generateProlog(args.output, ofcls, classes, args.testdir, args.testspec[0])
def testrun():
- test_args = "--classdir /fuego-core/engine/overlays/base/ --ovfiles /fuego-ro/conf/boards/qemu-arm.board --output prolog.sh --testplans /fuego-core/engine/overlays/testplans/testplan_default.json --debug 2".split()
- # test_args = "--classdir overlays-new/base/ --ovfiles overlays-new/boards/minnow.board --output prolog.sh --debug 2".split()
+ test_args = "--debug 2 --classdir /fuego-core/engine/overlays/base/ --ovfiles /fuego-ro/boards/qemu-arm.board --testdir Benchmark.Dhrystone --testspec default --output prolog.sh".split()
run(test_args)
run()
--
2.7.4
More information about the Fuego
mailing list