[Fuego] [PATCH 2/2] ftc: implement put-run for squad
Tim.Bird at sony.com
Tim.Bird at sony.com
Fri Jan 11 01:50:33 UTC 2019
A few comments inline below.
> -----Original Message-----
> From: Daniel Sangorrin
>
> This is an early implementation, more debugging is needed
> and there are several FIXTHIS included, but it works.
> You can try it on Ubuntu Xenial with the instructions below.
>
> Install server application
> host$ sudo apt-get install rabbitmq-server
> [Note] this is required for squad to communicate with celery
> worker tasks
> host$ git clone https://github.com/Linaro/squad
> host$ cd squad/
> host$ mkvirtualenv --python=python3 mysquad
> host$ cd squad
> host$ pip3 install -r requirements-dev.txt
> host$ ./manage.py test
> host$ ./manage.py migrate
> host$ ./manage.py createsuperuser
> host$ ./manage.py runserver
>
> [Note] The next time you use it use this:
> host$ cd squad
> host$ workon squad
> host$ ./manage.py runserver
>
> Configure a team and project in Squad
> host$ firefox http://localhost:8000/
> - click upright > settings
> - profile: write your name
> - API token: copy the token
> -> assign it to "server_squad_token" in
> fuego-ro/fuego.conf
> - click upright > Administration
> - Authentication and authorization
> - Add group (this is a user group!)
> - name: myusergroup
> - permissons: choose all
> - Core
> - Add group (this is a team)
> - slug=name=fuego
> -> assign it to
> "server_squad_team" in fuego-ro/fuego.conf
> - description: whatever
> - usergroups: myusergroup
> - Add project
> - group: fuego
> - slug=name=jessie
> -> assign it to
> "server_squad_project" in fuego-ro/fuego.conf
>
> Upload a run from Fuego
> fuego-docker# ftc list-runs -q
> Benchmark.IOzone-default-1-bbb
> fuego-docker# ftc put-run Benchmark.IOzone-default-1-bbb
> Packaging run 'Benchmark.IOzone-default-1-bbb'
> run/
> run/devlog.after.txt
> run/syslog.before.txt
> run/build.xml
> run/devlog.txt
> run/testlog.txt
> run/consolelog.txt
> run/run.json
> Run packaged successfully, and is at: /tmp/run-
> Benchmark.IOzone-default-1-on-fuegohost:bbb.frp
> WARNING: not adding attachment syslog.after
> WARNING: not adding attachment test_spec
> POSTING to
> http://localhost:8000/api/submit/fuego/jessie/a80be74d1629bdb2455b08db
> 453beba0-4.9.133/bbb-4.9
> Run package run-Benchmark.IOzone-default-1-on-
> fuegohost:bbb.frp was accepted by the server.
OK - yeah - this definitely needs a wiki page. Or, if we like squad enough
we preinstall it in the container and set all this up for at least the local
instance. (And we still need a wiki page for how people would change
the settings for a non-local or non-Fuego squad instance.)
>
> Signed-off-by: Daniel Sangorrin <daniel.sangorrin at toshiba.co.jp>
> ---
> engine/scripts/ftc | 131
> ++++++++++++++++++++++++++++++++++++++++++++++++-----
> 1 file changed, 119 insertions(+), 12 deletions(-)
>
> diff --git a/engine/scripts/ftc b/engine/scripts/ftc
> index 73fd98a..ab2d2cb 100755
> --- a/engine/scripts/ftc
> +++ b/engine/scripts/ftc
> @@ -519,9 +519,16 @@ class config_class:
> # set values from conf_map
> self.host = conf_map.get("host_name", "localhost")
> self.host_name = conf_map.get("host_name", "localhost")
> - self.fuego_server = conf_map.get("fuego_server", "fuegotest.org")
> - #self.SERVER_URL_BASE =
> "http://%s/server/Fuego_Server?action=Fuego." % self.fuego_server
> - self.SERVER_URL_BASE = "http://%s/fserver.py/?action=" %
> self.fuego_server
> + self.server_type = conf_map.get("server_type", "fuego")
> + self.server_domain = conf_map.get("server_domain", "fuegotest.org")
> + #self.SERVER_URL_BASE =
> "http://%s/server/Fuego_Server?action=Fuego." % self.server_domain
> + if self.server_type == "fuego":
> + self.SERVER_URL_BASE = "http://%s/fserver.py/?action=" %
> self.server_domain
> + elif self.server_type == "squad":
> + self.SERVER_URL_BASE = "http://%s/api" % self.server_domain
> + self.token = conf_map.get("server_squad_token",
> "WRITE_TOKEN_IN_FUEGO.CONF")
I like the stand-in value here. :-)
> + self.team = conf_map.get("server_squad_team", "fuego")
> + self.project_id = conf_map.get("server_squad_project",
> "myproject")
>
> # handle paths (including relative ones)
> conf_dir = os.path.dirname(os.path.abspath(config_path))
> @@ -3282,6 +3289,103 @@ def do_put_test(conf, options):
>
> print "Test package %s was accepted by the server." %
> os.path.basename(test_filepath)
>
> +def put_run_fuego(conf, run_filepath):
> + url = conf.SERVER_URL_BASE+"put_run"
> + run_files = {"file1": open(run_filepath, "rb")}
> + resp = requests.post(url, files=run_files)
> + return resp.text.split('\n', 1)
> +
> +def put_run_squad(conf, run_filepath):
> + # squad doesn't know about fuego run formats
> + dirpath = tempfile.mkdtemp()
> + try:
> + subprocess.check_call('tar -C %s -xf %s' % (dirpath, run_filepath),
> shell=True)
> + with open(dirpath + '/run/run.json') as f:
> + json_data = json.load(f)
> + # create submit url
> + # Ex:
> http://localhost:8000/api/submit/fuego/Benchmark.Dhrystone/713c5342/bb
> b
> + # project_id: this is a name for something you want to test periodically.
> + # For example, the major version of some software that will be
> updated
> + # with bug fixes periodically such as the stable kernel v4.4.y
> + # or a rootfs built with a specific yocto branch
> + # -> For now, the project_id is specified in fuego.conf using
> server_squad_project.
> + # [FIXTHIS] add a command line argument to override it (we don't
> have this concept in fuego)
This begs the question of possibly supporting a plugin architecture
for ftc, for settings and communications protocols for backends.
> + # build_id: this is something like an unique identifier for a snapshot
> + # of your project when you built it. For example, the commit hash of
> + # the kernel/yocto repo, the kernel minor version or the date. You
> + # can test the same build_id on multiple environments
> (boards/kernels)
interesting.
> + # for comparison.
> + # -> For now, just use the testsuite_version and kernel_version
> + # [FIXTHIS] add a command line argument to override it (we don't
> have this concept in fuego)
> + # env_id: this is the environment for the test
> + # -> For now, use the board name and kernel_major_version
> + # [FIXTHIS] add a command line argument to override it
> + project_id = conf.project_id
> + build_id = json_data['metadata']['testsuite_version'] + '-' +
> json_data['metadata']['kernel_version']
> + kernel_major_version =
> '.'.join(json_data['metadata']['kernel_version'].split('.')[:2])
> + env_id = json_data['metadata']['board'] + '-' + kernel_major_version
> + url = "%s/submit/%s/%s/%s/%s" % (conf.SERVER_URL_BASE,
> conf.team, project_id, build_id, env_id)
> +
> + # prepare a multipart files dict for posting
> + files = []
> + ## prepare metadata file
> + metadata = json_data['metadata']
> + # the job_id needs to be unique within the project, it identifies a run
fserver requires something like this as well. I don't recall if I'm using timestamp,
but I might be.
> + timestamp = metadata['timestamp'].replace(':', 'colon').replace('+',
> 'plus')
OK - I presume that ':' and '+' are illegal chars for squad job ids? this probably
produces pretty ugly job ids though. Could you just strip them?
> + metadata['job_id'] = '-'.join((metadata['job_name'],
> metadata['build_number'], metadata['kernel_version'], timestamp))
> + metadata['job_status'] = json_data['status']
> + metadata['datetime'] = metadata['timestamp']
> + files.append(('metadata', ('metadata', json.dumps(metadata))))
> + ## prepare tests and metrics files
> + test_sets = json_data['test_sets']
> + tests = dict()
> + metrics = dict()
> + suite = json_data['name'].split('.')[1]
> + #tests[suite] = json_data['status']
> + for test_set in test_sets:
> + test_set_key = os.path.join(suite, test_set['name'])
> + #tests[test_set_key] = test_set['status']
> + for test_case in test_set['test_cases']:
> + test_case_key = os.path.join(test_set_key, test_case['name'])
> + if 'measurements' in test_case:
> + for test_measure in test_case['measurements']:
> + test_measure_key = os.path.join(test_case_key,
> test_measure['name'])
> + # use the suite name to group all subtests
> + squad_key = suite + '/' + test_measure_key.replace('/','_')
> + tests[squad_key] = test_measure['status']
> + if test_measure['status'] != 'SKIP':
> + metrics[squad_key] = [test_measure['measure']]
> + else:
> + squad_key= suite + '/' + test_case_key.replace('/','_')
> + tests[squad_key] = test_case['status']
> + files.append(('tests', ('tests', json.dumps(tests))))
> + files.append(('metrics', ('metrics', json.dumps(metrics))))
> + ## prepare attachments
> + for attachment in metadata['attachments']:
> + abs_path = dirpath + '/run/' + attachment['path']
> + if os.path.isfile(abs_path):
> + files.append(('attachment', (attachment['path'], open(abs_path,
> 'rb'))))
> + else:
> + print("WARNING: not adding attachment " + attachment['name'])
> +
> + # set token
> + headers = {}
> + headers['Auth-Token'] = conf.token
> +
> + print("POSTING to " + url)
> + response = requests.post(url, headers=headers, files=files)
> + if str(response.status_code)[:1] == "2":
> + result = 'OK'
> + else:
> + result = 'NG'
> + content = response.text.split('\n', 1)
> + except Exception as e:
> + result = "NG"
> + content = 'Exception in ftc'
> + print_error(str(e))
> + finally:
> + shutil.rmtree(dirpath)
> + return result, content
>
> def do_put_run(conf, options):
> # FIXTHIS - could consolidate do_put_test and do_put_run into
> do_put_item
> @@ -3302,16 +3406,19 @@ def do_put_run(conf, options):
> created_package = True
> run_filepath = do_package_run(conf, [run_arg, "-o", "/tmp"])
>
> - url = conf.SERVER_URL_BASE+"put_run"
> - run_files = {"file1": open(run_filepath, "rb")}
> - resp = requests.post(url, files=run_files)
> - if created_package:
> - os.remove(run_filepath)
> - result, content = resp.text.split('\n', 1)
> - if result != "OK":
> - error_out("Can't put run to server\nServer returned message: %s" %
> content)
> + try:
> + if conf.server_type == "fuego":
> + result, content = put_run_fuego(conf, run_filepath)
> + elif conf.server_type == "squad":
> + result, content = put_run_squad(conf, run_filepath)
> + finally:
> + if created_package:
> + os.remove(run_filepath)
>
> - print "Run package %s was accepted by the server." %
> os.path.basename(run_filepath)
> + if result != "OK":
> + print("Can't put run to server\nServer returned message: %s" %
> content)
> + else:
> + print "Run package %s was accepted by the server." %
> os.path.basename(run_filepath)
>
> # returns a matching item (which must be unique)
> # from a list. A matching item is one with leading
> --
> 2.7.4
Looks good for a prototype.
I'm applying as is.
-- Tim
More information about the Fuego
mailing list