[Fuego] [PATCH 01/16] Add fuego-release Functional test
Tim.Bird at sony.com
Tim.Bird at sony.com
Fri Mar 30 22:17:23 UTC 2018
My strategy is to apply all your patches, but ask for modifications,
that you can provide as patches on top of this set.
If I make these changes, they will break other patches in the
series, which will be obnoxious and time-consuming to fix.
See below inline for comments.
-- Tim
> -----Original Message-----
> From: Guilherme Campos Camargo
> Allows Fuego to test new Fuego releases.
>
> The fuego and fuego-core repositores and branches to be tested may be
> configured on the specs.json file. Their defaults are:
>
> "FUEGOREPO":"https://bitbucket.org/tbird20d/fuego",
> "FUEGOBRANCH":"master",
> "FUEGOCOREREPO":"https://bitbucket.org/tbird20d/fuego-core",
> "FUEGOCOREBRANCH":"master"
>
Can you please add underscores to these names:
FUEGO_REPO, FUEGO_CORE_BRANCH, etc.
?
It's very nice to be able to customize these in the spec file.
> Signed-off-by: Guilherme Campos Camargo <guicc at profusion.mobi>
> ---
> engine/tests/Functional.fuegotest/fuego_test.sh | 29 ++
> engine/tests/Functional.fuegotest/spec.json | 11 +
> engine/tests/Functional.fuegotest/test_run.py | 427
> ++++++++++++++++++++++++
> 3 files changed, 467 insertions(+)
> create mode 100755 engine/tests/Functional.fuegotest/fuego_test.sh
> create mode 100644 engine/tests/Functional.fuegotest/spec.json
> create mode 100755 engine/tests/Functional.fuegotest/test_run.py
>
> diff --git a/engine/tests/Functional.fuegotest/fuego_test.sh
> b/engine/tests/Functional.fuegotest/fuego_test.sh
> new file mode 100755
> index 0000000..192c15b
> --- /dev/null
> +++ b/engine/tests/Functional.fuegotest/fuego_test.sh
> @@ -0,0 +1,29 @@
> +#!/bin/bash
> +
> +set -e
> +
> +readonly fuego_release_dir=fuego-release
> +
> +function test_build {
> + if [ -d ${fuego_release_dir} ]; then
> + rm -r ${fuego_release_dir}
> + fi
> + git clone --quiet --depth 1 --single-branch \
> + --branch "${FUNCTIONAL_FUEGOTEST_FUEGOBRANCH}" \
> + "${FUNCTIONAL_FUEGOTEST_FUEGOREPO}" \
> + "${fuego_release_dir}/fuego"
> + git clone --quiet --depth 1 --single-branch \
> + --branch "${FUNCTIONAL_FUEGOTEST_FUEGOCOREBRANCH}" \
> + "${FUNCTIONAL_FUEGOTEST_FUEGOCOREREPO}" \
> + "${fuego_release_dir}/fuego-core"
> + cd -
> +}
> +
> +function test_run {
> + sudo -n ${TEST_HOME}/test_run.py "${fuego_release_dir}/fuego"
> + report "echo ok 1 minimal test on target"
> +}
> +
> +function test_processing {
> + log_compare "$TESTDIR" "1" "^ok" "p"
> +}
> diff --git a/engine/tests/Functional.fuegotest/spec.json
> b/engine/tests/Functional.fuegotest/spec.json
> new file mode 100644
> index 0000000..cc345ba
> --- /dev/null
> +++ b/engine/tests/Functional.fuegotest/spec.json
> @@ -0,0 +1,11 @@
> +{
> + "testName": "Functional.fuegotest",
> + "specs": {
> + "default": {
> + "FUEGOREPO":"https://bitbucket.org/tbird20d/fuego",
> + "FUEGOBRANCH":"master",
> + "FUEGOCOREREPO":"https://bitbucket.org/tbird20d/fuego-core",
> + "FUEGOCOREBRANCH":"master"
> + }
Can you add specs for my 'test' and 'next' branches as well?
> + }
> +}
> diff --git a/engine/tests/Functional.fuegotest/test_run.py
> b/engine/tests/Functional.fuegotest/test_run.py
> new file mode 100755
> index 0000000..96fcfb7
> --- /dev/null
> +++ b/engine/tests/Functional.fuegotest/test_run.py
> @@ -0,0 +1,427 @@
> +#!/usr/bin/env python3
> +import argparse
> +import http
> +import http.client
> +import logging
> +import os
> +import re
> +import subprocess
> +import sys
> +import time
> +
> +import docker
> +import pexpect
> +import requests
> +from selenium import webdriver
> +
> +LOGGER = logging.getLogger('test_run')
> +STREAM_HANDLER = logging.StreamHandler()
> +STREAM_HANDLER.setFormatter(
> + logging.Formatter('%(name)s:%(levelname)s: %(message)s'))
> +LOGGER.setLevel(logging.DEBUG)
> +LOGGER.addHandler(STREAM_HANDLER)
> +
> +
> +def loop_until_timeout(func, timeout=10, num_tries=5):
> + LOGGER.debug('Running %s()', func.__name__)
> +
> + for _ in range(num_tries):
> + LOGGER.debug(' Try number %s...', _ + 1)
> + if func():
> + LOGGER.debug(' Success')
> + return True
> + time.sleep(timeout/num_tries)
> + LOGGER.debug(' Failure')
> +
> + return False
> +
> +
> +class SeleniumCommand:
> + def exec(self, selenium_ctx):
> + self.driver = selenium_ctx.driver
> + self.driver.refresh()
> + self.driver.implicitly_wait(3)
> + LOGGER.debug('Executing Selenium Command \'%s\'',
> + self.__class__.__name__)
> +
> +
> +class Visit(SeleniumCommand):
> + def __init__(self, url, timeout=10, expected_result=200):
> + self.url = url
> + self.timeout = timeout
> + self.expected_result = expected_result
> +
> + def exec(self, selenium_ctx):
> + super().exec(selenium_ctx)
> +
> + LOGGER.debug(' Visiting \'%s\'', self.url)
> + self.driver.get(self.url)
> +
> + r = requests.get(self.url)
> + if r.status_code != self.expected_result:
> + LOGGER.debug(' HTTP Status Code \'%s\' is different '
> + 'from the expected \'%s\'', r.status_cod, self.url)
> + return False
> +
> + LOGGER.debug(' HTTP Status Code is same as expected \'%s\'',
> + r.status_code)
> + return True
> +
> +
> +class CheckText(SeleniumCommand):
> + def __init__(self, _id, text, expected_result=True):
> + self._id = _id
> + self.text = text
> + self.expected_result = expected_result
> +
> + def exec(self, selenium_ctx):
> + super().exec(selenium_ctx)
> +
> + try:
> + text = self.driver.find_element_by_id(self._id).text
> + except Exception: # TODO: Use proper Exception
> + return False
> +
> + LOGGER.debug(
> + ' Searching for \'%s\' in \'id:%s\'', self.text, self._id)
> +
> + result = True
> + if self.text not in text:
> + LOGGER.error(
> + ' \'%s\' not found in id \'%s\' with text \'%s\'', self.text,
> + self._id, text)
> + result = False
> +
> + LOGGER.debug(' \'%s\' was found', self.text)
> +
> + return result == self.expected_result
> +
> +
> +class ShExpect():
> + BASH_PATTERN = 'test_run_pr1:#'
> + COMMAND_RESULT_PATTERN = re.compile('^([0-9]+)', re.M)
> + OUTPUT_VARIABLE = 'cmd_output'
> + COMMAND_OUTPUT_DELIM = ':test_run_cmd_out:'
> + COMMAND_OUTPUT_PATTERN = re.compile(
> + r'^{0}(.*){0}\s+{1}'.format(
> + COMMAND_OUTPUT_DELIM, BASH_PATTERN), re.M | re.S)
> +
> + def __init__(self, cmd, expected_output=None, expected_result=0):
> + self.cmd = cmd
> + self.expected_result = expected_result
> + self.expected_output = expected_output
> +
> + def exec(self, pexpect_ctx):
> + self.client = pexpect_ctx.client
> +
> + LOGGER.debug('Executing command \'%s\'', self.cmd)
> + try:
> + self.client.sendline('{}=$({} 2>&1)'.format(
> + self.OUTPUT_VARIABLE, self.cmd))
> + self.client.expect(self.BASH_PATTERN)
> +
> + self.client.sendline('echo $?')
> + self.client.expect(self.COMMAND_RESULT_PATTERN)
> + result = int(self.client.match.group(1))
> +
> + self.client.sendline(
> + 'echo "{0}${{{1}}}{0}"'.format(
> + self.COMMAND_OUTPUT_DELIM,
> + self.OUTPUT_VARIABLE))
> + self.client.expect(self.COMMAND_OUTPUT_PATTERN)
> + out = self.client.match.group(1)
> +
> + if result != self.expected_result:
> + LOGGER.error('The command \'%s\' returned the code \'%d\', '
> + 'but the expected code is \'%d\''
> + '\nCommand output: \'%s\'',
> + self.cmd, result, self.expected_result, out)
> + return False
> + if self.expected_output is not None and \
> + re.search(self.expected_output, out) is None:
> + LOGGER.error('Wrong output for command \'%s\'. '
> + 'Expected \'%s\'\nReceived \'%s\'',
> + self.cmd, self.expected_output, out)
> + return False
> + except pexpect.exceptions.TIMEOUT:
> + LOGGER.error('Timeout for command \'%s\'', self.cmd)
> + return False
> + except pexpect.exceptions.EOF:
> + LOGGER.error('Lost connection with docker. Aborting')
> + return False
> + return True
> +
> +
> +class FuegoContainer:
> + def __init__(self, install_script, image_name, container_name,
> + jenkins_port):
> + self.install_script = install_script
> + self.image_name = image_name
> + self.container_name = container_name
> + self.jenkins_port = jenkins_port
> +
> + self.docker_client = docker.APIClient()
> + self.container = self.setup_docker()
> +
> + def __del__(self):
> + if self.container:
> + LOGGER.debug('Removing Container')
> + self.container.remove(force=True)
> +
> + def stop(self):
> + self.container.remove(force=True)
> + self.container = None
> +
> + def setup_docker(self):
> + cmd = './{} {}'.format(self.install_script, self.image_name)
I'm not sure I like mixing string formatting styles.
I'm not sure which is more prevalent in this code, but the rest
of Fuego uses '%s'. Please change this to:
cmd = "./%s %s" % (self.install_script, self.image_name)
> + LOGGER.debug('Running \'%s\' to install the docker image. '
> + 'This may take a while....', cmd)
> + status = subprocess.call(cmd, shell=True)
> + if status != 0:
> + return None
> + docker_client = docker.from_env()
> + containers = docker_client.containers.list(
> + all=True, filters={'name': self.container_name})
> + if containers:
> + LOGGER.debug(
> + 'Erasing the container \'%s\', so a new one can be created',
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + self.container_name)
> + containers[0].remove(force=True)
> +
> + container = docker_client.containers.create(
> + self.image_name,
> + stdin_open=True, tty=True, network_mode='bridge',
> + name=self.container_name, command='/bin/bash')
> + LOGGER.debug('Container \'%s\' created', self.container_name)
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + return container
> +
> + def is_running(self):
> + try:
> + container_status = self.docker_client.\
> + inspect_container(self.container_name)['State']['Running']
> + except KeyError:
> + return False
> +
> + return container_status
> +
> + def get_ip(self):
> + container_addr = None
> +
> + if not self.is_running():
> + return None
> +
> + def fetch_ip():
> + nonlocal container_addr
> + try:
> + container_addr = self.docker_client.\
> + inspect_container(
> + self.container_name)['NetworkSettings']['IPAddress']
> + except KeyError:
> + return False
> +
> + return False if container_addr is None else True
I'd rather this was:
if container_addr is None:
return False
else
return True
I'm not a big fan of reordering the conditional logic, even to make the
code less verbose.
> +
> + if not loop_until_timeout(fetch_ip, timeout=10):
> + LOGGER.error('Could not fetch the container IP address')
> + return None
> +
> + return container_addr
> +
> + def get_url(self):
> + container_addr = self.get_ip()
> +
> + if container_addr:
> + return 'http://{}:{}/fuego/'.\
> + format(container_addr, self.jenkins_port)
Please use '%s" and the string format operator '%'.
> + else:
> + return None
> +
> +
> +class PexpectContainerSession():
> + def __init__(self, container, start_script, timeout):
> + self.container = container
> + self.start_script = start_script
> + self.timeout = timeout
> +
> + def start(self):
> + LOGGER.debug(
> + 'Starting container \'%s\'', self.container.container_name)
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + self.client = pexpect.spawnu(
> + '{} {}'.format(
> + self.start_script, self.container.container_name),
> + echo=False, timeout=self.timeout)
> +
> + PexpectContainerSession.set_ps1(self.client)
> +
> + if not self.wait_for_jenkins():
> + return False
> +
> + return True
> +
> + def __del__(self):
> + self.client.terminate(force=True)
> +
> + @staticmethod
> + def set_ps1(client):
> + client.sendline('export PS1="{}"'.format(ShExpect.BASH_PATTERN))
Please use '%s" and the string format operator '%'.
> + client.expect(ShExpect.BASH_PATTERN)
> +
> + def wait_for_jenkins(self):
> + def ping_jenkins():
> + try:
> + conn = http.client.HTTPConnection(container_addr,
> + self.container.jenkins_port,
> + timeout=30)
> + conn.request('HEAD', '/fuego/')
> + resp = conn.getresponse()
> + version = resp.getheader('X-Jenkins')
> + status = resp.status
> + conn.close()
> + LOGGER.debug(
> + ' HTTP Response code: \'%d\' - Jenkins Version: \'%s\'',
> + status, version)
> + if status == http.client.OK and version is not None:
> + return True
> + except (ConnectionRefusedError, OSError):
> + return False
> +
> + return False
> +
> + container_addr = self.container.get_ip()
> + if container_addr is None:
> + return False
> + LOGGER.debug('Trying to reach jenkins at container \'%s\' via '
> + 'the container\'s IP \'%s\' at port \'%d\'',
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + self.container.container_name,
> + container_addr, self.container.jenkins_port)
> + if not loop_until_timeout(ping_jenkins, 10):
> + LOGGER.error('Could not connect to jenkins')
> + return False
> +
> + return True
> +
> +
> +class SeleniumContainerSession():
> + def __init__(self, container):
> + self.container = container
> + self.driver = None
> + self.root_url = container.get_url()
> +
> + def start(self):
> + options = webdriver.ChromeOptions()
> + options.add_argument('headless')
> + options.add_argument('no-sandbox')
> + options.add_argument('window-size=1200x600')
> +
> + self.driver = webdriver.Chrome(chrome_options=options)
> + self.driver.get(self.root_url)
> +
> + self.driver.get(self.root_url)
> + LOGGER.debug('Started a Selenium Session on %s', self.root_url)
> + return True
> +
> +
> +def main():
> + DEFAULT_TIMEOUT = 120
> + DEFAULT_IMAGE_NAME = 'fuego-release'
> + DEFAULT_CONTAINER_NAME = 'fuego-release-container'
> + DEFAULT_INSTALL_SCRIPT = 'install.sh'
> + DEFAULT_START_SCRIPT = 'fuego-host-scripts/docker-start-container.sh'
> + DEFAULT_JENKINS_PORT = 8080
> +
> + def execute_tests(timeout):
> + LOGGER.debug('Starting tests')
> +
> + ctx_mapper = {
> + ShExpect: pexpect_session,
> + SeleniumCommand: selenium_session
> + }
> +
> + tests_ok = True
> + for cmd in COMMANDS_TO_TEST:
> + for base_class, ctx in ctx_mapper.items():
> + if isinstance(cmd, base_class):
> + if not cmd.exec(ctx):
> + tests_ok = False
> + break
> +
> + if tests_ok:
> + LOGGER.debug('Tests finished.')
> +
> + return tests_ok
> +
> + parser = argparse.ArgumentParser()
> + parser.add_argument('working_dir', help='The working directory',
> type=str)
> + parser.add_argument('-s', '--install-script',
> + help='The script that will be used to install the '
> + 'docker image. Defaults to \'{}\''
> + .format(DEFAULT_INSTALL_SCRIPT),
Please use + for strings concatenated at the end.
> + default=DEFAULT_INSTALL_SCRIPT,
> + type=str)
> + parser.add_argument('-a', '--start-script',
> + help='The script used to start the container. '
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + 'Defaults to \'{}\''
Please use + for strings concatenated at the end.
> + .format(DEFAULT_START_SCRIPT),
> + default=DEFAULT_START_SCRIPT,
> + type=str)
> + parser.add_argument('-i', '--image-name',
> default=DEFAULT_IMAGE_NAME,
> + help='The image name that should be used. '
> + 'Defaults to \'{}\''
> + .format(DEFAULT_IMAGE_NAME), type=str)
Please use + for strings concatenated at the end.
> + parser.add_argument('-c', '--container-name',
> + default=DEFAULT_CONTAINER_NAME,
> + help='The container name that should be used for the '
> + 'test. Defaults to \'{}\''
> + .format(DEFAULT_CONTAINER_NAME),
Please use + for strings concatenated at the end.
> + type=str)
> + parser.add_argument('-t', '--timeout', help='The timeout value for '
> + 'commands. Defaults to {}'
> + .format(DEFAULT_TIMEOUT),
Please use + for strings concatenated at the end.
> + default=DEFAULT_TIMEOUT, type=int)
> + parser.add_argument('-j', '--jenkins-port',
> + help='The port where the jenkins is running on the '
> + 'test container. Defaults to {}'
> + .format(DEFAULT_JENKINS_PORT),
Please use + for strings concatenated at the end.
> + default=DEFAULT_JENKINS_PORT, type=int)
> + args = parser.parse_args()
> +
> + LOGGER.debug('Changing working dir to \'%s\'', args.working_dir)
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes.
> + os.chdir(args.working_dir)
> +
> + container = FuegoContainer(args.install_script, args.image_name,
> + args.container_name, args.jenkins_port)
> +
> + pexpect_session = PexpectContainerSession(container, args.start_script,
> + args.timeout)
> + if not pexpect_session.start():
> + return 1
> +
> + selenium_session = SeleniumContainerSession(container)
> + if not selenium_session.start():
> + return 1
> +
> + COMMANDS_TO_TEST = [
> + ShExpect(
> + 'echo $\'hello\n\n\nfrom\n\n\ncontainer\'',
Please use " (double-quote) on the outside of this string, to avoid having
to escape the internal single-quotes. (or vice-versa)
> + r'hello\s+from\s+container'),
> + ShExpect(
> + 'cat -ThisOptionDoesNotExists', expected_result=1),
> + ShExpect('ftc add-nodes docker'),
> + ShExpect(
> + 'ftc list-nodes', r'Jenkins nodes in this system:\s*docker\s*.*'),
> + ShExpect('ftc add-jobs -b docker -p testplan_docker'),
> + ShExpect(
> + 'ftc list-jobs', r'Jenkins jobs in this system:(\s*docker\..*)+'),
> + Visit(url=container.get_url()),
> + CheckText(_id='executors', text='docker'),
> + CheckText(_id='executors', text='master')
> + ]
> +
> + if not execute_tests(args.timeout):
> + return 1
> +
> + return 0
> +
> +
> +if __name__ == '__main__':
> + sys.exit(main())
> --
> 2.16.2
Thanks.
-- Tim
More information about the Fuego
mailing list