[Openais] [PATCH 6/7] CTS: add votequorum test agent
Angus Salkeld
asalkeld at redhat.com
Mon Mar 29 19:32:06 PDT 2010
This is just the beginning of the quorum tests.
Signed-off-by: Angus Salkeld <asalkeld at redhat.com>
---
cts/agents/Makefile.am | 5 +-
cts/agents/common_test_agent.c | 1 -
cts/agents/common_test_agent.h | 6 +-
cts/agents/sam_test_agent.c | 2 +-
cts/agents/votequorum_test_agent.c | 222 ++++++++++++++++++++++++++++++++++++
cts/corosync.py | 27 ++++-
cts/corotests.py | 216 ++++++++++++++++++++++++++++++++++-
7 files changed, 466 insertions(+), 13 deletions(-)
create mode 100644 cts/agents/votequorum_test_agent.c
diff --git a/cts/agents/Makefile.am b/cts/agents/Makefile.am
index be301a3..9a1b31b 100644
--- a/cts/agents/Makefile.am
+++ b/cts/agents/Makefile.am
@@ -33,7 +33,7 @@ MAINTAINERCLEANFILES = Makefile.in
INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include
SOURCES =
-TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent
+TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent votequorum_test_agent
if INSTALL_TESTAGENTS
agentdir = $(datadir)/$(PACKAGE)/tests
@@ -66,6 +66,9 @@ sam_test_agent_SOURCES = sam_test_agent.c common_test_agent.c
sam_test_agent_LDADD = -lsam ../../exec/coropoll.o
sam_test_agent_LDFLAGS = -L../../lib
+votequorum_test_agent_SOURCES = votequorum_test_agent.c common_test_agent.c
+votequorum_test_agent_LDADD = -lvotequorum -lquorum -lcoroipcc ../../exec/coropoll.o
+votequorum_test_agent_LDFLAGS = -L../../lib
if BUILD_DARWIN
diff --git a/cts/agents/common_test_agent.c b/cts/agents/common_test_agent.c
index 6626103..2f1c675 100644
--- a/cts/agents/common_test_agent.c
+++ b/cts/agents/common_test_agent.c
@@ -47,7 +47,6 @@
#include <poll.h>
#include <unistd.h>
#include <fcntl.h>
-#include <corosync/hdb.h>
#include <corosync/totem/coropoll.h>
#include "common_test_agent.h"
diff --git a/cts/agents/common_test_agent.h b/cts/agents/common_test_agent.h
index b717290..ca510f1 100644
--- a/cts/agents/common_test_agent.h
+++ b/cts/agents/common_test_agent.h
@@ -32,14 +32,15 @@
* THE POSSIBILITY OF SUCH DAMAGE.
*/
+#ifndef CORO_COMMON_TEST_AGNET_H_DEFINED
+#define CORO_COMMON_TEST_AGNET_H_DEFINED
+#include <corosync/hdb.h>
#define OK_STR "OK"
#define FAIL_STR "FAIL"
#define NOT_SUPPORTED_STR "NOT_SUPPORTED"
-
-
extern int32_t parse_debug;
#define HOW_BIG_AND_BUF 4096
@@ -49,4 +50,5 @@ int test_agent_run(int server_port, ta_do_command_fn func);
hdb_handle_t ta_poll_handle_get(void);
+#endif /* CORO_COMMON_TEST_AGNET_H_DEFINED */
diff --git a/cts/agents/sam_test_agent.c b/cts/agents/sam_test_agent.c
index 6a96ad1..43c05df 100644
--- a/cts/agents/sam_test_agent.c
+++ b/cts/agents/sam_test_agent.c
@@ -52,9 +52,9 @@
#include <sys/un.h>
#include <sys/wait.h>
#include <syslog.h>
+
#include <corosync/corotypes.h>
#include <corosync/sam.h>
-#include <corosync/hdb.h>
#include "common_test_agent.h"
diff --git a/cts/agents/votequorum_test_agent.c b/cts/agents/votequorum_test_agent.c
new file mode 100644
index 0000000..ab9d96e
--- /dev/null
+++ b/cts/agents/votequorum_test_agent.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2010 Red Hat Inc
+ *
+ * All rights reserved.
+ *
+ * Author: Angus Salkeld <asalkeld at redhat.com>
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/un.h>
+#include <syslog.h>
+
+#include <corosync/corotypes.h>
+#include <corosync/votequorum.h>
+#include <corosync/quorum.h>
+#include "common_test_agent.h"
+
+
+static void getinfo (int sock)
+{
+ votequorum_callbacks_t callbacks;
+ int ret;
+ struct votequorum_info info;
+ char response[100];
+ votequorum_handle_t g_handle;
+
+ callbacks.votequorum_notify_fn = NULL;
+ callbacks.votequorum_expectedvotes_notify_fn = NULL;
+
+ ret = votequorum_initialize(&g_handle, &callbacks);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ ret = votequorum_getinfo(g_handle, 0, &info);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "votequorum_getinfo FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ snprintf (response, 100, "%d:%d:%d:%d:%d",
+ info.node_votes,
+ info.node_expected_votes,
+ info.highest_expected,
+ info.total_votes,
+ info.quorum);
+
+send_response:
+ votequorum_finalize (g_handle);
+ send (sock, response, strlen (response), 0);
+}
+
+
+static void setexpected (int sock, char *arg)
+{
+ votequorum_callbacks_t callbacks;
+ int ret;
+ char response[100];
+ votequorum_handle_t g_handle;
+
+ callbacks.votequorum_notify_fn = NULL;
+ callbacks.votequorum_expectedvotes_notify_fn = NULL;
+
+ ret = votequorum_initialize(&g_handle, &callbacks);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ ret = votequorum_setexpected (g_handle, atoi(arg));
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "set expected votes FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ snprintf (response, 100, "%s", OK_STR);
+
+send_response:
+ votequorum_finalize (g_handle);
+ send (sock, response, strlen (response) + 1, 0);
+}
+
+static void setvotes (int sock, char *arg)
+{
+ votequorum_callbacks_t callbacks;
+ int ret;
+ char response[100];
+ votequorum_handle_t g_handle;
+
+ callbacks.votequorum_notify_fn = NULL;
+ callbacks.votequorum_expectedvotes_notify_fn = NULL;
+
+ ret = votequorum_initialize(&g_handle, &callbacks);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ ret = votequorum_setvotes (g_handle, 0, atoi(arg));
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "set votes FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ snprintf (response, 100, "%s", OK_STR);
+
+send_response:
+ votequorum_finalize (g_handle);
+ send (sock, response, strlen (response), 0);
+}
+
+
+static void getquorate (int sock)
+{
+ int ret;
+ int quorate;
+ char response[100];
+ quorum_handle_t handle;
+
+ ret = quorum_initialize (&handle, NULL);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ ret = quorum_getquorate (handle, &quorate);
+ if (ret != CS_OK) {
+ snprintf (response, 100, "%s", FAIL_STR);
+ syslog (LOG_ERR, "getquorate FAILED: %d\n", ret);
+ goto send_response;
+ }
+
+ snprintf (response, 100, "%d", quorate);
+
+send_response:
+ quorum_finalize (handle);
+ send (sock, response, strlen (response), 0);
+}
+
+static void do_command (int sock, char* func, char*args[], int num_args)
+{
+ char response[100];
+
+ if (parse_debug)
+ syslog (LOG_DEBUG,"RPC:%s() called.", func);
+
+ if (strcmp ("votequorum_getinfo", func) == 0) {
+ getinfo (sock);
+ } else if (strcmp ("votequorum_setvotes", func) == 0) {
+ setvotes (sock, args[0]);
+ } else if (strcmp ("votequorum_setexpected", func) == 0) {
+ setexpected (sock, args[0]);
+ } else if (strcmp ("quorum_getquorate", func) == 0) {
+ getquorate (sock);
+ } else {
+ syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func);
+ snprintf (response, 100, "%s", NOT_SUPPORTED_STR);
+ send (sock, response, strlen (response), 0);
+ }
+}
+
+
+int main (int argc, char *argv[])
+{
+ int ret;
+
+ openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON);
+ syslog (LOG_ERR, "votequorum_test_agent STARTING");
+
+ parse_debug = 1;
+ ret = test_agent_run (9037, do_command);
+ syslog (LOG_ERR, "votequorum_test_agent EXITING");
+
+ return ret;
+}
+
+
diff --git a/cts/corosync.py b/cts/corosync.py
index 4dc884c..626f8dd 100644
--- a/cts/corosync.py
+++ b/cts/corosync.py
@@ -135,9 +135,10 @@ class corosync_flatiron(ClusterManager):
),
"LogFileName" : Environment["LogFileName"],
})
- self.cpg_agent={}
- self.confdb_agent={}
- self.sam_agent={}
+ self.cpg_agent = {}
+ self.confdb_agent = {}
+ self.sam_agent = {}
+ self.votequorum_agent = {}
self.config = CoroConfig ()
self.node_to_ip = {}
@@ -219,6 +220,8 @@ class corosync_flatiron(ClusterManager):
self.confdb_agent[node].restart()
if self.sam_agent.has_key(node):
self.sam_agent[node].restart()
+ if self.votequorum_agent.has_key(node):
+ self.votequorum_agent[node].restart()
return ret
def StopaCM(self, node):
@@ -230,6 +233,8 @@ class corosync_flatiron(ClusterManager):
self.cpg_agent[node].stop()
if self.sam_agent.has_key(node):
self.sam_agent[node].stop()
+ if self.votequorum_agent.has_key(node):
+ self.votequorum_agent[node].stop()
return ClusterManager.StopaCM(self, node)
def test_node_CM(self, node):
@@ -334,6 +339,8 @@ class TestAgentComponent(ScenarioComponent):
self.CM.confdb_agent[node].start()
self.CM.sam_agent[node] = SamTestAgent(node, CM.Env)
self.CM.sam_agent[node].start()
+ self.CM.votequorum_agent[node] = VoteQuorumTestAgent(node, CM.Env)
+ self.CM.votequorum_agent[node].start()
return 1
def TearDown(self, CM):
@@ -343,6 +350,7 @@ class TestAgentComponent(ScenarioComponent):
self.CM.cpg_agent[node].stop()
self.CM.confdb_agent[node].stop()
self.CM.sam_agent[node].stop()
+ self.CM.votequorum_agent[node].stop()
###################################################################
class TestAgent(object):
@@ -567,3 +575,16 @@ class SamTestAgent(TestAgent):
def cpg_local_get(self):
return 1
+###################################################################
+class VoteQuorumTestAgent(TestAgent):
+
+ def __init__(self, node, Env=None):
+ TestAgent.__init__(self, "votequorum_test_agent", node, 9037, env=Env)
+ self.initialized = False
+ self.nodeid = None
+ self.send_recv = True
+
+ def cpg_local_get(self):
+ return 1
+
+
diff --git a/cts/corotests.py b/cts/corotests.py
index c356d86..d0a4705 100644
--- a/cts/corotests.py
+++ b/cts/corotests.py
@@ -45,6 +45,7 @@ class CoroTest(CTSTest):
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.config = {}
+ self.need_all_up = True
def setup(self, node):
ret = CTSTest.setup(self, node)
@@ -67,11 +68,14 @@ class CoroTest(CTSTest):
# apply the config
self.CM.apply_new_config()
- # start any killed corosync's
+ # start/stop all corosyncs'
for n in self.CM.Env["nodes"]:
- if not self.CM.StataCM(n):
+ if self.need_all_up and not self.CM.StataCM(n):
self.incr("started")
self.start(n)
+ if not self.need_all_up and self.CM.StataCM(n):
+ self.incr("stopped")
+ self.stop(n)
return ret
@@ -192,6 +196,7 @@ class CpgCfgChgOnLowestNodeJoin(CTSTest):
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.config = {}
+ self.need_all_up = False
self.config['compatibility'] = 'none'
def lowest_ip_set(self):
@@ -263,7 +268,7 @@ class CpgCfgChgOnExecCrash(CpgConfigChangeBase):
self.name="CpgCfgChgOnExecCrash"
def failure_action(self):
- self.CM.log("sending SIGSEGV to corosync on " + self.wobbly)
+ self.CM.log("sending KILL to corosync on " + self.wobbly)
self.CM.rsh(self.wobbly, "killall -9 corosync")
self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid")
self.CM.ShouldBeStatus[self.wobbly] = "down"
@@ -639,6 +644,206 @@ class SamTest4(CoroTest):
return self.failure('sam test 4 failed')
+class QuorumState(object):
+ def __init__(self, cm, node):
+ self.node = node
+ self.CM = cm
+
+ def refresh(self):
+ info = self.CM.votequorum_agent[self.node].votequorum_getinfo()
+ assert(info != 'FAIL')
+ assert(info != 'NOT_SUPPORTED')
+
+ #self.CM.log('refresh: ' + info)
+ params = info.split(':')
+
+ self.node_votes = int(params[0])
+ self.expected_votes = int(params[1])
+ self.highest_expected = int(params[2])
+ self.total_votes = int(params[3])
+ self.quorum = int(params[4])
+ self.quorate = self.CM.votequorum_agent[self.node].quorum_getquorate()
+ assert(self.quorate != 'FAIL')
+ assert(self.quorate != 'NOT_SUPPORTED')
+ #self.CM.log('quorate: ' + str(self.quorate))
+
+###################################################################
+class VoteQuorumBase(CoroTest):
+ '''
+ '''
+
+ def setup(self, node):
+ ret = CoroTest.setup(self, node)
+ self.id_map = {}
+ self.listener = None
+ for n in self.CM.Env["nodes"]:
+ if self.listener is None:
+ self.listener = n
+ if self.need_all_up:
+ self.CM.cpg_agent[n].clean_start()
+ self.CM.cpg_agent[n].cpg_join(self.name)
+ self.id_map[n] = self.CM.cpg_agent[n].cpg_local_get()
+
+ #self.CM.votequorum_agent[self.listener].record_events()
+ return ret
+
+ def wait_for_quorum_change(self):
+ found = False
+ max_timeout = 5 * 60
+ waited = 0
+
+ printit = 0
+ self.CM.log("Waiting for quorum event on " + self.listener)
+ while not found:
+ try:
+ event = self.CM.votequorum_agent[self.listener].read_event()
+ except:
+ return self.failure('connection to test agent failed.')
+ if not event == None:
+ self.CM.debug("RECEIVED: " + str(event))
+ if event == None:
+ if waited >= max_timeout:
+ return self.failure("timedout(" + str(waited) + " sec) == no event!")
+ else:
+ time.sleep(1)
+ waited = waited + 1
+ printit = printit + 1
+ if printit is 60:
+ print 'waited 60 seconds'
+ printit = 0
+
+ elif str(event.node_id) in str(self.wobbly_id) and not event.is_member:
+ self.CM.log("Got the config change in " + str(waited) + " seconds")
+ found = True
+ else:
+ self.CM.debug("No match")
+ self.CM.debug("wobbly nodeid:" + str(self.wobbly_id))
+ self.CM.debug("event nodeid:" + str(event.node_id))
+ self.CM.debug("event.is_member:" + str(event.is_member))
+
+ if found:
+ return self.success()
+
+# repeat below with equal and uneven votes
+
+###################################################################
+class VoteQuorumGoDown(VoteQuorumBase):
+# all up
+# calc min expected votes to get Q
+# bring nodes down one-by-one
+# confirm cluster looses Q when V < EV
+#
+
+ def __init__(self, cm):
+ VoteQuorumBase.__init__(self, cm)
+ self.name="VoteQuorumGoDown"
+ self.victims = []
+ self.expected = len(self.CM.Env["nodes"])
+ self.config['quorum/provider'] = 'corosync_votequorum'
+ self.config['quorum/expected_votes'] = self.expected
+ #self.CM.log('set expected to %d' % (self.expected))
+
+ def __call__(self, node):
+ self.incr("calls")
+
+ state = QuorumState(self.CM, self.listener)
+ for n in self.CM.Env["nodes"]:
+ if n is self.listener:
+ continue
+
+ self.victims.append(n)
+ self.CM.StopaCM(n)
+
+ nodes_alive = len(self.CM.Env["nodes"]) - len(self.victims)
+ state.refresh()
+ #self.expected = self.expected - 1
+
+ if state.node_votes != 1:
+ self.failure('unexpected number of node_votes')
+
+ if state.expected_votes != self.expected:
+ self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected))
+ self.failure('unexpected number of expected_votes')
+
+ if state.total_votes != nodes_alive:
+ self.failure('unexpected number of total votes')
+
+ min = ((len(self.CM.Env["nodes"]) + 2) / 2)
+ if min != state.quorum:
+ self.failure('we should have %d (not %d) as quorum' % (min, state.quorum))
+
+ if nodes_alive < state.quorum:
+ if state.quorate == 1:
+ self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive))
+ else:
+ if state.quorate == 0:
+ self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive))
+
+ return self.success()
+
+
+# all down
+# calc min expected votes to get Q
+# bring nodes up one-by-one
+# confirm cluster gains Q when V >= EV
+#
+###################################################################
+class VoteQuorumGoUp(VoteQuorumBase):
+# all up
+# calc min expected votes to get Q
+# bring nodes down one-by-one
+# confirm cluster looses Q when V < EV
+#
+
+ def __init__(self, cm):
+ VoteQuorumBase.__init__(self, cm)
+ self.name="VoteQuorumGoUp"
+ self.need_all_up = False
+ self.expected = len(self.CM.Env["nodes"])
+ self.config['quorum/provider'] = 'corosync_votequorum'
+ self.config['quorum/expected_votes'] = self.expected
+ #self.CM.log('set expected to %d' % (self.expected))
+
+ def __call__(self, node):
+ self.incr("calls")
+
+ self.CM.StartaCM(self.listener)
+ nodes_alive = 1
+ state = QuorumState(self.CM, self.listener)
+ state.refresh()
+
+ for n in self.CM.Env["nodes"]:
+ if n is self.listener:
+ continue
+
+ if state.node_votes != 1:
+ self.failure('unexpected number of node_votes')
+
+ if state.expected_votes != self.expected:
+ self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected))
+ self.failure('unexpected number of expected_votes')
+
+ if state.total_votes != nodes_alive:
+ self.failure('unexpected number of total votes')
+
+ min = ((len(self.CM.Env["nodes"]) + 2) / 2)
+ if min != state.quorum:
+ self.failure('we should have %d (not %d) as quorum' % (min, state.quorum))
+
+ if nodes_alive < state.quorum:
+ if state.quorate == 1:
+ self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive))
+ else:
+ if state.quorate == 0:
+ self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive))
+
+ self.CM.StartaCM(n)
+ nodes_alive = nodes_alive + 1
+ state.refresh()
+
+ return self.success()
+
+
GenTestClasses = []
GenTestClasses.append(CpgMsgOrderBasic)
GenTestClasses.append(CpgMsgOrderZcb)
@@ -647,6 +852,8 @@ GenTestClasses.append(CpgCfgChgOnGroupLeave)
GenTestClasses.append(CpgCfgChgOnNodeLeave)
GenTestClasses.append(CpgCfgChgOnNodeIsolate)
GenTestClasses.append(CpgCfgChgOnLowestNodeJoin)
+GenTestClasses.append(VoteQuorumGoDown)
+GenTestClasses.append(VoteQuorumGoUp)
AllTestClasses = []
AllTestClasses.append(ConfdbReplaceTest)
@@ -657,10 +864,10 @@ AllTestClasses.append(SamTest1)
AllTestClasses.append(SamTest2)
AllTestClasses.append(SamTest3)
AllTestClasses.append(SamTest4)
-
AllTestClasses.append(ServiceLoadTest)
AllTestClasses.append(MemLeakObject)
AllTestClasses.append(MemLeakSession)
+
AllTestClasses.append(FlipTest)
AllTestClasses.append(RestartTest)
AllTestClasses.append(StartOnebyOne)
@@ -668,7 +875,6 @@ AllTestClasses.append(SimulStart)
AllTestClasses.append(StopOnebyOne)
AllTestClasses.append(SimulStop)
AllTestClasses.append(RestartOnebyOne)
-#AllTestClasses.append(PartialStart)
def CoroTestList(cm, audits):
--
1.6.6.1
More information about the Openais
mailing list