[PATCH][cr-tests] pthread3: Ensure thread state is consistent across C/R

Sukadev Bhattiprolu sukadev at linux.vnet.ibm.com
Fri Jan 8 17:30:40 PST 2010


From: Sukadev Bhattiprolu <sukadev at linux.vnet.ibm.com>
Date: Fri, 8 Jan 2010 11:30:23 -0800
Subject: [PATCH] pthread3: Ensure thread state is consistent across C/R

Specifically, this test ensures that concurrency, thread-specific-info and
scheduling parameters are consistent across checkpoint/restart.

Signed-off-by: Sukadev Bhattiprolu <sukadev at linux.vnet.ibm.com>
---
 process-tree/Makefile        |    2 +-
 process-tree/pthread3.c      |  376 ++++++++++++++++++++++++++++++++++++++++++
 process-tree/run-pthread3.sh |  205 +++++++++++++++++++++++
 3 files changed, 582 insertions(+), 1 deletions(-)
 create mode 100644 process-tree/pthread3.c
 create mode 100755 process-tree/run-pthread3.sh

diff --git a/process-tree/Makefile b/process-tree/Makefile
index 3d6010d..b043394 100644
--- a/process-tree/Makefile
+++ b/process-tree/Makefile
@@ -1,5 +1,5 @@
 
-targets		= ptree1 pthread1 pthread2
+targets		= ptree1 pthread1 pthread2 pthread3
 
 INCLUDE		= ../libcrtest
 LIBCRTEST	= ../libcrtest/common.o
diff --git a/process-tree/pthread3.c b/process-tree/pthread3.c
new file mode 100644
index 0000000..c025c9c
--- /dev/null
+++ b/process-tree/pthread3.c
@@ -0,0 +1,376 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <wait.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <libcrtest.h>
+#include <pthread.h>
+
+#define	ERROR_EXIT	((void *)1)
+#define MIN_STACK_SIZE	(64 *1024)
+#define LOG_PREFIX	"logs.d/pthread3"
+
+FILE *logfp;
+int num_threads = 4;
+int *tstatus;
+pthread_barrier_t barrier;
+pthread_mutex_t dump_lock;
+pthread_key_t key;
+
+struct thread_info {
+	int tid;
+	int concurrency;
+	void *specific;
+	sigset_t sigmask;
+	int sched_policy;
+	struct sched_param sched_param;
+};
+
+static void usage(char *argv[])
+{
+	printf("%s [h] [-n num-threads]\n", argv[0]);
+	printf("\t <num-threads> # of threads, default 5\n");
+	do_exit(1);
+}
+
+set_thread_info(int tnum)
+{
+	int rc;
+	void *specific;
+	void *sp2;
+
+	specific = (void *)pthread_self();
+
+	rc = pthread_setspecific(key, specific);
+	if (rc < 0) {
+		fprintf(logfp, "%d: pthread_setspecific() failed, rc %d, "
+				"error %s\n", tnum, rc, strerror(errno));
+		do_exit(1);
+	}
+
+	/*
+	 * TODO: Change other fields in tinfo to some non-default value
+	 */
+}
+
+get_thread_info(int tnum, struct thread_info *tinfo)
+{
+	int rc;
+
+	tinfo->tid = pthread_self();
+	tinfo->concurrency = pthread_getconcurrency();
+	tinfo->specific = pthread_getspecific(key);
+
+	if (tinfo->specific != (void *)tinfo->tid) {
+		fprintf(logfp, "%d: pthread_getspcific(): expected %p, actual "
+				"%p\n", tnum, (void *)tinfo->tid,
+				tinfo->specific);
+		do_exit(1);
+	}
+
+	rc = pthread_sigmask(SIG_SETMASK, NULL, &tinfo->sigmask);
+	if (rc < 0) {
+		fprintf(logfp, "%d: pthread_sigmask() failed, rc %d, "
+				"error %s\n", tnum, rc, strerror(errno));
+		do_exit(1);
+	}
+
+	rc = pthread_getschedparam(pthread_self(), &tinfo->sched_policy,
+				&tinfo->sched_param);
+	if (rc < 0) {
+		fprintf(logfp, "%d: pthread_getschedparam() failed, rc %d, "
+				"error %s\n", tnum, rc, strerror(errno));
+		do_exit(1);
+	}
+}
+
+compare_thread_info(int tnum, struct thread_info *exp_tinfo,
+		struct thread_info *act_tinfo)
+{
+	int rc;
+
+	rc = 0;
+	if (exp_tinfo->tid != act_tinfo->tid) {
+		rc = 1;
+		fprintf(logfp, "thread_info.tid miscompare: expected %p, "
+				"actual %p\n", (void *)exp_tinfo->tid,
+				(void *)act_tinfo->tid);
+	}
+
+	if (exp_tinfo->concurrency != act_tinfo->concurrency) {
+		rc = 1;
+		fprintf(logfp, "thread_info.concurrency miscompare: expected "
+				"%d, actual %d\n", exp_tinfo->concurrency,
+				act_tinfo->concurrency);
+	}
+
+	if (exp_tinfo->specific != act_tinfo->specific) {
+		rc = 1;
+		fprintf(logfp, "thread_info.specific miscompare: expected "
+				"%p, actual %p\n", exp_tinfo->specific,
+				act_tinfo->specific);
+	}
+
+	if (memcmp(&exp_tinfo->sigmask, &act_tinfo->sigmask, sizeof(sigset_t))) {
+		rc = 1;
+		fprintf(logfp, "thread_info.sigmask miscompare: \n");
+	}
+
+	if (exp_tinfo->sched_policy != act_tinfo->sched_policy) {
+		rc = 1;
+		fprintf(logfp, "thread_info.sched_policy miscompare: expected "
+				"%d, actual %d\n", exp_tinfo->sched_policy,
+				act_tinfo->sched_policy);
+	}
+
+	if (memcmp(&exp_tinfo->sched_param, &act_tinfo->sched_param,
+				sizeof(struct sched_param))) {
+		rc = 1;
+		fprintf(logfp, "thread_info.sched_param miscompare: expected "
+				"priority %d, actual %d\n",
+				exp_tinfo->sched_param.sched_priority,
+				act_tinfo->sched_param.sched_priority);
+	}
+
+	if (rc)
+		do_exit(1);
+}
+
+
+void *do_work(void *arg)
+{
+	int tnum = (int)arg;
+	int rc;
+	struct thread_info exp_tinfo, act_tinfo;
+
+	memset(&exp_tinfo, 0, sizeof(struct thread_info));
+	memset(&act_tinfo, 0, sizeof(struct thread_info));
+
+	set_thread_info(tnum);
+
+	get_thread_info(tnum, &exp_tinfo);
+
+	fprintf(logfp, "%d: Thread %lu: waiting for checkpoint\n", tnum,
+			pthread_self());
+	fflush(logfp);
+
+	/*
+	 * Inform main-thread we are ready for checkpoint.
+	 */
+	rc = pthread_barrier_wait(&barrier);
+	if (rc != PTHREAD_BARRIER_SERIAL_THREAD && rc != 0) {
+		fprintf(logfp, "%d: pthread_barrier_wait() failed, rc %d, "
+				"error %s\n", tnum, rc, strerror(errno));
+		do_exit(1);
+	}
+
+	/*
+	 * Wait for checkpoint/restart.
+	 */
+	while(!test_done())
+		sleep(1);
+
+	/*
+	 * Collect attributes after checkpoint/restart.
+	 */
+	get_thread_info(tnum, &act_tinfo);
+
+	/*
+	 * Compare attributes before and after C/R.
+	 */
+	compare_thread_info(tnum, &exp_tinfo, &act_tinfo);
+
+	fprintf(logfp, "%d: Thread %lu: exiting, rc 0\n", tnum,
+			pthread_self());
+	fflush(logfp);
+
+	tstatus[tnum] = 0;
+	pthread_exit((void *)&tstatus[tnum]);
+}
+
+static void create_key(pthread_key_t *key)
+{
+	int rc;
+
+	rc = pthread_key_create(key, NULL);
+	if (rc < 0) {
+		fprintf(logfp, "pthread_key_create() failed, rc %d, error %s\n",
+				rc, strerror(errno));
+		do_exit(1);
+	}
+}
+
+pthread_attr_t *alloc_thread_attr()
+{
+	int rc;
+	pthread_attr_t *attr;
+
+	attr = malloc(sizeof(pthread_attr_t));
+	if (!attr) {
+		fprintf(logfp, "malloc(attr): error %s\n", strerror(errno));
+		do_exit(1);
+	}
+
+	rc = pthread_attr_init(attr);
+	if (rc < 0) {
+		fprintf(logfp, "pthread_attr_init(): rc %d error %s\n", rc,
+				strerror(errno));
+		do_exit(1);
+	}
+
+	return attr;
+}
+
+pthread_t *create_threads(int n)
+{
+	int i;
+	int rc;
+	pthread_t *tid_list;
+	pthread_t tid;
+	pthread_attr_t *attr = NULL;
+
+	tid_list = (pthread_t *)malloc(n * sizeof(pthread_t));
+	tstatus = malloc(sizeof(int) * n);
+
+	if (!tid_list || !tstatus) {
+		fprintf(logfp, "malloc() failed, n %d, error %s\n",
+				n, strerror(errno));
+		do_exit(1);
+	}
+
+	for (i = 0; i < n; i++) {
+		attr = alloc_thread_attr();
+		if (!attr)
+			do_exit(1);
+
+		rc = pthread_create(&tid, attr, do_work, (void *)i);
+		if (rc < 0) {
+			fprintf(logfp, "pthread_create(): i %d, rc %d, "
+					"error %s\n", i, rc, strerror(errno));
+			do_exit(1);
+		}
+
+		tid_list[i] = tid;
+	}
+
+	fprintf(logfp, "Created %d threads\n", n);
+	fflush(logfp);
+
+	return tid_list;
+}
+
+int wait_for_threads(pthread_t *tid_list, int n)
+{
+	int i;
+	int rc;
+	int status;
+	int *statusp;
+	int exit_status;
+
+	exit_status = 0;
+	for (i = 0; i < n; i++) {
+		rc = pthread_join(tid_list[i], (void **)&statusp);
+		if (rc < 0) {
+			fprintf(logfp, "pthread_join() failed, i %d, rc %d "
+					"error %s\n", i, rc, strerror(errno));
+			do_exit(1);
+		}
+
+		fprintf(logfp, "i %d: *statusp %x\n", i, *statusp);
+		fflush(logfp);
+
+		if (*statusp)
+			exit_status = 1;
+	}
+
+	return exit_status;
+}
+
+main(int argc, char *argv[])
+{
+	int c;
+	int i;
+	int rc;
+	int status;
+	pthread_t *tid_list;
+	char log_file[256];
+
+	sprintf(log_file, "%s.log", LOG_PREFIX);
+
+	if (test_done()) {
+		fprintf(stderr, "Remove %s before running test\n", TEST_DONE);
+		do_exit(1);
+	}
+
+
+	while ((c = getopt(argc, argv, "hn:")) != EOF) {
+		switch (c) {
+		case 'n': num_threads = atoi(optarg); break;
+		case 'h':
+		default:
+			usage(argv);
+		}
+	};
+
+	logfp = fopen(log_file, "w");
+	if (!logfp) {
+		fprintf(stderr, "fopen(%s) failed, %s\n", log_file,
+					strerror(errno));
+		fflush(stderr);
+		do_exit(1);
+	}
+
+	fprintf(stderr, "Redirecting output to %s\n", log_file);
+	fflush(stderr);
+
+	for (i=0; i<100; i++)  {
+		if (fileno(logfp) != i)
+			close(i);
+	}
+
+
+	/*
+	 * Create a barrier which the main-thread can use to determine
+	 * when all threads are ready for checkpoint.
+	 */
+	rc = pthread_barrier_init(&barrier, NULL, num_threads+1);
+	if (rc < 0) {
+		fprintf(logfp, "pthread_barrier_init() failed, rc %d, "
+				"error %s\n", rc, strerror(errno));
+		do_exit(1);
+	}
+
+	rc = pthread_mutex_init(&dump_lock, NULL);
+	if (rc) {
+		fprintf(logfp, "pthread_mutex_init() failed, rc %d, error %s\n",
+				rc, strerror(errno));
+		do_exit(1);
+	}
+
+	create_key(&key);
+
+	tid_list = create_threads(num_threads);
+
+	/*
+	 * Wait for everyone to be ready for checkpoint
+	 */
+	pthread_barrier_wait(&barrier);
+	if (rc != PTHREAD_BARRIER_SERIAL_THREAD && rc != 0) {
+		fprintf(logfp, "main: pthread_barrier_wait() failed, rc %d, "
+				"error %s\n", rc, strerror(errno));
+		do_exit(1);
+	}
+
+	/*
+	 * Now that we closed the special files and created the threads,
+	 * tell any wrapper scripts, we are ready for checkpoint
+	 */
+	set_checkpoint_ready();
+
+	rc = wait_for_threads(tid_list, num_threads);
+
+	fprintf(logfp, "Exiting with status %d\n", rc);
+
+	do_exit(rc);
+}
diff --git a/process-tree/run-pthread3.sh b/process-tree/run-pthread3.sh
new file mode 100755
index 0000000..0cffe44
--- /dev/null
+++ b/process-tree/run-pthread3.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+
+source ../common.sh
+
+dir=`mktemp -p . -d -t cr_pthread3_XXXXXXX` || (echo "mktemp failed"; exit 1)
+mkdir $dir
+echo "Using output dir $dir"
+cd $dir
+
+# NOTE: As of ckpt-v15-dev, the --container option to 'ckpt' causes this
+#	test to fail with "container not isolated" message due to the
+#	log-file being shared between the application threads.
+#
+CHECKPOINT="`which checkpoint` --container"
+RESTART=`which restart`
+ECHO="/bin/echo -e"
+
+TEST_CMD="../pthread3"
+TEST_ARGS="-n 4"			# -n: number of threads
+SCRIPT_LOG="log-run-pthread3"
+TEST_PID_FILE="pid.pthread3";
+
+SNAPSHOT_DIR="snap1.d"
+
+TEST_DONE="test-done"
+CHECKPOINT_FILE="checkpoint-pthread3";
+CHECKPOINT_READY="checkpoint-ready"
+CHECKPOINT_DONE="checkpoint-done"
+
+LOGS_DIR="logs.d"
+
+NS_EXEC="../../ns_exec"
+NS_EXEC_ARGS="-cgpuimP $TEST_PID_FILE"
+
+checkpoint()
+{
+	local pid=$1
+
+	$ECHO "Checkpoint: $CHECKPOINT $pid \> $CHECKPOINT_FILE"
+	$CHECKPOINT $pid > $CHECKPOINT_FILE
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		$ECHO "***** FAIL: Checkpoint of $pid failed"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1;
+	fi
+}
+
+function wait_for_checkpoint_ready()
+{
+	# Wait for test to finish setup
+	while [ ! -f $CHECKPOINT_READY ]; do
+		$ECHO "\t- Waiting for $CHECKPOINT_READY"
+		sleep 1;
+	done;
+}
+
+function create_container()
+{
+	local pid;
+
+	cmdline="$NS_EXEC $NS_EXEC_ARGS -- $TEST_CMD $TEST_ARGS"
+
+	$ECHO "\t- Creating container:"
+	$ECHO "\t- $cmdline"
+
+	$cmdline &
+
+	wait_for_checkpoint_ready;
+
+	# Find global pid of container-init
+	pid=`cat $TEST_PID_FILE`;
+	if [  "x$pid" == "x" ]; then
+		$ECHO "***** FAIL: Invalid container-init pid $pid"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1
+	fi
+	$ECHO "Created container with pid $pid" >> $SCRIPT_LOG
+}
+
+function restart_container
+{
+	local ret;
+
+	cmdline="$RESTART --pids --pidns --wait"
+	$ECHO "\t- $cmdline"
+
+	sleep 1
+
+	$cmdline < $CHECKPOINT_FILE >> $SCRIPT_LOG 2>&1 &
+	ret=$?
+
+	if [ $ret -ne 0 ]; then
+		$ECHO "***** FAIL: Restart of $pid failed"
+		ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+		exit 1;
+	fi
+}
+
+function create_fs_snapshot()
+{
+	# Prepare for snapshot
+	if [ -d $SNAPSHOT_DIR ]; then
+		rm -rf ${SNAPSHOT_DIR}.prev
+		mv $SNAPSHOT_DIR ${SNAPSHOT_DIR}.prev
+		mkdir $SNAPSHOT_DIR
+	fi
+
+	# Snapshot the log files
+	cp ${LOGS_DIR}/* $SNAPSHOT_DIR
+}
+
+function restore_fs_snapshot()
+{
+	# Restore the snapshot after the main process has been killed
+	/bin/cp ${SNAPSHOT_DIR}/* $LOGS_DIR
+}
+
+# Make sure no stray pthread1 from another run is still going
+killall $TEST_CMD > $SCRIPT_LOG 2>&1
+
+if [ ! -d $LOGS_DIR ]; then
+	mkdir $LOGS_DIR
+fi
+
+if [ ! -d $DATA_DIR ]; then
+	mkdir $DATA_DIR
+fi
+
+if [ ! -d $SNAPSHOT_DIR ]; then
+	mkdir $SNAPSHOT_DIR
+fi
+
+if [ ! -f $INPUT_DATA ]; then
+	$FILEIO -C $INPUT_DATA
+fi
+
+> $SCRIPT_LOG;
+cnt=1
+while [ $cnt -lt 15 ]; do
+	$ECHO "===== Iteration $cnt"
+
+	# Remove any 'state' files, start the app and let it tell us
+	# when it is ready
+	rm -f $CHECKPOINT_READY $TEST_DONE $TEST_PID_FILE
+
+	create_container
+	wait_for_checkpoint_ready
+
+	pid=`cat $TEST_PID_FILE`
+
+	$ECHO "\t- Done creating container, cinit-pid $pid"
+
+	ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+
+	# override default freezerdir
+	if [ -d $freezerdir ]; then
+		rmdir $freezerdir
+	fi
+	freezerdir=$freezermountpoint/$pid
+	freeze_pid $pid
+
+	num_pids1=`ps -efL |grep $TEST_CMD | wc -l`
+
+	create_fs_snapshot
+
+	checkpoint $pid
+
+	touch $CHECKPOINT_DONE
+
+	killall -9 `basename $TEST_CMD`
+
+	thaw
+
+	sleep 3
+
+	restore_fs_snapshot
+
+	restart_container
+
+	sleep 3;
+
+	num_pids2=`ps -efL |grep $TEST_CMD | wc -l`
+	ps -efL |grep $TEST_CMD >> $SCRIPT_LOG
+	$ECHO "\t- num_pids1 $num_pids1, num_pids2 $num_pids2";
+
+	# ns_exec pid is parent-pid of restarted-container-init
+	nspid=`pidof restart`
+
+	if [ "x$nspid" == "x" ]; then
+		$ECHO "***** FAIL: Can't find pid of $RESTART"
+		exit 1;
+	fi
+
+	# End test gracefully
+	touch $TEST_DONE
+
+	$ECHO "\t- Waiting for restarted container to exit (gloabl-pid $nspid)"
+	wait $nspid;
+	ret=$?
+
+	$ECHO "\t- Container exited, status $ret"
+
+	cnt=$((cnt+1))
+done
-- 
1.6.0.4



More information about the Containers mailing list