[Lsb-messages] /var/www/bzr/lsb/devel/buildbot-config r317: Add script for running jobs on low-resource buildbot slaves nicely.
Jeff Licquia
licquia at linuxfoundation.org
Sun Feb 24 01:59:35 UTC 2013
------------------------------------------------------------
revno: 317
committer: Jeff Licquia <licquia at linuxfoundation.org>
branch nick: buildbot-config
timestamp: Sat 2013-02-23 20:59:35 -0500
message:
Add script for running jobs on low-resource buildbot slaves nicely.
added:
low-resource-jobs
-------------- next part --------------
=== added file 'low-resource-jobs'
--- a/low-resource-jobs 1970-01-01 00:00:00 +0000
+++ b/low-resource-jobs 2013-02-24 01:59:35 +0000
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+# low-resource-jobs - run jobs on low-resource buildbot slaves
+# Copyright 2013 Linux Foundation.
+# Written by Jeff Licquia <licquia at linuxfoundation.org>.
+
+# This script looks at available jobs on low-resource build slaves,
+# and runs one job per low-resource slave. It decides which job to
+# run by looking at the age of the current builds, and whether any
+# builds are so old as to never have been run, and picks the oldest.
+# For jobs that have been never run, or jobs with the same age, it
+# picks randomly. Beware: it runs one job per builder per run, so
+# your low-resource slave can start consuming lots of resources
+# fairly quickly if you run this script a lot of times. For testing
+# purposes, use --dry-run.
+
+import sys
+import random
+import json
+import urllib2
+
+toplevel_url = "http://www.linuxbase.org/buildbot/json"
+spool_dir = "/opt/buildbot/jobdir"
+
+# Ultimately, read this from the buildbot config. For now, though,
+# it's easier just to hard-code.
+low_resource_slave_archs = ["s390", "s390x"]
+
+# We use the x86 build list for now, as it should have one of every
+# build minus the arch-independent builds.
+canonical_slave = "lfbuild-x86"
+
+def get_build_list():
+ build_list_source = urllib2.urlopen("%s/slaves/%s" % (toplevel_url,
+ canonical_slave))
+ build_data = json.loads(build_list_source.read())
+ return [x[:x.rindex("-")] for x in build_data["builders"].keys()]
+
+def get_low_resource_slave_archs():
+ # XXX: Read this from the buildbot config someday.
+ return low_resource_slave_archs
+
+def get_last_build_time(arch, build):
+ try:
+ build_source = urllib2.urlopen("%s/builders/%s/builds/-1"
+ % (toplevel_url, build + "-" + arch))
+ build_data = json.loads(build_source.read())
+ return build_data["times"][-1]
+ except urllib2.URLError:
+ return 0
+
+def submit_build(arch, build):
+ jobfile = open(spool_dir + "/" + arch + "_lowresource_job", "w")
+ jobfile.write("""projects=%s
+branch_name=devel
+architectures=%s
+""" % (build, arch))
+ jobfile.close()
+
+def main():
+ random.seed()
+ builds = get_build_list()
+ last_run_times = {}
+ builds_to_run = []
+ for slave_arch in get_low_resource_slave_archs():
+ last_run_times[slave_arch] = {}
+ oldest_builds = []
+ for build in builds:
+ last_run_times[slave_arch][build] = \
+ get_last_build_time(slave_arch, build)
+ if len(oldest_builds) == 0:
+ oldest_builds.append(build)
+ elif last_run_times[slave_arch][build] == \
+ last_run_times[slave_arch][oldest_builds[0]]:
+ oldest_builds.append(build)
+ elif last_run_times[slave_arch][build] < \
+ last_run_times[slave_arch][oldest_builds[0]]:
+ oldest_builds = [build]
+ if len(oldest_builds) == 1:
+ builds_to_run.append((slave_arch, oldest_builds[0]))
+ else:
+ build_to_run = random.choice(oldest_builds)
+ builds_to_run.append((slave_arch, build_to_run))
+
+ if len(sys.argv) > 1 and sys.argv[1] == "--dry-run":
+ for (arch, build) in builds_to_run:
+ print arch + ": " + build
+ else:
+ for (arch, build) in builds_to_run:
+ submit_build(arch, build)
+
+if __name__ == "__main__":
+ main()
More information about the lsb-messages
mailing list