[Lsb-messages] /var/www/bzr/lsb/devel/buildbot-config r317: Add script for running jobs on low-resource buildbot slaves nicely.

Jeff Licquia licquia at linuxfoundation.org
Sun Feb 24 01:59:35 UTC 2013


------------------------------------------------------------
revno: 317
committer: Jeff Licquia <licquia at linuxfoundation.org>
branch nick: buildbot-config
timestamp: Sat 2013-02-23 20:59:35 -0500
message:
  Add script for running jobs on low-resource buildbot slaves nicely.
added:
  low-resource-jobs
-------------- next part --------------
=== added file 'low-resource-jobs'
--- a/low-resource-jobs	1970-01-01 00:00:00 +0000
+++ b/low-resource-jobs	2013-02-24 01:59:35 +0000
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+# low-resource-jobs - run jobs on low-resource buildbot slaves
+# Copyright 2013 Linux Foundation.
+# Written by Jeff Licquia <licquia at linuxfoundation.org>.
+
+# This script looks at available jobs on low-resource build slaves,
+# and runs one job per low-resource slave.  It decides which job to
+# run by looking at the age of the current builds, and whether any
+# builds are so old as to never have been run, and picks the oldest.
+# For jobs that have been never run, or jobs with the same age, it
+# picks randomly.  Beware: it runs one job per builder per run, so
+# your low-resource slave can start consuming lots of resources
+# fairly quickly if you run this script a lot of times.  For testing
+# purposes, use --dry-run.
+
+import sys
+import random
+import json
+import urllib2
+
+toplevel_url = "http://www.linuxbase.org/buildbot/json"
+spool_dir = "/opt/buildbot/jobdir"
+
+# Ultimately, read this from the buildbot config.  For now, though,
+# it's easier just to hard-code.
+low_resource_slave_archs = ["s390", "s390x"]
+
+# We use the x86 build list for now, as it should have one of every
+# build minus the arch-independent builds.
+canonical_slave = "lfbuild-x86"
+
+def get_build_list():
+    build_list_source = urllib2.urlopen("%s/slaves/%s" % (toplevel_url,
+                                                          canonical_slave))
+    build_data = json.loads(build_list_source.read())
+    return [x[:x.rindex("-")] for x in build_data["builders"].keys()]
+
+def get_low_resource_slave_archs():
+    # XXX: Read this from the buildbot config someday.
+    return low_resource_slave_archs
+
+def get_last_build_time(arch, build):
+    try:
+        build_source = urllib2.urlopen("%s/builders/%s/builds/-1" 
+                                       % (toplevel_url, build + "-" + arch))
+        build_data = json.loads(build_source.read())
+        return build_data["times"][-1]
+    except urllib2.URLError:
+        return 0
+
+def submit_build(arch, build):
+    jobfile = open(spool_dir + "/" + arch + "_lowresource_job", "w")
+    jobfile.write("""projects=%s
+branch_name=devel
+architectures=%s
+""" % (build, arch))
+    jobfile.close()
+
+def main():
+    random.seed()
+    builds = get_build_list()
+    last_run_times = {}
+    builds_to_run = []
+    for slave_arch in get_low_resource_slave_archs():
+        last_run_times[slave_arch] = {}
+        oldest_builds = []
+        for build in builds:
+            last_run_times[slave_arch][build] = \
+                get_last_build_time(slave_arch, build)
+            if len(oldest_builds) == 0:
+                oldest_builds.append(build)
+            elif last_run_times[slave_arch][build] == \
+                    last_run_times[slave_arch][oldest_builds[0]]:
+                oldest_builds.append(build)
+            elif last_run_times[slave_arch][build] < \
+                    last_run_times[slave_arch][oldest_builds[0]]:
+                oldest_builds = [build]
+        if len(oldest_builds) == 1:
+            builds_to_run.append((slave_arch, oldest_builds[0]))
+        else:
+            build_to_run = random.choice(oldest_builds)
+            builds_to_run.append((slave_arch, build_to_run))
+
+    if len(sys.argv) > 1 and sys.argv[1] == "--dry-run":
+        for (arch, build) in builds_to_run:
+            print arch + ": " + build
+    else:
+        for (arch, build) in builds_to_run:
+            submit_build(arch, build)
+
+if __name__ == "__main__":
+    main()



More information about the lsb-messages mailing list