[zorg] r273226 - Added automatic generation of SingleBranchSchedulers to track accurate source code dependencies.

[Galina Kistanova via llvm-commits]

Author: gkistanova
Date: Mon Jun 20 19:08:18 2016
New Revision: 273226

URL: http://llvm.org/viewvc/llvm-project?rev=273226&view=rev
Log:
Added automatic generation of SingleBranchSchedulers to track accurate source code dependencies.

Added:
    zorg/trunk/buildbot/osuosl/master/config/schedulers.py
Modified:
    zorg/trunk/buildbot/osuosl/master/config/__init__.py

Modified: zorg/trunk/buildbot/osuosl/master/config/__init__.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/__init__.py?rev=273226&r1=273225&r2=273226&view=diff
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/__init__.py (original)
+++ zorg/trunk/buildbot/osuosl/master/config/__init__.py Mon Jun 20 19:08:18 2016
@@ -5,5 +5,6 @@ options = ConfigParser.RawConfigParser()
 options.read(os.path.join(os.path.dirname(__file__), 'local.cfg'))
 
 import builders
+import schedulers
 import slaves
 import status

Added: zorg/trunk/buildbot/osuosl/master/config/schedulers.py
URL: http://llvm.org/viewvc/llvm-project/zorg/trunk/buildbot/osuosl/master/config/schedulers.py?rev=273226&view=auto
==============================================================================
--- zorg/trunk/buildbot/osuosl/master/config/schedulers.py (added)
+++ zorg/trunk/buildbot/osuosl/master/config/schedulers.py Mon Jun 20 19:08:18 2016
@@ -0,0 +1,63 @@
+from twisted.python import log
+
+from buildbot.schedulers.filter import ChangeFilter
+from buildbot.schedulers.basic import SingleBranchScheduler
+
+# Since we have many parametric builders, we dynamically build the minimum set
+# of schedulers, which covers all actually used combinations of dependencies.
+def getSingleBranchSchedulers(builders, schedulers, **kwargs):
+    """
+    I'm taking over all of not yet assigned builders with the
+    declared source code dependencies, and automatically generate
+    a minimum set of SingleBranchSchedulers to handle all the declared
+    source code dependency combinations.
+    """
+
+    # Prepare a list of builders which already have been assigned to schedulers.
+    builders_with_schedulers = {
+        b for s in schedulers for b in s.builderNames
+    }
+
+    builders_with_automatic_schedulers = []
+    for builder in builders:
+        # Only for the builders created with LLVMBuildFactory or similar.
+        if getattr(builder['factory'], 'depends_on_projects', None):
+            # And only if this builder does not yet have an assigned scheduler.
+            if builder['name'] not in builders_with_schedulers:
+                # This builder is a candidate for an automatic scheduler.
+                builders_with_automatic_schedulers.append(builder)
+
+    automatic_schedulers = []
+    # Do we have any to take care of?
+    if builders_with_automatic_schedulers:
+        # Let's reconsile first to get a unique set of dependencies.
+        # We need a set of unique sets of dependent projects.
+        set_of_dependencies = set([
+            frozenset(getattr(b['factory'], 'depends_on_projects'))
+            for b in builders_with_automatic_schedulers
+        ])
+
+        treeStableTimer = kwargs.get('treeStableTimer', 2*60)
+        automatic_schedulers = []
+        for projects in set_of_dependencies:
+            sch_builders = [
+                b['name']
+                for b in builders_with_automatic_schedulers
+                if frozenset(getattr(b['factory'], 'depends_on_projects')) == projects
+            ]
+
+            automatic_schedulers.append(
+                SingleBranchScheduler(
+                    name="auto_scheduler_%x" % hash(projects),
+                    treeStableTimer=treeStableTimer,
+                    builderNames=sch_builders,
+                    change_filter=ChangeFilter(project=list(projects))
+                )
+            )
+
+            log.msg(
+                "Generated SingleBranchScheduler: { name='auto_scheduler_%x'" % hash(projects),
+                ", builderNames=", sch_builders,
+                ", change_filter=", list(projects),
+                "}")
+    return automatic_schedulers