buildbot
Nested triggers not executing builds but marking them success
We execute steps.Trigger with multiple builders within builds executed by other step.Triggers. For some reason certain builds may not run at all, yet for some reason they become green before even triggering any steps.
On the following photo one can see one correctly executed level 3, step 2 - test-trigger4-asan and one incorrectly executed level 3, step 2 - test-trigger4-asan despite both being implemented in the same fashion:
For a test 4 workers without limits were used. Here is the relevant code:
from buildbot.process import factory
from buildbot.plugins import changes, util, schedulers, steps
from properties import testeroid
from main import c
################################################################################
### level 4
test_step_level4_step1 = steps.ShellCommand(
name=util.Interpolate('level 4, step 1 - %(prop:bsp)s %(prop:flavour)s %(prop:san)s'),
command=util.Interpolate('echo bsp:%(prop:bsp)s flavour:%(prop:flavour)s san:%(prop:san)s; sleep 5; echo done'),
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_step_level4_step2 = steps.ShellCommand(
name=util.Interpolate('level 4, step 2 - %(prop:bsp)s %(prop:flavour)s %(prop:san)s'),
command=util.Interpolate("echo FAIL; sleep 7; if [ '%(prop:san)s' = 'msan' -a '%(prop:flavour)s' = 'llvm' -a '%(prop:bsp)s' = 'BBBB' ]; then false; else true; fi"),
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
)
test_trigger_level4_steps = [test_step_level4_step1, test_step_level4_step2]
test_trigger_level4_factory = factory.BuildFactory()
test_trigger_level4_factory.addSteps(test_trigger_level4_steps)
c['builders'].append(
util.BuilderConfig(
name='test-trigger4',
factory=test_trigger_level4_factory,
workernames=testeroid['workers'],
tags=['test_trigger4'],
)
)
c['schedulers'].append(
schedulers.Triggerable(
name='test-trigger4',
builderNames=['test-trigger4'],
)
)
################################################################################
### level 3
test_step_level3_step1 = steps.ShellCommand(
name=util.Interpolate('level 3, step 1 - %(prop:bsp)s %(prop:flavour)s'),
command=util.Interpolate('echo bsp:%(prop:bsp)s flavour:%(prop:flavour)s; sleep 10; echo done'),
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level3_steps = [test_step_level3_step1]
for san in ['asan', 'msan']:
step_name = 'level 3, step 2 - test-trigger4-' + san
ts_step = steps.Trigger(
name=step_name,
schedulerNames=['test-trigger4'],
waitForFinish=False,
flunkOnWarnings=True,
set_properties={
'bsp': util.Property('bsp'),
'flavour': util.Property('flavour'),
'san': san,
},
)
test_trigger_level3_steps.append(ts_step)
test_step_level3_step3 = steps.ShellCommand(
name=util.Interpolate('level 3, step 3 - %(prop:bsp)s %(prop:flavour)s'),
command='ls -al; sleep 3; echo done',
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level3_steps.append(test_step_level3_step3)
test_trigger_level3_factory = factory.BuildFactory()
test_trigger_level3_factory.addSteps(test_trigger_level3_steps)
c['builders'].append(
util.BuilderConfig(
name='test-trigger3',
factory=test_trigger_level3_factory,
workernames=testeroid['workers'],
tags=['test_trigger3'],
)
)
c['schedulers'].append(
schedulers.Triggerable(
name='test-trigger3',
builderNames=['test-trigger3'],
)
)
################################################################################
### level 2
test_step_level2_step1 = steps.ShellCommand(
name=util.Interpolate('level 2, step 1 - %(prop:bsp)s'),
command=util.Interpolate('echo %(prop:bsp)s; sleep 7; echo done'),
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level2_steps = [test_step_level2_step1]
for flavour in ['gcc', 'llvm']:
step_name = 'level 2, step 2 - test-trigger3-' + flavour
ts_step = steps.Trigger(
name=step_name,
schedulerNames=['test-trigger3'],
waitForFinish=False,
flunkOnWarnings=True,
set_properties={
'bsp': util.Property('bsp'),
'flavour': flavour,
},
)
test_trigger_level2_steps.append(ts_step)
test_step_level2_step3 = steps.ShellCommand(
name=util.Interpolate('level 2, step 3 - %(prop:bsp)s'),
command='ls; sleep 11; echo done',
workdir=util.Interpolate('%(prop:builddir)s/%(prop:buildnumber)s'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level2_steps.append(test_step_level2_step3)
test_trigger_level2_factory = factory.BuildFactory()
test_trigger_level2_factory.addSteps(test_trigger_level2_steps)
c['builders'].append(
util.BuilderConfig(
name='test-trigger2',
factory=test_trigger_level2_factory,
workernames=testeroid['workers'],
tags=['test_trigger2'],
)
)
c['schedulers'].append(
schedulers.Triggerable(
name='test-trigger2',
builderNames=['test-trigger2'],
)
)
################################################################################
### level 1
test_step_level1_step1 = steps.ShellCommand(
name=util.Interpolate('level 1, step 1'),
command='ls; sleep 7; echo done',
workdir=util.Interpolate('%(prop:builddir)s/'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level1_steps = [test_step_level1_step1]
for bsp in ['AAAA', 'BBBB']:
step_name = 'level 1, step 2 - test-trigger2-' + bsp
ts_step = steps.Trigger(
name=step_name,
schedulerNames=['test-trigger2'],
waitForFinish=False,
flunkOnWarnings=True,
set_properties={
'bsp': bsp,
},
)
test_trigger_level1_steps.append(ts_step)
test_step_level1_step3 = steps.ShellCommand(
name=util.Interpolate('level 1, step 3'),
command='ls -al; sleep 3; echo done',
workdir=util.Interpolate('%(prop:builddir)s/'),
lazylogfiles=True,
logEnviron=False,
haltOnFailure=True,
)
test_trigger_level1_steps.append(test_step_level1_step3)
test_trigger_level1_factory = factory.BuildFactory()
test_trigger_level1_factory.addSteps(test_trigger_level1_steps)
c['builders'].append(
util.BuilderConfig(
name='test-trigger1',
factory=test_trigger_level1_factory,
workernames=testeroid['workers'],
tags=['test_trigger1'],
locks=[],
)
)
c['schedulers'].append(
schedulers.ForceScheduler(
name='test-trigger1',
buttonName='Run trigger test',
builderNames=['test-trigger1'],
properties=[],
codebases=[util.CodebaseParameter(codebase='', hide=True)],
)
)
The issue seems to be caused by build request becoming completed for some reason, but so far we were unable to track it. While it looks somewhat familiar to https://github.com/buildbot/buildbot/issues/6251, the patch suggested there does not affect the issue.
The issue is apparently caused by the default implementation of collapseRequests. This function does not compare properties and believes that the tasks are doing the same thing, and kills some of them. The following implementation of collapseRequests successfully resolves the issue:
def collapseRequests(master, builder, req1, req2):
return False
c['collapseRequests'] = collapseRequests
Finding a workaround took us some time, with the primary reason for that being no visual signature that the build was terminated by the collapser. Other CI providers (at least I remember that from Travis) usually assign canceled status in such case and additionally show the reason why the build was canceled, e.g. due to subsequent pushes.
Perhaps this issue can be changed to a feature request of a better visual confirmation when collapseRequests terminates a build?