run-tests.py 9.67 KB
Newer Older
1 2 3 4 5
#!/usr/bin/python3
import os
import re
import sys
import time
6
import stat
7 8 9 10 11 12
import argparse
import subprocess
from helperslib import BuildSpecs, BuildSystem, CommonUtils, EnvironmentHandler

# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility to execute tests for a build.')
13
parser.add_argument('--product', type=str, required=True)
14 15 16 17 18 19 20
parser.add_argument('--project', type=str, required=True)
parser.add_argument('--branchGroup', type=str, required=True)
parser.add_argument('--platform', type=str, required=True)
parser.add_argument('--usingInstall', type=str, required=True)
arguments = parser.parse_args()

# Load our build specification, which governs how we handle this build
21
buildSpecification = BuildSpecs.Loader( product=arguments.product, project=arguments.project, branchGroup=arguments.branchGroup, platform=arguments.platform )
22 23 24 25 26 27 28 29 30 31 32 33 34 35

# Determine the environment we need to provide for the installation process
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=arguments.usingInstall )

# Determine where our source code is checked out to and where we will be building it
# We'll assume that the directory we're running from is where the sources are located
sourcesLocation = os.getcwd()
buildLocation = CommonUtils.buildDirectoryForSources( sources=sourcesLocation, inSourceBuild=buildSpecification['in-source-build'] )

# First Check: Is this a CMake project, and is testing enabled for it?
if not BuildSystem.CMake.canConfigure(sourcesLocation) or not buildSpecification['run-tests']:
	# We either can't test (not CMake) or tests are disabled, so nothing for us to do here!
	sys.exit(0)

36 37 38 39 40
# Temporary Block: Don't run tests on FreeBSD at the moment as it causes lots of issues for us
if sys.platform == 'freebsd11':
	# Then don't continue any further
	sys.exit(0)

41 42 43 44 45 46 47 48 49 50 51
# On Windows, we need to pull bin\data\ over from the Craft prefix to ensure resources in it can be found by QStandardPaths
# This is a bit of a hack, but there isn't much we can do here as Qt doesn't give us any means of telling it to look elsewhere
if sys.platform == 'win32' and 'CRAFT_ROOT' in os.environ:
	# Determine where Craft is...
	craftRoot = os.path.realpath( os.environ['CRAFT_ROOT'] )
	sourceDirectory = os.path.join( craftRoot, 'bin\data' )
	# Determine where we want to deploy these files to
	destinationDirectory = os.path.join( buildLocation, 'bin\data' )
	# Do the copy!
	CommonUtils.recursiveDirectoryCopy( sourceDirectory, destinationDirectory )

52 53 54 55
# Get a count of the number of available tests
# This relies on the command "ctest -N" producing at least one line that matches "Total Tests: x" where x is the number of tests
process = subprocess.Popen( "ctest -N", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=buildEnvironment, cwd=buildLocation)
stdout, stderr = process.communicate()
Ben Cooksley's avatar
Ben Cooksley committed
56
testsFound = int( re.search( b'Total Tests: ([0-9]+)', stdout, re.MULTILINE ).group(1) )
57 58 59 60 61

# If we had less than 1 test then there is nothing for us to do - bail
if testsFound == 0:
	sys.exit(0)

62 63 64 65 66 67 68 69 70 71
# Looks like we have some tests to run
# Before we can get started, we should make sure the environment is ready for use
# On Linux/*BSD systems we need to make sure XDG_RUNTIME_DIR is set as lots of tools are reliant on it being set and existing
if sys.platform != 'win32' and sys.platform != 'darwin':
	# Set it in the environment
	buildEnvironment['XDG_RUNTIME_DIR'] = '/tmp/runtime-kdeci/'
	# And make sure it exists
	if not os.path.exists( buildEnvironment['XDG_RUNTIME_DIR'] ):
		# Create it!
		os.makedirs( buildEnvironment['XDG_RUNTIME_DIR'] )
72 73
		# And lock down it's permissions
		os.chmod( buildEnvironment['XDG_RUNTIME_DIR'], stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR )
74

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
# Are we being asked to ensure ASAN is forcibly injected?
# On FreeBSD their ASAN is different it behaves properly
# So we only need to force inject on Linux
if sys.platform =='linux' and buildSpecification['force-inject-asan']:
	# Where could the ASAN library live?
	knownAsanLocations = [
		'/lib64/',
		'/usr/lib64/',
		'/usr/lib/x86_64-linux-gnu/'
	]
	# And what names could it be hiding under?
	knownAsanNames = [
		'libasan.so.4',
		'libasan.so.3',
		'libasan.so.2'
	]
	# Knowing all of that, let's find ASAN's library and set it up to be injected
	buildEnvironment['LD_PRELOAD'] = CommonUtils.firstPresentFileInPaths( knownAsanLocations, knownAsanNames )

94 95
# We want Qt to be noisy about debug output to make debugging tests easier
buildEnvironment['QT_LOGGING_RULES'] = "*.debug=true"
96
# We want to force Qt to print to stderr, even on Windows
Ben Cooksley's avatar
Ben Cooksley committed
97
buildEnvironment['QT_LOGGING_TO_CONSOLE'] = '1'
98
buildEnvironment['QT_FORCE_STDERR_LOGGING'] = '1'
99 100
# We also want CMake to be noisy when tests fail
buildEnvironment['CTEST_OUTPUT_ON_FAILURE'] = '1'
101

102 103 104
# Cleanup the builder if needed
if sys.platform == 'freebsd11':
	subprocess.call("killall -9 dbus-daemon kded5 kioslave klauncher kdeinit5 kiod openbox Xvfb", shell=True)
105 106 107 108
	if os.path.exists('/tmp/.X90-lock'):
		os.remove('/tmp/.X90-lock')
	if os.path.exists('/tmp/.X11-unix/X90'):
		os.remove('/tmp/.X11-unix/X90')
109

110 111 112
# Spawn a X windowing system if needed
# We'll also launch a Window Manager at the same time as some tests often need or unknowingly rely on one being present
# As X doesn't belong on Windows or OSX we don't run it there
Ben Cooksley's avatar
Ben Cooksley committed
113
if buildSpecification['setup-x-environment'] and ( sys.platform != 'win32' and sys.platform != 'darwin' ):
114 115 116 117 118
	# Setup Xvfb
	buildEnvironment['DISPLAY'] = ':90'
	commandToRun = "Xvfb :90 -ac -screen 0 1600x1200x24+32"
	xvfbProcess = subprocess.Popen( commandToRun, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT, shell=True, env=buildEnvironment )

119 120 121
	# Give Xvfb a few moments to get on it's feet
	time.sleep( 5 )

122 123
	# Startup a Window Manager
	commandToRun = "openbox"
Ben Cooksley's avatar
Ben Cooksley committed
124
	wmProcess = subprocess.Popen( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
125 126 127

# Spawn D-Bus if needed
# Same rules apply for X in regards to Windows and OSX - it doesn't belong so we don't support it
Ben Cooksley's avatar
Ben Cooksley committed
128
if buildSpecification['launch-dbus-session'] and ( sys.platform != 'win32' and sys.platform != 'darwin' ):
129
	# Determine the command to run, then launch it and wait for it to exit
130
	commandToRun = 'dbus-launch'
Ben Cooksley's avatar
Ben Cooksley committed
131
	process = subprocess.Popen( commandToRun, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=buildEnvironment )
132 133 134
	process.wait()
	# Determine what environment variables need to be set, and ensure those are included in our build environment
	for variable in process.stdout:
135 136
		variable  = str(variable, 'utf-8')
		splitVars = variable.split('=', 1)
137 138
		buildEnvironment[ splitVars[0] ] = splitVars[1].strip()

139 140 141
# Do we need to run update-mime-database?
# First we need to determine what mime directory we will have
# On most platforms this will be $prefix/share/mime
142
mimeDirectory = os.path.realpath( os.path.join( arguments.usingInstall, 'share', 'mime' ) )
143 144
# Except on Windows... where it is bin/data/mime/
if sys.platform == 'win32':
145
	mimeDirectory = os.path.realpath( os.path.join( arguments.usingInstall, 'bin', 'data', 'mime' ) )
146 147 148 149 150

# Make sure the mime directory exists - otherwise there is no point to running update-mime-database
if os.path.exists( mimeDirectory ):
	# Let's run update-mime-database
	commandToRun = BuildSystem.substituteCommandTokens( 'update-mime-database "' + mimeDirectory + '"'  )
151 152
	process = subprocess.Popen( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
	process.wait()
153

154 155 156
# KDE Tests often need kdeinit running, in order to have klauncher, kded, etc available so let's spawn those now
# This is the case regardless of the platform
commandToRun = 'kdeinit5'
157
try:
158
	kdeinitProcess = subprocess.Popen( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
159 160
except OSError:
	pass
161

162 163 164 165
# As some of the above might still be getting themselves ready, wait for a few moments...
time.sleep( 5 )

# Now it's time to invoke CTest! Build up the command...
166
commandToRun = "ctest -T Test --output-on-failure --no-compress-output --test-output-size-passed 1048576 --test-output-size-failed 1048576 --timeout {timeLimit} {additionalCTestArguments}"
167
commandToRun = commandToRun.format( timeLimit=buildSpecification['per-test-timeout'], additionalCTestArguments=buildSpecification['ctest-arguments'] )
168 169 170 171 172 173 174
# And run it!
ctestProcess = subprocess.Popen( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildLocation, env=buildEnvironment )
ctestProcess.wait()

# Now that CTest is done, we convert it's output to JUnit format
junitOutput = BuildSystem.CMake.convertCTestResultsToJUnit( buildLocation )
junitFilename = os.path.join( sourcesLocation, 'JUnitTestResults.xml' )
175
with open(junitFilename, 'w', encoding='UTF-8') as junitFile:
176 177 178
	junitFile.write( str(junitOutput) )

# To ensure we don't hang, cleanup the Window Manager and X server if needed
Ben Cooksley's avatar
Ben Cooksley committed
179
if buildSpecification['setup-x-environment'] and ( sys.platform != 'win32' and sys.platform != 'darwin' ):
180 181 182
	wmProcess.terminate()
	xvfbProcess.terminate()

183 184
# Finally, we do some last cleanup
# This is particularly relevant on FreeBSD and Windows where the slaves are permanent
185
# We wait around on Windows before trying to kill off kioslave.exe as it likes to get into a weird state and not die
186
if sys.platform == 'win32':
187 188 189 190 191
	subprocess.call("taskkill /f /T /im kded5.exe", shell=True)
	subprocess.call("taskkill /f /T /im klauncher.exe", shell=True)
	subprocess.call("taskkill /f /T /im kdeinit5.exe", shell=True)
	subprocess.call("taskkill /f /T /im test_crasher.exe", shell=True)
	subprocess.call("taskkill /f /T /im dbus-daemon.exe", shell=True)
192
	time.sleep( 60 )
193
	subprocess.call("taskkill /f /T /im kioslave.exe", shell=True)
194 195 196 197

if sys.platform == 'freebsd11':
	subprocess.call("killall -9 dbus-daemon kded5 kioslave klauncher kdeinit5 kiod openbox Xvfb", shell=True)

198 199
# All done!
sys.exit(0)