gather-jobs.py 6.04 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!/usr/bin/python3
import os
import sys
import json
import argparse
from helperslib import BuildSpecs, CommonUtils, Buildable, Packages

# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility to determine which jobs need to be registered in Jenkins.')
10
parser.add_argument('--environment', type=str, required=True)
11 12 13 14 15 16 17
arguments = parser.parse_args()

# Initialise the dependnecy resolver
dependencyResolver = Buildable.DependencyResolver()
# Ask the resolver to load the list of projects...
projectsTreeLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects' )
dependencyResolver.loadProjectsFromTree( projectsTreeLocation )
18 19 20
# Load our local ignore file
ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'ignored-projects' )
dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation )
21 22 23
# Make sure the Platform specific ignore rules are loaded as well
ignoreRulesLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'project-ignore-rules.yaml' )
dependencyResolver.loadProjectsIgnoreRules( ignoreRulesLocation )
24

25 26
# Initialise the product handler
productHandler = Buildable.ProductHandler( dependencyResolver )
27
# Load the platform builds
28 29
productsFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'product-definitions.yaml' )
productHandler.loadProductInformation( productsFile )
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45

# Initialise the branch resolver
branchResolver = Buildable.BranchResolver( dependencyResolver )
# Ask the resolver to load the list branch mapping
lmsLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'logical-module-structure' )
branchResolver.loadProjectsToBranchesData( lmsLocation )

# Our output will be a list of Dictionaries, containing several keys:
# 1) The name
# 2) The repository url
# 3) The branch to be built
# 4) The branch group it belongs to
# 5) The platform it should be run against
# 6) The description for the resulting job
jobsGathered = []

46 47 48 49 50 51 52 53 54
# We also output a second set of data, which contains the structure (Products and the Platforms those are built on)
# This is used to produce some of the supporting / infrastructure jobs (for building all the dependencies of a Product for instance)
# As well as setup the views in Jenkins
jobsStructure = {
	'products': list( productHandler.knownProducts() ),
	'environment': arguments.environment,
	'combinations': []
}

55 56
# With everything now ready, we determine what products we have and start going from there
for product in productHandler.knownProducts():
57
	# We need the list of branch groups we want to be looking at here
58 59 60
	relevantBranchGroups = productHandler.branchGroupsFor( product )
	# As well as the platforms we are interested in
	platformsToBuild = productHandler.platformsFor( product )
61

62 63
	# Begin determining which jobs this product needs
	# As certain projects can be ignored on the Platform level we have to do this on a Platform by Platform basis for each Product
64
	for platform in platformsToBuild:
65 66 67 68
		# Determine which projects are built on this product/platform combination
		builtProjects = productHandler.projectsFor( product, platform )

		# Now that we know that, let's begin going over each branch group (as we need to store some platform/branch group combo information too)
69
		for branchGroup in relevantBranchGroups:
70
			# Prepare the job structure entry for this platform and branch group combination
71 72 73 74 75
			combinationEntry = {
				'product': product,
				'platform': platform,
				'branchGroup': branchGroup
			}
76
			# Now store that entry...
77 78
			jobsStructure['combinations'].append( combinationEntry )

79 80 81 82 83 84 85
			# We don't have to do any further checks at this point, so let's go over each project
			for project in builtProjects:
				# Determine the branch that would be built
				branchToBuild = branchResolver.branchFor( project, branchGroup )
				# Do we have a valid branch?
				if branchToBuild is None or branchToBuild == '':
					continue
86

87 88 89 90 91 92 93
				# Grab the list of email addresses we need to notify for build failures
				buildFailureEmails = productHandler.notificationsFor( product, project, failuresOnly = True )
				buildFailureEmails = ', '.join( buildFailureEmails )

				# Grab the list of email addresses we need to notify for unstable builds (failing tests)
				unstableBuildEmails = productHandler.notificationsFor( product, project, failuresOnly = False )
				unstableBuildEmails = ', '.join( unstableBuildEmails )
94

95 96
				# Determine the pipeline we should be using
				# Start with a generic template
97
				pipeline = "pipeline-templates/{platform}.template".format( platform=platform )
98 99 100

				# Do we have a product specific one?
				# If so, make sure we use it
101
				productPipeline = "pipeline-templates/{product}/{platform}.template".format( product=product, platform=platform )
102 103 104
				if os.path.exists(productPipeline):
					pipeline = productPipeline

105
				# We have a winner!
106 107
				jobEntry = {
					'name': project.name,
108 109
					'product': product,
					'repositoryUrl': 'git://anongit.kde.org/' + project.name,
110 111 112 113
					'browserUrl': 'https://cgit.kde.org/' + project.name + '.git',
					'branch': branchToBuild,
					'branchGroup': branchGroup,
					'platform': platform,
Ben Cooksley's avatar
Ben Cooksley committed
114
                                        'pipelineTemplate': pipeline,
115
					'description': project.description,
116
					'environment': arguments.environment,
117 118
					'buildFailureEmails': buildFailureEmails,
					'unstableBuildEmails': unstableBuildEmails
119
				}
120

121 122
				# Store it
				jobsGathered.append( jobEntry )
123 124 125 126 127 128 129

# Now output the jobs we've gathered in JSON to disk
# This will subsequently be read in by a Jenkins DSL script and turned into Jenkins Jobs
filePath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'gathered-jobs.json' )
with open(filePath, 'w') as jobsFile:
	json.dump( jobsGathered, jobsFile )

130 131 132 133 134 135
# Also output the structure data we've gathered in JSON to disk
# This will subsequently be read in by a Jenkins DSL script
filePath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'gathered-structure.json' )
with open(filePath, 'w') as structureFile:
	json.dump( jobsStructure, structureFile )

136 137
# All done!
sys.exit(0)