build-product-dependencies.py 11.4 KB
Newer Older
1 2 3 4
#!/usr/bin/python3
import os
import sys
import argparse
5
import subprocess
6
import collections
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
from helperslib import CommonUtils, Buildable

# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility build all dependencies of a given product.')
parser.add_argument('--product', type=str, required=True)
parser.add_argument('--branchGroup', type=str, required=True)
parser.add_argument('--platform', type=str, required=True)
parser.add_argument('--environment', type=str, required=True)
parser.add_argument('--installTo', type=str, required=True)
arguments = parser.parse_args()

# Initialise the Dependency Resolver
dependencyResolver = Buildable.DependencyResolver()

# Ask the Dependency Resolver to load the list of projects...
projectsTreeLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects' )
dependencyResolver.loadProjectsFromTree( projectsTreeLocation )

# Now ask it to load the list of projects to ignore
ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'build-script-ignore' )
dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation )
# As well as our local ignore file
ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'ignored-projects' )
dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation )

32 33
# Make sure the Platform specific ignore rules are loaded as well
ignoreRulesLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'project-ignore-rules.yaml' )
Ben Cooksley's avatar
Ben Cooksley committed
34
dependencyResolver.loadProjectsIgnoreRules( ignoreRulesLocation )
35

36 37 38 39 40 41
# Now get it to load the list of dependencies - we have first the common file...
dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'dependency-data-common' )
dependencyResolver.loadDependenciesFromFile( dependenciesFile )
# And then the branch group specific file...
dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'dependency-data-' + arguments.branchGroup )
dependencyResolver.loadDependenciesFromFile( dependenciesFile )
42 43 44

# If it exists, we should load an OS specific file as well
dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'dependency-' + sys.platform )
45 46 47 48 49 50 51 52 53 54
if os.path.exists( dependenciesFile ):
	dependencyResolver.loadDependenciesFromFile( dependenciesFile )

# Now that the Dependency Resolver is all setup and ready to work
# We should initialise the product handler
productHandler = Buildable.ProductHandler( dependencyResolver )
# Load the platform builds
productsFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'product-definitions.yaml' )
productHandler.loadProductInformation( productsFile )

55 56 57 58 59 60
# Initialise the branch resolver
branchResolver = Buildable.BranchResolver( dependencyResolver )
# Ask the resolver to load the list branch mapping
lmsLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'logical-module-structure' )
branchResolver.loadProjectsToBranchesData( lmsLocation )

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
# Before we proceed let's do a few sanity checks to make sure we have been called with a valid request
# Is this product known?
if arguments.product not in productHandler.knownProducts():
	print("Fatal exception: The provided product is not known")
	sys.exit(1)

# Is this branch group used with this product?
if arguments.branchGroup not in productHandler.branchGroupsFor( arguments.product ):
	print("Fatal exception: The provided branch group is not enabled for this product")
	sys.exit(1)

# Is this platform enabled for this product?
if arguments.platform not in productHandler.platformsFor( arguments.product ):
	print("Fatal exception: The provided platform is not enabled for this product")
	sys.exit(1)

# Now all the validation is out of the way...
print("KDE CI Product Dependency Builder")
print("Building For Product: " + arguments.product + "\n")

# Determine which projects are built on this platform
print("Determining Projects in this Product...")
83
builtProjects = productHandler.projectsFor( arguments.product, arguments.platform )
84 85 86 87

# Prepare to resolve the dependencies of all the projects
projectDependencies = {}
initialDependencies = []
88
cleanedDependencies = []
89
completeDependencies = []
90
reverseDependencies = collections.defaultdict(list)
91 92 93 94 95

# For each project we now resolve it's dependencies
print("Resolving Immediate Dependencies for Member Projects...")
for project in builtProjects:
	# Resolve them for this project...
96 97 98 99 100
	ourDependencies = dependencyResolver.forProject( project, arguments.platform )
	# Store them for our later sorting
	projectDependencies[ project.name ] = ourDependencies
	# Add them the list we've assembled
	initialDependencies += ourDependencies
101 102 103
	# Go over and add the necessary items to the reverse dependency map
	for dependency in ourDependencies:
		reverseDependencies[ dependency.name ].append( project.name )
104

105 106 107
# Determine which projects aren't depended on by anyone else
# We don't need to build these in order to allow for everything else to be built
cleanedDependencies = [ project for project in initialDependencies if project.name in reverseDependencies ]
108 109 110 111 112 113 114

# We will now resolve the full dependency graph of everything outside of the Product 
# This will ensure we can assemble a fully reliable order to build everything
# And won't fail because something outside of the Product depends on something inside the Product
# It is acceptable at this point to include something which is in the Product even though that really should be happening
# Developers are assumed to accept responsibility for the broken pieces if they break ABI, etc.
print("Resolving Dependencies for all Identified Immediate Dependencies")
115
for project in cleanedDependencies:
116
	# Resolve the dependencies for this project...
117
	ourDependencies = dependencyResolver.forProject( project, arguments.platform )
118 119 120 121
	# Store them for our later sorting
	projectDependencies[ project.name ] = ourDependencies
	# Add them to the complete list of items to build
	completeDependencies += ourDependencies
122 123
	# Also add ourselves
	completeDependencies.append( project )
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166

# Perform de-duplication on the list of complete dependencies to build
print("Eliminating Duplicates from the complete list of Dependencies")
completeDependencies = list(set(completeDependencies))

# Now it's time to sort them all into a build order
# Make preparations to do so...
orderToBuild = []

# Begin the sorting process
# To do this we basically compare the projects dependencies against the list of projects we have to build
# If all of the dependencies of the project are satisfied we can consider it suitable for building and add it to the list
print("Determining Build Order, To Be Built As Follows:")
while len(completeDependencies) != 0:
	# Go over all the products in our list
	for project in completeDependencies:
		# Grab the products dependencies
		ourDependencies = projectDependencies[ project.name ]
		# Eliminate them against what's in our list of things to build
		remainingDependencies = list( set(ourDependencies) - set(orderToBuild) )

		# Do we have anything left?
		if len(remainingDependencies) > 0:
			# Not it's turn unfortunately
			continue

		# We have a winner!
		completeDependencies.remove(project)
		orderToBuild.append(project)
		print("== " + project.name)

# Now it's time to do the actual process of building
# To make things easier, let's find a few things out
baseDirectory = os.getcwd()
# Make sure we have an installation prefix
if not os.path.isdir( arguments.installTo ):
	os.makedirs( arguments.installTo )

# Let's start!
for project in orderToBuild:
	# Print a notice of where we are at...
	print("\n\n\n**** Beginning Build For: " + project.name + " ****\n\n\n")

167
	# Determine the branch that would be built
Ben Cooksley's avatar
Ben Cooksley committed
168
	branchToBuild = branchResolver.branchFor( project, arguments.branchGroup )
169 170 171 172
	# Do we have a valid branch?
	if branchToBuild is None or branchToBuild == '':
		continue

173 174 175 176 177 178
	# Do we need to pass any special parameters to Git?
	gitCommandParameters = ""
	# On Windows we do not want Symlinks as many tools just can't handle them and die horribly
	if sys.platform == 'win32':
		gitCommandParameters = "-c core.symlinks=false"

179 180 181
	# All of the below will be run inside a try block
	try:
		# Clone the necessary Git repository
182
		gitCloneCommand = 'git clone {gitCommandParameters} "{repositoryUrl}" --branch "{branchToBuild}"'
Ben Cooksley's avatar
Ben Cooksley committed
183
		commandToRun = gitCloneCommand.format(
184 185 186 187 188
			repositoryUrl='git://anongit.kde.org/' + project.name,
			branchToBuild=branchToBuild,
			gitCommandParameters=gitCommandParameters
		)
		subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=baseDirectory )
189 190

		# Construct the path to the directory where the repository was cloned to
Ben Cooksley's avatar
Ben Cooksley committed
191
		repositoryPath = os.path.join( baseDirectory, project.name )
192 193

		# Configure It
194
		configureCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/configure-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --installTo "{installTo}"'
195 196 197 198 199 200 201 202
		commandToRun = configureCommand.format( 
			pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product,
			projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform,
			installTo=arguments.installTo
		)
		subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath )

		# Compile It
Ben Cooksley's avatar
Ben Cooksley committed
203
		compileCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/compile-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --usingInstall "{installTo}"'
Ben Cooksley's avatar
Ben Cooksley committed
204
		commandToRun = compileCommand.format(
205 206 207 208 209 210 211
			pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product,
			projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform,
			installTo=arguments.installTo
		)
		subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath )

		# Install It
212
		installCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/install-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --installTo "{installTo}" --divertTo "{baseDirectory}/{projectName}/install-divert/"'
213 214 215 216 217 218 219 220
		commandToRun = installCommand.format( 
			pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product,
			projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform,
			installTo=arguments.installTo
		)
		subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath )

		# Finally Capture It
221
		captureCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/capture-install.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --environment {ciEnvironment} --platform {currentPlatform} --divertedTo  "{baseDirectory}/{projectName}/install-divert/" --installedTo "{installTo}"'
222 223 224 225 226 227 228 229 230 231 232 233
		commandToRun = captureCommand.format( 
			pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product,
			projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform,
			installTo=arguments.installTo, ciEnvironment=arguments.environment
		)
		subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath )

	except Exception:
		sys.exit(1)

# All finished
sys.exit(0)