Members of the KDE Community are recommended to subscribe to the kde-community mailing list at https://mail.kde.org/mailman/listinfo/kde-community to allow them to participate in important discussions and receive other important announcements

Commit c658ac6b authored by Ben Cooksley's avatar Ben Cooksley

Introduce changes to start deploying all static (including Capacity based)...

Introduce changes to start deploying all static (including Capacity based) sites from the Binary Factory.
This is part of the system migration to Nicoda
parent 0a3ad279
......@@ -10,7 +10,7 @@ knownJobs.each {
def cronSchedule = "${it.cron}"
// Read in the necessary Pipeline script
def pipelineTemplate = readFileFromWorkspace("staticweb/${it.type}.pipeline")
def pipelineTemplate = readFileFromWorkspace("staticweb/pipeline-templates/${it.type}.pipeline")
// Now we can construct our Pipeline script
// We append a series of variables to the top of it to provide a variety of useful information to the otherwise templated script
......@@ -18,7 +18,7 @@ knownJobs.each {
def pipelineScript = """
|def name = "${it.name}"
|def deploypath = "${it.deploypath}"
|def gitUrl = "https://anongit.kde.org/${it.repository}.git"
|def repositoryUrl = "${it.repositoryUrl}"
|def gitBranch = "${it.branch}"
|${pipelineTemplate}""".stripMargin()
......
#!/usr/bin/python3
import os
import sys
import json
import yaml
import argparse
# Parse the command line arguments we've been given
parser = argparse.ArgumentParser(description='Utility to determine which jobs need to be registered in Jenkins.')
parser.add_argument('--static-jobs', type=str, required=True, dest='staticJobs')
parser.add_argument('--custom-jobs', type=str, required=True, dest='customJobs')
arguments = parser.parse_args()
# Grab the list of conventional, static jobs in order to commence processing it
with open(arguments.staticJobs, 'r') as dataFile:
# Parse the YAML file
jobsToCreate = yaml.load( dataFile )
# Our output will be a list of Dictionaries, containing several keys:
# 1) The name of the job
# 2) The repository (or Subversion path) to be checked out
# 3) The branch of the repository to be checked out (in the case of Subversion, this will be blank)
# 4) The schedule on which the repository
jobsGathered = []
# Let's get started processing conventional jobs!
for jobPipelineTemplate in jobsToCreate.keys():
# We now go over each website that uses this particular pipeline template
for website in jobsToCreate[ jobPipelineTemplate ]:
# Construct the basic empty template for the job that will publish this website
jobEntry = {
'name': '',
'repositoryUrl': '',
'branch': 'master',
'cron': '',
'type': jobPipelineTemplate,
'deploypath': ''
}
# For the job entry we will need a name
# To create this we take the domain this website is published at, and swap all dots with dashes
jobEntry['name'] = website['domain'].replace('.', '-')
# Does this website use Git for it's repository?
if 'repository' in website:
jobEntry['repositoryUrl'] = 'https://anongit.kde.org/' + website['repository']
# Otherwise could it be using SVN for it's repository?
if 'svnpath' in website:
jobEntry['repositoryUrl'] = 'svn://svn.kde.org/home/kde/' + website['svnpath']
# Along with the path it should be deployed at on the server
jobEntry['deploypath'] = '/srv/www/generated/{0}'.format( website['domain'] )
# Finally we can add it to the list
jobsGathered.append( jobEntry )
# Now we have to process custom jobs
# This is essentially done to calculate the Subversion/Git repository URL for them
with open(arguments.customJobs, 'r') as dataFile:
# Load the list of custom jobs in
jobsToCreate = json.load(dataFile)
# Process the custom jobs...
for customJob in jobsToCreate:
# Construct the standard empty template for this job which we'll then populate
jobEntry = {
'name': '',
'repositoryUrl': '',
'branch': '',
'cron': '',
'type': '',
'deploypath': ''
}
# Transfer across most of the details we can
jobEntry['name'] = customJob['name']
jobEntry['branch'] = customJob['branch']
jobEntry['cron'] = customJob['cron']
jobEntry['type'] = customJob['type']
jobEntry['deploypath'] = customJob['deploypath']
# Does this website use Git for it's repository?
if 'repository' in website:
jobEntry['repositoryUrl'] = 'https://anongit.kde.org/' + website['repository']
# Otherwise could it be using SVN for it's repository?
if 'svnpath' in website:
jobEntry['repositoryUrl'] = 'svn://svn.kde.org/home/kde/' + website['svnpath']
# Finally we can add it to the list
jobsGathered.append( jobEntry )
# Now output the jobs we've gathered in JSON to disk
# This will subsequently be read in by a Jenkins DSL script and turned into Jenkins Jobs
filePath = os.path.join( os.getcwd(), 'gathered-jobs.json')
with open(filePath, 'w') as jobsFile:
json.dump( jobsGathered, jobsFile, sort_keys=True, indent=2 )
# All done!
sys.exit(0)
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
// Pre-Generated Docs to be included and published as well
checkout changelog: true, poll: true, scm: [
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
// Manifests for Inqlude
checkout changelog: true, poll: true, scm: [
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
// Request a node to be allocated to us
node( "StaticWeb" ) {
// We want Timestamps on everything
timestamps {
// We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed
catchError {
// First Thing: Checkout Sources
stage('Checkout Sources') {
// Make sure we have a clean slate to begin with
deleteDir()
// Code
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: repositoryUrl]]
]
}
// Let's build website now
stage('Website build') {
sh """
rm -rf .git/
"""
}
// Deploy the website!
stage('Publishing Website') {
sh """
rsync -Hav --delete -e "ssh -i $HOME/WebsitePublishing/website-upload.key" ./ sitedeployer@nicoda.kde.org:${deploypath}
"""
}
}
}
}
// Request a node to be allocated to us
node( "StaticWeb" ) {
// We want Timestamps on everything
timestamps {
// We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed
catchError {
// First Thing: Checkout Sources
stage('Checkout Sources') {
// Make sure we have a clean slate to begin with
deleteDir()
// Code
checkout( [
$class: 'SubversionSCM',
filterChangelog: false,
ignoreDirPropChanges: false,
includedRegions: '',
locations: [[cancelProcessOnExternalsFail: true, depthOption: 'infinity', ignoreExternalsOption: false, local: '.', remote: repositoryUrl]],
quietOperation: true,
workspaceUpdater: [$class: 'UpdateUpdater']
])
}
// Let's build website now
stage('Website build') {
sh """
find -name '.svn' -type d -exec rm -rf '{}' \;
"""
}
// Deploy the website!
stage('Publishing Website') {
sh """
rsync -Hav --delete -e "ssh -i $HOME/WebsitePublishing/website-upload.key" ./ sitedeployer@nicoda.kde.org:${deploypath}
"""
}
}
}
}
......@@ -12,7 +12,7 @@ timestamps {
checkout changelog: true, poll: true, scm: [
$class: 'GitSCM',
branches: [[name: gitBranch]],
userRemoteConfigs: [[url: gitUrl]]
userRemoteConfigs: [[url: repositoryUrl]]
]
}
......
'static-git':
- domain: akademy2010.kde.org
repository: websites/akademy2010-kde-org
- domain: akademy2012.kde.org
repository: websites/akademy2012-kde-org
- domain: autoconfig.kde.org
repository: websites/autoconfig-kde-org
- domain: buzz.kde.org
repository: websites/buzz-kde-org
- domain: conf.qtcon.org
repository: websites/conf-qtcon-org
- domain: desktopsummit.org
repository: websites/desktopsummit-org
- domain: edu.kde.org
repository: websites/edu-kde-org
- domain: ev.kde.org
repository: websites/ev-kde-org
- domain: freebsd.kde.org
repository: websites/freebsd-kde-org
- domain: games.kde.org
repository: websites/kdeitalia-it
- domain: jp.kde.org
repository: websites/jp-kde-org
- domain: juk.kde.org
repository: websites/juk-kde-org
- domain: kdesrc-build.kde.org
repository: websites/kdesrc-build-kde-org
- domain: kmymoney.org
repository: websites/kmymoney-org
- domain: konqueror.org
repository: websites/konqueror-org
- domain: konversation.kde.org
repository: websites/konversation-kde-org
- domain: krusader.org
repository: websites/krusader-org
- domain: kube.kde.org
repository: websites/kube-kde-org/_site
- domain: lakademy.kde.org
repository: websites/lakademy-kde-org
- domain: marble.kde.org
repository: websites/marble-kde-org
- domain: multimedia.kde.org
repository: websites/multimedia-kde-org
- domain: necessitas.kde.org
repository: websites/necessitas-kde-org
- domain: neon.kde.org
repository: websites/neon-kde-org
- domain: okular.kde.org
repository: websites/okular-kde-org
- domain: pe.kde.org
repository: websites/pe-kde-org
- domain: peruse.kde.org
repository: websites/peruse-kde-org
- domain: phonon.kde.org
repository: websites/phonon-kde-org
- domain: rekonq.kde.org
repository: websites/rekonq-kde-org
- domain: releases.neon.kde.org
repository: websites/releases-neon-kde-org
- domain: solid.kde.org
repository: websites/solid-kde-org
- domain: timeline.kde.org
repository: websites/timeline-kde-org
- domain: umbrello.kde.org
repository: websites/umbrello-kde-org
- domain: utils.kde.org
repository: websites/utils-kde-org
- domain: vdesign.kde.org
repository: websites/vdesign-kde-org
- domain: vvave.kde.org
repository: websites/vvave-kde-org
- domain: wiki.desktopsummit.org
repository: websites/wiki-desktopsummit-org
- domain: wiki.kde.org
repository: websites/wiki-kde-org
- domain: windows.kde.org
repository: websites/windows-kde-org
- domain: yakuake.kde.org
repository: websites/yakuake-kde-org
'static-svn':
- domain: conference2004.kde.org
repository: /trunk/www/areas/events/info/conference2004
- domain: conference2005.kde.org
repository: /trunk/www/areas/events/info/conference2005
- domain: conference2006.kde.org
repository: /trunk/www/areas/events/info/conference2006
- domain: conference2007.kde.org
repository: /trunk/www/areas/events/info/conference2007
- domain: conference2008.kde.org
repository: /trunk/www/areas/events/info/conference2008
- domain: conference2009.kde.org
repository: /trunk/www/areas/events/info/conference2009
- domain: czechia.kde.org
repository: /trunk/www/international/czechia
- domain: events.kde.org
repository: /trunk/www/areas/events/
- domain: extragear.kde.org
repository: /trunk/www/areas/extragear/
- domain: il.kde.org
repository: /trunk/www/international/israel/
- domain: kmplayer.kde.org
repository: /trunk/www/apps/kmplayer/
- domain: kpdf.kde.org
repository: /trunk/www/apps/kpdf
- domain: kphotoalbum.org
repository: /trunk/www/sites/kphotoalbum
- domain: kst-plot.kde.org
repository: /trunk/www/areas/kst-plot/
- domain: ro.kde.org
repository: /trunk/www/international/romania
- domain: usability.kde.org
repository: /trunk/www/sites/usability
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment