Commit 02884f97 authored by Hannah von Reth's avatar Hannah von Reth

Craft: Add src cache piplines (src mirror)

parent 2d886aa7
// Where will the craftmaster configuration be found (relative to the working directory that is)?
def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/"
// Request a node to be allocated to us
node( "Ubuntu1804Craft" ) {
// We want Timestamps on everything
timestamps {
// We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed
catchError {
// First we want to make sure Craft is ready to go
stage('Preparing Craft') {
// Grab our tooling which we will need in a few moments
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']]
]
// We are also reliant on the CI System Tooling for parts of this, so grab it too
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']]
]
// Make sure that Craftmaster is ready to go
sh """
python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into ~/Craft/BC/
python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into ~/Craft/BC/
"""
}
// Now that is done, it's time to rebuild the Craft Cache!
stage('Build Craft Cache') {
sh """
cd ~/Craft/BC/
python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft
python3.6 craftmaster/CraftMaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=~/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list
"""
}
}
// And finally, we'll upload it to the master server
stage('Publish Craft Cache') {
sh """
cd "$WORKSPACE/ci-tooling/"
python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft
"""
}
}
}
// Where will the craftmaster configuration be found (relative to the working directory that is)?
def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/"
// Request a node to be allocated to us
node( "macOS" ) {
// We want Timestamps on everything
timestamps {
// We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed
catchError {
// First we want to make sure Craft is ready to go
stage('Preparing Craft') {
// Make sure we start with a clean slate
deleteDir()
// Grab our tooling which we will need in a few moments
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']]
]
// We are also reliant on the CI System Tooling for parts of this, so grab it too
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']]
]
// Make sure that Craftmaster is ready to go
sh """
/usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into ~/Craft/BC/
/usr/local/bin/python3 "$WORKSPACE/bf-tooling/craft/checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into ~/Craft/BC/
"""
}
// Now that is done, it's time to rebuild the Craft Cache!
stage('Build Craft Cache') {
sh """
cd ~/Craft/BC/
/usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft
/usr/local/bin/python3 craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=~/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list
"""
}
}
// We need to catch errors here to ensure that the cleanup always runs, even if the upload fails
// While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk!
catchError {
// And finally, we'll upload it to the master server
stage('Publish Craft Cache') {
sh """
cd "$WORKSPACE/ci-tooling/"
/usr/local/bin/python3 helpers/sftp-directory-mirror.py --source ~/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft
"""
}
}
// Then we can cleanup everything we did
stage('Cleaning Up') {
sh """
rm -rf ~/Craft/BC/
"""
}
// As the macOS Slaves are permanent ones, we erase the Workspace as the last thing we do
deleteDir()
}
}
// Where will the craftmaster configuration be found (relative to the working directory that is)?
def craftmasterConfigDir = "binary-factory-tooling/craft/configs/experimental/"
// Request a node to be allocated to us
node( "WindowsMSVC" ) {
// We want Timestamps on everything
timestamps {
// We want to catch any errors that occur to allow us to send out notifications (ie. emails) if needed
catchError {
// First things first, make sure everything that could be using up disk space is removed
// This ensures that WebEngine builds don't get tripped up...
stage('Cleaning Up Craft') {
bat """
rmdir /s /q C:\\Craft\\BC\\
rmdir /s /q C:\\Craft\\BinaryFactory\\
rmdir /s /q C:\\Craft\\Downloads\\
rmdir /s /q C:\\_\\
exit /b 0
"""
}
// First we want to make sure Craft is ready to go
stage('Preparing Craft') {
// Make sure we start with a clean slate
deleteDir()
// Grab our tooling which we will need in a few moments
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'bf-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/binary-factory-tooling.git']]
]
// We are also reliant on the CI System Tooling for parts of this, so grab it too
checkout changelog: false, poll: false, scm: [
$class: 'GitSCM',
branches: [[name: 'master']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'ci-tooling/']],
userRemoteConfigs: [[url: 'https://invent.kde.org/sysadmin/ci-tooling.git']]
]
// Make sure that Craftmaster is ready to go
bat """
python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "https://invent.kde.org/packaging/craftmaster.git" --into "C:/Craft/BC/"
python "%WORKSPACE%\\bf-tooling\\craft\\checkout-repository.py" --repository "https://invent.kde.org/sysadmin/binary-factory-tooling.git" --into "C:/Craft/BC/"
"""
}
// Now that is done, it's time to rebuild the Craft Cache!
stage('Build Craft Cache') {
bat """
setlocal enabledelayedexpansion
cd C:/Craft/BC/
python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables CreateCache=True UseCache=True -c -i craft
if errorlevel 1 exit /b %errorlevel%
python craftmaster/Craftmaster.py --config ${craftmasterConfigDir}/CraftBinaryCache.ini --target ${craftPlatform} --variables "DownloadDir=C:/Craft/BC/src" -c --create-download-cache --list-file ${craftmasterConfigDir}/BinaryCachePackages.list
if errorlevel 1 exit /b %errorlevel%
"""
}
}
// We need to catch errors here to ensure that the cleanup always runs, even if the upload fails
// While this is a bit wasteful it at least determines that errors in the publishing process leave a builder with a very full disk!
catchError {
// Upload it to the master server
stage('Publish Craft Cache') {
bat """
cd "%WORKSPACE%\\ci-tooling"
python helpers/sftp-directory-mirror.py --source C:/Craft/BC/src/ --destination /srv/archives/files/craft/src/ --server milonia.kde.org --username craft
"""
}
}
// Then we can cleanup everything we did
// It is critical this always happens, otherwise builders can run out of disk space!
stage('Cleaning Up') {
bat """
rmdir /s /q C:\\Craft\\BC\\
"""
}
// As the Windows Slaves are permanent ones, we erase the Workspace as the last thing we do
deleteDir()
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment