Commit a0f790ca authored by Harald Sitter's avatar Harald Sitter

import

parents
# Tools
## appstream.rb
Collects **desktop application** appdata from CI install dirs and as a fallback
from Git.
Requires valid screenshots and icons to be available!
Limited functionality with Git fallback. All software should be CI'd really.
Twiddles the following dirs:
- `../appdata/` appdata cache. contains json blobs converted from appdata
- `../icons/` icons cache. contains icons named from their appid (org.kde.foo.svg)
- `../thumbnails` thumbnails cache. scaled to 540 width. subdir per appid
# appstream_mkindex.rb
Iters `../appdata/` and generates an `../appdata/index.json` mapping category
names to appids. The icons for the categories are generated by appstream.rb
and in `../icons/categories/` as downcased version of the name.
# old_apps_compat.rb
Iters `../apps/` and generates compatibility rigging to preserve names from
v1 of the backend, this allows old app urls to remain working.
This diff is collapsed.
#!/usr/bin/env ruby
#
# Copyright 2018 Harld Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'json'
Dir.chdir(__dir__) if Dir.pwd != __dir__
appdata_dir = '../appdata'
index ||= {}
Dir.glob("#{appdata_dir}/*.json") do |file|
data = JSON.parse(File.read(file))
next unless data['Categories'] # not appdata
id = data.fetch('ID')
data.fetch('Categories').uniq.each do |category|
# NB: category is the Name value. Should l10n for categoresi get implemented
# this maybe need rethinking as mapping by name is somewhat fragile.
index[category] ||= []
index[category] << id
end
end
index.values.each(&:sort!)
# Old-ish structure
# Hash of categories and their apps as array inside
File.write("../index.json", JSON.generate(index))
# Copyright 2017-2018 Harld Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'tty/command'
require 'yaml'
require 'tempfile'
class AppData
attr_reader :path
def initialize(path)
@path = path
end
def read_as_yaml
Tempfile.create do |tmpfile|
TTY::Command.new.run("appstreamcli convert #{path} #{tmpfile.path}")
_dep11, appdata = YAML.load_stream(File.read(tmpfile.path))
appdata
end
end
end
# Copyright 2017-2018 Harld Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require_relative 'xdg/desktop'
# This is technically codified in /xdg/menus/kf5-applications.menu but I have
# no parser for that, and breaking it down to it's primary category map is a
# lot more work anyway.
CATEGORY_DESKTOPS_MAP = {
'AudioVideo' => 'kf5-multimedia.directory',
'Audio' => 'kf5-multimedia.directory',
'Video' => 'kf5-multimedia.directory',
'Development' => 'kf5-development.directory',
'Education' => 'kf5-education.directory',
'Game' => 'kf5-games.directory',
'Graphics' => 'kf5-graphics.directory',
'Network' => 'kf5-internet.directory',
'Office' => 'kf5-office.directory',
'Settings' => 'kf5-settingsmenu.directory',
'System' => 'kf5-system.directory',
'Utility' => 'kf5-utilities.directory'
}
MAIN_CATEGORIES = CATEGORY_DESKTOPS_MAP.keys
module Category
module_function
def category_desktops
@category_desktops ||= {}
end
def to_name(category)
desktop = category_desktops.fetch(category)
desktop.config['Desktop Entry']['Name']
end
end
# CATEGORY_DESKTOPS = CATEGORY_DESKTOPS_MAP.map do |category, dir|
# # FIXME: this is currently using system information expecting plasma-workspace to be installed
# [category, XDG::DesktopDirectoryLoader.new(dir).find]
# end.to_h
#
# MAIN_CATEGORIES = CATEGORY_DESKTOPS.keys
#
# MAIN_CATEGORIES_TO_NAMES = CATEGORY_DESKTOPS.collect do |category, desktop|
# raise "couldnt find desktop file for #{category}" unless desktop
# [category, desktop.config['Desktop Entry']['Name']]
# end.to_h
#
# NAMES_TO_MAIN_CATEGORIES = CATEGORY_DESKTOPS.collect do |category, desktop|
# raise "couldnt find desktop file for #{category}" unless desktop
# [desktop.config['Desktop Entry']['Name'], category]
# end.to_h
# Copyright 2017 Harald Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require_relative 'fnmatchpattern'
module CITooling
class RepositoryGroup
attr_reader :repositories
attr_reader :platforms
def initialize(hash)
p hash
@repositories = hash['repositories'].collect { |x| FNMatchPattern.new(x) }
@platforms = hash['platforms']
end
end
class Product
class RepositoryFilter
def initialize(branch_groups:, platforms:)
@branch_groups = branch_groups
@platforms = platforms
end
def filter(product)
return [] unless @branch_groups.all? { |x| product.branch_groups.include?(x) }
repos = product.includes.collect do |repo_group|
next unless @platforms.all? { |x| repo_group.platforms.include?(x) }
repo_group.repositories
end
repos.flatten.compact.uniq
end
end
attr_reader :name
alias to_s name
attr_reader :branch_groups
attr_reader :includes
def initialize(name, data)
@name = name || raise
@branch_groups = data['branchGroups'] || []
@includes = data['includes'].collect { |x| RepositoryGroup.new(x) }
end
def self.list
data = YAML.load_file('ci-tooling/local-metadata/product-definitions.yaml')
data.collect do |name, hash|
new(name, hash)
end
end
end
end
# Copyright 2017-2018 Harld Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'concurrent'
require 'tty/command'
require 'pp'
require 'json'
require 'open-uri'
require 'tmpdir'
require_relative 'app_data'
require_relative 'category'
require_relative 'ci_tooling'
require_relative 'icon_fetcher'
require_relative 'kde_project'
require_relative 'thumbnailer'
require_relative 'xml_languages'
require_relative 'xdg/desktop'
require_relative 'xdg/icon'
class InvalidError < StandardError; end
class AppStreamCollector
attr_reader :dir
attr_reader :path
attr_reader :project
attr_reader :appdata
attr_reader :appid
attr_reader :tmpdir
def initialize(dir, path:, project:)
@dir = dir
@path = path
@project = project
@appdata = AppData.new(path).read_as_yaml
# Sanitize for our purposes (can be either foo.desktop or foo, we always
# use the latter internally).
# The value in the appdata must stay as it is or appstream:// urls do not
# work!
@appid = appdata['ID'].gsub('.desktop', '')
# Mutate in the raw data as well. This way subsequent tooling can expecting
# the id to be standardized.
appdata['ID'] = @appid
end
def xdg_data_dir
"#{dir}/share/"
end
def desktop_file
@desktop_file ||=
XDG::ApplicationsLoader.new(appid, extra_data_dirs: ["#{xdg_data_dir}/applications"]).find
end
def theme
@icon_theme ||=
XDG::IconTheme.new('breeze', extra_data_dirs: ["#{xdg_data_dir}/icons"])
end
# FIXME: overridden for git crawling
def grab_icon
iconname = desktop_file.icon
return unless iconname
IconFetcher.new(iconname, theme).extend_appdata(appdata, cachename: appid)
end
def grab_categories
desktop_categories = desktop_file.categories & MAIN_CATEGORIES
unless desktop_categories
# FIXME: record into log
raise InvalidError, "#{appid} has no main categories. only has #{desktop_categories}"
end
appdata['Categories'] ||= []
# Iff the categories were defined in the appdata as well make sure to
# filter all !main categories.
appdata['Categories'] = appdata['Categories'] & MAIN_CATEGORIES
appdata['Categories'] += desktop_categories
appdata['Categories'].uniq!
appdata['Categories'].collect! { |x| Category.to_name(x) }
end
def grab_generic_name
appdata['X-KDE-GenericName'] =
XMLLanguages.from_desktop_entry(desktop_file, 'GenericName')
end
def grab_project
appdata['X-KDE-Project'] = project.id
appdata['X-KDE-Repository'] = project.repo
end
def grab
raise InvalidError, "no desktop file for #{appid}" unless desktop_file
unless desktop_file.show_in?('KDE') && desktop_file.display? && !desktop_file.hidden?
raise InvalidError, "desktop file for #{appid} not meant for display"
end
# FIXME: thumbnailer should not mangle the appdata, it should generate
# the thumbnails and we do the mangling...
Thumbnailer.thumbnail!(appdata)
grab_icon
grab_categories
grab_generic_name
grab_project
FileUtils.mkpath('../appdata', verbose: true)
File.write("../appdata/#{appdata.fetch('ID').gsub('.desktop', '')}.yaml", YAML.dump(appdata))
File.write("../appdata/#{appdata.fetch('ID').gsub('.desktop', '')}.json", JSON.generate(appdata))
# FIXME: we should put EVERYTHING into a well defined tree in a tmpdir,
# then move it into place in ONE place. so we can easily change where
# stuff ends up in the end and know where it is while we are working on
# the data
true
end
def self.grab(dir, project:)
any_good = false
Dir.glob("#{dir}/**/**.appdata.xml").each do |path|
warn " Grabbing #{path}"
# FIXME: broken blocking all of calligra
# # https://bugs.kde.org/show_bug.cgi?id=388687
next if path.include?('org.kde.calligragemini')
begin
good = new(dir, path: path, project: project).grab
any_good ||= good
rescue InvalidError => e
warn e
end
end
any_good
end
end
class GitAppStreamCollector < AppStreamCollector
def xdg_data_dir
"#{Dir.pwd}/breeze-icons/share/"
end
# FIXME: deferring to appstream via xdg_data_dir
# def grab_icon
# raise 'not implemented'
# # a) should look in breeze-icon unpack via theme
# # b) should try to find in tree?
# # c) maybe an override system?
# end
def desktop_file
files = Dir.glob("#{dir}/**/#{appid}.desktop")
raise "#{appid}.desktop: #{files.inspect}" unless files.size == 1
XDG::Desktop.new(files[0])
end
end
# frozen_string_literal: true
#
# Copyright (C) 2017 Harald Sitter <sitter@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) version 3, or any
# later version accepted by the membership of KDE e.V. (or its
# successor approved by the membership of KDE e.V.), which shall
# act as a proxy defined in Section 6 of version 3 of the license.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
class FNMatchPattern
attr_reader :pattern
def initialize(pattern)
@pattern = pattern
end
def ==(other)
File.fnmatch(@pattern, other)
end
end
# Copyright 2017 Harld Sitter <sitter@kde.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License or (at your option) version 3 or any later version
# accepted by the membership of KDE e.V. (or its successor approved
# by the membership of KDE e.V.), which shall act as a proxy
# defined in Section 14 of version 3 of the license.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
require 'fileutils'
require_relative 'xdg/icon'
class IconFetcher
attr_reader :iconname
attr_reader :theme
def initialize(iconname, theme)
@iconname = iconname
@theme = theme
end
def extend_appdata(appdata, cachename:, subdir: '')
icon = XDG::Icon.find_path(iconname, 48, theme)
raise InvalidError, "Couldn't find icon #{iconname}" unless icon
cachefile = cachename + File.extname(icon)
FileUtils.mkpath("../icons/#{subdir}")
FileUtils.cp(icon, "../icons/#{subdir}/#{cachefile}", verbose: true)
appdata['Icon'] ||= {}
icon = appdata['Icon']
icon['local'] ||= []
icon['local'] << { 'name' => cachefile }
end
end
# frozen_string_literal: true
#
# Copyright (C) 2017-2018 Harald Sitter <sitter@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) version 3, or any
# later version accepted by the membership of KDE e.V. (or its
# successor approved by the membership of KDE e.V.), which shall
# act as a proxy defined in Section 6 of version 3 of the license.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
require 'open-uri'
module KDE
# A project definition.
class Project
attr_reader :id
attr_reader :repo
def initialize(id, hash)
@id = id
@repo = hash['repo']
end
def self.get(id)
data = open("https://projects.kde.org/api/v1/project/#{id}").read
new(id, JSON.parse(data))
end
def self.list
data = open('https://projects.kde.org/api/v1/projects').read
JSON.parse(data)
end
end
end
# frozen_string_literal: true
#
# Copyright (C) 2017 Harald Sitter <sitter@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) version 3, or any
# later version accepted by the membership of KDE e.V. (or its
# successor approved by the membership of KDE e.V.), which shall
# act as a proxy defined in Section 6 of version 3 of the license.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
require 'fileutils'
require 'open-uri'
require 'tempfile'
require 'tty/command'
class Thumbnailer
class Thumbnail
# Width of the thumbnail.
attr_reader :width
# File path of the thumbnail file.
attr_reader :thumb_file
attr_reader :url
attr_reader :thumb_dir_relative
attr_reader :thumb_file_relative
attr_reader :last_modified_path
def initialize(url:, appid:)
@width = 540
@url = url
@thumb_dir = File.join('/thumbnails', appid)
@thumb_file = File.join(@thumb_dir, File.basename(@url))
@thumb_dir_relative = File.join('../', @thumb_dir)
@thumb_file_relative = File.join('../', @thumb_file)
@last_modified_path = "#{@thumb_file_relative}.last_modified"
@cmd = TTY::Command.new
end
def headers
return {} unless File.exist?(@last_modified_path)
{ 'If-Modified-Since' => File.read(last_modified_path).strip }
end
# Yields [tempfile, openuri::meta]
def open_as_tempfile(url, headers)
ext = File.extname(File.basename(url))
filename = File.basename(File.basename(url), ext)
open(url, headers) do |img|
# open for tiny files gives a stringio, read it and force it into
# a file for conversion. this is really fucked up api behavior...
Tempfile.open([filename, ext]) do |f|
f.write(img.read)
f.close # done writing, close the file to force a sync
yield f, img
end
end
end
# assignment branch cond is high because of transietn debug warnings
def open_orig
warn url
open_as_tempfile(url, headers) do |tmpfile, openuri|
yield tmpfile
# date is an array, nobody knows why.
File.write(last_modified_path, openuri.metas.fetch('date').fetch(0))
end
rescue OpenURI::HTTPError => e
code, _msg = e.io.status
warn [url, code, e]
raise e unless code == '304' # NotModified
rescue => e
raise "#{url} -> #{e}"
end
def generate
open_orig do |f|
FileUtils.mkpath(thumb_dir_relative, verbose: true)
# Borrowed from https://www.smashingmagazine.com/2015/06/efficient-image-resizing-with-imagemagick/
@cmd.run("mogrify -write #{thumb_file_relative} -filter Triangle" \
" -define filter:support=2 -thumbnail #{width} -unsharp 0.25x0.25+8+0.065" \
' -dither None -posterize 136 -quality 82 -define jpeg:fancy-upsampling=off' \
' -define png:compression-filter=5 -define png:compression-level=9 ' \
' -define png:compression-strategy=1 -define png:exclude-chunk=all ' \
" -interlace none -colorspace sRGB -strip #{f.path}")
end
end
end
def self.thumbnail!(appdata)
# FIXME: grab icon out of breeze OR the tree
return unless appdata['Screenshots'] && !appdata['Screenshots'].empty?
appdata.fetch('Screenshots').each do |screenshot|
next if screenshot.fetch('source-image').fetch('lang') != 'C'
thumb = Thumbnail.new(appid: appdata.fetch('ID'),
url: screenshot.fetch('source-image').fetch('url'))
thumb.generate
# FIXME: maybe should only set to name of file and resolve in php?
screenshot['thumbnails'] << { 'url' => thumb.thumb_file,
'width' => thumb.width }
end
end
end
# frozen_string_literal: true
#
# Copyright (C) 2017 Harald Sitter <sitter@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) version 3, or any
# later version accepted by the membership of KDE e.V. (or its
# successor approved by the membership of KDE e.V.), which shall
# act as a proxy defined in Section 6 of version 3 of the license.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
module XDG
class DesktopFileLoaderBase
def self.subdir
'applications'
end
def self.file_extension
'.desktop'
end
def self.xdg_data_dirs
# FIXME: lots of code dupe from icons
@xdg_data_dirs ||= begin
dirs = ENV.fetch('XDG_DATA_HOME', '~/.local/share')
dirs += ':' + ENV.fetch('XDG_DATA_DIRS',
'/usr/local/share/:/usr/share/')
dirs.split(':').collect do |path|
next if path.include?('..') # relative, skip as per spec
File.join(File.expand_path(path), subdir)
end.uniq.compact
end
end
attr_reader :id
attr_reader :extra_data_dirs
def initialize(id, extra_data_dirs: [])
@id = id
ext = self.class.file_extension
@id += ext unless @id.end_with?(ext)
@extra_data_dirs = extra_data_dirs
end
def find
path = find_path
return path unless path
Desktop.new(path)
end
def find_path
ids = [id]
ids += Array.new(id.count('-')) do |i|
# Run through each hyphen element from the front AND from the back AND
# both at the same time. Incremently replace them with slashes.
# We'll drop dupes later.
# a-b-c-d
# i=0 => [a/b-c-d, a-b-c/d, a/b-c/d]
# i=1 => [a/b/c-d, a-b/c/d, a/b/c/d]
# i=3 => [a/b/c/d, a/b/c/d, a/b/c/d]
[
id.split('-', 2 + i).join('/'),
id.reverse.split('-', 2 + i).join('/').reverse,
id.split('-', 2 + i).join('/').reverse.split('-', 2 + i).join('/').reverse,
]
end.flatten.uniq.compact
ids.each do |id|
file = find_by_id(id)
return file if file
end