commit 86e0da39bb261bf896e99cd2a4514a7729516bf1 Author: Nicolas Chauvet <kwizart(a)gmail.com&gt; Date: Wed Jun 17 16:56:40 2020 +0200 Add packages roles/packages3/bugz.fp.o/files/bugz.conf | 5 + roles/packages3/bugz.fp.o/tasks/main.yml | 9 + roles/packages3/web/files/distmappings.py | 41 ++++ .../packages3/web/files/fedmsg-hub-forward-compat | 9 + roles/packages3/web/files/package_128x128.png | Bin 0 -> 12266 bytes roles/packages3/web/files/packages-httpd.conf | 76 +++++++ roles/packages3/web/tasks/main.yml | 167 +++++++++++++++ roles/packages3/web/templates/fedoracommunity.py | 7 + roles/packages3/web/templates/packages-app.ini.j2 | 226 ++++++++++++++++++++ 9 files changed, 540 insertions(+), 0 deletions(-) --- diff --git a/roles/packages3/bugz.fp.o/files/bugz.conf b/roles/packages3/bugz.fp.o/files/bugz.conf new file mode 100644 index 0000000..bcd126c --- /dev/null +++ b/roles/packages3/bugz.fp.o/files/bugz.conf @@ -0,0 +1,5 @@ +RewriteEngine On +RewriteMap lowercase int:tolower + +RewriteRule ^/(.+) https://apps.rpmfusion.org/packages/$1/bugs/all [R,L] +RewriteRule ^/$ https://bugzilla.rpmfusion.org/ [R,L] diff --git a/roles/packages3/bugz.fp.o/tasks/main.yml b/roles/packages3/bugz.fp.o/tasks/main.yml new file mode 100644 index 0000000..675232e --- /dev/null +++ b/roles/packages3/bugz.fp.o/tasks/main.yml @@ -0,0 +1,9 @@ +- copy: > + src=bugz.conf dest=/etc/httpd/conf.d/{{website}}/bugz.conf + owner=root group=root mode=0644 + notify: + - reload httpd + tags: + - packages + - packages/proxy + - packages/bugz.fp.o diff --git a/roles/packages3/web/files/distmappings.py b/roles/packages3/web/files/distmappings.py new file mode 100644 index 0000000..ebf4b30 --- /dev/null +++ b/roles/packages3/web/files/distmappings.py @@ -0,0 +1,41 @@ +# Global list of koji tags we care about +tags = ({'name': 'Rawhide Free', 'tag': 'f33-free'}, + {'name': 'Rawhide NonFree', 'tag': 'f33-nonfree'}, + + {'name': 'Fedora 32 Free', 'tag': 'f32-free-updates'}, + {'name': 'Fedora 32 Free', 'tag': 'f32-free'}, + {'name': 'Fedora 32 Free Testing', 'tag': 'f32-free-updates-testing'}, + + {'name': 'Fedora 32 NonFree', 'tag': 'f32-nonfree-updates'}, + {'name': 'Fedora 32 NonFree', 'tag': 'f32-nonfree'}, + {'name': 'Fedora 32 NonFree Testing', 'tag': 'f32-nonfree-updates-testing'}, + + {'name': 'Fedora 31 Free', 'tag': 'f31-free-updates'}, + {'name': 'Fedora 31 Free', 'tag': 'f31-free'}, + {'name': 'Fedora 31 Free Testing', 'tag': 'f31-free-updates-testing'}, + + {'name': 'Fedora 31 NonFree', 'tag': 'f31-nonfree-updates'}, + {'name': 'Fedora 31 NonFree', 'tag': 'f31-nonfree'}, + {'name': 'Fedora 31 NonFree Testing', 'tag': 'f31-nonfree-updates-testing'}, + + + {'name': 'EPEL 8 Free', 'tag': 'el8-free'}, + {'name': 'EPEL 8 Free Testing', 'tag': 'el8-free-testing'}, + {'name': 'EPEL 8 NonFree', 'tag': 'el8-nonfree'}, + {'name': 'EPEL 8 NonFree Testing', 'tag': 'el8-nonfree-testing'}, + + {'name': 'EPEL 7 Free', 'tag': 'el7-free'}, + {'name': 'EPEL 7 Free Testing', 'tag': 'el7-free-testing'}, + {'name': 'EPEL 7 NonFree', 'tag': 'el7-nonfree'}, + {'name': 'EPEL 7 NonFree Testing', 'tag': 'el7-nonfree-testing'}, + + {'name': 'EPEL 6 Free', 'tag': 'el6-free'}, + {'name': 'EPEL 6 Free Testing', 'tag': 'el6-free-testing'}, + {'name': 'EPEL 6 NonFree', 'tag': 'el6-nonfree'}, + {'name': 'EPEL 6 NonFree Testing', 'tag': 'el6-nonfree-testing'}, + + ) + +tags_to_name_map = {} +for t in tags: + tags_to_name_map[t['tag']] = t['name'] diff --git a/roles/packages3/web/files/fedmsg-hub-forward-compat b/roles/packages3/web/files/fedmsg-hub-forward-compat new file mode 100644 index 0000000..239f620 --- /dev/null +++ b/roles/packages3/web/files/fedmsg-hub-forward-compat @@ -0,0 +1,9 @@ +#!/usr/bin/python +__requires__ = ['fedmsg>=0.13.1', 'WebOb>=0.9.7', 'sqlalchemy>=0.7'] +import sys +from pkg_resources import load_entry_point + +if __name__ == '__main__': + sys.exit( + load_entry_point('fedmsg', 'console_scripts', 'fedmsg-hub')() + ) diff --git a/roles/packages3/web/files/package_128x128.png b/roles/packages3/web/files/package_128x128.png new file mode 100644 index 0000000..d566f36 Binary files /dev/null and b/roles/packages3/web/files/package_128x128.png differ diff --git a/roles/packages3/web/files/packages-httpd.conf b/roles/packages3/web/files/packages-httpd.conf new file mode 100644 index 0000000..14348c1 --- /dev/null +++ b/roles/packages3/web/files/packages-httpd.conf @@ -0,0 +1,76 @@ +LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +LoadModule deflate_module modules/mod_deflate.so + +ExpiresActive On +#ExpiresDefault "access plus 300 seconds" + +ErrorLog logs/fedoracommunity_error_log +CustomLog logs/fedoracommunity_access_log combined + +AddOutputFilterByType DEFLATE text/html text/plain text/xml text/javascript text/css application/x-javascript + +# This caching may not necessarily be ideal, or even correct. +# However, it was the only I could get firebug to show me 302's for +# my ToscaWidget resources. +<filesmatch ".(gif|jpe?g|png|css|js)$"> + Header unset Cache-Control + Header unset Etag + Header add Cache-Control "max-age=2592000" + #ExpiresDefault A2592000 +</filesmatch> + +# we are deploying the packager app that is part of the new RPM Fusion Community +Alias /packages/css /usr/share/fedoracommunity/public/css +Alias /packages/javascript /usr/share/fedoracommunity/public/javascript +Alias /packages/images/icons /var/cache/fedoracommunity/packages/icons +Alias /packages/images /usr/share/fedoracommunity/public/images +Alias /packages/_res /usr/share/fedoracommunity/public/toscawidgets/resources/ + +Alias /packages/tw2/resources/tw2.jqplugins.ui/static /usr/lib/python2.7/site-packages/tw2/jqplugins/ui/static +Alias /packages/tw2/resources/tw2.jquery/static /usr/lib/python2.7/site-packages/tw2/jquery/static +Alias /packages/tw2/resources/fedoracommunity.connectors.widgets.widgets/static /usr/lib/python2.7/site-packages/fedoracommunity/connectors/widgets/static + +<Directory /usr/lib/python2.7/site-packages/tw2/jqplugins/ui/static> + Require all granted +</Directory> + +<Directory /usr/lib/python2.7/site-packages/tw2/jquery/static> + Require all granted +</Directory> + +<Directory /usr/lib/python2.7/site-packages/fedoracommunity/connectors/widgets/static> + Require all granted +</Directory> + +# Temporarily disabled until we can figure out how to get the moksha +# javascript resources pulled in with `python setup.py archive_tw_resources` +#Alias /community/toscawidgets /usr/share/fedoracommunity/public/toscawidgets + +WSGIPythonEggs /var/cache/fedoracommunity/.python-eggs +WSGIDaemonProcess fedoracommunity user=apache maximum-requests=50000 display-name=fedoracommunity processes=8 threads=4 +WSGISocketPrefix run/wsgi +WSGIRestrictStdout Off +WSGIRestrictSignal Off +WSGIPythonOptimize 1 + +WSGIScriptAlias /packages /usr/share/fedoracommunity/fedora-packages.wsgi + +<Directory /usr/share/fedoracommunity/> + Require all granted + WSGIProcessGroup fedoracommunity +</Directory> + +<Directory /var/cache/fedoracommunity/git.fedoraproject.org> + Require all granted +</Directory> + +<Directory /var/cache/fedoracommunity/packages/icons/> + # If someone tries to access an icon that doesn't exist, + # then send them to the default icon. This is used by + # fedmenu, which will request icons for packages that + # don't necessarily have them. The UI will look weird + # unless those get magically redirected to a nice default. + Require all granted + ErrorDocument 404 https://apps.rpmfusion.org/packages/images/icons/package_128x128.png +</Directory> diff --git a/roles/packages3/web/tasks/main.yml b/roles/packages3/web/tasks/main.yml new file mode 100644 index 0000000..1e6d1ae --- /dev/null +++ b/roles/packages3/web/tasks/main.yml @@ -0,0 +1,167 @@ +--- +# +# Configuration for the fedora-packages webapp +- name: install needed packages + package: name={{ item }} state=present + with_items: + - fedora-packages + - python-memcached + tags: + - packages + - packages/web + +- name: Create some directories + file: + path={{ item }} + state=directory + owner=apache + group=fedmsg + mode=775 + with_items: + - /etc/fedoracommunity + - /var/cache/fedoracommunity # the gluster role usually creates this one + - /var/tmp/fedoracommunity + tags: + - packages + - packages/web + +- name: Create some more locked down directories + file: path={{ item }} state=directory owner=apache group=apache mode=700 + with_items: + - /etc/pki/fedoracommunity + tags: + - packages + - packages/web + +- name: Copy over the app config + template: > + src=packages-app.ini.j2 + dest=/etc/fedoracommunity/production.ini + owner=apache group=fedmsg mode=0440 + notify: + - reload httpd + - restart fedmsg-hub + tags: + - packages + - packages/web + +- name: Copy over the httpd config + copy: > + src=packages-httpd.conf + dest=/etc/httpd/conf.d/fedora-packages.conf + owner=root group=root mode=644 + notify: + - reload httpd + tags: + - packages + - packages/web + +- name: Copy over the fedmsg consumer config + template: > + src=fedoracommunity.py + dest=/etc/fedmsg.d/fedoracommunity.py + owner=root group=root mode=644 + notify: + - restart fedmsg-hub + tags: + - packages + - packages/web + +- name: Copy over certs/public-keys, probably for talking with koji. + copy: > + src="{{private}}/files/packages/fedora-ca.cert" + dest="/etc/pki/fedoracommunity/{{item}}" + owner=apache + group=apache + mode=0644 + with_items: + - fedora-server-ca.cert + - fedora-upload-ca.cert + tags: + - packages + - packages/web + +- name: permanently hotfix the distmappings file + copy: > + src=distmappings.py + dest="{{pythonsitelib}}/fedoracommunity/search/distmappings.py" + owner=root group=root mode=0644 + notify: + - reload httpd + - restart fedmsg-hub + tags: + - packages + - packages/web + when: ansible_distribution_major_version|int < 7 and ansible_distribution == 'RedHat' + +# Here's the indexer stuff +- name: Create cache structure + file: > + dest="/var/cache/fedoracommunity/{{item}}" + state=directory + owner=apache + group=fedmsg + mode=0775 + with_items: + - packages + - rpm_cache + - git.fedoraproject.org + - packages/icons + - packages/xapian + - packages/xapian/search + - packages/tmp + - packages/tmp/icons + - packages/tmp/search + - packages/tmp/var + when: install_packages_indexer + tags: + - packages + - packages/web + +- name: Copy over the default icon + copy: > + src=package_128x128.png + dest=/var/cache/fedoracommunity/packages/icons/package_128x128.png + owner=root mode=644 setype=fusefs_t + when: install_packages_indexer + tags: + - packages + - packages/web + - icon + +- name: hotfix the fedmsg-hub executable to pick up forward compat packages + copy: src=fedmsg-hub-forward-compat dest=/usr/bin/fedmsg-hub + tags: + - packages + - packages/web + - hotfix + when: ansible_distribution_major_version|int < 7 and ansible_distribution == 'RedHat' + +# Lastly, here's some selinux stuff. +- name: set some selinux booleans + seboolean: name={{item}} persistent=yes state=yes + with_items: + - httpd_tmp_exec + - httpd_can_network_memcache + - httpd_can_network_connect + - httpd_use_fusefs + - httpd_use_nfs + - httpd_execmem + tags: + - packages + - packages/web + - selinux + +- name: /var/cache/fedoracommunity/git.fedoraproject.org file contexts + sefcontext: + target: '/var/cache/fedoracommunity/git.fedoraproject.org(/.*)?' + setype: httpd_sys_rw_content_t + state: present + +- name: Build the database the first time. This takes a while + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps{{env_suffix}}.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass + tags: + - packages + - packages/web + - selinux + when: ansible_distribution_major_version|int > 6 and (ansible_distribution == 'RedHat' or ansible_distribution == 'Fedora') diff --git a/roles/packages3/web/templates/fedoracommunity.py b/roles/packages3/web/templates/fedoracommunity.py new file mode 100644 index 0000000..4bb5213 --- /dev/null +++ b/roles/packages3/web/templates/fedoracommunity.py @@ -0,0 +1,7 @@ +config = { + {% if install_packages_indexer %} + 'fedoracommunity.fedmsg.consumer.enabled': True, + {% else %} + 'fedoracommunity.fedmsg.consumer.enabled': False, + {% endif %} +} diff --git a/roles/packages3/web/templates/packages-app.ini.j2 b/roles/packages3/web/templates/packages-app.ini.j2 new file mode 100644 index 0000000..1973c72 --- /dev/null +++ b/roles/packages3/web/templates/packages-app.ini.j2 @@ -0,0 +1,226 @@ +## +## Fedora Community Production configuration +## +## $Id: fedoracommunity-prod.ini.erb,v 1.0 2009/05/03 23:38:07 johnp Exp $ +## + +[DEFAULT] +profile = false +debug = false +profile.connectors = false +profile.dir = /var/log/fedoracommunity/profile + +# This is required to avoid a 404 error on, e.g. /packages/python-webob1.2 +disable_request_extensions = True + +#email_to = root(a)rpmfusion.org +#smtp_server = gateway +#error_email_from = fedoracommunity(a)rpmfusion.org + +fedoracommunity.extensions_dir = {{ pythonsitelib }}/fedoracommunity/plugins/extensions + +fedoracommunity.script_name = /packages +fedoracommunity.connector.kojihub.baseurl = https://koji{{env_suffix}}.rpmfusion.org/kojihub +fedoracommunity.connector.bodhi.baseurl = https://bodhi{{env_suffix}}.rpmfusion.org/ +fedoracommunity.connector.mdapi.baseurl = https://apps{{env_suffix}}.fedoraproject.org/mdapi +fedoracommunity.connector.fas.baseurl = https://admin{{env_suffix}}.rpmfusion.org/accounts/ +fedoracommunity.connector.icons.baseurl = http://download01.phx2.fedoraproject.org/pub/alt/screenshots +fedoracommunity.connector.bugzilla.baseurl = https://bugzilla.rpmfusion.org/xmlrpc.cgi +fedoracommunity.connector.bugzilla.cookiefile = /var/cache/fedoracommunity/bugzillacookies +fedoracommunity.connector.xapian.package-search.db = /var/cache/fedoracommunity/packages/xapian/search + +fedoracommunity.resource_path_prefix = /packages/_res/ + +# Git settings +git_repo_path = /var/cache/fedoracommunity/git.rpmfusion.org + +# FAS is locked down so we need a minimal user inorder to get public user info +# to unauthenticated users. You need to get a locked down account for this +# and fill in the user info here. Never check this file into git with +# this information filled in +fedoracommunity.connector.fas.minimal_user_name={{ fcommFasUser }} +fedoracommunity.connector.fas.minimal_user_password={{ fcommFasPassword }} + +# This is insecure, use only for testing +fedora.clients.check_certs = True + +# URL for getting message history +datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw + +## +## Moksha-specific configuration options +## + +# Where to store the feed caches. +# +feed_cache = postgres://fedoracommunity:{{ fcommFeedCacheDBPassword }}@db-community/fedoracommunity_feed_cache + +# +# Feed Streamer settings +# +# Max age (in seconds) of each feed in the cache +feed.max_age = 900 + +# Timeout in seconds for the web request +feed.timeout = 30 + +# The number of simultaneous connections +feed.deferred_groups = 10 + +# Where to initialize and store our application databases. %s is the app name. +app_db = sqlite:///%s.db + +# The location of our Orbited server +orbited_host = localhost +orbited_port = 9000 + +# Stomp broker configuration. +stomp_broker = localhost +stomp_port = 61613 +stomp_user = guest +stomp_pass = guest + +# Optional AMQP Broker. +#amqp_broker = guest/guest@localhost + +# Documentation directory +docs_dir = /srv/moksha/docs + +# Moksha chat configuration + +# Use a built-in IRC server +#chat.backend = irc://localhost:9999 +#chat.builtin = true +#chat.backend = irc://irc.freenode.net:6667 +#chat.rooms = default +#chat.default.staticRoomName = moksha +#chat.default.roomAssignmentMode = static +#chat.default.display.greeting = Moksha Chat +#chat.default.display.floating = true +#chat.default.display.floatingToggle = false +#chat.default.display.width = 400 +#chat.default.display.height = 300 +#chat.default.display.theme = simple +#chat.default.display.resizable = true + +moksha.extensionpoints=True +moksha.csrf_protection = False +moksha.csrf.login_handler = /login_handler +moksha.csrf.trusted_domains = admin.rpmfusion.org + +moksha.use_tw2 = True +moksha.livesocket = False + +cache.bugzilla.backend=dogpile.cache.memcached +cache.bugzilla.expiration_time=300 +cache.bugzilla.arguments.url=memcached02:11211 +cache.bugzilla.arguments.distributed_lock=False +cache.connectors.backend=dogpile.cache.memcached +cache.connectors.expiration_time=300 +cache.connectors.arguments.url=memcached02:11211 +cache.connectors.arguments.distributed_lock=False + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 8080 + +[app:main] +use = egg:fedoracommunity +full_stack = true +#lang = ru +#cache_dir = /var/cache/fedoracommunity/data +beaker.session.key = fedoracommunity +beaker.session.secret = {{ fcommBeakerSessionSecret }} + +beaker.cache.type = ext:memcached +beaker.cache.url = memcached01:11211 +beaker.cache.lock_dir = /var/cache/fedoracommunity/beaker + +# If you'd like to fine-tune the individual locations of the cache data dirs +# for the Cache data, or the Session saves, un-comment the desired settings +# here: +#beaker.cache.data_dir = %(here)s/data/cache +#beaker.session.data_dir = %(here)s/data/sessions + +# pick the form for your database +# %(here) may include a ':' character on Windows environments; this can +# invalidate the URI when specifying a SQLite db via path name +sqlalchemy.url=postgres://moksha:m0ksh4@localhost/moksha +# sqlalchemy.url=mysql://username:password@hostname:port/databasename + +# If you have sqlite, here's a simple default to get you started +# in development + +#sqlalchemy.url = sqlite:///%(here)s/devdata.db +sqlalchemy.echo = true +sqlalchemy.echo_pool = false +sqlalchemy.pool_recycle = 3600 + +sqlalchemy.pool_size=1 +sqlalchemy.max_overflow=2 + +# WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* +# Debug mode will enable the interactive debugging tool, allowing ANYONE to +# execute malicious code after an exception is raised. +set debug = false + +# Logging configuration +# Add additional loggers, handlers, formatters here +# Uses python's logging config file format +# http://docs.python.org/lib/logging-config-fileformat.html + +[loggers] +keys = root, moksha, sqlalchemy, tg, auth, pylons + +[handlers] +keys = console + +[formatters] +keys = generic + +# If you create additional loggers, add them as a key to [loggers] +[logger_root] +level = WARN +handlers = console + +[logger_moksha] +level = WARN +handlers = +qualname = moksha + +[logger_tg] +level = WARN +handlers = +qualname = tg + +# repoze.who is noisy by default +[logger_auth] +level = WARN +handlers = +qualname = auth + +[logger_pylons] +level = WARN +handlers = +qualname = pylons + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) + +# If you create additional handlers, add them as a key to [handlers] +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +# If you create additional formatters, add them as a key to [formatters] +[formatter_generic] +format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S