commit 240796e0c447f1b1d192fae72453ceb4ef8fba52 Author: Nicolas Chauvet <kwizart(a)gmail.com&gt; Date: Wed Nov 23 17:40:46 2016 +0100 Add mailman roles/mailman/defaults/main.yml | 12 + roles/mailman/files/django_fedora.py | 15 + roles/mailman/files/fedmsg-plugin-conf.py | 11 + roles/mailman/files/hyperkitty.Fedora.repo | 31 ++ roles/mailman/files/hyperkitty.RedHat.repo | 31 ++ roles/mailman/files/hyperkitty.lists-dev.repo | 31 ++ roles/mailman/files/hyperkitty.logrotate.conf | 6 + roles/mailman/files/import-mm2.py | 148 +++++++ roles/mailman/files/mailman-hyperkitty.cfg | 20 + .../files/mailman3-fedmsg-plugin.RedHat.repo | 17 + roles/mailman/files/memcached.sysconfig | 5 + roles/mailman/files/periodic.py | 23 + roles/mailman/files/pg-give-rights.py | 86 ++++ roles/mailman/files/post-update.sh | 39 ++ roles/mailman/files/settings_test.py | 47 +++ roles/mailman/files/urls.py | 18 + roles/mailman/files/webui.wsgi | 44 ++ roles/mailman/files/yamlget | 110 +++++ roles/mailman/handlers/main.yml | 9 + roles/mailman/tasks/main.yml | 435 ++++++++++++++++++++ roles/mailman/templates/apache-dummy.conf.j2 | 1 + roles/mailman/templates/apache.conf.j2 | 42 ++ roles/mailman/templates/bottom.html | 15 + roles/mailman/templates/crontab-mailman3.j2 | 1 + roles/mailman/templates/crontab-webui.j2 | 22 + roles/mailman/templates/initial-data.json.j2 | 30 ++ .../mailman/templates/mailman-migration-path.sh.j2 | 3 + roles/mailman/templates/mailman-migration.conf.j2 | 2 + roles/mailman/templates/mailman.cfg.j2 | 183 ++++++++ roles/mailman/templates/post-transaction.action.j2 | 8 + roles/mailman/templates/settings.py.j2 | 390 ++++++++++++++++++ roles/mailman/templates/settings_admin.py.j2 | 23 + 32 files changed, 1858 insertions(+), 0 deletions(-) --- diff --git a/roles/mailman/defaults/main.yml b/roles/mailman/defaults/main.yml new file mode 100644 index 0000000..ac6cf2a --- /dev/null +++ b/roles/mailman/defaults/main.yml @@ -0,0 +1,12 @@ +--- +mailman_webui_basedir: /srv/webui +mailman_webui_confdir: "{{ mailman_webui_basedir }}/config" +mailman_db_server: localhost +mailman_mailman_db_pass: changeme +mailman_hyperkitty_admin_db_pass: changeme +mailman_hyperkitty_db_pass: changeme +mailman_hyperkitty_cookie_key: changeme +mailman_domains: +- lists.example.com +- lists.example.org +mailman_social_login: [] diff --git a/roles/mailman/files/django_fedora.py b/roles/mailman/files/django_fedora.py new file mode 100644 index 0000000..b8cda8a --- /dev/null +++ b/roles/mailman/files/django_fedora.py @@ -0,0 +1,15 @@ +from django.http import UnreadablePostError +from pylibmc import Error as MemcachedError + +EXCLUDED = ( + UnreadablePostError, + MemcachedError, +) + +def exclude_useless_errors(record): + if record.exc_info: + exc_type, exc_value = record.exc_info[:2] + for excluded_class in EXCLUDED: + if isinstance(exc_value, excluded_class): + return False + return True diff --git a/roles/mailman/files/fedmsg-plugin-conf.py b/roles/mailman/files/fedmsg-plugin-conf.py new file mode 100644 index 0000000..92793a5 --- /dev/null +++ b/roles/mailman/files/fedmsg-plugin-conf.py @@ -0,0 +1,11 @@ +config = { + # These are mailing lists that we don't publish to fedmsg (because spam or privacy) + 'mailman.excluded_lists': [ + 'scm-commits', # too much traffic + 'council-private', # private list + 'cwg-private', # private list + 'fesco', # private list + 'security-private', # private list + 'diversity-private', # private list + ], +} diff --git a/roles/mailman/files/hyperkitty.Fedora.repo b/roles/mailman/files/hyperkitty.Fedora.repo new file mode 100644 index 0000000..39c339d --- /dev/null +++ b/roles/mailman/files/hyperkitty.Fedora.repo @@ -0,0 +1,31 @@ +[hyperkitty] +name=HyperKitty archiver and its dependencies +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/fedora-$releasever/$basearch/ +enabled=1 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-source] +name=HyperKitty archiver and its dependencies - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/fedora-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel] +name=HyperKitty archiver and its dependencies (devel versions) +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/fedora-$releasever/$basearch/ +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel-source] +name=HyperKitty archiver and its dependencies (devel versions) - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/fedora-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc diff --git a/roles/mailman/files/hyperkitty.RedHat.repo b/roles/mailman/files/hyperkitty.RedHat.repo new file mode 100644 index 0000000..0debff4 --- /dev/null +++ b/roles/mailman/files/hyperkitty.RedHat.repo @@ -0,0 +1,31 @@ +[hyperkitty] +name=HyperKitty archiver and its dependencies +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/$basearch/ +enabled=1 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-source] +name=HyperKitty archiver and its dependencies - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel] +name=HyperKitty archiver and its dependencies (devel versions) +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/$basearch/ +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel-source] +name=HyperKitty archiver and its dependencies (devel versions) - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc diff --git a/roles/mailman/files/hyperkitty.lists-dev.repo b/roles/mailman/files/hyperkitty.lists-dev.repo new file mode 100644 index 0000000..3f175db --- /dev/null +++ b/roles/mailman/files/hyperkitty.lists-dev.repo @@ -0,0 +1,31 @@ +[hyperkitty] +name=HyperKitty archiver and its dependencies +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/$basearch/ +enabled=1 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-source] +name=HyperKitty archiver and its dependencies - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel] +name=HyperKitty archiver and its dependencies (devel versions) +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/$basearch/ +enabled=1 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc + +[hyperkitty-devel-source] +name=HyperKitty archiver and its dependencies (devel versions) - Source +baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/SRPMS +enabled=0 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc diff --git a/roles/mailman/files/hyperkitty.logrotate.conf b/roles/mailman/files/hyperkitty.logrotate.conf new file mode 100644 index 0000000..05c9113 --- /dev/null +++ b/roles/mailman/files/hyperkitty.logrotate.conf @@ -0,0 +1,6 @@ +/var/log/hyperkitty/*.log { + missingok + notifempty + delaycompress + su root apache +} diff --git a/roles/mailman/files/import-mm2.py b/roles/mailman/files/import-mm2.py new file mode 100755 index 0000000..fa88df4 --- /dev/null +++ b/roles/mailman/files/import-mm2.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +from __future__ import unicode_literals, absolute_import, print_function + +import os +import sys +import subprocess +import pickle +from optparse import OptionParser +from locale import getpreferredencoding +import yaml + +MAILMAN_BIN = subprocess.check_output(["which", "mailman3"]).decode("ascii").strip() + +from mailman.commands.cli_import import Bouncer +sys.modules["Mailman.Bouncer"] = Bouncer + +def call(command): + print(" ".join(command)) + subprocess.check_call(command, env=os.environ) + +def cmdget(command): + print(" ".join(command)) + out = subprocess.check_output(command, env=os.environ) + return out.decode(getpreferredencoding()).strip() + + +class Importer(object): + + def __init__(self, opts, config): + self.opts = opts + self.config = config + self.index_path = self._get_index_path() + self.existing_lists = [ l.strip() for l in + cmdget(["sudo", "-u", "mailman", + MAILMAN_BIN, "lists", "-q"]).split("\n") ] + if opts.exclude: + self.excluded = opts.exclude.strip().split(",") + else: + self.excluded = [] + if opts.include: + self.included = opts.include.strip().split(",") + else: + self.included = [] + + def _get_index_path(self): + return None + sys.path.append(self.config["confdir"]) + settings = __import__("settings") + sys.path.pop() + return settings.KITTYSTORE_SEARCH_INDEX + + def import_dir(self, mm2libdir): + all_listnames = [ d for d in os.listdir( + os.path.join(mm2libdir, 'lists')) + if not d.startswith(".") ] + all_listnames.sort() + for index, listname in enumerate(all_listnames): + listaddr = "%s@%s" % (listname, self.opts.domain.strip()) + if listname in self.excluded or listaddr in self.excluded: + print("Skipping excluded list %s" % listaddr) + continue + if self.included and ( + listname not in self.included and + listaddr not in self.included): + print("Skipping not included list %s" % listaddr) + continue + print(listaddr, "(%d/%d)" % (index+1, len(all_listnames))) + confpickle = os.path.join(mm2libdir, 'lists', listname, + 'config.pck') + if not os.path.exists(confpickle): + print("Missing configuration pickle:", confpickle) + continue + list_is_new = bool(listaddr not in self.existing_lists) + if self.opts.recreate and not list_is_new: + call(["sudo", "-u", "mailman", MAILMAN_BIN, "remove", + listaddr]) + list_is_new = True + if list_is_new: + call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d", + listaddr]) + call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21", + listaddr, confpickle]) + if not self.opts.no_archives: + archivefile = os.path.join( + mm2libdir, "archives", "private", + "%s.mbox" % listname, "%s.mbox" % listname) + archive_policy = bool(pickle.load(open(confpickle, "rb"), + encoding="utf-8", errors="ignore").get('archive')) + if not archive_policy: + print("List %s wants no archiving" % listname) + continue + if os.path.exists(archivefile) and \ + (list_is_new or not self.opts.new_only): + call(["sudo", "django-admin", "hyperkitty_import", + "--pythonpath", self.config["confdir"], + "--settings", "settings", "-l", listaddr, + "--no-sync-mailman", archivefile]) + if self.index_path: + call(["sudo", "chown", "mailman:apache", "-R", self.index_path]) + call(["sudo", "chmod", "g+w", self.index_path]) + if not self.opts.no_sync: + call(["sudo", "django-admin", "mailman_sync", + "--pythonpath", self.config["confdir"], + "--settings", "settings"]) + + + +def main(): + parser = OptionParser() + parser.add_option("-n", "--new-only", action="store_true", + help="Only import the archives when the list is new") + parser.add_option("-A", "--no-archives", action="store_true", + help="Don't import the archives, only import the list config") + parser.add_option("-c", "--config", default="/etc/mailman-migration.conf", + help="Configuration file (default: %defaults)") + parser.add_option("-d", "--domain", + help="Domain for the mailing-lists") + parser.add_option("-x", "--exclude", default="", + help="Comma-separated list of lists to exclude") + parser.add_option("-i", "--include", default="", + help="Comma-separated list of lists to include, no other " + "list will be imported") + parser.add_option("-R", "--recreate", action="store_true", + help="Recreate the lists and re-import their configuration") + parser.add_option("-S", "--no-sync", action="store_true", + help="Don't run the mailman_sync admin command") + opts, args = parser.parse_args() + if len(args) != 1: + parser.error("Only one arg: the Mailman 2.1 lib dir to import") + if opts.include and opts.exclude: + parser.error("Only one of 'include' or 'exclude' may be used") + if not opts.domain: + parser.error("You must provide a domain name for the lists (--domain)") + + mm2libdir = args[0] + if not os.path.exists(mm2libdir): + parser.error("No such directory: %s" % mm2libdir) + + with open(opts.config) as conffile: + config = yaml.safe_load(conffile) + + importer = Importer(opts, config) + importer.import_dir(mm2libdir) + + +if __name__ == "__main__": + main() diff --git a/roles/mailman/files/mailman-hyperkitty.cfg b/roles/mailman/files/mailman-hyperkitty.cfg new file mode 100644 index 0000000..ad9d6b1 --- /dev/null +++ b/roles/mailman/files/mailman-hyperkitty.cfg @@ -0,0 +1,20 @@ +# This is the mailman extension configuration file to enable HyperKitty as an +# archiver. Remember to add the following lines in the mailman.cfg file: +# +# [archiver.hyperkitty] +# class: mailman_hyperkitty.Archiver +# enable: yes +# configuration: /path/to/here/hyperkitty.cfg +# + +[general] + +# This is your HyperKitty installation, preferably on the localhost. This +# address will be used by Mailman to forward incoming emails to HyperKitty +# for archiving. It does not need to be publicly available, in fact it's +# better if it is not. +base_url: http://localhost/archives/ + +# Shared API key, must be the identical to the value in HyperKitty's +# settings. +api_key: SecretArchiverAPIKey diff --git a/roles/mailman/files/mailman3-fedmsg-plugin.RedHat.repo b/roles/mailman/files/mailman3-fedmsg-plugin.RedHat.repo new file mode 100644 index 0000000..2f0bad0 --- /dev/null +++ b/roles/mailman/files/mailman3-fedmsg-plugin.RedHat.repo @@ -0,0 +1,17 @@ +[mailman3-fedmsg-plugin] +name=fedmsg plugin for mailman3, and its deps +baseurl=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/epel-7-$basearch/ +enabled=1 +skip_if_unavailable=1 +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/pubkey.gpg + +## COPR doesn't provide a source repo as far as I can tell. +## Please fix this file if I'm wrong. +#[mailman3-fedmsg-plugin-source] +#name=fedmsg plugin for mailman3, and its deps - Source +#baseurl=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/epel-7-source/ +#enabled=0 +#skip_if_unavailable=1 +#gpgcheck=1 +#gpgkey=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/pubkey.gpg diff --git a/roles/mailman/files/memcached.sysconfig b/roles/mailman/files/memcached.sysconfig new file mode 100644 index 0000000..8987176 --- /dev/null +++ b/roles/mailman/files/memcached.sysconfig @@ -0,0 +1,5 @@ +PORT="11211" +USER="memcached" +MAXCONN="1024" +CACHESIZE="4096" +OPTIONS="" diff --git a/roles/mailman/files/periodic.py b/roles/mailman/files/periodic.py new file mode 100755 index 0000000..73dcee0 --- /dev/null +++ b/roles/mailman/files/periodic.py @@ -0,0 +1,23 @@ +#!/usr/bin/python3 + +import os +import sys + +from mailman.core.initialize import initialize +from mailman.config import config +from mailman.interfaces.pending import IPendings +from mailman.interfaces.requests import IListRequests, RequestType +from zope.component import getUtility + + +def clean_pended(): + getUtility(IPendings).evict() + + +if __name__ == '__main__': + if os.getuid() == 0: + print("This script must be run as the mailman user", file=sys.stderr) + sys.exit(1) + initialize(config_path="/etc/mailman.cfg") + clean_pended() + config.db.commit() diff --git a/roles/mailman/files/pg-give-rights.py b/roles/mailman/files/pg-give-rights.py new file mode 100755 index 0000000..48fd220 --- /dev/null +++ b/roles/mailman/files/pg-give-rights.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python2 +# vim: et ts=4 sw=4 fileencoding=utf-8 + +""" +Give non-admin rights to the database app user. +""" + +CONFFILE = "/etc/mailman-migration.conf" + + +import site +import re +import yaml +import psycopg2 + + +def give_rights(dbhost, dbuser, dbpasswd, dbname, dbreguser=None): + if dbreguser is None: + dbreguser = dbname + "app" + conn = psycopg2.connect(host=dbhost, user=dbuser, password=dbpasswd, + database=dbname) + cur = conn.cursor() + # Database permissions + dbrightsquery = "GRANT CONNECT,TEMP ON DATABASE %s TO %s;" % (dbname, dbreguser) + print dbrightsquery + cur.execute(dbrightsquery) + # Table permissions + cur.execute(""" + SELECT 'GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON "' || relname || '" TO %s;' + FROM pg_class + JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE nspname = 'public' AND relkind IN ('r', 'v'); + """ % dbreguser) + queries = [ q[0] for q in cur ] + for query in queries: + print query + cur.execute(query) + # Sequence permissions + cur.execute(""" + SELECT 'GRANT USAGE,SELECT,UPDATE ON ' || relname || ' TO %s;' + FROM pg_class + JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE nspname = 'public' AND relkind = 'S'; + """ % dbreguser) + queries = [ q[0] for q in cur ] + for query in queries: + print query + cur.execute(query) + conn.commit() + cur.close() + conn.close() + + +def main(): + with open(CONFFILE) as conffile: + conf = yaml.safe_load(conffile) + site.addsitedir(conf["confdir"]) + import settings_admin + + ## KittyStore + #dbspec = re.match(""" + # postgresql:// + # (?P<user>[a-z]+) + # : + # (?P<password>[^@]+) + # @ + # (?P<host>[^/]+) + # / + # (?P<database>[^/?]+) + # """, settings_admin.KITTYSTORE_URL, re.X) + #give_rights(dbspec.group("host"), + # dbspec.group("user"), + # dbspec.group("password"), + # dbspec.group("database") + # ) + + # HyperKitty + give_rights( + settings_admin.DATABASES["default"]["HOST"], + settings_admin.DATABASES["default"]["USER"], + settings_admin.DATABASES["default"]["PASSWORD"], + settings_admin.DATABASES["default"]["NAME"], + ) + + +if __name__ == "__main__": main() diff --git a/roles/mailman/files/post-update.sh b/roles/mailman/files/post-update.sh new file mode 100755 index 0000000..6dd58fa --- /dev/null +++ b/roles/mailman/files/post-update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +CONFFILE=/etc/mailman-migration.conf + +set -e + +export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available + +BASEDIR=`yamlget basedir $CONFFILE` +CONFDIR=`yamlget confdir $CONFFILE` +INDEXDIR=$BASEDIR/fulltext_index + +# Give database rights to the non-admin user (must be done before loading initial data) +sleep $[ ( $RANDOM % 10 ) + 1 ]s # avoid simultaneous lockups on parallel servers. Yes, this is dirty. +$BASEDIR/bin/pg-give-rights.py > /dev/null + +echo "static files" +django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings +django-admin compress --pythonpath $CONFDIR --settings settings +echo "db migration" +django-admin migrate --pythonpath $CONFDIR --settings settings_admin --noinput +echo "load initial data" +django-admin loaddata $CONFDIR/initial-data.json --pythonpath $CONFDIR --settings settings +mkdir -p $INDEXDIR +chown apache:apache -R $INDEXDIR + +# SELinux contexts +echo "SELinux contexts" +restorecon -r "$BASEDIR" + +# Run unit tests +echo "unit tests" +django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty postorius + +# Reload Apache to flush the python cache +systemctl reload httpd + +# Clean the cache +systemctl restart memcached diff --git a/roles/mailman/files/settings_test.py b/roles/mailman/files/settings_test.py new file mode 100644 index 0000000..1b5c79b --- /dev/null +++ b/roles/mailman/files/settings_test.py @@ -0,0 +1,47 @@ +#-*- coding: utf-8 -*- + +""" +Copy of the Django settings file, but with the database set for unit tests. +""" + +from settings import * +try: + from settings_local import * +except ImportError: + pass + +TESTING = True + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } +} + +# Mailman API credentials for testing Postorius +MAILMAN_REST_API_URL = 'http://localhost:9001' +MAILMAN_REST_API_USER = 'restadmin' +MAILMAN_REST_API_PASS = 'restpass' + +VCR_RECORD_MODE = 'once' +USE_SSL = False + +COMPRESS_ENABLED = False +# Empty the precompilers mapping for testing: django-compressor will run them +# even if compress_enabled is false, no idea why +COMPRESS_PRECOMPILERS = () + +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': ':memory:', + 'STORAGE': 'ram', + }, +} +HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' + +LOGGING = {} diff --git a/roles/mailman/files/urls.py b/roles/mailman/files/urls.py new file mode 100644 index 0000000..5e4b405 --- /dev/null +++ b/roles/mailman/files/urls.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from django.conf.urls import include, url +from django.contrib import admin + +from django.core.urlresolvers import reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + url(r'^$', RedirectView.as_view( + url=reverse_lazy('hyperkitty.views.index.index'), + permanent=True)), + url(r'^admin/', include('postorius.urls')), + url(r'^archives/', include('hyperkitty.urls')), + url(r'', include('django_mailman3.urls')), + url(r'^accounts/', include('allauth.urls')), + url(r'^django-admin/', include(admin.site.urls)), +] diff --git a/roles/mailman/files/webui.wsgi b/roles/mailman/files/webui.wsgi new file mode 100644 index 0000000..f9a875e --- /dev/null +++ b/roles/mailman/files/webui.wsgi @@ -0,0 +1,44 @@ +""" +WSGI config for hyperkitty_standalone project. + +This module contains the WSGI application used by Django's development server +and any production WSGI deployments. It should expose a module-level variable +named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover +this application via the ``WSGI_APPLICATION`` setting. + +Usually you will have the standard Django WSGI application here, but it also +might make sense to replace the whole Django WSGI application with a custom one +that later delegates to the Django one. For example, you could introduce WSGI +middleware here, or combine a Django application with an application of another +framework. + +""" +import os +import sys +import site + + +## For some unknown reason, sometimes mod_wsgi fails to set the python paths to +## the virtualenv, with the 'python-path' option. You can do it here too. +## +## Remember original sys.path. +#prev_sys_path = list(sys.path) +## Add here, for the settings module +#site.addsitedir(os.path.abspath(os.path.dirname(__file__))) +## Add the virtualenv +#venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'lib', 'python2.6', 'site-packages') +#site.addsitedir(venv) +# Reorder sys.path so new directories at the front. +#new_sys_path = [] +#for item in list(sys.path): +# if item not in prev_sys_path: +# new_sys_path.append(item) +# sys.path.remove(item) +# sys.path[:0] = new_sys_path + +site.addsitedir(os.path.abspath(os.path.dirname(__file__))) + +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' + +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() diff --git a/roles/mailman/files/yamlget b/roles/mailman/files/yamlget new file mode 100755 index 0000000..68a5876 --- /dev/null +++ b/roles/mailman/files/yamlget @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent: + +u""" +yamlget +------- + +Output any key in a YAML-formatted file. The aim is to make such a +configuration file accessible to shell scripts. + +.. :Authors: + Aurélien Bompard <aurelien(a)bompard.org&gt; <http://aurelien.bompard.org> + +.. :License: + GNU GPL v3 or later +""" + +from __future__ import print_function + +import os +import sys +from optparse import OptionParser + +import yaml + + +def get_key(fullkey, data): + """ + Get the requested key from the parsed data. + :param fullkey: the key to get, nested values can be accessed by using a + colon (":") as the separator + :param data: the parsed data, from yaml.load() + + Examples: + + >>> data = { + ... 'bool': [True, False, True, False], + ... 'dict': {'hp': 13, 'sp': 5}, + ... 'float': 3.14159, + ... 'int': 42, + ... 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'], + ... 'none': [None, None], + ... 'text': "The Set of Gauntlets 'Pauraegen'", + ... } + >>> get_key('bool', data) + [True, False, True, False] + >>> get_key('bool:2', data) + False + >>> get_key('dict', data) + {'hp': 13, 'sp': 5} + >>> get_key('dict:hp', data) + 13 + >>> get_key('float', data) + 3.14159 + >>> get_key('int', data) + 42 + >>> get_key('list', data) + ['LITE', 'RES_ACID', 'SUS_DEXT'] + >>> get_key('list:2', data) + 'RES_ACID' + >>> get_key('list:2:5', data) + 'RES_ACID' + >>> get_key('none', data) + [None, None] + >>> get_key('none:1', data) + >>> get_key('text', data) + "The Set of Gauntlets 'Pauraegen'" + >>> get_key('2', ['item1', 'item2', 'item3']) + 'item2' + """ + + value = data + while value is not None: + key, _sep, fullkey = fullkey.partition(":") + if isinstance(value, list): + try: + key = int(key) + except TypeError: + print("Wrong key format: %s, it should be an integer" % key, + file=sys.stderr) + sys.exit(1) + value = value[key - 1] # start at 1, not 0 + elif isinstance(value, dict): + value = value.get(key) + else: + break # we've got the value now + if not fullkey: + break # can't go any further + return value + +def main(): + parser = OptionParser(usage="%prog <key> <yaml-file>") + args = parser.parse_args()[1] + if len(args) != 2: + parser.error("wrong number of arguments") + fullkey, filepath = args + if not os.path.exists(filepath): + parser.error("no such file: %s" % filepath) + + with open(filepath) as yamlfile: + data = yaml.safe_load_all(yamlfile).next() + + #from pprint import pprint; pprint(data) + value = get_key(fullkey, data) + if value is not None: + print(value) + + +if __name__ == "__main__": + main() diff --git a/roles/mailman/handlers/main.yml b/roles/mailman/handlers/main.yml new file mode 100644 index 0000000..8efdefd --- /dev/null +++ b/roles/mailman/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: restart mailman3 + service: name=mailman3 state=restarted + +- name: reload apache + service: name=httpd state=reloaded + +- name: restart memcached + service: name=memcached state=restarted diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml new file mode 100644 index 0000000..bdf5b23 --- /dev/null +++ b/roles/mailman/tasks/main.yml @@ -0,0 +1,435 @@ +--- +# Configuration for Mailman 3 +# PostgreSQL initialization must have been done already + + +# +# SELinux +# TODO: switch to the sefcontext module when we update Ansible to 2.2+ +# +- name: install semanage + yum: pkg=policycoreutils-python state=present + tags: + - mailman + - selinux + +- name: set the SELinux policy for the configuration directory + command: semanage fcontext -a -t etc_t "{{ mailman_webui_confdir }}(/.*)?" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the fulltext index + command: semanage fcontext -a -t httpd_sys_rw_content_t "{{ mailman_webui_basedir }}/fulltext_index(/.*)?" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the static files directory + command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/static(/.*)?" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the templates override directory + command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/templates(/.*)?" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the log directory + command: semanage fcontext -a -t httpd_log_t "/var/log/hyperkitty(/.*)?" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the generated postfix databases + command: semanage fcontext -a -t etc_aliases_t "{{ mailman_webui_basedir }}/var/data/postfix_.*" + tags: + - mailman + - selinux + +- name: set the SELinux policy for the old static archives + command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/old-archives/pipermail(/.*)?" + tags: + - mailman + - selinux + +- name: allow Apache to remotely connect to PostgreSQL + seboolean: name=httpd_can_network_connect_db state=yes persistent=yes + tags: + - mailman + - selinux + +- name: allow Apache to remotely connect to Mailman + seboolean: name=httpd_can_network_connect state=yes persistent=yes + tags: + - mailman + - selinux + +- name: allow Apache to remotely connect to Memcached + seboolean: name=httpd_can_network_memcache state=yes persistent=yes + tags: + - mailman + - selinux + + +# +# Packages +# +- name: setup the hyperkitty repo + copy: src={{item}} dest=/etc/yum.repos.d/hyperkitty.repo + with_first_found: + - hyperkitty.{{ansible_hostname}}.repo + - hyperkitty.{{ansible_distribution}}.repo + - hyperkitty.repo + tags: mailman + +- name: install GPG to validate the key + yum: state=present name=gnupg + tags: mailman + +- name: add the GPG key + rpm_key: state=present key=https://repos.fedorapeople.org/repos/abompard/abompard.asc + tags: mailman + +- name: install needed packages + yum: pkg={{ item }} state=present + with_items: + - python-psycopg2 + - python34-psycopg2 + - hyperkitty + - hyperkitty-selinux + - postorius + - memcached + - python-pylibmc + - python-django-haystack-xapian + - yum-plugin-post-transaction-actions + # to run the test suite: + - python-beautifulsoup4 + - python-mock + - python-whoosh + - python-tox + - python-vcrpy + # scripts + - python34-PyYAML + # mailman soft dep to convert html to plaintext + - lynx + tags: + - packages + - mailman + +- name: setup the mailman3-fedmsg repo + copy: src=mailman3-fedmsg-plugin.RedHat.repo + dest=/etc/yum.repos.d/mailman3-fedmsg-plugin.repo + when: ansible_distribution == 'RedHat' + tags: mailman + +- name: add the GPG key for the mailman3-fedmsg-plugin repo + rpm_key: state=present key=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34... + when: ansible_distribution == 'RedHat' + tags: mailman + +- name: install the mailman3 fedmsg plugin rpm + yum: pkg=mailman3-fedmsg-plugin + notify: restart mailman3 + when: ansible_distribution == 'RedHat' + tags: mailman + +- name: copy in our fedmsg-plugin conf file + copy: src=fedmsg-plugin-conf.py dest=/etc/fedmsg.d/fedmsg-plugin-conf.py + tags: + - fedmsgdconfig + - mailman + notify: restart mailman3 + +- name: install packages + yum: pkg={{ item }} state=present + with_items: + - mailman3 + - mailman3-selinux + - mailman3-hyperkitty + tags: + - packages + - mailman + + +# +# Initialize mailman (must be done after settings up the DBs) +# +- name: add mailman to the apache group + user: name=mailman groups=apache append=yes + tags: + - config + - mailman + notify: + - restart mailman3 + +# access to the aliases files generated by mailman +- name: add postfix to the mailman group + user: name=postfix groups=mailman append=yes + tags: + - config + - mailman + notify: + - restart postfix + +# for access to the full-text index +- name: add apache to the mailman group + user: name=apache groups=mailman append=yes + tags: + - config + - mailman + notify: + - reload httpd + +- name: set the mailman conffile + template: src={{ item }} dest=/etc/mailman.cfg + owner=root group=mailman mode=0640 + with_first_found: + - mailman.cfg.{{ ansible_hostname }}.j2 + - mailman.cfg.j2 + tags: + - config + - mailman + notify: + - restart mailman3 + + +# +# Crontab +# +- name: set the hyperkitty crontab + template: src=crontab-webui.j2 dest=/etc/cron.d/hyperkitty + tags: + - config + - mailman +- name: set the mailman crontab + template: src=crontab-mailman3.j2 dest=/etc/cron.d/mailman3 + tags: + - config + - mailman + +# +# Logging +# +- name: hyperkitty logging -- directory + file: path=/var/log/hyperkitty state=directory + owner=root group=apache mode=2775 + tags: mailman +- name: hyperkitty logging -- file creation + copy: content="" dest=/var/log/hyperkitty/hyperkitty.log + force=no + tags: mailman +- name: hyperkitty logging -- file permissions + file: path=/var/log/hyperkitty/hyperkitty.log state=file + owner=root group=apache mode=664 + tags: mailman +- name: hyperkitty logging -- rotation + copy: src=hyperkitty.logrotate.conf + dest=/etc/logrotate.d/hyperkitty + tags: mailman + + +# +# HyperKitty + Postorius setup +# + +- name: create the configuration directory + file: path={{ mailman_webui_confdir }} state=directory + tags: mailman + +- name: install the hyperkitty settings file + template: src=settings.py.j2 + dest="{{ mailman_webui_confdir }}/settings.py" + owner=root group=apache mode=0640 + tags: + - config + - mailman + notify: + - reload apache + #- restart mailman3 + +- name: install the hyperkitty settings admin file + template: src=settings_admin.py.j2 + dest="{{ mailman_webui_confdir }}/settings_admin.py" + owner=root group=root mode=0600 + tags: + - config + - mailman + +- name: install the hyperkitty settings test file + copy: src=settings_test.py + dest="{{ mailman_webui_confdir }}/settings_test.py" + owner=root group=root mode=0644 + tags: + - config + - mailman + +- name: install the django_fedora module + copy: src=django_fedora.py + dest="{{ mailman_webui_confdir }}/django_fedora.py" + owner=root group=root mode=0644 + tags: + - config + - mailman + notify: + - reload apache + +- name: install the hyperkitty urls file + copy: src=urls.py + dest="{{ mailman_webui_confdir }}/urls.py" + owner=root group=root mode=0644 + tags: + - config + - mailman + notify: + - reload apache + +- name: install the hyperkitty wsgi file + copy: src=webui.wsgi + dest="{{ mailman_webui_confdir }}/webui.wsgi" + owner=root group=root mode=0644 + tags: + - config + - mailman + notify: + - reload apache + +- name: install the hyperkitty/postorius dummy httpd conf file + template: src=apache-dummy.conf.j2 + dest=/etc/httpd/conf.d/{{ item }}.conf + with_items: + - hyperkitty + - postorius + tags: + - config + - mailman + notify: + - reload apache + +- name: install the hyperkitty httpd conf file + template: src=apache.conf.j2 + dest=/etc/httpd/conf.d/mailman-webui.conf + tags: + - config + - mailman + notify: + - reload apache + +- name: create the fulltext index dir + file: path="{{ mailman_webui_basedir }}/fulltext_index" + state=directory owner=apache group=apache mode=0755 + tags: mailman + +- name: create the hyperkitty static files dir + file: path="{{ mailman_webui_basedir }}/static" + state=directory owner=root group=root mode=0755 + tags: mailman + +- name: create the hyperkitty templates override dir + file: path="{{ mailman_webui_basedir }}/templates/hyperkitty" + state=directory owner=root group=root mode=0755 + tags: mailman + +- name: Install our fedmenu js hook + template: src=bottom.html + dest="{{mailman_webui_basedir }}/templates/hyperkitty/bottom.html" + tags: mailman + +# +# Plug HyperKitty into Mailman +# +- name: copy the mailman-hyperkitty conffile + copy: src=mailman-hyperkitty.cfg + dest="/etc/mailman3.d/hyperkitty.cfg" + owner=root group=mailman mode=0640 + tags: + - config + - mailman + notify: + - restart mailman3 + +# +# Scripts +# +- name: install the migration conffile + template: src=mailman-migration.conf.j2 + dest=/etc/mailman-migration.conf + owner=root group=root mode=0644 + tags: mailman + +- name: create the scripts dir + file: path="{{ mailman_webui_basedir }}/bin" + state=directory owner=root group=root mode=0755 + tags: mailman + +- name: install the migration environment + template: src=mailman-migration-path.sh.j2 + dest=/etc/profile.d/mailman-migration-path.sh + owner=root group=root mode=0644 + tags: mailman + +- name: install the scripts + copy: src={{ item }} dest="{{ mailman_webui_basedir }}/bin/{{ item }}" + owner=root group=root mode=0755 + tags: mailman + with_items: + - yamlget + - pg-give-rights.py + - post-update.sh + - import-mm2.py + - periodic.py + +- name: copy the initial user fixture + template: src=initial-data.json.j2 + dest={{ mailman_webui_basedir }}/config/initial-data.json + owner=root group=apache mode=0640 + when: inventory_hostname.startswith('mailman01') + tags: mailman + + +# Sync databases and collect static files on RPM install/upgrade +- name: install the post-transaction trigger + template: src=post-transaction.action.j2 + dest=/etc/yum/post-actions/hyperkitty.action + tags: mailman + +# +# Only run this on mailman01 for now. +# TODO: run it on lists-dev too +# + +# The post-update scripts needs memcached to be up (django-compressor will +# store the timestamps there) +- name: start services + service: state=started enabled=yes name=memcached + tags: mailman + +- name: run the post-update script + command: "{{ mailman_webui_basedir }}/bin/post-update.sh" + tags: mailman + + +## Postfix +#- name: create the postfix aliases +# command: su mailman -s /bin/sh -c "mailman3 aliases" +# creates=/var/lib/mailman3/data/postfix_lmtp.db + + +# Memcached +- name: set the memcached sysconfig file + copy: src=memcached.sysconfig dest=/etc/sysconfig/memcached + tags: mailman + notify: + - restart memcached + + +# Start services +- name: start services + service: state=started enabled=yes name={{ item }} + with_items: + - httpd + - mailman3 + - postfix + tags: mailman diff --git a/roles/mailman/templates/apache-dummy.conf.j2 b/roles/mailman/templates/apache-dummy.conf.j2 new file mode 100644 index 0000000..584a4f0 --- /dev/null +++ b/roles/mailman/templates/apache-dummy.conf.j2 @@ -0,0 +1 @@ +# See {{ mailman_webui_confdir }} diff --git a/roles/mailman/templates/apache.conf.j2 b/roles/mailman/templates/apache.conf.j2 new file mode 100644 index 0000000..43a2c2b --- /dev/null +++ b/roles/mailman/templates/apache.conf.j2 @@ -0,0 +1,42 @@ +Alias /favicon.ico {{ mailman_webui_basedir }}/static/hyperkitty/img/favicon.ico +Alias /static {{ mailman_webui_basedir }}/static + +#ErrorLog /var/log/httpd/webui_error.log +#CustomLog /var/log/httpd/webui_access.log combined + +WSGIScriptAlias / {{ mailman_webui_confdir }}/webui.wsgi +WSGIDaemonProcess webui display-name=webui maximum-requests=1000 processes=4 threads=10 +WSGISocketPrefix run/wsgi +WSGIRestrictStdout On +WSGIRestrictSignal Off +WSGIPythonOptimize 1 + +<Directory "{{ mailman_webui_confdir }}"> + <Files webui.wsgi> + Order deny,allow + Allow from all + Require all granted + </Files> + WSGIProcessGroup webui +</Directory> + +<Directory "{{ mailman_webui_basedir }}/static"> + Order deny,allow + Allow from all + Require all granted +</Directory> + + +# Old static archives + +Alias /pipermail/ {{ mailman_webui_basedir }}/old-archives/pipermail/ +<Directory {{ mailman_webui_basedir }}/old-archives/pipermail> + Options MultiViews FollowSymLinks + AllowOverride None + Require all granted + AddDefaultCharset Off +</Directory> + +RedirectMatch ^/pipermail[/]*$ / +RedirectMatch ^/mailman/listinfo/$ / +RedirectMatch ^/mailman$ / diff --git a/roles/mailman/templates/bottom.html b/roles/mailman/templates/bottom.html new file mode 100644 index 0000000..a892972 --- /dev/null +++ b/roles/mailman/templates/bottom.html @@ -0,0 +1,15 @@ +{% if env == 'staging' %} +<script src="https://apps.stg.fedoraproject.org/fedmenu/js/fedmenu.js"> </script> +{% else %} +<script src="https://apps.fedoraproject.org/fedmenu/js/fedmenu.js"> </script> +{% endif %} +<script> + fedmenu({ +{% if env == 'staging' %} + 'url': 'https://apps.stg.fedoraproject.org/js/data.js', +{% else %} + 'url': 'https://apps.fedoraproject.org/js/data.js', +{% endif %} + 'position': 'bottom-left' + }); +</script> diff --git a/roles/mailman/templates/crontab-mailman3.j2 b/roles/mailman/templates/crontab-mailman3.j2 new file mode 100644 index 0000000..5be2306 --- /dev/null +++ b/roles/mailman/templates/crontab-mailman3.j2 @@ -0,0 +1 @@ +42 * * * * mailman {{ mailman_webui_basedir }}/bin/periodic.py diff --git a/roles/mailman/templates/crontab-webui.j2 b/roles/mailman/templates/crontab-webui.j2 new file mode 100644 index 0000000..12ecdc6 --- /dev/null +++ b/roles/mailman/templates/crontab-webui.j2 @@ -0,0 +1,22 @@ +# This goes in /etc/cron.d/. +# Replace "apache" by your webserver user ("www-data" on Debian systems) and +# set the path to the Django project directory + +{% if ansible_hostname == 'mailman01' %} +@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings +@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings +@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings +@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings +@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings +2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings +* * * * * apache django-admin runjobs minutely --pythonpath {{ mailman_webui_confdir }} --settings settings +{% else %} +# These are only active on mailman01 (primary) server +#@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings +#@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings +#@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings +#@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings +#@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings +#2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings +#* * * * * apache django-admin runjobs minutely --pythonpath {{ mailman_webui_confdir }} --settings settings +{% endif %} diff --git a/roles/mailman/templates/initial-data.json.j2 b/roles/mailman/templates/initial-data.json.j2 new file mode 100644 index 0000000..91f73ba --- /dev/null +++ b/roles/mailman/templates/initial-data.json.j2 @@ -0,0 +1,30 @@ +[ +{% for host in mailman_domains %} +{ + "fields": { + "domain": "{{ host }}", + "name": "RPM Fusion mailing-lists" + }, + "model": "sites.site", + "pk": {{ loop.index }} +}, +{% endfor %} +{% for service_name, service_data in mailman_login.items() %} +{ + "fields": { + "name": "{{ service_data.display_name }}", + "sites": [ + {% for host in mailman_domains %} + {{ loop.index }}{% if not loop.last %},{% endif %} + {% endfor %} + ], + "client_id": "{{ mailman_login_secrets[env][service_name].client_id }}", + "secret": "{{ mailman_login_secrets[env][service_name].secret }}", + "key": "", + "provider": "{{ service_data.provider }}" + }, + "model": "socialaccount.socialapp", + "pk": {{ loop.index }} +}{% if not loop.last %},{% endif %} +{% endfor %} +] diff --git a/roles/mailman/templates/mailman-migration-path.sh.j2 b/roles/mailman/templates/mailman-migration-path.sh.j2 new file mode 100644 index 0000000..4c2cb5e --- /dev/null +++ b/roles/mailman/templates/mailman-migration-path.sh.j2 @@ -0,0 +1,3 @@ +PATH=$PATH:{{ mailman_webui_basedir }}/bin +export PATH + diff --git a/roles/mailman/templates/mailman-migration.conf.j2 b/roles/mailman/templates/mailman-migration.conf.j2 new file mode 100644 index 0000000..96f852b --- /dev/null +++ b/roles/mailman/templates/mailman-migration.conf.j2 @@ -0,0 +1,2 @@ +basedir: {{ mailman_webui_basedir }} +confdir: {{ mailman_webui_confdir }} diff --git a/roles/mailman/templates/mailman.cfg.j2 b/roles/mailman/templates/mailman.cfg.j2 new file mode 100644 index 0000000..8c1f3a1 --- /dev/null +++ b/roles/mailman/templates/mailman.cfg.j2 @@ -0,0 +1,183 @@ +# This is the absolute bare minimum base configuration file. User supplied +# configurations are pushed onto this. + +[mailman] +# This address is the "site owner" address. Certain messages which must be +# delivered to a human, but which can't be delivered to a list owner (e.g. a +# bounce from a list owner), will be sent to this address. It should point to +# a human. +site_owner: root(a)rpmfusion.org + +# The local URL part to the administration interface (Postorius). +# The full URL will be constructed by prepending the domain URL set in the +# list's domain properties. +listinfo_url = /admin/ + +# Set the paths to be Fedora-compliant +layout: fhs + +[paths.dev] +var_dir = {{ mailman_webui_basedir }}/var + +[paths.fhs] +bin_dir: /usr/libexec/mailman3 +var_dir: /var/lib/mailman3 +queue_dir: /var/spool/mailman3 +log_dir: /var/log/mailman3 +lock_dir: /run/lock/mailman3 +ext_dir: /etc/mailman3.d +pid_file: /run/mailman3/master.pid + +[database] +class: mailman.database.postgresql.PostgreSQLDatabase +url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman + +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman3.d/hyperkitty.cfg + +[archiver.fedmsg] +class: mailman3_fedmsg_plugin.Archiver +enable: yes + +[archiver.prototype] +enable: yes + + +[antispam] +# This section defines basic antispam detection settings. + +# This value contains lines which specify RFC 822 headers in the email to +# check for spamminess. Each line contains a `key: value` pair, where the key +# is the header to check and the value is a Python regular expression to match +# against the header's value. E.g.: +# +# X-Spam: (yes|maybe) +# +# The header value and regular expression are always matched +# case-insensitively. +header_checks: + X-Spam: yes + X-Spam-Flag: Yes + X-Spam-Status: ^Yes, + +# The chain to jump to if any of the header patterns matches. This must be +# the name of an existing chain such as 'discard', 'reject', 'hold', or +# 'accept', otherwise 'hold' will be used. +jump_chain: discard + + +[mta] +# Email is sent on the submission port to bypass spam checking. +smtp_port: 587 + + +[language.en] +# Change the english language to be UTF-8 (it defaults to ascii). +description: English (USA) +charset: utf-8 +enabled: yes + + +# http://www.lingoes.net/en/translator/langcode.htm + +[language.pt] +description: Protuguese +charset: iso-8859-15 +enabled: yes + +[language.cs] +description: Czech +charset: utf-8 +enabled: yes + +[language.ca] +description: Catalan +charset: utf-8 +enabled: yes + +[language.ja] +description: Japanese +charset: utf-8 +enabled: yes + +[language.ar] +description: Arabic +charset: utf-8 +enabled: yes + +[language.nl] +description: Dutch +charset: utf-8 +enabled: yes + +[language.pl] +description: Polish +charset: utf-8 +enabled: yes + +[language.es] +description: Spanish +charset: utf-8 +enabled: yes + +[language.pt_BR] +description: Protuguese (Brazil) +charset: iso-8859-15 +enabled: yes + +[language.zh_CN] +description: Chinese (S) +charset: utf-8 +enabled: yes + +[language.zh_TW] +description: Chinese (T) +charset: utf-8 +enabled: yes + +[language.ru] +description: Russian +charset: utf-8 +enabled: yes + +[language.vi] +description: Vietnamese +charset: utf-8 +enabled: yes + +[language.it] +description: Italian +charset: utf-8 +enabled: yes + +[language.fr] +description: French +charset: utf-8 +enabled: yes + +[language.ro] +description: Romanian +charset: utf-8 +enabled: yes + +[language.de] +description: German +charset: utf-8 +enabled: yes + +[language.hu] +description: Hungarian +charset: utf-8 +enabled: yes + +[language.ko] +description: Korean +charset: utf-8 +enabled: yes + +[language.uk] +description: Ukrainian +charset: utf-8 +enabled: yes diff --git a/roles/mailman/templates/post-transaction.action.j2 b/roles/mailman/templates/post-transaction.action.j2 new file mode 100644 index 0000000..a344ebc --- /dev/null +++ b/roles/mailman/templates/post-transaction.action.j2 @@ -0,0 +1,8 @@ +# Run the post-update script + +hyperkitty:install:{{ mailman_webui_basedir }}/bin/post-update.sh +hyperkitty:update:{{ mailman_webui_basedir }}/bin/post-update.sh +postorius:install:{{ mailman_webui_basedir }}/bin/post-update.sh +postorius:update:{{ mailman_webui_basedir }}/bin/post-update.sh +python-django-mailman3:install:{{ mailman_webui_basedir }}/bin/post-update.sh +python-django-mailman3:update:{{ mailman_webui_basedir }}/bin/post-update.sh diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 new file mode 100644 index 0000000..d0f96d3 --- /dev/null +++ b/roles/mailman/templates/settings.py.j2 @@ -0,0 +1,390 @@ +#-*- coding: utf-8 -*- +""" +Django settings for HyperKitty + Postorius +""" + +import os +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +import django_fedora + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '{{ mailman_hyperkitty_cookie_key }}' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ADMINS = ( + ('HyperKitty Admin', &#39;kwizart(a)rpmfusion.org&#39;), +) +SERVER_EMAIL = &#39;root(a)rpmfusion.org&#39; +DEFAULT_FROM_EMAIL = &quot;admin(a)rpmfusion.org&quot; + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts +ALLOWED_HOSTS = [ +{% for host in mailman_domains %} + "{{ host }}", +{% endfor %} + ".rpmfusion.org", + "localhost", # Archiving API from Mailman + "127.0.0.1", # HAProxy ping + "{{ ansible_hostname }}", # Varnish ping +] + +# Mailman API credentials +MAILMAN_REST_API_URL = 'http://localhost:8001' +MAILMAN_REST_API_USER = 'restadmin' +MAILMAN_REST_API_PASS = 'restpass' +MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey' +MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1') + +# Application definition + +INSTALLED_APPS = ( + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'hyperkitty', + 'rest_framework', + 'django_gravatar', + 'paintstore', + 'compressor', + 'haystack', + 'django_extensions', + 'postorius', + 'django_mailman3', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', + 'django_mailman3.lib.auth.fedora', + {% for service_name, service_data in mailman_login.items() %} + 'allauth.socialaccount.providers.{{ service_data.provider }}', + {% endfor %} + 'allauth.socialaccount.providers.openid', +) + + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + #'hyperkitty.middleware.SSLRedirect', + 'django_mailman3.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + '{{ mailman_webui_basedir }}/templates', + ], + 'OPTIONS': { + 'loaders': [ + # https://docs.djangoproject.com/en/1.8/ref/templates/api/#django.template.... + ('django.template.loaders.cached.Loader', [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ]), + ], + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'hyperkitty.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'hyperkitty', + 'USER': 'hyperkittyapp', + 'PASSWORD': '{{ mailman_hyperkitty_db_pass }}', + 'HOST': '{{ mailman_db_server }}', + 'PORT': '', # Set to empty string for default. + } +} + + + +# Security & production settings +# https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +CSRF_COOKIE_SECURE = True +CSRF_COOKIE_HTTPONLY = True +SESSION_COOKIE_SECURE = True +SECURE_CONTENT_TYPE_NOSNIFF = True +SECURE_BROWSER_XSS_FILTER = True +X_FRAME_OPTIONS = 'DENY' +# We're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True +# In the Fedora infra, requests are systematically redirected to HTTPS. +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https') +#SECURE_SSL_REDIRECT = True +#SECURE_HSTS_SECONDS = 3600 +#SECURE_HSTS_INCLUDE_SUBDOMAINS = True + + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'America/Chicago' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.8/howto/static-files/ + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/var/www/example.com/static/" +#STATIC_ROOT = '' +STATIC_ROOT = "{{ mailman_webui_basedir }}/static/" + +# URL prefix for static files. +# Example: "http://example.com/static/", "http://static.example.com/" +STATIC_URL = '/static/' + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +# 'django.contrib.staticfiles.finders.DefaultStorageFinder', + 'compressor.finders.CompressorFinder', +) + + +# Sessions +# https://docs.djangoproject.com/en/1.8/topics/http/sessions/ + +SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db' + + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages +MESSAGE_TAGS = { + messages.ERROR: 'danger' + } + + +# +# Authentication +# + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'hk_root' +LOGOUT_URL = 'account_logout' + +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + 'openid': { + 'SERVERS': [ + dict(id='yahoo', + name='Yahoo', + openid_url='http://me.yahoo.com'), + ], + }, + 'google': { + 'SCOPE': ['profile', 'email'], + 'AUTH_PARAMS': {'access_type': 'online'}, + }, + 'facebook': { + 'METHOD': 'oauth2', + 'SCOPE': ['email'], + 'FIELDS': [ + 'email', + 'name', + 'first_name', + 'last_name', + 'locale', + 'timezone', + ], + 'VERSION': 'v2.4', + }, + 'stackexchange': { + 'SITE': 'stackoverflow', + }, +} + + +# +# Gravatar +# https://github.com/twaddington/django-gravatar +# +# Gravatar base url. +GRAVATAR_URL = 'http://cdn.libravatar.org/' +# Gravatar base secure https url. +GRAVATAR_SECURE_URL = 'https://seccdn.libravatar.org/' +# Gravatar size in pixels. +#GRAVATAR_DEFAULT_SIZE = '80' +# An image url or one of the following: 'mm', 'identicon', 'monsterid', 'wavatar', 'retro'. +GRAVATAR_DEFAULT_IMAGE = 'retro' +# One of the following: 'g', 'pg', 'r', 'x'. +#GRAVATAR_DEFAULT_RATING = 'g' +# True to use https by default, False for plain http. +GRAVATAR_DEFAULT_SECURE = True + +# +# django-compressor +# https://pypi.python.org/pypi/django_compressor +# +COMPRESS_PRECOMPILERS = ( + ('text/less', 'lessc {infile} {outfile}'), + ('text/x-scss', 'sassc -t compressed {infile} {outfile}'), + ('text/x-sass', 'sassc -t compressed {infile} {outfile}'), +) +COMPRESS_OFFLINE = True +# needed for debug mode +#INTERNAL_IPS = ('127.0.0.1',) + + +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'xapian_backend.XapianEngine', + 'PATH': "{{ mailman_webui_basedir }}/fulltext_index", + }, +} + + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + }, + 'exclude_useless_errors': { + '()': 'django.utils.log.CallbackFilter', + 'callback': django_fedora.exclude_useless_errors, + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false', 'exclude_useless_errors'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'file':{ + 'level': 'DEBUG', + #'class': 'logging.handlers.RotatingFileHandler', + 'class': 'logging.handlers.WatchedFileHandler', + 'filename': '/var/log/hyperkitty/hyperkitty.log', + 'formatter': 'verbose', + }, + 'null': { + 'class': 'logging.NullHandler', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': [ + 'file', + {% if env == 'production' %} + 'mail_admins', + {% endif %} + ], + 'level': 'DEBUG', + }, + 'django.security.DisallowedHost': { + 'handlers': ['null'], + 'propagate': False, + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + 'root': { + 'handlers': ['file'], + 'level': 'INFO', + }, +} + + +# Cache: use the local memcached server +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', + 'LOCATION': '127.0.0.1:11211', + } +} + + +# Only display mailing-lists from the same virtual host as the webserver +FILTER_VHOST = False diff --git a/roles/mailman/templates/settings_admin.py.j2 b/roles/mailman/templates/settings_admin.py.j2 new file mode 100644 index 0000000..41f2567 --- /dev/null +++ b/roles/mailman/templates/settings_admin.py.j2 @@ -0,0 +1,23 @@ +#-*- coding: utf-8 -*- + +""" +Copy of the Django settings file, but with database admin credentials (for +schema modifications) +""" + +from settings import * +try: + from settings_local import * +except ImportError: + pass + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'hyperkitty', + 'USER': 'hyperkittyadmin', + 'PASSWORD': '{{ mailman_hyperkitty_admin_db_pass }}', + 'HOST': '{{ mailman_db_server }}', + 'PORT': '', + } +}