commit eae5e13f47661fd44594f2f32d3d6a01c701cc05 Author: Nicolas Chauvet <kwizart(a)gmail.com&gt; Date: Wed Nov 9 10:50:40 2016 +0100 Add awstats roles/awstats/files/combineHttp.cron | 1 + roles/awstats/files/combineHttpLogs.sh | 70 ++++++++++++++++++++++++++++++++ roles/awstats/tasks/main.yml | 29 +++++++++++++ 3 files changed, 100 insertions(+), 0 deletions(-) --- diff --git a/roles/awstats/README b/roles/awstats/README new file mode 100644 index 0000000..e69de29 diff --git a/roles/awstats/files/combineHttp.cron b/roles/awstats/files/combineHttp.cron new file mode 100644 index 0000000..016a796 --- /dev/null +++ b/roles/awstats/files/combineHttp.cron @@ -0,0 +1 @@ +0 06 * * * root /usr/local/bin/combineHttpLogs.sh diff --git a/roles/awstats/files/combineHttpLogs.sh b/roles/awstats/files/combineHttpLogs.sh new file mode 100644 index 0000000..60d38f1 --- /dev/null +++ b/roles/awstats/files/combineHttpLogs.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# This file is part of Fedora Project Infrastructure Ansible +# Repository. +# +# Fedora Project Infrastructure Ansible Repository is free software: +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# Fedora Project Infrastructure Ansible Repository is distributed in +# the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with Fedora Project Infrastructure Ansible Repository. If +# not, see <http://www.gnu.org/licenses/>;. + +# Because sync-http may not get all logs for 3 days, we only merge +# things after 4 days. + +NUMDAYS=4 +YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y) +MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m) +DAY=$(/bin/date -d "-${NUMDAYS} days" +%d) + +LOGDIR=/var/log/hosts +NFSDIR=/mnt/rpmfusion_stats/combined-http +PROXYLOG=${LOGDIR}/proxy*/${YEAR}/${MONTH}/${DAY}/http/ +DL_LOG=${LOGDIR}/download*/${YEAR}/${MONTH}/${DAY}/http/ + +TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY} + +LOGMERGE=/usr/share/awstats/tools/logresolvemerge.pl + +FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}' | sort -u ) + +mkdir -p ${TARGET} + +for FILE in ${FILES}; do + TEMP=$(echo ${FILE} | sed 's/\.xz$//') + perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP} +done + +FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}' | sort -u ) + +mkdir -p ${TARGET} + +for FILE in ${FILES}; do + TEMP=$(echo ${FILE} | sed 's/\.xz$//') + perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP} +done + +# Now we link up the files into latest directory +# 1. make sure the latest directory exists +# 2. go into it. +# 3. remove the old links +# 4. link up all the files we merged over + +if [[ -d ${NFSDIR}/latest ]]; then + pushd ${NFSDIR}/latest &> /dev/null + /bin/rm -f * + for file in ../${YEAR}/${MONTH}/${DAY}/*; do + ln -s ${file} . + done + popd &> /dev/null +fi diff --git a/roles/awstats/tasks/main.yml b/roles/awstats/tasks/main.yml new file mode 100644 index 0000000..0d915e1 --- /dev/null +++ b/roles/awstats/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: install awstats package + yum: state=present name=awstats + tags: + - packages + - awstats + +- name: make the awstats directory + file: path=/var/www/html/awstats-reports state=directory + when: inventory_hostname.startswith('data-analysis') + tags: + - awstats + +- name: awstats proxy merge script (log01) + copy: src=combineHttpLogs.sh dest=/usr/local/bin/ mode=0755 + when: inventory_hostname.startswith('log') + tags: + - config + - awstats + +- name: awstats daily cron job to merge old logs + copy: src=combineHttp.cron dest=/etc/cron.d/ mode=0644 + when: inventory_hostname.startswith('log') + tags: + - cron + - awstats + +#- name: set up http configs for awstats on dataserver +