commit b3757e3260fd214eccdf30a3648f0b8619b765c1 Author: Nicolas Chauvet <kwizart(a)gmail.com&gt; Date: Thu Nov 10 09:36:14 2016 +0100 Add scripts files/scripts/create-filelist | 130 ++++++++++++++++++++++++++++ files/scripts/update-fullfiletimelist | 150 +++++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 0 deletions(-) --- diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist new file mode 100755 index 0000000..eeba9d0 --- /dev/null +++ b/files/scripts/create-filelist @@ -0,0 +1,130 @@ +#!/usr/bin/python +from __future__ import print_function + +# A simple script to generate a file list in a format easily consumable by a +# shell script. + +# Originally written by Jason Tibbitts <tibbs(a)math.uh.edu&gt; in 2016. +# Donated to the public domain. If you require a statement of license, please +# consider this work to be licensed as "CC0 Universal", any version you choose. + +import argparse +import hashlib +import os +import sys +from scandir import scandir + + +def get_ftype(entry): + """Return a simple indicator of the file type.""" + if entry.is_symlink(): + return 'l' + if entry.is_dir(): + return 'd' + return 'f' + + +def sha1(fname): + """Return the SHA1 checksum of a file in hex.""" + fh = open(fname, 'rb') + sha1 = hashlib.sha1() + block = fh.read(2 ** 16) + while len(block) > 0: + sha1.update(block) + block = fh.read(2 ** 16) + + return sha1.hexdigest() + + +def recursedir(path='.', skip=[], alwaysskip=['.~tmp~']): + """Just like scandir, but recursively. + + Will skip everything in the skip array, but only at the top level + directory. + """ + for entry in scandir(path): + if entry.name in skip: + continue + if entry.name in alwaysskip: + continue + if entry.is_dir(follow_symlinks=False): + # Don't pass skip here, because we only skip in the top level + for rentry in recursedir(entry.path, alwaysskip=alwaysskip): + yield rentry + yield entry + + +def parseopts(): + null = open(os.devnull, 'w') + p = argparse.ArgumentParser( + description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror.') + p.add_argument('-c', '--checksum', action='store_true', + help='Include checksums of all repomd.xml files in the file list.') + p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files', + help='Include checksums of all instances of the specified file.') + p.add_argument('-s', '--skip', action='store_true', + help='Skip the file lists in the top directory') + p.add_argument('-S', '--skip-file', action='append', dest='skip_files', + help='Skip the specified file in the top directory.') + + p.add_argument('-d', '--dir', help='Directory to scan (default: .).') + + p.add_argument('-t', '--timelist', type=argparse.FileType('w'), default=sys.stdout, + help='Filename of the file list with times (default: stdout).') + p.add_argument('-f', '--filelist', type=argparse.FileType('w'), default=null, + help='Filename of the file list without times (default: no plain file list is generated).') + + opts = p.parse_args() + + if not opts.dir: + opts.dir = '.' + + opts.checksum_files = opts.checksum_files or [] + if opts.checksum: + opts.checksum_files += ['repomd.xml'] + + opts.skip_files = opts.skip_files or [] + if opts.skip: + if not opts.timelist.name == '<stdout>': + opts.skip_files += [opts.timelist.name] + if not opts.filelist.name == '<stdout>': + opts.skip_files += [opts.filelist.name] + + return opts + + +def main(): + opts = parseopts() + checksums = {} + + os.chdir(opts.dir) + + print('[Version]', file=opts.timelist) + print('2', file=opts.timelist) + print(file=opts.timelist) + print('[Files]', file=opts.timelist) + + for entry in recursedir(skip=opts.skip_files): + # opts.filelist.write(entry.path + '\n') + print(entry.path, file=opts.filelist) + if entry.name in opts.checksum_files: + checksums[entry.path[2:]] = True + info = entry.stat(follow_symlinks=False) + modtime = max(info.st_mtime, info.st_ctime) + size = info.st_size + ftype = get_ftype(entry) + # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:])) + print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist) + + print('\n[Checksums SHA1]', file=opts.timelist) + + # It's OK if the checksum section is empty, but we should include it anyway + # as the client expects it. + for f in sorted(checksums): + print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist) + + print('\n[End]', file=opts.timelist) + + +if __name__ == '__main__': + main() diff --git a/files/scripts/update-fullfiletimelist b/files/scripts/update-fullfiletimelist new file mode 100755 index 0000000..016ca8e --- /dev/null +++ b/files/scripts/update-fullfiletimelist @@ -0,0 +1,150 @@ +#!/bin/bash + +# Note: this is only an example of how you'd call create-filelist. Edit to fit +# your requirements. Note that you must supply a valid path for the lockfile, +# and it must be outside of your repository unless you want that lockfile to +# show up in your file lists. + +# Takes a list of module names. Generates file lists for all of them and them +# moves them into place at once. If you are creating hardlinks between rsync +# modules, it is required that you update the file lists of both mirrors at the +# same time. Otherwise the clients may make separate copies of the files. + +# The directory where all of the modules live +# Or pass it with -t +TOPD=/srv/mirror/pub + +# The modules to process. Or pass them on the command line. +MODS=() + +# Path to the create-filelist program. +# Or specify it with -p. +CREATE=/usr/local/bin/create-filelist + +# These strings will be eval'ed later with $mod replaced by its value in +# context. +FILELIST=fullfilelist +TIMELIST='fullfiletimelist-$mod' + +usage () { + echo + echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module [module ...]" + echo + echo " -l: Path to the lock file" + echo " -p: Path to the create-filelist program" + echo " -t: Path to directory containing modules" + echo + echo "At least one module to process must be provided." + echo "All paths must be absolute." +} + +while [[ $# > 0 ]]; do + opt=$1 + case $opt in + -l) + LOCKFILE=$(realpath $2) + shift + ;; + -p) + CREATE=$(realpath $2) + shift + ;; + -t) + TOPD=$(realpath $2) + shift + ;; + -*) + (>&2 echo "Unknown option $opt."; usage) + exit 1 + ;; + *) # Remaining args are modules + MODS+=($opt) + ;; + esac + shift +done + +if [[ -z $LOCKFILE ]]; then + (>&2 echo "Must specify LOCKFILE, either by editing the source or via the -l option."; usage) + exit 2 +fi +if [[ ! -d $(dirname $LOCKFILE) ]]; then + (>&2 echo "Given directory $(dirname $LOCKFILE) does not exist."; usage) + exit 2 +fi +if [[ ! -f $CREATE ]]; then + (>&2 echo "Specified executable $CREATE does not exist."; usage) + exit 2 +fi + +if [[ ! -d $TOPD ]]; then + (>&2 echo "Provided directory $TOPD does not exist."; usage) + exit 2 +fi + +if [[ ${#MODS[@]} -eq 0 ]]; then + (>&2 echo "No modules specified"; usage) + exit 2 +fi + +tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX) +if [[ $? -ne 0 ]]; then + (>&2 echo "Creating temporary directory failed?") + exit 1 +fi +trap "rm -rf $tmpd" EXIT +cd $tmpd + +( + # We want to wait forever until we can do what we're asked + flock -x 9 + + # If you don't want to wait forever, try one of the following: + # flock -n 9 || exit 1 - Gives up immediately + # flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up + # Don't change the '9', unless you change the last line of this script. + + for mod in $MODS; do + currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} + currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} + flname=$(basename $currentfl) + tlname=$(basename $currenttl) + + $CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname + + # If a file list exsts and doesn't differ from what we just generated, + # delete the latter. + if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null; then + rm -f $flname + fi + if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null; then + rm -f $tlname + fi + done + + # Now we have the new file lists but in a temporary directory which + # probably isn't on the same filesystem. Copy them to temporary files in + # the right place. + for mod in $MODS; do + currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} + currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} + flname=$(basename $currentfl) + fldir=$(dirname $currentfl) + tlname=$(basename $currenttl) + tldir=$(dirname $currenttl) + + if [[ -f $flname ]]; then + tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX) + cp -p $flname $tmpf + chmod 644 $tmpf + mv $tmpf $currentfl + fi + if [[ -f $tlname ]]; then + tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX) + cp -p $tlname $tmpf + chmod 644 $tmpf + mv $tmpf $currenttl + fi + done + +) 9>$LOCKFILE