8:36 a.m.
commit b3757e3260fd214eccdf30a3648f0b8619b765c1
Author: Nicolas Chauvet <kwizart(a)gmail.com>
Date: Thu Nov 10 09:36:14 2016 +0100
Add scripts
files/scripts/create-filelist | 130 ++++++++++++++++++++++++++++
files/scripts/update-fullfiletimelist | 150 +++++++++++++++++++++++++++++++++
2 files changed, 280 insertions(+), 0 deletions(-)
---
diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist
new file mode 100755
index 0000000..eeba9d0
--- /dev/null
+++ b/files/scripts/create-filelist
@@ -0,0 +1,130 @@
+#!/usr/bin/python
+from __future__ import print_function
+
+# A simple script to generate a file list in a format easily consumable by a
+# shell script.
+
+# Originally written by Jason Tibbitts <tibbs(a)math.uh.edu> in 2016.
+# Donated to the public domain. If you require a statement of license, please
+# consider this work to be licensed as "CC0 Universal", any version you
choose.
+
+import argparse
+import hashlib
+import os
+import sys
+from scandir import scandir
+
+
+def get_ftype(entry):
+ """Return a simple indicator of the file type."""
+ if entry.is_symlink():
+ return 'l'
+ if entry.is_dir():
+ return 'd'
+ return 'f'
+
+
+def sha1(fname):
+ """Return the SHA1 checksum of a file in hex."""
+ fh = open(fname, 'rb')
+ sha1 = hashlib.sha1()
+ block = fh.read(2 ** 16)
+ while len(block) > 0:
+ sha1.update(block)
+ block = fh.read(2 ** 16)
+
+ return sha1.hexdigest()
+
+
+def recursedir(path='.', skip=[], alwaysskip=['.~tmp~']):
+ """Just like scandir, but recursively.
+
+ Will skip everything in the skip array, but only at the top level
+ directory.
+ """
+ for entry in scandir(path):
+ if entry.name in skip:
+ continue
+ if entry.name in alwaysskip:
+ continue
+ if entry.is_dir(follow_symlinks=False):
+ # Don't pass skip here, because we only skip in the top level
+ for rentry in recursedir(entry.path, alwaysskip=alwaysskip):
+ yield rentry
+ yield entry
+
+
+def parseopts():
+ null = open(os.devnull, 'w')
+ p = argparse.ArgumentParser(
+ description='Generate a list of files and times, suitable for consumption by
quick-fedora-mirror.')
+ p.add_argument('-c', '--checksum', action='store_true',
+ help='Include checksums of all repomd.xml files in the file
list.')
+ p.add_argument('-C', '--checksum-file', action='append',
dest='checksum_files',
+ help='Include checksums of all instances of the specified
file.')
+ p.add_argument('-s', '--skip', action='store_true',
+ help='Skip the file lists in the top directory')
+ p.add_argument('-S', '--skip-file', action='append',
dest='skip_files',
+ help='Skip the specified file in the top directory.')
+
+ p.add_argument('-d', '--dir', help='Directory to scan (default:
.).')
+
+ p.add_argument('-t', '--timelist',
type=argparse.FileType('w'), default=sys.stdout,
+ help='Filename of the file list with times (default:
stdout).')
+ p.add_argument('-f', '--filelist',
type=argparse.FileType('w'), default=null,
+ help='Filename of the file list without times (default: no plain
file list is generated).')
+
+ opts = p.parse_args()
+
+ if not opts.dir:
+ opts.dir = '.'
+
+ opts.checksum_files = opts.checksum_files or []
+ if opts.checksum:
+ opts.checksum_files += ['repomd.xml']
+
+ opts.skip_files = opts.skip_files or []
+ if opts.skip:
+ if not opts.timelist.name == '<stdout>':
+ opts.skip_files += [opts.timelist.name]
+ if not opts.filelist.name == '<stdout>':
+ opts.skip_files += [opts.filelist.name]
+
+ return opts
+
+
+def main():
+ opts = parseopts()
+ checksums = {}
+
+ os.chdir(opts.dir)
+
+ print('[Version]', file=opts.timelist)
+ print('2', file=opts.timelist)
+ print(file=opts.timelist)
+ print('[Files]', file=opts.timelist)
+
+ for entry in recursedir(skip=opts.skip_files):
+ # opts.filelist.write(entry.path + '\n')
+ print(entry.path, file=opts.filelist)
+ if entry.name in opts.checksum_files:
+ checksums[entry.path[2:]] = True
+ info = entry.stat(follow_symlinks=False)
+ modtime = max(info.st_mtime, info.st_ctime)
+ size = info.st_size
+ ftype = get_ftype(entry)
+ # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype,
entry.path[2:]))
+ print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]),
file=opts.timelist)
+
+ print('\n[Checksums SHA1]', file=opts.timelist)
+
+ # It's OK if the checksum section is empty, but we should include it anyway
+ # as the client expects it.
+ for f in sorted(checksums):
+ print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist)
+
+ print('\n[End]', file=opts.timelist)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/files/scripts/update-fullfiletimelist
b/files/scripts/update-fullfiletimelist
new file mode 100755
index 0000000..016ca8e
--- /dev/null
+++ b/files/scripts/update-fullfiletimelist
@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# Note: this is only an example of how you'd call create-filelist. Edit to fit
+# your requirements. Note that you must supply a valid path for the lockfile,
+# and it must be outside of your repository unless you want that lockfile to
+# show up in your file lists.
+
+# Takes a list of module names. Generates file lists for all of them and them
+# moves them into place at once. If you are creating hardlinks between rsync
+# modules, it is required that you update the file lists of both mirrors at the
+# same time. Otherwise the clients may make separate copies of the files.
+
+# The directory where all of the modules live
+# Or pass it with -t
+TOPD=/srv/mirror/pub
+
+# The modules to process. Or pass them on the command line.
+MODS=()
+
+# Path to the create-filelist program.
+# Or specify it with -p.
+CREATE=/usr/local/bin/create-filelist
+
+# These strings will be eval'ed later with $mod replaced by its value in
+# context.
+FILELIST=fullfilelist
+TIMELIST='fullfiletimelist-$mod'
+
+usage () {
+ echo
+ echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module
[module ...]"
+ echo
+ echo " -l: Path to the lock file"
+ echo " -p: Path to the create-filelist program"
+ echo " -t: Path to directory containing modules"
+ echo
+ echo "At least one module to process must be provided."
+ echo "All paths must be absolute."
+}
+
+while [[ $# > 0 ]]; do
+ opt=$1
+ case $opt in
+ -l)
+ LOCKFILE=$(realpath $2)
+ shift
+ ;;
+ -p)
+ CREATE=$(realpath $2)
+ shift
+ ;;
+ -t)
+ TOPD=$(realpath $2)
+ shift
+ ;;
+ -*)
+ (>&2 echo "Unknown option $opt."; usage)
+ exit 1
+ ;;
+ *) # Remaining args are modules
+ MODS+=($opt)
+ ;;
+ esac
+ shift
+done
+
+if [[ -z $LOCKFILE ]]; then
+ (>&2 echo "Must specify LOCKFILE, either by editing the source or via the
-l option."; usage)
+ exit 2
+fi
+if [[ ! -d $(dirname $LOCKFILE) ]]; then
+ (>&2 echo "Given directory $(dirname $LOCKFILE) does not exist.";
usage)
+ exit 2
+fi
+if [[ ! -f $CREATE ]]; then
+ (>&2 echo "Specified executable $CREATE does not exist."; usage)
+ exit 2
+fi
+
+if [[ ! -d $TOPD ]]; then
+ (>&2 echo "Provided directory $TOPD does not exist."; usage)
+ exit 2
+fi
+
+if [[ ${#MODS[@]} -eq 0 ]]; then
+ (>&2 echo "No modules specified"; usage)
+ exit 2
+fi
+
+tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
+if [[ $? -ne 0 ]]; then
+ (>&2 echo "Creating temporary directory failed?")
+ exit 1
+fi
+trap "rm -rf $tmpd" EXIT
+cd $tmpd
+
+(
+ # We want to wait forever until we can do what we're asked
+ flock -x 9
+
+ # If you don't want to wait forever, try one of the following:
+ # flock -n 9 || exit 1 - Gives up immediately
+ # flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up
+ # Don't change the '9', unless you change the last line of this script.
+
+ for mod in $MODS; do
+ currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
+ currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+ flname=$(basename $currentfl)
+ tlname=$(basename $currenttl)
+
+ $CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname
+
+ # If a file list exsts and doesn't differ from what we just generated,
+ # delete the latter.
+ if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null;
then
+ rm -f $flname
+ fi
+ if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null;
then
+ rm -f $tlname
+ fi
+ done
+
+ # Now we have the new file lists but in a temporary directory which
+ # probably isn't on the same filesystem. Copy them to temporary files in
+ # the right place.
+ for mod in $MODS; do
+ currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
+ currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+ flname=$(basename $currentfl)
+ fldir=$(dirname $currentfl)
+ tlname=$(basename $currenttl)
+ tldir=$(dirname $currenttl)
+
+ if [[ -f $flname ]]; then
+ tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX)
+ cp -p $flname $tmpf
+ chmod 644 $tmpf
+ mv $tmpf $currentfl
+ fi
+ if [[ -f $tlname ]]; then
+ tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX)
+ cp -p $tlname $tmpf
+ chmod 644 $tmpf
+ mv $tmpf $currenttl
+ fi
+ done
+
+) 9>$LOCKFILE