diff options
Diffstat (limited to 'server/mrrepo')
| -rwxr-xr-x | server/mrrepo | 220 |
1 files changed, 163 insertions, 57 deletions
diff --git a/server/mrrepo b/server/mrrepo index 029ce35..1bb3989 100755 --- a/server/mrrepo +++ b/server/mrrepo @@ -6,14 +6,15 @@ # # Will first download (via http or https if -s specified) the manifest file # from git.example.org which should list all publicly available repositories. -# It will then mirror each remote repository in /var/scm locally, using the -# git protocol. +# It will then pull-mirror each remote repository locally in /var/scm using +# the git protocol. # -# Afterwards it may mirror it further to a remote repository that can be -# specified in the manifest file. If mirroring via https, then you also most -# likely need to provide credentials for the remote https URLs in the -# mrrepo-config file. This file should be placed next to and will be sourced -# by the mrrepo script (remember to adjust its permissions). +# Afterwards it may push-mirror them as well as local repositories (specified +# in the local manifest) further to a remote repository that can be specified +# in the manifest files. If mirroring via https, then you also most likely +# need to provide credentials for the remote https URLs in the mrrepo-config +# file. This file should be placed next to and will be sourced by the mrrepo +# script (remember to adjust its permissions). # # The manifest file line format (lines starting with # are ignored): # @@ -120,9 +121,14 @@ function fetch () # <url> [<curl-options>] curl "${curl_ops[@]}" "$@" "$u" } -fetch "$prot://$host/manifest" -z manifest -o manifest +fetch "$prot://$host/manifest" -z remote.manifest -o remote.manifest -function field () # <line> <num> [<name>] +function manifest_filter () # <file> +{ + sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' "$1" +} + +function manifest_field () # <line> <num> [<name>] { local r r="$(echo "$1 " | cut -d ' ' -f "$2")" @@ -133,66 +139,141 @@ function field () # <line> <num> [<name>] echo "$r" } -# Collect new repositories (in the new array) and while at it fix up remote -# URLs with credentials (in the auth_remotes map). Note that we still save -# original remote URLs to use them for diagnostics not to expose credentials -# (think about cron job diagnostics sent by email). +# Collect remote repositories (in the remote array) and while at it fix up +# push URLs with credentials (in the push_auth map). Note that we also save +# the original push URLs (in push_orig) to use them for diagnostics so that we +# don't expose credentials (think about cron job diagnostics sent by email). # -new=() -declare -A orig_remotes -declare -A auth_remotes +remote=() +declare -A push_orig +declare -A push_auth + +function push_add () # <rep> <url> +{ + # Note that currently we only support adding credentials for https URLs. + # + local r="$1" + local u="$2" + + push_orig["$r"]="$u" + + local p c + for p in "${!credentials[@]}"; do + if [[ "$u" == "$p"* ]]; then + c="${credentials[$p]}" + u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')" + break; + fi + done + + push_auth["$r"]="$u" +} while read l || [ -n "$l" ]; do - r=$(field "$l" 1 'path') - u=$(field "$l" 2) + r=$(manifest_field "$l" 1 'path') + u=$(manifest_field "$l" 2) - new+=("$r") + remote+=("$r") - # If the remote URL is specified then add credentials into it, if found. - # Note that currently we only support adding credentials for https URLs. + # If the push URL is specified then add it to auth/orig maps. # if [ -n "$u" ]; then - orig_remotes["$r"]="$u" + push_add "$r" "$u" + fi +done < <(manifest_filter remote.manifest) + +# Find all the existing repositories (directories that end with .git) and sort +# them out into mirrored and local public. Note that local private will end up +# in the mirrored array and will require ad hoc handling. +# +all=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -)) + +mirror=() +local=() + +# If we have local manifest, load its repositories and also verify they are +# not in remotes. Also add their push URLs similar to remotes. +# +if test -f manifest; then - for p in "${!credentials[@]}"; do - if [[ "$u" == "$p"* ]]; then - c="${credentials[$p]}" - u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')" - break; + while read l || [ -n "$l" ]; do + r=$(manifest_field "$l" 1 'path') + u=$(manifest_field "$l" 2) + + for i in "${remote[@]}"; do + if [ "$i" = "$r" ]; then + error "attempt to mirror into local public repository $r" fi done - auth_remotes["$r"]="$u" - fi -done < <(sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' manifest) + local+=("$r") -# Find all the existing repositories (directories that end with .git). -# -old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -)) + # If the push URL is specified then add it to auth/orig maps. + # + if [ -n "$u" ]; then + push_add "$r" "$u" + fi + done < <(manifest_filter manifest) + + # Everything that is not in local is mirrored (or local private). + # + for r in "${all[@]}"; do + + for i in "${local[@]}"; do + if [ "$i" = "$r" ]; then + if [ "$verb" -ge 1 ]; then + info "local public repository $r" + fi + r= + break + fi + done + + if [ -n "$r" ]; then + mirror+=("$r") + fi + done +else + mirror=("${all[@]}") +fi git_ops=() if [ "$verb" -eq 0 ]; then git_ops+=(-q) fi -for r in "${new[@]}"; do +for r in "${remote[@]}"; do + + # Zap empty directories. + # if [ -d "$r" ]; then if [ -z "$(ls -A "$r")" ]; then rm -r "$r" fi fi + if [ ! -d "$r" ]; then + if [ "$verb" -ge 1 ]; then - info "new repository $r in manifest, cloning" + info "new repository $r in remote manifest, cloning" info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r" fi + mkdir -p "$r" git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r" # Also copy the description file. # fetch "$prot://$host/$r/description" -o "$r/description" + else + + # Make sure it is not a local private repository. + # + if test ! -f "$r/git-daemon-export-ok"; then + error "attempt to mirror into local private repository $r" + fi + if [ "$verb" -ge 1 ]; then info "existing repository $r, fetching" info git -C "$r" fetch "${git_ops[@]}" --prune --tags @@ -204,9 +285,46 @@ for r in "${new[@]}"; do fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description" fi - # Mirror to the remote URL, if present. + # Mark as public. # - au="${auth_remotes[$r]}" + if test ! -f "$r/git-daemon-export-ok"; then + touch "$r/git-daemon-export-ok" + fi +done + +# Remove old mirrored repositories. +# +for o in "${mirror[@]}"; do + + # Don't touch if it's local private repository. + # + if test ! -f "$o/git-daemon-export-ok"; then + if [ "$verb" -ge 1 ]; then + info "skipping local private repository $o" + fi + continue + fi + + for i in "${remote[@]}"; do + if [ "$i" = "$o" ]; then + o= + break + fi + done + + if [ -n "$o" ]; then + if [ "$verb" -ge 1 ]; then + info "repository $o is no longer in remote manifest, removing" + fi + rm -rf "$o" + fi +done + +# Mirror to the push URLs. +# +for r in "${!push_auth[@]}"; do + + au="${push_auth[$r]}" if [ -n "$au" ]; then cmd=( git -C "$r" push "${git_ops[@]}" --mirror "$au" ) @@ -225,30 +343,18 @@ for r in "${new[@]}"; do # credentials. It may potentially appear in git's STDERR, so we replace all # its occurrences with the original one, not containing credentials. # - ou="${orig_remotes[$r]}" + ou="${push_orig[$r]}" if [ "$au" != "$ou" ]; then - GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s%$au%$ou%g" >&2 - else - GIT_TERMINAL_PROMPT=0 "${cmd[@]}" - fi - fi -done -# Remove old repositories. -# -for o in "${old[@]}"; do - for n in "${new[@]}"; do - if [ "$o" = "$n" ]; then - o= - break - fi - done + # Escape special characters in sed pattern/substitution. + # + au="$(sed -e 's/[].*/\[]/\\&/g' <<<"$au")" + ou="$(sed -e 's/[&/\]/\\&/g' <<<"$ou")" - if [ -n "$o" ]; then - if [ "$verb" -ge 1 ]; then - info "repository $o is no longer in manifest, removing" + GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s/$au/$ou/g" >&2 + else + GIT_TERMINAL_PROMPT=0 "${cmd[@]}" fi - rm -rf "$o" fi done |