#!/usr/bin/env bash #< # Download all public repositories for a given user or organization on GitHub. # # Usage: download-github-repos [<option>...] <who> -- <git arguments> # download-github-repos -h # # Options: # -d <outdir> Save downloaded repositories under <outdir>. # -f Force downloading into non-empty directory. # -p <parallelism> Run <parallelism> instances of git clone. # # When arguments to git clone are not provided, this passes --quiet. # # Commands: # -h Show this help. # # Example: # download-github-repos lockss -- -q --recurse-submodules #> declare -r GITHUB_API=https://api.github.com declare -r EX_USAGE=64 EX_DATAERR=65 fail() { printf "%s\\n" "$2" >&2 exit $1 } get_repo_urls() { local org=$1 n page # It seems that the "users" API includes both users and organizations; # whereas the "orgs" API includes only the latter. n=$(curl -s "$GITHUB_API/users/$org" | jq .public_repos) [[ ! $n =~ ^[0-9]+$ ]] && exit 1 # Math is hard. for ((page = 1; page <= (n + 99) / 100; page++)); do curl -s "$GITHUB_API/users/$org/repos?page=$page&per_page=100" | jq -r ".[].ssh_url" done } set -euo pipefail shopt -s nullglob dotglob # # Process arguments. # declare force=0 declare parallelism=4 declare outdir="" declare org="" while getopts d:fp:h opt; do case $opt in d) outdir=$OPTARG ;; f) force=1 ;; p) [[ $OPTARG =~ ^[0-9]+$ ]] || fail $EX_USAGE "Bad value for -p." parallelism=$OPTARG ;; h|*) sed -ne '/^#</,/^#>/ { /^#\(<\|>\)/d; s/^# \?//; p; }' -- "$0" [[ $opt == "?" ]] && exit $EX_USAGE exit 0 ;; esac done shift $((OPTIND - 1)) [[ $# == 1 || $# -gt 1 && $2 == "--" ]] || fail $EX_USAGE "Wrong number of arguments." org=$1; shift [[ $org =~ ^[A-Za-z0-9_./-]+$ ]] || fail $EX_USAGE "Bad org name." : "${outdir:=$org}" if [[ ! -d $outdir ]]; then mkdir -p -- "$outdir" elif ((!force)); then if compgen -G "$outdir/*" >/dev/null; then fail $EX_DATAERR \ "Output directory ${outdir@Q} is not empty. (Use -f to force.)" fi fi if [[ ${1-} == "--" ]]; then shift else set -- --quiet fi type jq &>/dev/null || fail "Could not find required executable jq." # # Main. # cd -- "$outdir" get_repo_urls "$org" | xargs -r -I"{}" -P "$parallelism" git clone "{}" "$@"