Skip to content
Snippets Groups Projects
Commit 10a45cee authored by McConahy, Renee Margaret's avatar McConahy, Renee Margaret
Browse files

Add script to clone entire GitHub organizations.

I found this to be most useful when I was trying to find things in the
LOCKSS codebase.
parent 8efb46c6
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/env bash
#<
# Download all public repositories for a given user or organization on GitHub.
#
# Usage: download-github-repos [<option>...] <who> -- <git arguments>
# download-github-repos -h
#
# Options:
# -d <outdir> Save downloaded repositories under <outdir>.
# -f Force downloading into non-empty directory.
# -p <parallelism> Run <parallelism> instances of git clone.
#
# When arguments to git clone are not provided, this passes --quiet.
#
# Commands:
# -h Show this help.
#
# Example:
# download-github-repos lockss -- -q --recurse-submodules
#>
declare -r GITHUB_API=https://api.github.com
declare -r EX_USAGE=64 EX_DATAERR=65
fail() {
printf "%s\\n" "$2" >&2
exit $1
}
get_repo_urls() {
local org=$1 n page
# It seems that the "users" API includes both users and organizations;
# whereas the "orgs" API includes only the latter.
n=$(curl -s "$GITHUB_API/users/$org" | jq .public_repos)
[[ ! $n =~ ^[0-9]+$ ]] && exit 1
# Math is hard.
for ((page = 1; page <= (n + 99) / 100; page++)); do
curl -s "$GITHUB_API/users/$org/repos?page=$page&per_page=100" |
jq -r ".[].ssh_url"
done
}
set -euo pipefail
shopt -s nullglob dotglob
#
# Process arguments.
#
declare force=0
declare parallelism=4
declare outdir=""
declare org=""
while getopts d:fp:h opt; do
case $opt in
d) outdir=$OPTARG ;;
f) force=1 ;;
p)
[[ $OPTARG =~ ^[0-9]+$ ]] || fail $EX_USAGE "Bad value for -p."
parallelism=$OPTARG ;;
h|*)
sed -ne '/^#</,/^#>/ { /^#\(<\|>\)/d; s/^# \?//; p; }' -- "$0"
[[ $opt == "?" ]] && exit $EX_USAGE
exit 0 ;;
esac
done
shift $((OPTIND - 1))
[[ $# == 1 || $# -gt 1 && $2 == "--" ]] ||
fail $EX_USAGE "Wrong number of arguments."
org=$1; shift
[[ $org =~ ^[A-Za-z0-9_./-]+$ ]] || fail $EX_USAGE "Bad org name."
: "${outdir:=$org}"
if [[ ! -d $outdir ]]; then
mkdir -p -- "$outdir"
elif ((!force)); then
if compgen -G "$outdir/*" >/dev/null; then
fail $EX_DATAERR \
"Output directory ${outdir@Q} is not empty. (Use -f to force.)"
fi
fi
if [[ ${1-} == "--" ]]; then
shift
else
set -- --quiet
fi
type jq &>/dev/null || fail "Could not find required executable jq."
#
# Main.
#
cd -- "$outdir"
get_repo_urls "$org" | xargs -r -I"{}" -P "$parallelism" git clone "{}" "$@"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment