Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/usr/bin/env bash
#<
# Download all public repositories for a given user or organization on GitHub.
#
# Usage: download-github-repos [<option>...] <who> -- <git arguments>
# download-github-repos -h
#
# Options:
# -d <outdir> Save downloaded repositories under <outdir>.
# -f Force downloading into non-empty directory.
# -p <parallelism> Run <parallelism> instances of git clone.
#
# When arguments to git clone are not provided, this passes --quiet.
#
# Commands:
# -h Show this help.
#
# Example:
# download-github-repos lockss -- -q --recurse-submodules
#>
declare -r GITHUB_API=https://api.github.com
declare -r EX_USAGE=64 EX_DATAERR=65
fail() {
printf "%s\\n" "$2" >&2
exit $1
}
get_repo_urls() {
local org=$1 n page
# It seems that the "users" API includes both users and organizations;
# whereas the "orgs" API includes only the latter.
n=$(curl -s "$GITHUB_API/users/$org" | jq .public_repos)
[[ ! $n =~ ^[0-9]+$ ]] && exit 1
# Math is hard.
for ((page = 1; page <= (n + 99) / 100; page++)); do
curl -s "$GITHUB_API/users/$org/repos?page=$page&per_page=100" |
jq -r ".[].ssh_url"
done
}
set -euo pipefail
shopt -s nullglob dotglob
#
# Process arguments.
#
declare force=0
declare parallelism=4
declare outdir=""
declare org=""
while getopts d:fp:h opt; do
case $opt in
d) outdir=$OPTARG ;;
f) force=1 ;;
p)
[[ $OPTARG =~ ^[0-9]+$ ]] || fail $EX_USAGE "Bad value for -p."
parallelism=$OPTARG ;;
h|*)
sed -ne '/^#</,/^#>/ { /^#\(<\|>\)/d; s/^# \?//; p; }' -- "$0"
[[ $opt == "?" ]] && exit $EX_USAGE
exit 0 ;;
esac
done
shift $((OPTIND - 1))
[[ $# == 1 || $# -gt 1 && $2 == "--" ]] ||
fail $EX_USAGE "Wrong number of arguments."
org=$1; shift
[[ $org =~ ^[A-Za-z0-9_./-]+$ ]] || fail $EX_USAGE "Bad org name."
: "${outdir:=$org}"
if [[ ! -d $outdir ]]; then
mkdir -p -- "$outdir"
elif ((!force)); then
if compgen -G "$outdir/*" >/dev/null; then
fail $EX_DATAERR \
"Output directory ${outdir@Q} is not empty. (Use -f to force.)"
fi
fi
if [[ ${1-} == "--" ]]; then
shift
else
set -- --quiet
fi
type jq &>/dev/null || fail "Could not find required executable jq."
#
# Main.
#
cd -- "$outdir"
get_repo_urls "$org" | xargs -r -I"{}" -P "$parallelism" git clone "{}" "$@"