githubで特定のownerの持つリポジトリを全部cloneしたい

github上で特定のownerの持つrepositoryを全部cloneしたくなった。

具体的にはjupyter関連のコードを雑に手元に持っておきたかった。以下のワンライナーで済む(ただし100個以下の場合)。

$ TARGETNAME=jupyter
$ http -b --pretty=format https://api.github.com/users/${TARGETNAME}/repos sort==updated direct==desc per_page==100 | jqfpy '[f"""git clone --depth=1 {d["ssh_url"]} """ for d in get()]' --squash -r | bash

詳細

以下詳細

httpieを使っている。curlでも良い。

apiの情報は以下を見れば分かる。更新日順でsortしている。

-Repositories | GitHub Developer Guide

paginationなどのoptionは以下。一度に取れるのは100個が限界。

-Traversing with Pagination | GitHub Developer Guide

ssh_urlにclone用のURLが入っている。(e.g.)

[
    {
        "archive_url": "https://api.github.com/repos/jupyter/jupyter/{archive_format}{/ref}",
        "archived": false,
        "assignees_url": "https://api.github.com/repos/jupyter/jupyter/assignees{/user}",
        "blobs_url": "https://api.github.com/repos/jupyter/jupyter/git/blobs{/sha}",
        "branches_url": "https://api.github.com/repos/jupyter/jupyter/branches{/branch}",
        "clone_url": "https://github.com/jupyter/jupyter.git",
        "collaborators_url": "https://api.github.com/repos/jupyter/jupyter/collaborators{/collaborator}",
        "comments_url": "https://api.github.com/repos/jupyter/jupyter/comments{/number}",
        "commits_url": "https://api.github.com/repos/jupyter/jupyter/commits{/sha}",
        "compare_url": "https://api.github.com/repos/jupyter/jupyter/compare/{base}...{head}",
        "contents_url": "https://api.github.com/repos/jupyter/jupyter/contents/{+path}",
        "contributors_url": "https://api.github.com/repos/jupyter/jupyter/contributors",
        "created_at": "2015-06-04T21:05:00Z",
        "default_branch": "master",
        "deployments_url": "https://api.github.com/repos/jupyter/jupyter/deployments",
        "description": "Jupyter metapackage for installation, docs and chat",
        "downloads_url": "https://api.github.com/repos/jupyter/jupyter/downloads",
        "events_url": "https://api.github.com/repos/jupyter/jupyter/events",
        "fork": false,
        "forks": 1175,
        "forks_count": 1175,
        "forks_url": "https://api.github.com/repos/jupyter/jupyter/forks",
        "full_name": "jupyter/jupyter",
        "git_commits_url": "https://api.github.com/repos/jupyter/jupyter/git/commits{/sha}",
        "git_refs_url": "https://api.github.com/repos/jupyter/jupyter/git/refs{/sha}",
        "git_tags_url": "https://api.github.com/repos/jupyter/jupyter/git/tags{/sha}",
        "git_url": "git://github.com/jupyter/jupyter.git",
        "has_downloads": true,
        "has_issues": true,
        "has_pages": false,
        "has_projects": true,
        "has_wiki": true,
        "homepage": "https://jupyter.readthedocs.io/",
        "hooks_url": "https://api.github.com/repos/jupyter/jupyter/hooks",
        "html_url": "https://github.com/jupyter/jupyter",
        "id": 36895421,
        "issue_comment_url": "https://api.github.com/repos/jupyter/jupyter/issues/comments{/number}",
        "issue_events_url": "https://api.github.com/repos/jupyter/jupyter/issues/events{/number}",
        "issues_url": "https://api.github.com/repos/jupyter/jupyter/issues{/number}",
        "keys_url": "https://api.github.com/repos/jupyter/jupyter/keys{/key_id}",
        "labels_url": "https://api.github.com/repos/jupyter/jupyter/labels{/name}",
        "language": "Python",
        "languages_url": "https://api.github.com/repos/jupyter/jupyter/languages",
        "license": {
            "key": "bsd-3-clause",
            "name": "BSD 3-Clause \"New\" or \"Revised\" License",
            "node_id": "MDc6TGljZW5zZTU=",
            "spdx_id": "BSD-3-Clause",
            "url": "https://api.github.com/licenses/bsd-3-clause"
        },
        "merges_url": "https://api.github.com/repos/jupyter/jupyter/merges",
        "milestones_url": "https://api.github.com/repos/jupyter/jupyter/milestones{/number}",
        "mirror_url": null,
        "name": "jupyter",
        "node_id": "MDEwOlJlcG9zaXRvcnkzNjg5NTQyMQ==",
        "notifications_url": "https://api.github.com/repos/jupyter/jupyter/notifications{?since,all,participating}",
        "open_issues": 102,
        "open_issues_count": 102,
        "owner": {
            "avatar_url": "https://avatars1.githubusercontent.com/u/7388996?v=4",
            "events_url": "https://api.github.com/users/jupyter/events{/privacy}",
            "followers_url": "https://api.github.com/users/jupyter/followers",
            "following_url": "https://api.github.com/users/jupyter/following{/other_user}",
            "gists_url": "https://api.github.com/users/jupyter/gists{/gist_id}",
            "gravatar_id": "",
            "html_url": "https://github.com/jupyter",
            "id": 7388996,
            "login": "jupyter",
            "node_id": "MDEyOk9yZ2FuaXphdGlvbjczODg5OTY=",
            "organizations_url": "https://api.github.com/users/jupyter/orgs",
            "received_events_url": "https://api.github.com/users/jupyter/received_events",
            "repos_url": "https://api.github.com/users/jupyter/repos",
            "site_admin": false,
            "starred_url": "https://api.github.com/users/jupyter/starred{/owner}{/repo}",
            "subscriptions_url": "https://api.github.com/users/jupyter/subscriptions",
            "type": "Organization",
            "url": "https://api.github.com/users/jupyter"
        },
        "private": false,
        "pulls_url": "https://api.github.com/repos/jupyter/jupyter/pulls{/number}",
        "pushed_at": "2018-06-08T20:59:12Z",
        "releases_url": "https://api.github.com/repos/jupyter/jupyter/releases{/id}",
        "size": 4784,
        "ssh_url": "git@github.com:jupyter/jupyter.git",
        "stargazers_count": 5130,
        "stargazers_url": "https://api.github.com/repos/jupyter/jupyter/stargazers",
        "statuses_url": "https://api.github.com/repos/jupyter/jupyter/statuses/{sha}",
        "subscribers_url": "https://api.github.com/repos/jupyter/jupyter/subscribers",
        "subscription_url": "https://api.github.com/repos/jupyter/jupyter/subscription",
        "svn_url": "https://github.com/jupyter/jupyter",
        "tags_url": "https://api.github.com/repos/jupyter/jupyter/tags",
        "teams_url": "https://api.github.com/repos/jupyter/jupyter/teams",
        "trees_url": "https://api.github.com/repos/jupyter/jupyter/git/trees{/sha}",
        "updated_at": "2018-07-11T06:51:28Z",
        "url": "https://api.github.com/repos/jupyter/jupyter",
        "watchers": 5130,
        "watchers_count": 5130
    },
...
]

邪魔なので切る。jqでも良い

cat jupyter.json | jqfpy '[{"ssh_url": d["ssh_url"], "stargazers_count": d["stargazers_count"]} for d in get()]' --squash --compact | head -n 10
{"ssh_url": "git@github.com:jupyter/jupyter.git", "stargazers_count": 5130}
{"ssh_url": "git@github.com:jupyter/notebook.git", "stargazers_count": 4381}
{"ssh_url": "git@github.com:jupyter/docker-stacks.git", "stargazers_count": 2972}
{"ssh_url": "git@github.com:jupyter/jupyter_kernel_test.git", "stargazers_count": 29}
{"ssh_url": "git@github.com:jupyter/jupyter.github.io.git", "stargazers_count": 85}
{"ssh_url": "git@github.com:jupyter/colaboratory.git", "stargazers_count": 652}
{"ssh_url": "git@github.com:jupyter/nbdime.git", "stargazers_count": 678}
{"ssh_url": "git@github.com:jupyter/nbconvert.git", "stargazers_count": 433}
{"ssh_url": "git@github.com:jupyter/nbgrader.git", "stargazers_count": 518}
{"ssh_url": "git@github.com:jupyter/nbviewer.git", "stargazers_count": 1225}

実行したいシェルスクリプトを出力できたらpipeでbashにリダイレクトする。

作業用のMakefile

memo 作業用のMakefile

TARGETNAME ?= jupyter
URL := https://api.github.com/users/${TARGETNAME}/repos
BASH ?= cat

default: parse
fetch: $(addsuffix .json,$(addprefix data/,${TARGETNAME}))
parse: $(addsuffix .parsed.json,$(addprefix data/,${TARGETNAME}))

data/${TARGETNAME}.json:
    mkdir -p data
    http -b --pretty=format ${URL} sort==updated direct==desc per_page==100 | tee $@

data/${TARGETNAME}.parsed.json: data/${TARGETNAME}.json
    jqfpy '[h.pick("html_url", "homepage", "stargazers_count", "updated_at", d=d) for d in get()]' $^ --squash | tee $@

clean:
  rm -rf data/*

clone:
  http -b --pretty=format ${URL} sort==updated direct==desc per_page==100 | jqfpy '[f"""git clone --depth=1 {d["ssh_url"]} """ for d in get()]' --squash -r | ${BASH}