rewrite

.github/workflows/archive.yaml

@@ -1,27 +1,35 @@
-name: Run Archive
+name: Archive
 
 on:
-  # once a month
+  # run once a month
   schedule:
     - cron: "0 0 1 * *"
-  # on demand
+  # run on manual trigger
   workflow_dispatch:
 
+env:
+  FORCE_COLOR: true
+  GITHUB_USER: "${{ secrets.GITHUB_USER }}"
+  GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+  SOFTWARE_HERITAGE_TOKEN: "${{ secrets.SOFTWARE_HERITAGE_TOKEN }}"
+
 jobs:
-  run-bot:
+  archive:
+    timeout-minutes: 60
     runs-on: ubuntu-latest
-      # secrets to environment variables
-    environment: backup
-    env:
-      SOFTWARE_HERITAGE_TOKEN: ${{ secrets.SOFTWARE_HERITAGE_TOKEN }}
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
+
       - name: Set up Node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: "12.x"
+          node-version: "18"
+
       - name: Install packages
-        run: yarn install
+        run: npm install
+
       - name: Run archive
-        run: yarn archive
+        run: npm run archive
+        timeout-minutes: 60

.gitignore

@@ -1,4 +1,2 @@
 node_modules
 .env
-clone
-clone.sh

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "plugins": ["@ianvs/prettier-plugin-sort-imports"],
+  "importOrder": ["^[a-zA-Z]", "^@[a-zA-Z]", "^./"]
+}

README.md

@@ -1,22 +1,50 @@
-# GitHub Org Backup
+# GitHub Backup
 
-This repo contains methods to look up all of a GitHub organization's repos and wikis, and then either **🗄️ archive** them by submitting them to [Software Heritage](https://softwareheritage.org/), or **⬇️ clone** them locally.
+This repo contains scripts to look up all of a GitHub user's or organization's repos and wikis, and then either **🗄️ archive** them by submitting them to [Software Heritage](https://softwareheritage.org/), or **⬇️ clone** them locally.
+
+## Better Alternative
+
+**Before you use these scripts**, see if this **better alternative works** for you.
+This has the following limitations:
+
+- Only works for GitHub organizations, not users.
+- Doesn't backup wikis.
+- Only submits to Software Heritage.
+
+Software Heritage [has an API endpoint](https://docs.softwareheritage.org/user/save_code_now/webhooks/index.html#github) that listens for a webhook event and automatically submits the associated repo to be archived.
+GitHub also allows you to [configure webhooks at an organization level](https://docs.github.com/en/webhooks-and-events/webhooks/creating-webhooks), so that an event is fired any time there is a push (or other activity) in any of the org's repos.
+
+Combining these two features, you can easily set up automatic archival for all of your org's repos:
+
+1. Go to your GitHub organization's settings.
+2. Look for the "webhooks" settings.
+3. Add a new webhook.
+   1. Set the "payload URL" to be [Software Heritage's API endpoint](https://docs.softwareheritage.org/user/save_code_now/webhooks/index.html#github).
+   2. Set the "content type" as "application/json".
+   3. Change what events trigger the webhook as needed.
+
+This has several benefits over the scripts contained in this repo:
+
+- Only runs when there are changes to a repo, not periodically.
+- Only runs on the repo that has changed, instead of all of the repos in the org.
+- Don't have to worry about GitHub eventually stopping periodic runs in stale repos (see note below).
+- Don't need a dedicated fork of this repo in your org, just need to configure a few org settings.
 
 ## Usage
 
 ### Run Locally
 
 1. [Install Node](https://nodejs.org/en).
 2. Clone this repo locally and `cd` into the directory.
-3. Run `npm install` to install necessary Node packages.
+3. Run `npm install` to install necessary packages.
 4. Run `npm run archive` or `npm run clone` to **archive** or **clone**.
 
 ### Run on GitHub Actions
 
-1. Fork this repository to your own organization.
+1. Fork this repository to your own user or organization.
 2. The GitHub Actions configuration included in this repo will automatically [run the **archive** script periodically](../../actions).
 
-**Note**: GitHub eventually stops running periodic Actions if your repo hasn't had any activity in a few months.
+**Note**: GitHub eventually stops running periodic (cron) workflows if your repo hasn't had any activity in a few months.
 To get around this, you can either make a commit to your repo every so often, or you can [manually trigger the run of the **archive** workflow](https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow).
 
 ## Environment Variables
@@ -26,20 +54,17 @@ For these scripts to work properly, you need to provide some information and acc
 When running locally, put these variables in a `.env` file in the root of the repo:
 
 ```
-GITHUB_ORGANIZATION=greenelab
+GITHUB_USER=greenelab
 GITHUB_TOKEN=XXXXXXXX
 SOFTWARE_HERITAGE_TOKEN=XXXXXXXX
 ```
 
-When running on GitHub actions:
-
-1. [Create an environment](https://docs.github.com/en/actions/reference/environments#creating-an-environment) with the name `backup`.
-2. [Create an environment secret](https://docs.github.com/en/actions/reference/environments#environment-secrets) for each variable you want to set.
+When running on GitHub actions, [create a repository secret](https://docs.github.com/en/actions/security-guides/encrypted-secrets) for each variable you want to set.
 
-#### `GITHUB_ORGANIZATION`
+#### `GITHUB_USER`
 
-The GitHub organization you want to backup, e.g., "greenelab". 
-When running on GitHub Actions, this can be omitted to automatically use the current org running the workflow.
+The GitHub user or organization you want to backup, e.g., "greenelab".
+When running on GitHub Actions, this can be omitted to automatically use the current user or organization running the workflow.
 
 #### `GITHUB_TOKEN`
 
@@ -48,4 +73,4 @@ When running on GitHub Actions, this can be omitted to automatically use the [Gi
 
 #### `SOFTWARE_HERITAGE_TOKEN`
 
-A [Software Heritage authentication/API token](https://archive.softwareheritage.org/api/#authentication).
+A [Software Heritage authentication/API token](https://archive.softwareheritage.org/oidc/profile/#tokens).

archive.ts

@@ -0,0 +1,10 @@
+import { getRepos } from "./github";
+import { archiveRepos } from "./software-heritage";
+
+/** archive script */
+const archive = async () => {
+  const repos = await getRepos();
+  await archiveRepos(repos);
+};
+
+archive();

clone.ts

@@ -0,0 +1,24 @@
+import { execSync } from "child_process";
+import { githubUser } from "./env";
+import { getRepos } from "./github";
+import { info } from "./util";
+
+/** clone directory */
+const dir = `../${githubUser}-clone`;
+
+/** clone repos locally */
+const cloneRepos = async (urls: string[]) => {
+  info(`Cloning repos to ${dir}`);
+
+  /** run commands to clone */
+  execSync(`mkdir -p ${dir}`);
+  for (const url of urls) execSync(`git clone ${url}`, { cwd: dir });
+};
+
+/** clone script */
+const clone = async () => {
+  const repos = await getRepos();
+  await cloneRepos(repos);
+};
+
+clone();

env.ts

@@ -0,0 +1,30 @@
+import { info } from "./util";
+
+/** load env vars from local .env file */
+require("dotenv").config();
+
+/** github user or organization name, e.g. "greenelab" */
+const githubUser =
+  /** read from explicitly set env var */
+  process.env.GITHUB_USER ||
+  /** or, read from env var set by github actions */
+  (process.env.GITHUB_REPOSITORY || "").split("/")[0] ||
+  "";
+
+/** github auth token */
+const githubToken = process.env.GITHUB_TOKEN || "";
+/** software heritage auth/api token */
+const softwareHeritageToken = process.env.SOFTWARE_HERITAGE_TOKEN || "";
+
+/** log key */
+const logKey = (name = "", value = "", truncate = true) =>
+  info(
+    `${name}: ${truncate && value ? `"...${value.slice(-4)}"` : `"${value}"`}`,
+  );
+
+/** log keys */
+logKey("GITHUB_USER", githubUser, false);
+logKey("GITHUB_TOKEN", githubToken);
+logKey("SOFTWARE_HERITAGE_TOKEN", softwareHeritageToken);
+
+export { githubUser, githubToken, softwareHeritageToken };