historic docs v2

docs/website/.gitignore

@@ -23,3 +23,9 @@ jaffle_shop
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# ignore all versions, there are generated dynamically
+versions.json
+versioned_docs
+versioned_sidebars
+.dlt-repo

docs/website/README.md

@@ -64,4 +64,29 @@ The site is deployed using `netlify`. The `netlify` build command is:
 npm run build:netlify
 ```
 
-It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`.
+It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`.
+
+## Docs versions
+
+We keep a few additional versions of our docs for the users to be able read about how former and future versions of dlt work. We use docusaurus versions for this but we do not check the historical versions into the repo but rather use a script to build the former versions on deployment. To locally build the versions run:
+
+```
+npm run update-versions
+```
+
+This will execute the script at tools/update_versions.js. This tool will do the following:
+
+* Find all the highest minor versions the tags of the repo (e.g. 0.4.13, 0.5.22, 1.1.3)
+* It will create a version for all of these tags that are larger than the minimum version defined in MINIMUM_SEMVER_VERSION in the script.
+* It will NOT create a version for the highest version, we assume the most up to date docs for the highest versions are the tip of master
+* It will NOT create any docs versions for pre-releases.
+* It will create a future version called "devel" from the current commit of this repo.
+* It will set up docusaurus to display all of these versions correctly.
+
+You can clear these versions with 
+
+```
+npm run clear-versions
+```
+
+The netflify deployment of these docs need to happen from the master branch so that the current version gets properly selected.

docs/website/docusaurus.config.js

@@ -1,11 +1,43 @@
 // @ts-check
 // Note: type annotations allow type checking and IDEs autocompletion
+const fs = require("fs")
 require('dotenv').config()
 
 const lightCodeTheme = require('prism-react-renderer/themes/dracula');
 // const lightCodeTheme = require('prism-react-renderer/themes/github');
 const darkCodeTheme = require('prism-react-renderer/themes/dracula');
 
+// create versions config
+const versions = {"current": {
+  label: 'devel', 
+  path: 'devel',
+  noIndex: true
+}}
+
+// inject master version renaming only if versions present
+if (fs.existsSync("versions.json")) {
+  let latestLabel = "latest"
+  if (process.env.DOCUSAURUS_DLT_VERSION) {
+    latestLabel = `${process.env.DOCUSAURUS_DLT_VERSION} (latest)`
+  }
+
+
+  versions["master"] = {
+    label: latestLabel,
+    path: '/'
+  }
+  // disable indexing for all known versions
+  for (let v of JSON.parse(fs.readFileSync("versions.json"))) {
+    if (v == "master") {
+      continue;
+    }
+    versions[v] = {
+      noIndex: true
+    }
+  }
+
+}
+
 /** @type {import('@docusaurus/types').Config} */
 const config = {
   title: 'dlt Docs',
@@ -30,8 +62,6 @@ const config = {
     locales: ['en'],
   },
 
-
-
   presets: [
     [
       '@docusaurus/preset-classic',
@@ -50,12 +80,7 @@ const config = {
           editUrl: (params) => {
             return "https://github.com/dlt-hub/dlt/tree/devel/docs/website/docs/" + params.docPath;
           },
-          versions: {
-            current: {
-              label: 'current',
-            },
-          },
-          lastVersion: 'current',
+          versions: versions,
           showLastUpdateAuthor: true,
           showLastUpdateTime: true,
         },
@@ -83,7 +108,9 @@ const config = {
           href: 'https://dlthub.com'
         },
         items: [
-          { label: 'dlt ' + (process.env.IS_MASTER_BRANCH ? "stable ": "devel ") + (process.env.DOCUSAURUS_DLT_VERSION || "0.0.1"), position: 'left', href: 'https://github.com/dlt-hub/dlt', className: 'version-navbar'  },
+          {
+            type: 'docsVersionDropdown',
+          },
           {
             type: 'doc',
             docId: 'intro',
@@ -196,15 +223,4 @@ const config = {
   ],
 };
 
-if (!process.env.IS_MASTER_BRANCH && config.themeConfig) {
-  config.themeConfig.announcementBar = {
-    id: 'devel docs',
-    content:
-      'This is the development version of the dlt docs. <a target="_blank" rel="noopener noreferrer" href="https://dlthub.com/docs/intro">Go to the stable docs.</a>',
-    backgroundColor: '#4c4898',
-    textColor: '#fff',
-    isCloseable: false,
-  }
-}
-
 module.exports = config;

docs/website/package.json

@@ -4,16 +4,18 @@
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
-    "start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"",
-    "build": "node tools/preprocess_docs.js  && PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && docusaurus build",
-    "build:netlify": "node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && node tools/update_version_env.js && docusaurus build --out-dir build/docs",
+    "start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"",
+    "build": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. poetry run pydoc-markdown && docusaurus build",
+    "build:netlify": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && docusaurus build --out-dir build/docs",
     "swizzle": "docusaurus swizzle",
     "clear": "docusaurus clear",
     "serve": "docusaurus serve",
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "preprocess-docs": "node tools/preprocess_docs.js",
-    "generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown"
+    "generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown",
+    "clear-versions": "node tools/clear_versions.js",
+    "update-versions": "node tools/update_versions.js"
   },
   "dependencies": {
     "@docusaurus/core": "2.4.3",
@@ -28,6 +30,7 @@
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-twitter-embed": "^4.0.4",
+    "semver": "^7.6.3",
     "string-dedent": "^3.0.1",
     "sync-fetch": "^0.5.2",
     "toml": "^3.0.0"

docs/website/sidebars.js

@@ -356,21 +356,4 @@ if (fs.existsSync('./docs_processed/api_reference/sidebar.json')) {
   }
 }
 
-// on the master branch link to devel and vice versa
-if (process.env.IS_MASTER_BRANCH) {
-  sidebars.tutorialSidebar.push(    {
-    type: 'link',
-    label: 'Switch to Devel Docs',
-    href: 'https://dlthub.com/devel/intro',
-    className: 'learn-more-link',
-  })
-} else {
-  sidebars.tutorialSidebar.push(    {
-    type: 'link',
-    label: 'Switch to Stable Docs',
-    href: 'https://dlthub.com/docs/intro',
-    className: 'learn-more-link',
-  })
-}
-
 module.exports = sidebars;

docs/website/tools/clear_versions.js

@@ -0,0 +1,11 @@
+const fs = require('fs');
+
+version_files = [
+    "versions.json",
+    "versioned_docs",
+    "versioned_sidebars"
+]
+
+for (let f of version_files) {
+    fs.rmSync(f, { recursive: true, force: true })
+}

docs/website/tools/update_versions.js

@@ -0,0 +1,107 @@
+const proc = require('child_process')
+const fs = require('fs');
+const semver = require('semver')
+
+
+// const
+const REPO_DIR = ".dlt-repo"
+const REPO_DOCS_DIR = REPO_DIR + "/docs/website"
+const REPO_URL = "https://github.com/dlt-hub/dlt.git"
+const VERSIONED_DOCS_FOLDER = "versioned_docs"
+const VERSIONED_SIDEBARS_FOLDER = "versioned_sidebars"
+const ENV_FILE = '.env'
+
+// no doc versions below this version will be deployed
+const MINIMUM_SEMVER_VERSION = "0.5.0"
+
+// clear old repo version
+fs.rmSync(REPO_DIR, { recursive: true, force: true })
+
+// checkout fresh
+console.log("Checking out dlt repo")
+proc.execSync(`git clone ${REPO_URL} ${REPO_DIR}`)
+
+// find tags
+console.log("Discovering versions")
+const tags = proc.execSync(`cd ${REPO_DIR} && git tag`).toString().trim().split("\n");
+console.log(`Found ${tags.length} tags`)
+
+// parse and filter invalid tags
+let versions = tags.map(v => semver.valid(v)).filter(v => v != null)
+
+// remove all tags below the min version and sort
+min_version = semver.valid(MINIMUM_SEMVER_VERSION)
+versions = semver.rsort(versions.filter(v => semver.gt(v, min_version)))
+
+// remove prelease versions
+versions.filter(v => semver.prerelease(v) == null)
+
+console.log(`Found ${versions.length} elligible versions`)
+if (versions.length < 2) {
+    console.error("Sanity check failed, not enough elligble version tags found")
+    process.exit(1)
+}
+
+// write last version into env file
+const envFileContent = `DOCUSAURUS_DLT_VERSION=${versions[0]}`;
+fs.writeFileSync(ENV_FILE, envFileContent, 'utf8');
+
+// go through the versions and find all newest versions of any major version
+// the newest version is replace by the master branch here so the master head
+// always is the "current" doc
+const selectedVersions = ["master"];
+let lastVersion = versions[0];
+for (let ver of versions) {
+    if ( semver.major(ver) != semver.major(lastVersion)) {
+        selectedVersions.push(ver)
+    }
+    lastVersion = ver;
+}
+
+console.log(`Will create docs versions for ${selectedVersions}`)
+
+// create folders
+fs.rmSync(VERSIONED_DOCS_FOLDER, { recursive: true, force: true })
+fs.rmSync(VERSIONED_SIDEBARS_FOLDER, { recursive: true, force: true })
+fs.rmSync("versions.json", { force: true })
+
+fs.mkdirSync(VERSIONED_DOCS_FOLDER);
+fs.mkdirSync(VERSIONED_SIDEBARS_FOLDER);
+
+// check that checked out repo is on devel
+console.log("Checking branch")
+const branch = proc.execSync(`cd ${REPO_DIR} && git rev-parse --abbrev-ref HEAD`).toString().trim()
+
+// sanity check
+if (branch != "devel") {
+    console.error("Could not check out devel branch")
+    process.exit(1)
+}
+
+selectedVersions.reverse()
+for (const version of selectedVersions) {
+
+    // checkout verison and verify we have the right tag
+    console.log(`Generating version ${version}, switching to tag:`)
+    proc.execSync(`cd ${REPO_DIR} && git checkout ${version}`)
+
+    // const tag = proc.execSync(`cd ${REPO_DIR} && git describe --exact-match --tags`).toString().trim()
+    // if (tag != version) {
+    //     console.error(`Could not checkout version ${version}`)
+    //     process.exit(1)
+    // }
+
+    // build doc version, we also run preprocessing and markdown gen for each doc version
+    console.log(`Building docs...`)
+    proc.execSync(`cd ${REPO_DOCS_DIR} && npm run preprocess-docs && PYTHONPATH=. pydoc-markdown`)
+
+    console.log(`Snapshotting version...`)
+    proc.execSync(`cd ${REPO_DOCS_DIR} && npm run docusaurus docs:version ${version}`)
+
+    console.log(`Moving snapshot`)
+    fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_DOCS_FOLDER, VERSIONED_DOCS_FOLDER, {recursive: true})
+    fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_SIDEBARS_FOLDER, VERSIONED_SIDEBARS_FOLDER, {recursive: true})   
+
+}
+
+fs.cpSync(REPO_DOCS_DIR+"/versions.json", "versions.json")