From 799e715b5e0119fe8dafc873da137ea98c088f4b Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Mon, 29 Jul 2024 15:51:40 -0700 Subject: [PATCH] ci[patch]: Adds CI action validating new notebooks (#6263) * Adds CI action validating new notebooks * Update job * Hard throw * Fix script * Make sync * Fail * Fix * Format * Fix * Fix * Remove test notebook * Fix * Fix --- .github/workflows/validate_new_notebooks.yml | 49 ++++++++++++++++++++ docs/core_docs/scripts/quarto-build.js | 2 +- docs/core_docs/scripts/validate_notebook.ts | 39 +++++++++------- package.json | 1 + 4 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/validate_new_notebooks.yml diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml new file mode 100644 index 000000000000..9bd7ae6b153a --- /dev/null +++ b/.github/workflows/validate_new_notebooks.yml @@ -0,0 +1,49 @@ +name: Validate new notebooks + +# If another push to the same PR or branch happens while this workflow is still running, +# cancel the earlier run in favor of the next run. +# +# There's no point in testing an outdated version of the code. GitHub only allows +# a limited number of job runners to be active at the same time, so it's better to cancel +# pointless jobs early so that more useful jobs can run sooner. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - 'docs/core_docs/**' + workflow_dispatch: + +jobs: + validate-new-notebooks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + - name: Validate new notebooks + run: | + new_notebooks=$(echo '${{ steps.changed-files.outputs.added_files }}' | tr ' ' '\n' | grep '\.ipynb$' || true) + echo "New notebooks: $new_notebooks" + if [ -n "$new_notebooks" ]; then + for notebook in $new_notebooks; do + yarn notebook:validate "$notebook" + done + else + echo "No new notebooks to validate." + fi \ No newline at end of file diff --git a/docs/core_docs/scripts/quarto-build.js b/docs/core_docs/scripts/quarto-build.js index 9397d972b592..e502be91dd7d 100644 --- a/docs/core_docs/scripts/quarto-build.js +++ b/docs/core_docs/scripts/quarto-build.js @@ -2,7 +2,7 @@ const fs = require("node:fs"); const { glob } = require("glob"); const { execSync } = require("node:child_process"); -const IGNORED_CELL_REGEX = /```\w*?\n\/\/ ?@ls-docs-hide-cell\n[\s\S]*?```/g; +const IGNORED_CELL_REGEX = /```\w*?\n\/\/ ?@lc-docs-hide-cell\n[\s\S]*?```/g; async function main() { const allIpynb = await glob("./docs/**/*.ipynb"); diff --git a/docs/core_docs/scripts/validate_notebook.ts b/docs/core_docs/scripts/validate_notebook.ts index d5483c8c2333..00ba61aa30d0 100644 --- a/docs/core_docs/scripts/validate_notebook.ts +++ b/docs/core_docs/scripts/validate_notebook.ts @@ -1,9 +1,8 @@ -import * as fs from "node:fs/promises"; +import * as fs from "node:fs"; import * as ts from "typescript"; -import { v4 as uuidv4 } from "uuid"; -export async function extract(filepath: string) { - const cells = JSON.parse((await fs.readFile(filepath)).toString()).cells; +export function extract(filepath: string) { + const cells = JSON.parse(fs.readFileSync(filepath).toString()).cells; const code = cells .map((cell: Record) => { if (cell.cell_type === "code") { @@ -33,13 +32,11 @@ const run = async () => { [pathname.split("/").length - 1].replace(".ipynb", ".mts"); const tempFilepath = `./tmp/${filename}`; try { - const typescriptSource = await extract(pathname); - try { - await fs.access("./tmp", fs.constants.F_OK); - } catch (err) { - await fs.mkdir("./tmp"); + const typescriptSource = extract(pathname); + if (!fs.existsSync("./tmp")) { + fs.mkdirSync("./tmp"); } - await fs.writeFile(tempFilepath, typescriptSource); + fs.writeFileSync(tempFilepath, typescriptSource); const program = ts.createProgram([tempFilepath], { module: ts.ModuleKind.NodeNext, moduleResolution: ts.ModuleResolutionKind.NodeNext, @@ -48,6 +45,7 @@ const run = async () => { skipLibCheck: true, }); const diagnostics = ts.getPreEmitDiagnostics(program); + const issueStrings: string[] = []; if (diagnostics.length === 0) { console.log("No type errors found."); } else { @@ -59,27 +57,36 @@ const run = async () => { diagnostic.messageText, "\n" ); - console.log( + issueStrings.push( `${diagnostic.file.fileName} (${line + 1},${ character + 1 }): ${message}` ); } else { - console.log( + issueStrings.push( ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") ); } }); } - } catch (e) { - console.log(e); + if (issueStrings.length) { + const issues = issueStrings.join("\n"); + console.error(issues); + const err = new Error("Found type errors in new notebook."); + (err as any).details = issues; + throw err; + } } finally { try { - await fs.rm(tempFilepath); + fs.rmSync(tempFilepath); } catch (e) { // Do nothing } } }; -run(); +try { + run(); +} catch { + process.exit(1); +} diff --git a/package.json b/package.json index 7c45191ba7be..1f8124266a93 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:standard:int": "turbo test:standard:int", "test:standard": "yarn test:standard:unit && yarn test:standard:int", "example": "yarn workspace examples start", + "notebook:validate": "yarn workspace core_docs validate", "precommit": "turbo precommit", "docs": "yarn workspace core_docs start", "docs:api_refs": "yarn workspace api_refs start",