better concurrency debugging, --concurrency flag for npx quartz build

.gitignore

@@ -5,3 +5,4 @@ public
 tsconfig.tsbuildinfo
 .obsidian
 .quartz-cache
+private/

content/build.md

@@ -24,3 +24,4 @@ Once you're happy with it, let's see how to [[hosting|deploy Quartz to the web]]
 > - `-o` or `--output`: the output folder. This is normally just `public`
 > - `--serve`: run a local hot-reloading server to preview your Quartz
 > - `--port`: what port to run the local preview server on
+> - `--concurrency`: how many threads to use to parse notes

quartz/bootstrap-cli.mjs

@@ -84,6 +84,10 @@ const BuildArgv = {
     default: false,
     describe: "show detailed bundle information",
   },
+  concurrency: {
+    number: true,
+    describe: "how many threads to use to parse notes"
+  }
 }
 
 function escapePath(fp) {

quartz/build.ts

@@ -1,19 +1,5 @@
 import sourceMapSupport from "source-map-support"
-sourceMapSupport.install({
+sourceMapSupport.install(options)
-  retrieveSourceMap(source) {
-    // source map hack to get around query param
-    // import cache busting
-    if (source.includes(".quartz-cache")) {
-      let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
-      return {
-        map: fs.readFileSync(realSource, "utf8"),
-      }
-    } else {
-      return null
-    }
-  },
-})
-
 import path from "path"
 import { PerfTimer } from "./perf"
 import { rimraf } from "rimraf"
@@ -23,14 +9,13 @@ import { parseMarkdown } from "./processors/parse"
 import { filterContent } from "./processors/filter"
 import { emitContent } from "./processors/emit"
 import cfg from "../quartz.config"
-import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path"
+import { FilePath, joinSegments, slugifyFilePath } from "./path"
 import chokidar from "chokidar"
 import { ProcessedContent } from "./plugins/vfile"
 import { Argv, BuildCtx } from "./ctx"
 import { glob, toPosixPath } from "./glob"
 import { trace } from "./trace"
-import { fileURLToPath } from "url"
+import { options } from "./sourcemap"
-import fs from "fs"
 
 async function buildQuartz(argv: Argv, clientRefresh: () => void) {
   const ctx: BuildCtx = {

quartz/ctx.ts

@@ -7,6 +7,7 @@ export interface Argv {
   output: string
   serve: boolean
   port: number
+  concurrency?: number
 }
 
 export interface BuildCtx {

quartz/processors/parse.ts

@@ -56,6 +56,8 @@ async function transpileWorkerScript() {
     platform: "node",
     format: "esm",
     packages: "external",
+    sourcemap: true,
+    sourcesContent: false,
     plugins: [
       {
         name: "css-and-scripts-as-text",
@@ -116,7 +118,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
   const log = new QuartzLogger(argv.verbose)
 
   const CHUNK_SIZE = 128
-  let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
+  let concurrency = ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
 
   let res: ProcessedContent[] = []
   log.start(`Parsing input files using ${concurrency} threads`)
@@ -142,7 +144,11 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
       childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
     }
 
-    const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
+    const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
+      const errString = err.toString().slice("Error:".length)
+      console.error(errString)
+      process.exit(1)
+    })
     res = results.flat()
     await pool.terminate()
   }

quartz/sourcemap.ts

@@ -0,0 +1,19 @@
+import fs from "fs"
+import sourceMapSupport from "source-map-support"
+import { fileURLToPath } from "url"
+
+export const options: sourceMapSupport.Options = {
+  // source map hack to get around query param
+  // import cache busting
+  retrieveSourceMap(source) {
+    if (source.includes(".quartz-cache")) {
+      let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
+      return {
+        map: fs.readFileSync(realSource, "utf8"),
+      }
+    } else {
+      return null
+    }
+  },
+}
+

quartz/trace.ts

@@ -1,17 +1,22 @@
 import chalk from "chalk"
 import process from "process"
+import { isMainThread } from "workerpool"
 
 const rootFile = /.*at file:/
 export function trace(msg: string, err: Error) {
   const stack = err.stack
-  console.log()
+
-  console.log(
+  const lines: string[] = []
+
+  lines.push("")
+  lines.push(
     "\n" +
     chalk.bgRed.black.bold(" ERROR ") +
     "\n" +
     chalk.red(` ${msg}`) +
     (err.message.length > 0 ? `: ${err.message}` : ""),
   )
+
   if (!stack) {
     return
   }
@@ -23,11 +28,20 @@ export function trace(msg: string, err: Error) {
     }
 
     if (!line.includes("node_modules")) {
-      console.log(` ${line}`)
+      lines.push(` ${line}`)
       if (rootFile.test(line)) {
         reachedEndOfLegibleTrace = true
       }
     }
   }
+
+  const traceMsg = lines.join("\n")
+  if (!isMainThread) {
+    // gather lines and throw
+    throw new Error(traceMsg)
+  } else {
+    // print and exit
+    console.error(traceMsg)
     process.exit(1)
+  }
 }

quartz/worker.ts

@@ -1,7 +1,10 @@
+import sourceMapSupport from "source-map-support"
+sourceMapSupport.install(options)
 import cfg from "../quartz.config"
 import { Argv, BuildCtx } from "./ctx"
 import { FilePath, ServerSlug } from "./path"
 import { createFileParser, createProcessor } from "./processors/parse"
+import { options } from "./sourcemap"
 
 // only called from worker thread
 export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {