profiling, better concurrency heuristics

.gitignore

@@ -2,6 +2,7 @@
 .gitignore
 node_modules
 public
+prof
 tsconfig.tsbuildinfo
 .obsidian
 .quartz-cache

content/advanced/making plugins.md

@@ -2,7 +2,7 @@
 title: Making your own plugins
 ---
 
-This part of the documentation will assume you have some basic coding knowledge and will include code snippets that describe the interface of what Quartz plugins should look like.
+This part of the documentation will assume you have some basic coding knowledge in TypeScript and will include code snippets that describe the interface of what Quartz plugins should look like.
 
 ![[quartz-transform-pipeline.png]]
 

package.json

@@ -14,7 +14,8 @@
   "scripts": {
     "check": "tsc --noEmit && npx prettier . --check",
     "format": "npx prettier . --write",
-    "test": "tsx ./quartz/path.test.ts"
+    "test": "tsx ./quartz/path.test.ts",
+    "profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1"
   },
   "keywords": [
     "site generator",

quartz/processors/parse.ts

@@ -9,7 +9,6 @@ import { PerfTimer } from "../perf"
 import { read } from "to-vfile"
 import { FilePath, QUARTZ, slugifyFilePath } from "../path"
 import path from "path"
-import os from "os"
 import workerpool, { Promise as WorkerPromise } from "workerpool"
 import { QuartzLogger } from "../log"
 import { trace } from "../trace"
@@ -82,6 +81,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
     const res: ProcessedContent[] = []
     for (const fp of fps) {
       try {
+        const perf = new PerfTimer()
         const file = await read(fp)
 
         // strip leading and trailing whitespace
@@ -101,7 +101,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
         res.push([newAst, file])
 
         if (argv.verbose) {
-          console.log(`[process] ${fp} -> ${file.data.slug}`)
+          console.log(`[process] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
         }
       } catch (err) {
         trace(`\nFailed to process \`${fp}\``, err as Error)
@@ -112,14 +112,15 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
   }
 }
 
+const clamp = (num: number, min: number, max: number) => Math.min(Math.max(Math.round(num), min), max);
 export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
   const { argv } = ctx
   const perf = new PerfTimer()
   const log = new QuartzLogger(argv.verbose)
 
+  // rough heuristics: 128 gives enough time for v8 to JIT and optimize parsing code paths
   const CHUNK_SIZE = 128
-  let concurrency =
-    ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
+  const concurrency = ctx.argv.concurrency ?? clamp(fps.length / CHUNK_SIZE, 1, 4)
 
   let res: ProcessedContent[] = []
   log.start(`Parsing input files using ${concurrency} threads`)