tomoyanonymous/quartz-research-note
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

profiling, better concurrency heuristics

Browse Source
This commit is contained in:
Jacky Zhao 2023-08-09 00:26:33 -07:00
parent 486bd9358c
commit 1b37945b13
4 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
.gitignore
node_modules
public
prof
tsconfig.tsbuildinfo
.obsidian
.quartz-cache

2
content/advanced/making plugins.md
View File

@ -2,7 +2,7 @@
title: Making your own plugins
---
This part of the documentation will assume you have some basic coding knowledge and will include code snippets that describe the interface of what Quartz plugins should look like.
This part of the documentation will assume you have some basic coding knowledge in TypeScript and will include code snippets that describe the interface of what Quartz plugins should look like.
![[quartz-transform-pipeline.png]]

3
package.json
View File

@ -14,7 +14,8 @@
"scripts": {
"check": "tsc --noEmit && npx prettier . --check",
"format": "npx prettier . --write",
"test": "tsx ./quartz/path.test.ts"
"test": "tsx ./quartz/path.test.ts",
"profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1"
},
"keywords": [
"site generator",

9
quartz/processors/parse.ts
View File

@ -9,7 +9,6 @@ import { PerfTimer } from "../perf"
import { read } from "to-vfile"
import { FilePath, QUARTZ, slugifyFilePath } from "../path"
import path from "path"
import os from "os"
import workerpool, { Promise as WorkerPromise } from "workerpool"
import { QuartzLogger } from "../log"
import { trace } from "../trace"
@ -82,6 +81,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
const res: ProcessedContent[] = []
for (const fp of fps) {
try {
const perf = new PerfTimer()
const file = await read(fp)
// strip leading and trailing whitespace
@ -101,7 +101,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
res.push([newAst, file])
if (argv.verbose) {
console.log(`[process] ${fp} -> ${file.data.slug}`)
console.log(`[process] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
}
} catch (err) {
trace(`\nFailed to process \`${fp}\``, err as Error)
@ -112,14 +112,15 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
}
}
const clamp = (num: number, min: number, max: number) => Math.min(Math.max(Math.round(num), min), max);
export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
const { argv } = ctx
const perf = new PerfTimer()
const log = new QuartzLogger(argv.verbose)
// rough heuristics: 128 gives enough time for v8 to JIT and optimize parsing code paths
const CHUNK_SIZE = 128
let concurrency =
ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
const concurrency = ctx.argv.concurrency ?? clamp(fps.length / CHUNK_SIZE, 1, 4)
let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`)