better concurrency debugging, --concurrency flag for npx quartz build

This commit is contained in:
Jacky Zhao 2023-08-08 22:52:49 -07:00
parent d159375804
commit cfdb71d00d
9 changed files with 62 additions and 28 deletions

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ public
tsconfig.tsbuildinfo
.obsidian
.quartz-cache
private/

View File

@ -24,3 +24,4 @@ Once you're happy with it, let's see how to [[hosting|deploy Quartz to the web]]
> - `-o` or `--output`: the output folder. This is normally just `public`
> - `--serve`: run a local hot-reloading server to preview your Quartz
> - `--port`: what port to run the local preview server on
> - `--concurrency`: how many threads to use to parse notes

View File

@ -84,6 +84,10 @@ const BuildArgv = {
default: false,
describe: "show detailed bundle information",
},
concurrency: {
number: true,
describe: "how many threads to use to parse notes"
}
}
function escapePath(fp) {

View File

@ -1,19 +1,5 @@
import sourceMapSupport from "source-map-support"
sourceMapSupport.install({
retrieveSourceMap(source) {
// source map hack to get around query param
// import cache busting
if (source.includes(".quartz-cache")) {
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
return {
map: fs.readFileSync(realSource, "utf8"),
}
} else {
return null
}
},
})
sourceMapSupport.install(options)
import path from "path"
import { PerfTimer } from "./perf"
import { rimraf } from "rimraf"
@ -23,14 +9,13 @@ import { parseMarkdown } from "./processors/parse"
import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit"
import cfg from "../quartz.config"
import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path"
import { FilePath, joinSegments, slugifyFilePath } from "./path"
import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile"
import { Argv, BuildCtx } from "./ctx"
import { glob, toPosixPath } from "./glob"
import { trace } from "./trace"
import { fileURLToPath } from "url"
import fs from "fs"
import { options } from "./sourcemap"
async function buildQuartz(argv: Argv, clientRefresh: () => void) {
const ctx: BuildCtx = {

View File

@ -7,6 +7,7 @@ export interface Argv {
output: string
serve: boolean
port: number
concurrency?: number
}
export interface BuildCtx {

View File

@ -56,6 +56,8 @@ async function transpileWorkerScript() {
platform: "node",
format: "esm",
packages: "external",
sourcemap: true,
sourcesContent: false,
plugins: [
{
name: "css-and-scripts-as-text",
@ -116,7 +118,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
const log = new QuartzLogger(argv.verbose)
const CHUNK_SIZE = 128
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
let concurrency = ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`)
@ -142,7 +144,11 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
}
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
const errString = err.toString().slice("Error:".length)
console.error(errString)
process.exit(1)
})
res = results.flat()
await pool.terminate()
}

19
quartz/sourcemap.ts Normal file
View File

@ -0,0 +1,19 @@
import fs from "fs"
import sourceMapSupport from "source-map-support"
import { fileURLToPath } from "url"
export const options: sourceMapSupport.Options = {
// source map hack to get around query param
// import cache busting
retrieveSourceMap(source) {
if (source.includes(".quartz-cache")) {
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
return {
map: fs.readFileSync(realSource, "utf8"),
}
} else {
return null
}
},
}

View File

@ -1,17 +1,22 @@
import chalk from "chalk"
import process from "process"
import { isMainThread } from "workerpool"
const rootFile = /.*at file:/
export function trace(msg: string, err: Error) {
const stack = err.stack
console.log()
console.log(
const lines: string[] = []
lines.push("")
lines.push(
"\n" +
chalk.bgRed.black.bold(" ERROR ") +
"\n" +
chalk.red(` ${msg}`) +
(err.message.length > 0 ? `: ${err.message}` : ""),
chalk.bgRed.black.bold(" ERROR ") +
"\n" +
chalk.red(` ${msg}`) +
(err.message.length > 0 ? `: ${err.message}` : ""),
)
if (!stack) {
return
}
@ -23,11 +28,20 @@ export function trace(msg: string, err: Error) {
}
if (!line.includes("node_modules")) {
console.log(` ${line}`)
lines.push(` ${line}`)
if (rootFile.test(line)) {
reachedEndOfLegibleTrace = true
}
}
}
process.exit(1)
const traceMsg = lines.join("\n")
if (!isMainThread) {
// gather lines and throw
throw new Error(traceMsg)
} else {
// print and exit
console.error(traceMsg)
process.exit(1)
}
}

View File

@ -1,7 +1,10 @@
import sourceMapSupport from "source-map-support"
sourceMapSupport.install(options)
import cfg from "../quartz.config"
import { Argv, BuildCtx } from "./ctx"
import { FilePath, ServerSlug } from "./path"
import { createFileParser, createProcessor } from "./processors/parse"
import { options } from "./sourcemap"
// only called from worker thread
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {