better concurrency debugging, --concurrency flag for npx quartz build

This commit is contained in:
Jacky Zhao 2023-08-08 22:52:49 -07:00
parent e4950e06a1
commit 49bd6bc3ff
9 changed files with 62 additions and 28 deletions

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ public
tsconfig.tsbuildinfo tsconfig.tsbuildinfo
.obsidian .obsidian
.quartz-cache .quartz-cache
private/

View File

@ -24,3 +24,4 @@ Once you're happy with it, let's see how to [[hosting|deploy Quartz to the web]]
> - `-o` or `--output`: the output folder. This is normally just `public` > - `-o` or `--output`: the output folder. This is normally just `public`
> - `--serve`: run a local hot-reloading server to preview your Quartz > - `--serve`: run a local hot-reloading server to preview your Quartz
> - `--port`: what port to run the local preview server on > - `--port`: what port to run the local preview server on
> - `--concurrency`: how many threads to use to parse notes

View File

@ -84,6 +84,10 @@ const BuildArgv = {
default: false, default: false,
describe: "show detailed bundle information", describe: "show detailed bundle information",
}, },
concurrency: {
number: true,
describe: "how many threads to use to parse notes"
}
} }
function escapePath(fp) { function escapePath(fp) {

View File

@ -1,19 +1,5 @@
import sourceMapSupport from "source-map-support" import sourceMapSupport from "source-map-support"
sourceMapSupport.install({ sourceMapSupport.install(options)
retrieveSourceMap(source) {
// source map hack to get around query param
// import cache busting
if (source.includes(".quartz-cache")) {
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
return {
map: fs.readFileSync(realSource, "utf8"),
}
} else {
return null
}
},
})
import path from "path" import path from "path"
import { PerfTimer } from "./perf" import { PerfTimer } from "./perf"
import { rimraf } from "rimraf" import { rimraf } from "rimraf"
@ -23,14 +9,13 @@ import { parseMarkdown } from "./processors/parse"
import { filterContent } from "./processors/filter" import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit" import { emitContent } from "./processors/emit"
import cfg from "../quartz.config" import cfg from "../quartz.config"
import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path" import { FilePath, joinSegments, slugifyFilePath } from "./path"
import chokidar from "chokidar" import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile" import { ProcessedContent } from "./plugins/vfile"
import { Argv, BuildCtx } from "./ctx" import { Argv, BuildCtx } from "./ctx"
import { glob, toPosixPath } from "./glob" import { glob, toPosixPath } from "./glob"
import { trace } from "./trace" import { trace } from "./trace"
import { fileURLToPath } from "url" import { options } from "./sourcemap"
import fs from "fs"
async function buildQuartz(argv: Argv, clientRefresh: () => void) { async function buildQuartz(argv: Argv, clientRefresh: () => void) {
const ctx: BuildCtx = { const ctx: BuildCtx = {

View File

@ -7,6 +7,7 @@ export interface Argv {
output: string output: string
serve: boolean serve: boolean
port: number port: number
concurrency?: number
} }
export interface BuildCtx { export interface BuildCtx {

View File

@ -56,6 +56,8 @@ async function transpileWorkerScript() {
platform: "node", platform: "node",
format: "esm", format: "esm",
packages: "external", packages: "external",
sourcemap: true,
sourcesContent: false,
plugins: [ plugins: [
{ {
name: "css-and-scripts-as-text", name: "css-and-scripts-as-text",
@ -116,7 +118,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
const log = new QuartzLogger(argv.verbose) const log = new QuartzLogger(argv.verbose)
const CHUNK_SIZE = 128 const CHUNK_SIZE = 128
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism() let concurrency = ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
let res: ProcessedContent[] = [] let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`) log.start(`Parsing input files using ${concurrency} threads`)
@ -142,7 +144,11 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs])) childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
} }
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises) const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
const errString = err.toString().slice("Error:".length)
console.error(errString)
process.exit(1)
})
res = results.flat() res = results.flat()
await pool.terminate() await pool.terminate()
} }

19
quartz/sourcemap.ts Normal file
View File

@ -0,0 +1,19 @@
import fs from "fs"
import sourceMapSupport from "source-map-support"
import { fileURLToPath } from "url"
export const options: sourceMapSupport.Options = {
// source map hack to get around query param
// import cache busting
retrieveSourceMap(source) {
if (source.includes(".quartz-cache")) {
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
return {
map: fs.readFileSync(realSource, "utf8"),
}
} else {
return null
}
},
}

View File

@ -1,17 +1,22 @@
import chalk from "chalk" import chalk from "chalk"
import process from "process" import process from "process"
import { isMainThread } from "workerpool"
const rootFile = /.*at file:/ const rootFile = /.*at file:/
export function trace(msg: string, err: Error) { export function trace(msg: string, err: Error) {
const stack = err.stack const stack = err.stack
console.log()
console.log( const lines: string[] = []
lines.push("")
lines.push(
"\n" + "\n" +
chalk.bgRed.black.bold(" ERROR ") + chalk.bgRed.black.bold(" ERROR ") +
"\n" + "\n" +
chalk.red(` ${msg}`) + chalk.red(` ${msg}`) +
(err.message.length > 0 ? `: ${err.message}` : ""), (err.message.length > 0 ? `: ${err.message}` : ""),
) )
if (!stack) { if (!stack) {
return return
} }
@ -23,11 +28,20 @@ export function trace(msg: string, err: Error) {
} }
if (!line.includes("node_modules")) { if (!line.includes("node_modules")) {
console.log(` ${line}`) lines.push(` ${line}`)
if (rootFile.test(line)) { if (rootFile.test(line)) {
reachedEndOfLegibleTrace = true reachedEndOfLegibleTrace = true
} }
} }
} }
process.exit(1)
const traceMsg = lines.join("\n")
if (!isMainThread) {
// gather lines and throw
throw new Error(traceMsg)
} else {
// print and exit
console.error(traceMsg)
process.exit(1)
}
} }

View File

@ -1,7 +1,10 @@
import sourceMapSupport from "source-map-support"
sourceMapSupport.install(options)
import cfg from "../quartz.config" import cfg from "../quartz.config"
import { Argv, BuildCtx } from "./ctx" import { Argv, BuildCtx } from "./ctx"
import { FilePath, ServerSlug } from "./path" import { FilePath, ServerSlug } from "./path"
import { createFileParser, createProcessor } from "./processors/parse" import { createFileParser, createProcessor } from "./processors/parse"
import { options } from "./sourcemap"
// only called from worker thread // only called from worker thread
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) { export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {