better concurrency debugging, --concurrency flag for npx quartz build
This commit is contained in:
parent
d159375804
commit
cfdb71d00d
1
.gitignore
vendored
1
.gitignore
vendored
@ -5,3 +5,4 @@ public
|
||||
tsconfig.tsbuildinfo
|
||||
.obsidian
|
||||
.quartz-cache
|
||||
private/
|
||||
|
@ -24,3 +24,4 @@ Once you're happy with it, let's see how to [[hosting|deploy Quartz to the web]]
|
||||
> - `-o` or `--output`: the output folder. This is normally just `public`
|
||||
> - `--serve`: run a local hot-reloading server to preview your Quartz
|
||||
> - `--port`: what port to run the local preview server on
|
||||
> - `--concurrency`: how many threads to use to parse notes
|
||||
|
@ -84,6 +84,10 @@ const BuildArgv = {
|
||||
default: false,
|
||||
describe: "show detailed bundle information",
|
||||
},
|
||||
concurrency: {
|
||||
number: true,
|
||||
describe: "how many threads to use to parse notes"
|
||||
}
|
||||
}
|
||||
|
||||
function escapePath(fp) {
|
||||
|
@ -1,19 +1,5 @@
|
||||
import sourceMapSupport from "source-map-support"
|
||||
sourceMapSupport.install({
|
||||
retrieveSourceMap(source) {
|
||||
// source map hack to get around query param
|
||||
// import cache busting
|
||||
if (source.includes(".quartz-cache")) {
|
||||
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
|
||||
return {
|
||||
map: fs.readFileSync(realSource, "utf8"),
|
||||
}
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
sourceMapSupport.install(options)
|
||||
import path from "path"
|
||||
import { PerfTimer } from "./perf"
|
||||
import { rimraf } from "rimraf"
|
||||
@ -23,14 +9,13 @@ import { parseMarkdown } from "./processors/parse"
|
||||
import { filterContent } from "./processors/filter"
|
||||
import { emitContent } from "./processors/emit"
|
||||
import cfg from "../quartz.config"
|
||||
import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path"
|
||||
import { FilePath, joinSegments, slugifyFilePath } from "./path"
|
||||
import chokidar from "chokidar"
|
||||
import { ProcessedContent } from "./plugins/vfile"
|
||||
import { Argv, BuildCtx } from "./ctx"
|
||||
import { glob, toPosixPath } from "./glob"
|
||||
import { trace } from "./trace"
|
||||
import { fileURLToPath } from "url"
|
||||
import fs from "fs"
|
||||
import { options } from "./sourcemap"
|
||||
|
||||
async function buildQuartz(argv: Argv, clientRefresh: () => void) {
|
||||
const ctx: BuildCtx = {
|
||||
|
@ -7,6 +7,7 @@ export interface Argv {
|
||||
output: string
|
||||
serve: boolean
|
||||
port: number
|
||||
concurrency?: number
|
||||
}
|
||||
|
||||
export interface BuildCtx {
|
||||
|
@ -56,6 +56,8 @@ async function transpileWorkerScript() {
|
||||
platform: "node",
|
||||
format: "esm",
|
||||
packages: "external",
|
||||
sourcemap: true,
|
||||
sourcesContent: false,
|
||||
plugins: [
|
||||
{
|
||||
name: "css-and-scripts-as-text",
|
||||
@ -116,7 +118,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
|
||||
const log = new QuartzLogger(argv.verbose)
|
||||
|
||||
const CHUNK_SIZE = 128
|
||||
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
|
||||
let concurrency = ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
|
||||
|
||||
let res: ProcessedContent[] = []
|
||||
log.start(`Parsing input files using ${concurrency} threads`)
|
||||
@ -142,7 +144,11 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
|
||||
childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
|
||||
}
|
||||
|
||||
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
|
||||
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
|
||||
const errString = err.toString().slice("Error:".length)
|
||||
console.error(errString)
|
||||
process.exit(1)
|
||||
})
|
||||
res = results.flat()
|
||||
await pool.terminate()
|
||||
}
|
||||
|
19
quartz/sourcemap.ts
Normal file
19
quartz/sourcemap.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import fs from "fs"
|
||||
import sourceMapSupport from "source-map-support"
|
||||
import { fileURLToPath } from "url"
|
||||
|
||||
export const options: sourceMapSupport.Options = {
|
||||
// source map hack to get around query param
|
||||
// import cache busting
|
||||
retrieveSourceMap(source) {
|
||||
if (source.includes(".quartz-cache")) {
|
||||
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
|
||||
return {
|
||||
map: fs.readFileSync(realSource, "utf8"),
|
||||
}
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
},
|
||||
}
|
||||
|
@ -1,17 +1,22 @@
|
||||
import chalk from "chalk"
|
||||
import process from "process"
|
||||
import { isMainThread } from "workerpool"
|
||||
|
||||
const rootFile = /.*at file:/
|
||||
export function trace(msg: string, err: Error) {
|
||||
const stack = err.stack
|
||||
console.log()
|
||||
console.log(
|
||||
|
||||
const lines: string[] = []
|
||||
|
||||
lines.push("")
|
||||
lines.push(
|
||||
"\n" +
|
||||
chalk.bgRed.black.bold(" ERROR ") +
|
||||
"\n" +
|
||||
chalk.red(` ${msg}`) +
|
||||
(err.message.length > 0 ? `: ${err.message}` : ""),
|
||||
chalk.bgRed.black.bold(" ERROR ") +
|
||||
"\n" +
|
||||
chalk.red(` ${msg}`) +
|
||||
(err.message.length > 0 ? `: ${err.message}` : ""),
|
||||
)
|
||||
|
||||
if (!stack) {
|
||||
return
|
||||
}
|
||||
@ -23,11 +28,20 @@ export function trace(msg: string, err: Error) {
|
||||
}
|
||||
|
||||
if (!line.includes("node_modules")) {
|
||||
console.log(` ${line}`)
|
||||
lines.push(` ${line}`)
|
||||
if (rootFile.test(line)) {
|
||||
reachedEndOfLegibleTrace = true
|
||||
}
|
||||
}
|
||||
}
|
||||
process.exit(1)
|
||||
|
||||
const traceMsg = lines.join("\n")
|
||||
if (!isMainThread) {
|
||||
// gather lines and throw
|
||||
throw new Error(traceMsg)
|
||||
} else {
|
||||
// print and exit
|
||||
console.error(traceMsg)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
@ -1,7 +1,10 @@
|
||||
import sourceMapSupport from "source-map-support"
|
||||
sourceMapSupport.install(options)
|
||||
import cfg from "../quartz.config"
|
||||
import { Argv, BuildCtx } from "./ctx"
|
||||
import { FilePath, ServerSlug } from "./path"
|
||||
import { createFileParser, createProcessor } from "./processors/parse"
|
||||
import { options } from "./sourcemap"
|
||||
|
||||
// only called from worker thread
|
||||
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {
|
||||
|
Loading…
Reference in New Issue
Block a user