diff --git a/package-lock.json b/package-lock.json index e53a3e58..a635676a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "@napi-rs/simple-git": "^0.1.8", "chalk": "^4.1.2", "cli-spinner": "^0.2.10", - "env-paths": "^3.0.0", "esbuild-sass-plugin": "^2.9.0", "github-slugger": "^2.0.0", "globby": "^13.1.4", @@ -34,29 +33,29 @@ "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "remark-obsidian-callout": "^1.1.3", "remark-parse": "^10.0.1", "remark-rehype": "^10.1.0", "remark-smartypants": "^2.0.0", - "require-from-string": "^2.0.2", "rimraf": "^5.0.1", "serve-handler": "^6.1.5", - "shiki": "^0.14.2", "to-vfile": "^7.2.4", "unified": "^10.1.2", "unist-util-visit": "^4.1.2", "vfile": "^5.3.7", + "workerpool": "^6.4.0", "yargs": "^17.7.2" }, "bin": { - "quartz": "quartz/bootstrap.mjs" + "quartz": "quartz/bootstrap-cli.mjs" }, "devDependencies": { "@types/cli-spinner": "^0.2.1", "@types/hast": "^2.3.4", "@types/node": "^20.1.2", "@types/pretty-time": "^1.1.2", - "@types/require-from-string": "^1.2.1", "@types/serve-handler": "^6.1.1", + "@types/workerpool": "^6.4.0", "@types/yargs": "^17.0.24", "esbuild": "^0.17.18", "typescript": "^5.0.4" @@ -938,12 +937,6 @@ "integrity": "sha512-4i+Y+O5H80Rh01lY/3Z0hB/UWc4R64ReE83joEpVsIG3iQWpYx66k6pQh1amJNZquKtJQyu/RcfkTtvL0KwssA==", "dev": true }, - "node_modules/@types/require-from-string": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/require-from-string/-/require-from-string-1.2.1.tgz", - "integrity": "sha512-mIDK7lTHc0uW67SxPIqkwCrxmdKBV5aAET560hyZnT8c6Ekp9Aah3GPqe8Pl1Yzn/i2NMYmYv+HiMLwjGDCIAQ==", - "dev": true - }, "node_modules/@types/serve-handler": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/@types/serve-handler/-/serve-handler-6.1.1.tgz", @@ -958,6 +951,15 @@ "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz", "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==" }, + "node_modules/@types/workerpool": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@types/workerpool/-/workerpool-6.4.0.tgz", + "integrity": "sha512-SIF2/169pDsLKeM8GQGHkOFifGalDbZgiBSaLUnnlVSRsAOenkAvQ6h4uhV2W+PZZczS+8LQxACwNkSykdT91A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/yargs": { "version": "17.0.24", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", @@ -998,7 +1000,8 @@ "node_modules/ansi-sequence-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.0.tgz", - "integrity": "sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==" + "integrity": "sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==", + "peer": true }, "node_modules/ansi-styles": { "version": "4.3.0", @@ -1347,17 +1350,6 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/env-paths": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", - "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/esbuild": { "version": "0.17.19", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz", @@ -2114,7 +2106,8 @@ "node_modules/jsonc-parser": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", - "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==" + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "peer": true }, "node_modules/katex": { "version": "0.16.7", @@ -3412,6 +3405,14 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-obsidian-callout": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/remark-obsidian-callout/-/remark-obsidian-callout-1.1.3.tgz", + "integrity": "sha512-q2jZgPMHP5Wyq0WgeGdwboPoepqN7Ib77iWagEKThZjtl1wUd7/f2tdErdPr/LeZu9K7I6r52XylwhLs5GJi9g==", + "dependencies": { + "unist-util-visit": "^4.1.2" + } + }, "node_modules/remark-parse": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.2.tgz", @@ -3476,14 +3477,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve": { "version": "1.22.2", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", @@ -3717,6 +3710,7 @@ "version": "0.14.2", "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.2.tgz", "integrity": "sha512-ltSZlSLOuSY0M0Y75KA+ieRaZ0Trf5Wl3gutE7jzLuIcWxLp5i/uEnLoQWNvgKXQ5OMpGkJnVMRLAuzjc0LJ2A==", + "peer": true, "dependencies": { "ansi-sequence-parser": "^1.1.0", "jsonc-parser": "^3.2.0", @@ -4158,12 +4152,14 @@ "node_modules/vscode-oniguruma": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", - "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==" + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", + "peer": true }, "node_modules/vscode-textmate": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", - "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==" + "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", + "peer": true }, "node_modules/web-namespaces": { "version": "2.0.1", @@ -4188,6 +4184,11 @@ "node": ">= 8" } }, + "node_modules/workerpool": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.4.0.tgz", + "integrity": "sha512-i3KR1mQMNwY2wx20ozq2EjISGtQWDIfV56We+yGJ5yDs8jTwQiLLaqHlkBHITlCuJnYlVRmXegxFxZg7gqI++A==" + }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", diff --git a/package.json b/package.json index 83ddd542..c3bf0e2d 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "@jackyzha0/quartz", "description": "🌱 publish your digital garden and notes as a website", "version": "4.0.3", + "type": "module", "author": "jackyzha0 ", "license": "MIT", "homepage": "https://quartz.jzhao.xyz", @@ -21,14 +22,13 @@ "quartz" ], "bin": { - "quartz": "./quartz/bootstrap.mjs" + "quartz": "./quartz/bootstrap-cli.mjs" }, "dependencies": { "@inquirer/prompts": "^1.0.3", "@napi-rs/simple-git": "^0.1.8", "chalk": "^4.1.2", "cli-spinner": "^0.2.10", - "env-paths": "^3.0.0", "esbuild-sass-plugin": "^2.9.0", "github-slugger": "^2.0.0", "globby": "^13.1.4", @@ -49,17 +49,17 @@ "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "remark-obsidian-callout": "^1.1.3", "remark-parse": "^10.0.1", "remark-rehype": "^10.1.0", "remark-smartypants": "^2.0.0", - "require-from-string": "^2.0.2", "rimraf": "^5.0.1", "serve-handler": "^6.1.5", - "shiki": "^0.14.2", "to-vfile": "^7.2.4", "unified": "^10.1.2", "unist-util-visit": "^4.1.2", "vfile": "^5.3.7", + "workerpool": "^6.4.0", "yargs": "^17.7.2" }, "devDependencies": { @@ -67,8 +67,8 @@ "@types/hast": "^2.3.4", "@types/node": "^20.1.2", "@types/pretty-time": "^1.1.2", - "@types/require-from-string": "^1.2.1", "@types/serve-handler": "^6.1.1", + "@types/workerpool": "^6.4.0", "@types/yargs": "^17.0.24", "esbuild": "^0.17.18", "typescript": "^5.0.4" diff --git a/quartz/build.ts b/quartz/build.ts index 73cebce3..db8e4a97 100644 --- a/quartz/build.ts +++ b/quartz/build.ts @@ -24,10 +24,10 @@ export default async function buildQuartz(argv: Argv, version: string) { const perf = new PerfTimer() const output = argv.output + const pluginCount = Object.values(cfg.plugins).flat().length + const pluginNames = (key: 'transformers' | 'filters' | 'emitters') => cfg.plugins[key].map(plugin => plugin.name) + console.log(`Loaded ${pluginCount} plugins`) if (argv.verbose) { - const pluginCount = Object.values(cfg.plugins).flat().length - const pluginNames = (key: 'transformers' | 'filters' | 'emitters') => cfg.plugins[key].map(plugin => plugin.name) - console.log(`Loaded ${pluginCount} plugins`) console.log(` Transformers: ${pluginNames('transformers').join(", ")}`) console.log(` Filters: ${pluginNames('filters').join(", ")}`) console.log(` Emitters: ${pluginNames('emitters').join(", ")}`) @@ -37,9 +37,7 @@ export default async function buildQuartz(argv: Argv, version: string) { if (argv.clean) { perf.addEvent('clean') await rimraf(output) - if (argv.verbose) { - console.log(`Cleaned output directory \`${output}\` in ${perf.timeSince('clean')}`) - } + console.log(`Cleaned output directory \`${output}\` in ${perf.timeSince('clean')}`) } // glob @@ -49,11 +47,7 @@ export default async function buildQuartz(argv: Argv, version: string) { ignore: cfg.configuration.ignorePatterns, gitignore: true, }) - - if (argv.verbose) { - console.log(`Found ${fps.length} input files in ${perf.timeSince('glob')}`) - - } + console.log(`Found ${fps.length} input files in ${perf.timeSince('glob')}`) const filePaths = fps.map(fp => `${argv.directory}${path.sep}${fp}`) const parsedFiles = await parseMarkdown(cfg.plugins.transformers, argv.directory, filePaths, argv.verbose) diff --git a/quartz/log.ts b/quartz/log.ts new file mode 100644 index 00000000..f96cfe4c --- /dev/null +++ b/quartz/log.ts @@ -0,0 +1,26 @@ +import { Spinner } from 'cli-spinner' + +export class QuartzLogger { + verbose: boolean + spinner: Spinner | undefined + constructor(verbose: boolean) { + this.verbose = verbose + } + + start(text: string) { + if (this.verbose) { + console.log(text) + } else { + this.spinner = new Spinner(`%s ${text}`) + this.spinner.setSpinnerString(18) + this.spinner.start() + } + } + + success(text: string) { + if (!this.verbose) { + this.spinner!.stop(true) + } + console.log(text) + } +} diff --git a/quartz/processors/emit.ts b/quartz/processors/emit.ts index f9155749..3407de21 100644 --- a/quartz/processors/emit.ts +++ b/quartz/processors/emit.ts @@ -7,6 +7,7 @@ import { EmitCallback } from "../plugins/types" import { ProcessedContent } from "../plugins/vfile" import { QUARTZ, slugify } from "../path" import { globbyStream } from "globby" +import chalk from "chalk" export async function emitContent(contentFolder: string, output: string, cfg: QuartzConfig, content: ProcessedContent[], verbose: boolean) { const perf = new PerfTimer() @@ -23,13 +24,18 @@ export async function emitContent(contentFolder: string, output: string, cfg: Qu let emittedFiles = 0 for (const emitter of cfg.plugins.emitters) { - const emitted = await emitter.emit(cfg.configuration, content, staticResources, emit) - emittedFiles += emitted.length + try { + const emitted = await emitter.emit(cfg.configuration, content, staticResources, emit) + emittedFiles += emitted.length - if (verbose) { - for (const file of emitted) { - console.log(`[emit:${emitter.name}] ${file}`) + if (verbose) { + for (const file of emitted) { + console.log(`[emit:${emitter.name}] ${file}`) + } } + } catch (err) { + console.log(chalk.red(`Failed to emit from plugin \`${emitter.name}\`: `) + err) + process.exit(1) } } diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts index 715a4e97..173675ec 100644 --- a/quartz/processors/parse.ts +++ b/quartz/processors/parse.ts @@ -12,9 +12,11 @@ import path from 'path' import os from 'os' import workerpool, { Promise as WorkerPromise } from 'workerpool' import { QuartzTransformerPlugin } from '../plugins/types' +import { QuartzLogger } from '../log' +import chalk from 'chalk' export type QuartzProcessor = Processor -export function createProcessor(transformers: QuartzTransformerPlugin[]): any { +export function createProcessor(transformers: QuartzTransformerPlugin[]): QuartzProcessor { // base Markdown -> MD AST let processor = unified().use(remarkParse) @@ -41,15 +43,11 @@ function* chunks(arr: T[], n: number) { } } -async function transpileWorkerScript(verbose: boolean) { +async function transpileWorkerScript() { // transpile worker script const cacheFile = "./.quartz-cache/transpiled-worker.mjs" const fp = "./quartz/worker.ts" - if (verbose) { - console.log("Transpiling worker script") - } - - await esbuild.build({ + return esbuild.build({ entryPoints: [fp], outfile: path.join("quartz", cacheFile), bundle: true, @@ -75,31 +73,50 @@ async function transpileWorkerScript(verbose: boolean) { }) } +export function createFileParser(baseDir: string, fps: string[], verbose: boolean) { + return async (processor: QuartzProcessor) => { + const res: ProcessedContent[] = [] + for (const fp of fps) { + try { + const file = await read(fp) + + // base data properties that plugins may use + file.data.slug = slugify(path.relative(baseDir, file.path)) + file.data.filePath = fp + + const ast = processor.parse(file) + const newAst = await processor.run(ast, file) + res.push([newAst, file]) + + if (verbose) { + console.log(`[process] ${fp} -> ${file.data.slug}`) + } + } catch (err) { + console.log(chalk.red(`Failed to process \`${fp}\`: `) + err) + process.exit(1) + } + } + + return res + } +} + export async function parseMarkdown(transformers: QuartzTransformerPlugin[], baseDir: string, fps: string[], verbose: boolean): Promise { const perf = new PerfTimer() + const log = new QuartzLogger(verbose) const CHUNK_SIZE = 128 let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism() - const res: ProcessedContent[] = [] + let res: ProcessedContent[] = [] + + log.start(`Parsing input files using ${concurrency} threads`) if (concurrency === 1) { // single-thread const processor = createProcessor(transformers) - for (const fp of fps) { - const file = await read(fp) - - // base data properties that plugins may use - file.data.slug = slugify(path.relative(baseDir, file.path)) - file.data.filePath = fp - - const ast = processor.parse(file) - res.push([await processor.run(ast, file), file]) - - if (verbose) { - console.log(`[process] ${fp} -> ${file.data.slug}`) - } - } + const parse = createFileParser(baseDir, fps, verbose) + res = await parse(processor) } else { - await transpileWorkerScript(verbose) + await transpileWorkerScript() const pool = workerpool.pool( './quartz/bootstrap-worker.mjs', { @@ -113,11 +130,12 @@ export async function parseMarkdown(transformers: QuartzTransformerPlugin[], bas for (const chunk of chunks(fps, CHUNK_SIZE)) { childPromises.push(pool.exec('parseFiles', [baseDir, chunk, verbose])) } + const results: ProcessedContent[][] = await WorkerPromise.all(childPromises) - res.push(...results.flat()) + res = results.flat() await pool.terminate() } - console.log(`Parsed and transformed ${res.length} Markdown files with ${concurrency} cores in ${perf.timeSince()}`) + log.success(`Parsed ${res.length} Markdown files in ${perf.timeSince()}`) return res } diff --git a/quartz/worker.ts b/quartz/worker.ts index 71678b8e..e2c278fa 100644 --- a/quartz/worker.ts +++ b/quartz/worker.ts @@ -1,30 +1,11 @@ -import { read } from "to-vfile" import config from "../quartz.config" -import { createProcessor } from "./processors/parse" -import { slugify } from "./path" -import path from "path" -import { ProcessedContent } from "./plugins/vfile" +import { createFileParser, createProcessor } from "./processors/parse" const transformers = config.plugins.transformers const processor = createProcessor(transformers) // only called from worker thread export async function parseFiles(baseDir: string, fps: string[], verbose: boolean) { - const res: ProcessedContent[] = [] - for (const fp of fps) { - const file = await read(fp) - - // base data properties that plugins may use - file.data.slug = slugify(path.relative(baseDir, file.path)) - file.data.filePath = fp - - const ast = processor.parse(file) - res.push([await processor.run(ast, file), file]) - - if (verbose) { - console.log(`[process] ${fp} -> ${file.data.slug}`) - } - } - - return res + const parse = createFileParser(baseDir, fps, verbose) + return parse(processor) }