fix watch-mode batching
This commit is contained in:
parent
145e57d4fc
commit
c60f17d101
@ -4,7 +4,6 @@ draft: true
|
|||||||
|
|
||||||
## high priority
|
## high priority
|
||||||
|
|
||||||
- back button doesn't work sometimes
|
|
||||||
- images in same folder are broken on shortest path mode
|
- images in same folder are broken on shortest path mode
|
||||||
- https://help.obsidian.md/Editing+and+formatting/Tags#Nested+tags nested tags?? and big tag listing
|
- https://help.obsidian.md/Editing+and+formatting/Tags#Nested+tags nested tags?? and big tag listing
|
||||||
- watch mode for config/source code
|
- watch mode for config/source code
|
||||||
|
@ -10,7 +10,7 @@ import { parseMarkdown } from "./processors/parse"
|
|||||||
import { filterContent } from "./processors/filter"
|
import { filterContent } from "./processors/filter"
|
||||||
import { emitContent } from "./processors/emit"
|
import { emitContent } from "./processors/emit"
|
||||||
import cfg from "../quartz.config"
|
import cfg from "../quartz.config"
|
||||||
import { FilePath } from "./path"
|
import { FilePath, slugifyFilePath } from "./path"
|
||||||
import chokidar from "chokidar"
|
import chokidar from "chokidar"
|
||||||
import { ProcessedContent } from "./plugins/vfile"
|
import { ProcessedContent } from "./plugins/vfile"
|
||||||
import WebSocket, { WebSocketServer } from "ws"
|
import WebSocket, { WebSocketServer } from "ws"
|
||||||
@ -20,6 +20,7 @@ async function buildQuartz(argv: Argv, version: string) {
|
|||||||
const ctx: BuildCtx = {
|
const ctx: BuildCtx = {
|
||||||
argv,
|
argv,
|
||||||
cfg,
|
cfg,
|
||||||
|
allSlugs: [],
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`))
|
console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`))
|
||||||
@ -51,6 +52,8 @@ async function buildQuartz(argv: Argv, version: string) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
const filePaths = fps.map((fp) => `${argv.directory}${path.sep}${fp}` as FilePath)
|
const filePaths = fps.map((fp) => `${argv.directory}${path.sep}${fp}` as FilePath)
|
||||||
|
ctx.allSlugs = fps.map((fp) => slugifyFilePath(fp as FilePath))
|
||||||
|
|
||||||
const parsedFiles = await parseMarkdown(ctx, filePaths)
|
const parsedFiles = await parseMarkdown(ctx, filePaths)
|
||||||
const filteredContent = filterContent(ctx, parsedFiles)
|
const filteredContent = filterContent(ctx, parsedFiles)
|
||||||
await emitContent(ctx, filteredContent)
|
await emitContent(ctx, filteredContent)
|
||||||
@ -74,18 +77,40 @@ async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
|
|||||||
contentMap.set(vfile.data.filePath!, content)
|
contentMap.set(vfile.data.filePath!, content)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rebuild(fp: string, action: "add" | "change" | "unlink") {
|
let timeoutId: ReturnType<typeof setTimeout> | null = null
|
||||||
const perf = new PerfTimer()
|
let toRebuild: Set<FilePath> = new Set()
|
||||||
|
let toRemove: Set<FilePath> = new Set()
|
||||||
|
async function rebuild(fp: string, action: "add" | "change" | "delete") {
|
||||||
if (!ignored(fp)) {
|
if (!ignored(fp)) {
|
||||||
console.log(chalk.yellow(`Detected change in ${fp}, rebuilding...`))
|
const filePath = `${argv.directory}${path.sep}${fp}` as FilePath
|
||||||
const fullPath = `${argv.directory}${path.sep}${fp}` as FilePath
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (action === "add" || action === "change") {
|
if (action === "add" || action === "change") {
|
||||||
const [parsedContent] = await parseMarkdown(ctx, [fullPath])
|
toRebuild.add(filePath)
|
||||||
contentMap.set(fullPath, parsedContent)
|
} else if (action === "delete") {
|
||||||
} else if (action === "unlink") {
|
toRemove.add(filePath)
|
||||||
contentMap.delete(fullPath)
|
}
|
||||||
|
|
||||||
|
if (timeoutId) {
|
||||||
|
clearTimeout(timeoutId)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeoutId = setTimeout(async () => {
|
||||||
|
const perf = new PerfTimer()
|
||||||
|
console.log(chalk.yellow("Detected change, rebuilding..."))
|
||||||
|
try {
|
||||||
|
const filesToRebuild = [...toRebuild].filter((fp) => !toRemove.has(fp))
|
||||||
|
|
||||||
|
ctx.allSlugs = [...new Set([...contentMap.keys(), ...toRebuild])]
|
||||||
|
.filter((fp) => !toRemove.has(fp))
|
||||||
|
.map((fp) => slugifyFilePath(path.relative(argv.directory, fp) as FilePath))
|
||||||
|
|
||||||
|
const parsedContent = await parseMarkdown(ctx, filesToRebuild)
|
||||||
|
for (const content of parsedContent) {
|
||||||
|
const [_tree, vfile] = content
|
||||||
|
contentMap.set(vfile.data.filePath!, content)
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const fp of toRemove) {
|
||||||
|
contentMap.delete(fp)
|
||||||
}
|
}
|
||||||
|
|
||||||
await rimraf(argv.output)
|
await rimraf(argv.output)
|
||||||
@ -96,8 +121,10 @@ async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
|
|||||||
} catch {
|
} catch {
|
||||||
console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`))
|
console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`))
|
||||||
}
|
}
|
||||||
|
|
||||||
connections.forEach((conn) => conn.send("rebuild"))
|
connections.forEach((conn) => conn.send("rebuild"))
|
||||||
|
toRebuild.clear()
|
||||||
|
toRemove.clear()
|
||||||
|
}, 250)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -110,7 +137,7 @@ async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
|
|||||||
watcher
|
watcher
|
||||||
.on("add", (fp) => rebuild(fp, "add"))
|
.on("add", (fp) => rebuild(fp, "add"))
|
||||||
.on("change", (fp) => rebuild(fp, "change"))
|
.on("change", (fp) => rebuild(fp, "change"))
|
||||||
.on("unlink", (fp) => rebuild(fp, "unlink"))
|
.on("unlink", (fp) => rebuild(fp, "delete"))
|
||||||
|
|
||||||
const server = http.createServer(async (req, res) => {
|
const server = http.createServer(async (req, res) => {
|
||||||
await serveHandler(req, res, {
|
await serveHandler(req, res, {
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
import { QuartzConfig } from "./cfg"
|
import { QuartzConfig } from "./cfg"
|
||||||
|
import { ServerSlug } from "./path"
|
||||||
|
|
||||||
export interface Argv {
|
export interface Argv {
|
||||||
directory: string
|
directory: string
|
||||||
@ -11,4 +12,5 @@ export interface Argv {
|
|||||||
export interface BuildCtx {
|
export interface BuildCtx {
|
||||||
argv: Argv
|
argv: Argv
|
||||||
cfg: QuartzConfig
|
cfg: QuartzConfig
|
||||||
|
allSlugs: ServerSlug[]
|
||||||
}
|
}
|
||||||
|
@ -20,10 +20,10 @@ type ComponentResources = {
|
|||||||
afterDOMLoaded: string[]
|
afterDOMLoaded: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
function getComponentResources(plugins: PluginTypes): ComponentResources {
|
function getComponentResources(ctx: BuildCtx): ComponentResources {
|
||||||
const allComponents: Set<QuartzComponent> = new Set()
|
const allComponents: Set<QuartzComponent> = new Set()
|
||||||
for (const emitter of plugins.emitters) {
|
for (const emitter of ctx.cfg.plugins.emitters) {
|
||||||
const components = emitter.getQuartzComponents()
|
const components = emitter.getQuartzComponents(ctx)
|
||||||
for (const component of components) {
|
for (const component of components) {
|
||||||
allComponents.add(component)
|
allComponents.add(component)
|
||||||
}
|
}
|
||||||
@ -127,7 +127,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => ({
|
|||||||
},
|
},
|
||||||
async emit(ctx, _content, resources, emit): Promise<FilePath[]> {
|
async emit(ctx, _content, resources, emit): Promise<FilePath[]> {
|
||||||
// component specific scripts and styles
|
// component specific scripts and styles
|
||||||
const componentResources = getComponentResources(ctx.cfg.plugins)
|
const componentResources = getComponentResources(ctx)
|
||||||
// important that this goes *after* component scripts
|
// important that this goes *after* component scripts
|
||||||
// as the "nav" event gets triggered here and we should make sure
|
// as the "nav" event gets triggered here and we should make sure
|
||||||
// that everyone else had the chance to register a listener for it
|
// that everyone else had the chance to register a listener for it
|
||||||
|
@ -2,7 +2,7 @@ import { QuartzFilterPlugin } from "../types"
|
|||||||
|
|
||||||
export const RemoveDrafts: QuartzFilterPlugin<{}> = () => ({
|
export const RemoveDrafts: QuartzFilterPlugin<{}> = () => ({
|
||||||
name: "RemoveDrafts",
|
name: "RemoveDrafts",
|
||||||
shouldPublish([_tree, vfile]) {
|
shouldPublish(_ctx, [_tree, vfile]) {
|
||||||
const draftFlag: boolean = vfile.data?.frontmatter?.draft ?? false
|
const draftFlag: boolean = vfile.data?.frontmatter?.draft ?? false
|
||||||
return !draftFlag
|
return !draftFlag
|
||||||
},
|
},
|
||||||
|
@ -2,7 +2,7 @@ import { QuartzFilterPlugin } from "../types"
|
|||||||
|
|
||||||
export const ExplicitPublish: QuartzFilterPlugin = () => ({
|
export const ExplicitPublish: QuartzFilterPlugin = () => ({
|
||||||
name: "ExplicitPublish",
|
name: "ExplicitPublish",
|
||||||
shouldPublish([_tree, vfile]) {
|
shouldPublish(_ctx, [_tree, vfile]) {
|
||||||
const publishFlag: boolean = vfile.data?.frontmatter?.publish ?? false
|
const publishFlag: boolean = vfile.data?.frontmatter?.publish ?? false
|
||||||
return publishFlag
|
return publishFlag
|
||||||
},
|
},
|
||||||
|
@ -1,15 +1,15 @@
|
|||||||
import { StaticResources } from "../resources"
|
import { StaticResources } from "../resources"
|
||||||
import { PluginTypes } from "./types"
|
|
||||||
import { FilePath, ServerSlug } from "../path"
|
import { FilePath, ServerSlug } from "../path"
|
||||||
|
import { BuildCtx } from "../ctx"
|
||||||
|
|
||||||
export function getStaticResourcesFromPlugins(plugins: PluginTypes) {
|
export function getStaticResourcesFromPlugins(ctx: BuildCtx) {
|
||||||
const staticResources: StaticResources = {
|
const staticResources: StaticResources = {
|
||||||
css: [],
|
css: [],
|
||||||
js: [],
|
js: [],
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const transformer of plugins.transformers) {
|
for (const transformer of ctx.cfg.plugins.transformers) {
|
||||||
const res = transformer.externalResources ? transformer.externalResources() : {}
|
const res = transformer.externalResources ? transformer.externalResources(ctx) : {}
|
||||||
if (res?.js) {
|
if (res?.js) {
|
||||||
staticResources.js.push(...res.js)
|
staticResources.js.push(...res.js)
|
||||||
}
|
}
|
||||||
@ -29,7 +29,6 @@ declare module "vfile" {
|
|||||||
// inserted in processors.ts
|
// inserted in processors.ts
|
||||||
interface DataMap {
|
interface DataMap {
|
||||||
slug: ServerSlug
|
slug: ServerSlug
|
||||||
allSlugs: ServerSlug[]
|
|
||||||
filePath: FilePath
|
filePath: FilePath
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,7 +29,7 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
|
|||||||
const opts = { ...defaultOptions, ...userOpts }
|
const opts = { ...defaultOptions, ...userOpts }
|
||||||
return {
|
return {
|
||||||
name: "LinkProcessing",
|
name: "LinkProcessing",
|
||||||
htmlPlugins() {
|
htmlPlugins(ctx) {
|
||||||
return [
|
return [
|
||||||
() => {
|
() => {
|
||||||
return (tree, file) => {
|
return (tree, file) => {
|
||||||
@ -40,11 +40,8 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
|
|||||||
if (opts.markdownLinkResolution === "relative") {
|
if (opts.markdownLinkResolution === "relative") {
|
||||||
return targetSlug as RelativeURL
|
return targetSlug as RelativeURL
|
||||||
} else if (opts.markdownLinkResolution === "shortest") {
|
} else if (opts.markdownLinkResolution === "shortest") {
|
||||||
// https://forum.obsidian.md/t/settings-new-link-format-what-is-shortest-path-when-possible/6748/5
|
|
||||||
const allSlugs = file.data.allSlugs!
|
|
||||||
|
|
||||||
// if the file name is unique, then it's just the filename
|
// if the file name is unique, then it's just the filename
|
||||||
const matchingFileNames = allSlugs.filter((slug) => {
|
const matchingFileNames = ctx.allSlugs.filter((slug) => {
|
||||||
const parts = slug.split(path.posix.sep)
|
const parts = slug.split(path.posix.sep)
|
||||||
const fileName = parts.at(-1)
|
const fileName = parts.at(-1)
|
||||||
return targetCanonical === fileName
|
return targetCanonical === fileName
|
||||||
|
@ -119,7 +119,7 @@ export const ObsidianFlavoredMarkdown: QuartzTransformerPlugin<Partial<Options>
|
|||||||
const opts = { ...defaultOptions, ...userOpts }
|
const opts = { ...defaultOptions, ...userOpts }
|
||||||
return {
|
return {
|
||||||
name: "ObsidianFlavoredMarkdown",
|
name: "ObsidianFlavoredMarkdown",
|
||||||
textTransform(src) {
|
textTransform(_ctx, src) {
|
||||||
// pre-transform wikilinks (fix anchors to things that may contain illegal syntax e.g. codeblocks, latex)
|
// pre-transform wikilinks (fix anchors to things that may contain illegal syntax e.g. codeblocks, latex)
|
||||||
if (opts.wikilinks) {
|
if (opts.wikilinks) {
|
||||||
src = src.toString()
|
src = src.toString()
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
import { PluggableList } from "unified"
|
import { PluggableList } from "unified"
|
||||||
import { StaticResources } from "../resources"
|
import { StaticResources } from "../resources"
|
||||||
import { ProcessedContent } from "./vfile"
|
import { ProcessedContent } from "./vfile"
|
||||||
import { GlobalConfiguration } from "../cfg"
|
|
||||||
import { QuartzComponent } from "../components/types"
|
import { QuartzComponent } from "../components/types"
|
||||||
import { FilePath, ServerSlug } from "../path"
|
import { FilePath, ServerSlug } from "../path"
|
||||||
import { BuildCtx } from "../ctx"
|
import { BuildCtx } from "../ctx"
|
||||||
@ -18,10 +17,10 @@ export type QuartzTransformerPlugin<Options extends OptionType = undefined> = (
|
|||||||
) => QuartzTransformerPluginInstance
|
) => QuartzTransformerPluginInstance
|
||||||
export type QuartzTransformerPluginInstance = {
|
export type QuartzTransformerPluginInstance = {
|
||||||
name: string
|
name: string
|
||||||
textTransform?: (src: string | Buffer) => string | Buffer
|
textTransform?: (ctx: BuildCtx, src: string | Buffer) => string | Buffer
|
||||||
markdownPlugins?: () => PluggableList
|
markdownPlugins?: (ctx: BuildCtx) => PluggableList
|
||||||
htmlPlugins?: () => PluggableList
|
htmlPlugins?: (ctx: BuildCtx) => PluggableList
|
||||||
externalResources?: () => Partial<StaticResources>
|
externalResources?: (ctx: BuildCtx) => Partial<StaticResources>
|
||||||
}
|
}
|
||||||
|
|
||||||
export type QuartzFilterPlugin<Options extends OptionType = undefined> = (
|
export type QuartzFilterPlugin<Options extends OptionType = undefined> = (
|
||||||
@ -29,7 +28,7 @@ export type QuartzFilterPlugin<Options extends OptionType = undefined> = (
|
|||||||
) => QuartzFilterPluginInstance
|
) => QuartzFilterPluginInstance
|
||||||
export type QuartzFilterPluginInstance = {
|
export type QuartzFilterPluginInstance = {
|
||||||
name: string
|
name: string
|
||||||
shouldPublish(content: ProcessedContent): boolean
|
shouldPublish(ctx: BuildCtx, content: ProcessedContent): boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
export type QuartzEmitterPlugin<Options extends OptionType = undefined> = (
|
export type QuartzEmitterPlugin<Options extends OptionType = undefined> = (
|
||||||
@ -43,7 +42,7 @@ export type QuartzEmitterPluginInstance = {
|
|||||||
resources: StaticResources,
|
resources: StaticResources,
|
||||||
emitCallback: EmitCallback,
|
emitCallback: EmitCallback,
|
||||||
): Promise<FilePath[]>
|
): Promise<FilePath[]>
|
||||||
getQuartzComponents(): QuartzComponent[]
|
getQuartzComponents(ctx: BuildCtx): QuartzComponent[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EmitOptions {
|
export interface EmitOptions {
|
||||||
|
@ -24,7 +24,7 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let emittedFiles = 0
|
let emittedFiles = 0
|
||||||
const staticResources = getStaticResourcesFromPlugins(cfg.plugins)
|
const staticResources = getStaticResourcesFromPlugins(ctx)
|
||||||
for (const emitter of cfg.plugins.emitters) {
|
for (const emitter of cfg.plugins.emitters) {
|
||||||
try {
|
try {
|
||||||
const emitted = await emitter.emit(ctx, content, staticResources, emit)
|
const emitted = await emitter.emit(ctx, content, staticResources, emit)
|
||||||
|
@ -1,16 +1,13 @@
|
|||||||
import { BuildCtx } from "../ctx"
|
import { BuildCtx } from "../ctx"
|
||||||
import { PerfTimer } from "../perf"
|
import { PerfTimer } from "../perf"
|
||||||
import { QuartzFilterPluginInstance } from "../plugins/types"
|
|
||||||
import { ProcessedContent } from "../plugins/vfile"
|
import { ProcessedContent } from "../plugins/vfile"
|
||||||
|
|
||||||
export function filterContent(
|
export function filterContent(ctx: BuildCtx, content: ProcessedContent[]): ProcessedContent[] {
|
||||||
{ cfg, argv }: BuildCtx,
|
const { cfg, argv } = ctx
|
||||||
content: ProcessedContent[],
|
|
||||||
): ProcessedContent[] {
|
|
||||||
const perf = new PerfTimer()
|
const perf = new PerfTimer()
|
||||||
const initialLength = content.length
|
const initialLength = content.length
|
||||||
for (const plugin of cfg.plugins.filters) {
|
for (const plugin of cfg.plugins.filters) {
|
||||||
const updatedContent = content.filter(plugin.shouldPublish)
|
const updatedContent = content.filter((item) => plugin.shouldPublish(ctx, item))
|
||||||
|
|
||||||
if (argv.verbose) {
|
if (argv.verbose) {
|
||||||
const diff = content.filter((x) => !updatedContent.includes(x))
|
const diff = content.filter((x) => !updatedContent.includes(x))
|
||||||
|
@ -7,23 +7,24 @@ import { Root as HTMLRoot } from "hast"
|
|||||||
import { ProcessedContent } from "../plugins/vfile"
|
import { ProcessedContent } from "../plugins/vfile"
|
||||||
import { PerfTimer } from "../perf"
|
import { PerfTimer } from "../perf"
|
||||||
import { read } from "to-vfile"
|
import { read } from "to-vfile"
|
||||||
import { FilePath, QUARTZ, ServerSlug, slugifyFilePath } from "../path"
|
import { FilePath, QUARTZ, slugifyFilePath } from "../path"
|
||||||
import path from "path"
|
import path from "path"
|
||||||
import os from "os"
|
import os from "os"
|
||||||
import workerpool, { Promise as WorkerPromise } from "workerpool"
|
import workerpool, { Promise as WorkerPromise } from "workerpool"
|
||||||
import { QuartzTransformerPluginInstance } from "../plugins/types"
|
|
||||||
import { QuartzLogger } from "../log"
|
import { QuartzLogger } from "../log"
|
||||||
import { trace } from "../trace"
|
import { trace } from "../trace"
|
||||||
import { BuildCtx } from "../ctx"
|
import { BuildCtx } from "../ctx"
|
||||||
|
|
||||||
export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void>
|
export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void>
|
||||||
export function createProcessor(transformers: QuartzTransformerPluginInstance[]): QuartzProcessor {
|
export function createProcessor(ctx: BuildCtx): QuartzProcessor {
|
||||||
|
const transformers = ctx.cfg.plugins.transformers
|
||||||
|
|
||||||
// base Markdown -> MD AST
|
// base Markdown -> MD AST
|
||||||
let processor = unified().use(remarkParse)
|
let processor = unified().use(remarkParse)
|
||||||
|
|
||||||
// MD AST -> MD AST transforms
|
// MD AST -> MD AST transforms
|
||||||
for (const plugin of transformers.filter((p) => p.markdownPlugins)) {
|
for (const plugin of transformers.filter((p) => p.markdownPlugins)) {
|
||||||
processor = processor.use(plugin.markdownPlugins!())
|
processor = processor.use(plugin.markdownPlugins!(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
// MD AST -> HTML AST
|
// MD AST -> HTML AST
|
||||||
@ -31,7 +32,7 @@ export function createProcessor(transformers: QuartzTransformerPluginInstance[])
|
|||||||
|
|
||||||
// HTML AST -> HTML AST transforms
|
// HTML AST -> HTML AST transforms
|
||||||
for (const plugin of transformers.filter((p) => p.htmlPlugins)) {
|
for (const plugin of transformers.filter((p) => p.htmlPlugins)) {
|
||||||
processor = processor.use(plugin.htmlPlugins!())
|
processor = processor.use(plugin.htmlPlugins!(ctx))
|
||||||
}
|
}
|
||||||
|
|
||||||
return processor
|
return processor
|
||||||
@ -73,7 +74,8 @@ async function transpileWorkerScript() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSlugs: ServerSlug[]) {
|
export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
|
||||||
|
const { argv, cfg } = ctx
|
||||||
return async (processor: QuartzProcessor) => {
|
return async (processor: QuartzProcessor) => {
|
||||||
const res: ProcessedContent[] = []
|
const res: ProcessedContent[] = []
|
||||||
for (const fp of fps) {
|
for (const fp of fps) {
|
||||||
@ -85,12 +87,11 @@ export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSl
|
|||||||
|
|
||||||
// Text -> Text transforms
|
// Text -> Text transforms
|
||||||
for (const plugin of cfg.plugins.transformers.filter((p) => p.textTransform)) {
|
for (const plugin of cfg.plugins.transformers.filter((p) => p.textTransform)) {
|
||||||
file.value = plugin.textTransform!(file.value)
|
file.value = plugin.textTransform!(ctx, file.value)
|
||||||
}
|
}
|
||||||
|
|
||||||
// base data properties that plugins may use
|
// base data properties that plugins may use
|
||||||
file.data.slug = slugifyFilePath(path.relative(argv.directory, file.path) as FilePath)
|
file.data.slug = slugifyFilePath(path.relative(argv.directory, file.path) as FilePath)
|
||||||
file.data.allSlugs = allSlugs
|
|
||||||
file.data.filePath = fp
|
file.data.filePath = fp
|
||||||
|
|
||||||
const ast = processor.parse(file)
|
const ast = processor.parse(file)
|
||||||
@ -111,24 +112,19 @@ export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSl
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
|
export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
|
||||||
const { argv, cfg } = ctx
|
const { argv } = ctx
|
||||||
const perf = new PerfTimer()
|
const perf = new PerfTimer()
|
||||||
const log = new QuartzLogger(argv.verbose)
|
const log = new QuartzLogger(argv.verbose)
|
||||||
|
|
||||||
const CHUNK_SIZE = 128
|
const CHUNK_SIZE = 128
|
||||||
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
|
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
|
||||||
|
|
||||||
// get all slugs ahead of time as each thread needs a copy
|
|
||||||
const allSlugs = fps.map((fp) =>
|
|
||||||
slugifyFilePath(path.relative(argv.directory, path.resolve(fp)) as FilePath),
|
|
||||||
)
|
|
||||||
|
|
||||||
let res: ProcessedContent[] = []
|
let res: ProcessedContent[] = []
|
||||||
log.start(`Parsing input files using ${concurrency} threads`)
|
log.start(`Parsing input files using ${concurrency} threads`)
|
||||||
if (concurrency === 1) {
|
if (concurrency === 1) {
|
||||||
try {
|
try {
|
||||||
const processor = createProcessor(cfg.plugins.transformers)
|
const processor = createProcessor(ctx)
|
||||||
const parse = createFileParser(ctx, fps, allSlugs)
|
const parse = createFileParser(ctx, fps)
|
||||||
res = await parse(processor)
|
res = await parse(processor)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.end()
|
log.end()
|
||||||
@ -144,7 +140,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
|
|||||||
|
|
||||||
const childPromises: WorkerPromise<ProcessedContent[]>[] = []
|
const childPromises: WorkerPromise<ProcessedContent[]>[] = []
|
||||||
for (const chunk of chunks(fps, CHUNK_SIZE)) {
|
for (const chunk of chunks(fps, CHUNK_SIZE)) {
|
||||||
childPromises.push(pool.exec("parseFiles", [argv, chunk, allSlugs]))
|
childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
|
||||||
}
|
}
|
||||||
|
|
||||||
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
|
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
|
||||||
|
@ -3,16 +3,14 @@ import { Argv, BuildCtx } from "./ctx"
|
|||||||
import { FilePath, ServerSlug } from "./path"
|
import { FilePath, ServerSlug } from "./path"
|
||||||
import { createFileParser, createProcessor } from "./processors/parse"
|
import { createFileParser, createProcessor } from "./processors/parse"
|
||||||
|
|
||||||
const transformers = cfg.plugins.transformers
|
|
||||||
const processor = createProcessor(transformers)
|
|
||||||
|
|
||||||
// only called from worker thread
|
// only called from worker thread
|
||||||
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {
|
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {
|
||||||
const ctx: BuildCtx = {
|
const ctx: BuildCtx = {
|
||||||
cfg,
|
cfg,
|
||||||
argv,
|
argv,
|
||||||
|
allSlugs,
|
||||||
}
|
}
|
||||||
|
const processor = createProcessor(ctx)
|
||||||
const parse = createFileParser(ctx, fps, allSlugs)
|
const parse = createFileParser(ctx, fps)
|
||||||
return parse(processor)
|
return parse(processor)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user