tomoyanonymous/quartz-research-note
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

base path refactor, more docs

Browse Source
This commit is contained in:
Jacky Zhao 2023-07-13 00:19:35 -07:00
parent 08f8e3b4a4
commit 906f91f8ee
37 changed files with 1861 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/deploy.yaml vendored
View File

@ -30,12 +30,8 @@ jobs:
- run: npm ci
- name: Build Quartz
run: npx quartz build
- name: Test
run: npm test
# TODO: when we actually release
# - name: Deploy
# uses: peaceiris/actions-gh-pages@v3
# with:
# github_token: ${{ secrets.GITHUB_TOKEN }}
# publish_dir: ./public
- name: Ensure Quartz builds
run: npx quartz build

4
content/configuration.md
View File

@ -45,6 +45,8 @@ This part of the configuration concerns anything that can affect the whole site.
## Plugins
You can think of Quartz plugins as a series of transformations over content.
![[quartz-transform-pipeline.png]]
```ts
plugins: {
transformers: [...],
@ -72,6 +74,6 @@ transformers: [
```
### Layout
Certain emitters may also output [HTML](https://developer.mozilla.org/en-US/docs/Web/HTML) files.
Certain emitters may also output [HTML](https://developer.mozilla.org/en-US/docs/Web/HTML) files. To make sure that
### Components

1
content/features/upcoming features.md
View File

@ -1,4 +1,5 @@
- fixes
- CLI
- update
- push

BIN
content/quartz-transform-pipeline.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 72 KiB

2
index.d.ts vendored
View File

@ -5,7 +5,7 @@ declare module '*.scss' {
// dom custom event
interface CustomEventMap {
"nav": CustomEvent<{ url: string }>;
"nav": CustomEvent<{ url: CanonicalSlug }>;
}
declare const fetchData: Promise<ContentIndex>

1413
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@ -12,7 +12,8 @@
"url": "https://github.com/jackyzha0/quartz.git"
},
"scripts": {
"typecheck": "tsc --noEmit"
"typecheck": "tsc --noEmit",
"test": "tsx ./quartz/path.test.ts"
},
"keywords": [
"site generator",
@ -83,6 +84,7 @@
"@types/workerpool": "^6.4.0",
"@types/yargs": "^17.0.24",
"esbuild": "^0.18.11",
"tsx": "^3.12.7",
"typescript": "^5.0.4"
}
}

2
quartz.config.ts
View File

@ -9,7 +9,7 @@ const generalConfiguration: GlobalConfiguration = {
analytics: {
provider: 'plausible',
},
canonicalUrl: "quartz.jzhao.xyz",
baseUrl: "quartz.jzhao.xyz",
ignorePatterns: ["private", "templates"],
theme: {
typography: {

2
quartz/cfg.ts
View File

@ -25,7 +25,7 @@ export interface GlobalConfiguration {
/** Base URL to use for CNAME files, sitemaps, and RSS feeds that require an absolute URL.
* Quartz will avoid using this as much as possible and use relative URLs most of the time
*/
canonicalUrl?: string,
baseUrl?: string,
theme: Theme
}

4
quartz/components/Backlinks.tsx
View File

@ -1,7 +1,7 @@
import { QuartzComponentConstructor, QuartzComponentProps } from "./types"
import style from "./styles/backlinks.scss"
import { relativeToRoot } from "../path"
import { stripIndex } from "./scripts/util"
import { clientSideSlug } from "./scripts/util"
function Backlinks({ fileData, allFiles }: QuartzComponentProps) {
const slug = fileData.slug!
@ -10,7 +10,7 @@ function Backlinks({ fileData, allFiles }: QuartzComponentProps) {
<h3>Backlinks</h3>
<ul class="overflow">
{backlinkFiles.length > 0 ?
backlinkFiles.map(f => <li><a href={stripIndex(relativeToRoot(slug, f.slug!))} class="internal">{f.frontmatter?.title}</a></li>)
backlinkFiles.map(f => <li><a href={clientSideSlug(relativeToRoot(slug, f.slug!))} class="internal">{f.frontmatter?.title}</a></li>)
: <li>No backlinks found</li>}
</ul>
</div>

6
quartz/components/Head.tsx
View File

@ -1,14 +1,14 @@
import { clientSideSlug, resolveToRoot } from "../path"
import { toServerSlug, pathToRoot } from "../path"
import { JSResourceToScriptElement } from "../resources"
import { QuartzComponentConstructor, QuartzComponentProps } from "./types"
export default (() => {
function Head({ fileData, externalResources }: QuartzComponentProps) {
const slug = clientSideSlug(fileData.slug!)
const slug = toServerSlug(fileData.slug!)
const title = fileData.frontmatter?.title ?? "Untitled"
const description = fileData.description ?? "No description provided"
const { css, js } = externalResources
const baseDir = resolveToRoot(slug)
const baseDir = pathToRoot(slug)
const iconPath = baseDir + "/static/icon.png"
const ogImagePath = baseDir + "/static/og-image.png"

4
quartz/components/PageList.tsx
View File

@ -1,7 +1,7 @@
import { relativeToRoot } from "../path"
import { QuartzPluginData } from "../plugins/vfile"
import { Date } from "./Date"
import { stripIndex } from "./scripts/util"
import { clientSideSlug } from "./scripts/util"
import { QuartzComponentProps } from "./types"
function byDateAndAlphabetical(f1: QuartzPluginData, f2: QuartzPluginData): number {
@ -34,7 +34,7 @@ export function PageList({ fileData, allFiles }: QuartzComponentProps) {
<Date date={page.dates.modified} />
</p>}
<div class="desc">
<h3><a href={stripIndex(relativeToRoot(slug, pageSlug))} class="internal">{title}</a></h3>
<h3><a href={clientSideSlug(relativeToRoot(slug, pageSlug))} class="internal">{title}</a></h3>
</div>
<ul class="tags">
{tags.map(tag => <li><a class="internal" href={relativeToRoot(slug, `tags/${tag}`)}>#{tag}</a></li>)}

4
quartz/components/PageTitle.tsx
View File

@ -1,10 +1,10 @@
import { resolveToRoot } from "../path"
import { pathToRoot } from "../path"
import { QuartzComponentConstructor, QuartzComponentProps } from "./types"
function PageTitle({ fileData, cfg }: QuartzComponentProps) {
const title = cfg?.pageTitle ?? "Untitled Quartz"
const slug = fileData.slug!
const baseDir = resolveToRoot(slug)
const baseDir = pathToRoot(slug)
return <h1 class="page-title"><a href={baseDir}>{title}</a></h1>
}

4
quartz/components/TagList.tsx
View File

@ -1,11 +1,11 @@
import { resolveToRoot } from "../path"
import { pathToRoot } from "../path"
import { QuartzComponentConstructor, QuartzComponentProps } from "./types"
import { slug as slugAnchor } from 'github-slugger'
function TagList({ fileData }: QuartzComponentProps) {
const tags = fileData.frontmatter?.tags
const slug = fileData.slug!
const baseDir = resolveToRoot(slug)
const baseDir = pathToRoot(slug)
if (tags && tags.length > 0) {
return <ul class="tags">{tags.map(tag => {
const display = `#${tag}`

4
quartz/components/pages/FolderContent.tsx
View File

@ -5,11 +5,11 @@ import path from "path"
import style from '../styles/listPage.scss'
import { PageList } from "../PageList"
import { clientSideSlug } from "../../path"
import { toServerSlug } from "../../path"
function FolderContent(props: QuartzComponentProps) {
const { tree, fileData, allFiles } = props
const folderSlug = clientSideSlug(fileData.slug!)
const folderSlug = toServerSlug(fileData.slug!)
const allPagesInFolder = allFiles.filter(file => {
const fileSlug = file.slug ?? ""
const prefixed = fileSlug.startsWith(folderSlug)

4
quartz/components/pages/TagContent.tsx
View File

@ -3,14 +3,14 @@ import { Fragment, jsx, jsxs } from 'preact/jsx-runtime'
import { toJsxRuntime } from "hast-util-to-jsx-runtime"
import style from '../styles/listPage.scss'
import { PageList } from "../PageList"
import { clientSideSlug } from "../../path"
import { toServerSlug } from "../../path"
function TagContent(props: QuartzComponentProps) {
const { tree, fileData, allFiles } = props
const slug = fileData.slug
if (slug?.startsWith("tags/")) {
const tag = clientSideSlug(slug.slice("tags/".length))
const tag = toServerSlug(slug.slice("tags/".length))
const allPagesWithTag = allFiles.filter(file => (file.frontmatter?.tags ?? []).includes(tag))
const listProps = {
...props,

8
quartz/components/renderPage.tsx
View File

@ -3,7 +3,7 @@ import { QuartzComponent, QuartzComponentProps } from "./types";
import HeaderConstructor from "./Header"
import BodyConstructor from "./Body"
import { JSResourceToScriptElement, StaticResources } from "../resources";
import { resolveToRoot } from "../path";
import { CanonicalSlug, pathToRoot } from "../path";
interface RenderComponents {
head: QuartzComponent
@ -15,8 +15,8 @@ interface RenderComponents {
footer: QuartzComponent,
}
export function pageResources(slug: string, staticResources: StaticResources): StaticResources {
const baseDir = resolveToRoot(slug)
export function pageResources(slug: CanonicalSlug, staticResources: StaticResources): StaticResources {
const baseDir = pathToRoot(slug)
const contentIndexPath = baseDir + "/static/contentIndex.json"
const contentIndexScript = `const fetchData = fetch(\`${contentIndexPath}\`).then(data => data.json())`
@ -32,7 +32,7 @@ export function pageResources(slug: string, staticResources: StaticResources): S
}
}
export function renderPage(slug: string, componentData: QuartzComponentProps, components: RenderComponents, pageResources: StaticResources): string {
export function renderPage(slug: CanonicalSlug, componentData: QuartzComponentProps, components: RenderComponents, pageResources: StaticResources): string {
const { head: Head, header, beforeBody, pageBody: Content, left, right, footer: Footer } = components
const Header = HeaderConstructor()
const Body = BodyConstructor()

15
quartz/components/scripts/graph.inline.ts
View File

@ -1,24 +1,25 @@
import { ContentDetails } from "../../plugins/emitters/contentIndex"
import * as d3 from 'd3'
import { registerEscapeHandler, relative, removeAllChildren } from "./util"
import { registerEscapeHandler, clientSideRelativePath, removeAllChildren } from "./util"
import { CanonicalSlug } from "../../path"
type NodeData = {
id: string,
id: CanonicalSlug,
text: string,
tags: string[]
} & d3.SimulationNodeDatum
type LinkData = {
source: string,
target: string
source: CanonicalSlug,
target: CanonicalSlug
}
const localStorageKey = "graph-visited"
function getVisited(): Set<string> {
function getVisited(): Set<CanonicalSlug> {
return new Set(JSON.parse(localStorage.getItem(localStorageKey) ?? "[]"))
}
function addToVisited(slug: string) {
function addToVisited(slug: CanonicalSlug) {
const visited = getVisited()
visited.add(slug)
localStorage.setItem(localStorageKey, JSON.stringify([...visited]))
@ -167,7 +168,7 @@ async function renderGraph(container: string, slug: string) {
.attr("fill", color)
.style("cursor", "pointer")
.on("click", (_, d) => {
const targ = relative(slug, d.id)
const targ = clientSideRelativePath(slug, d.id)
window.spaNavigate(new URL(targ))
})
.on("mouseover", function(_, d) {

13
quartz/components/scripts/search.inline.ts
View File

@ -1,9 +1,10 @@
import { Document } from "flexsearch"
import { ContentDetails } from "../../plugins/emitters/contentIndex"
import { registerEscapeHandler, relative, removeAllChildren } from "./util"
import { registerEscapeHandler, clientSideRelativePath, removeAllChildren } from "./util"
import { CanonicalSlug } from "../../path"
interface Item {
slug: string,
slug: CanonicalSlug,
title: string,
content: string,
}
@ -100,7 +101,7 @@ document.addEventListener("nav", async (e: unknown) => {
}
}
const formatForDisplay = (term: string, slug: string) => ({
const formatForDisplay = (term: string, slug: CanonicalSlug) => ({
slug,
title: highlight(term, data[slug].title ?? ""),
content: highlight(term, data[slug].content ?? "", true),
@ -112,7 +113,7 @@ document.addEventListener("nav", async (e: unknown) => {
button.id = slug
button.innerHTML = `<h3>${title}</h3><p>${content}</p>`
button.addEventListener('click', () => {
const targ = relative(currentSlug, slug)
const targ = clientSideRelativePath(currentSlug, slug)
window.spaNavigate(new URL(targ))
})
return button
@ -142,7 +143,7 @@ document.addEventListener("nav", async (e: unknown) => {
}
// order titles ahead of content
const allIds: Set<string> = new Set([...getByField("title"), ...getByField("content")])
const allIds: Set<CanonicalSlug> = new Set([...getByField("title"), ...getByField("content")])
const finalResults = [...allIds].map(id => formatForDisplay(term, id))
displayResults(finalResults)
}
@ -178,7 +179,7 @@ document.addEventListener("nav", async (e: unknown) => {
for (const [slug, fileData] of Object.entries<ContentDetails>(data)) {
await index.addAsync(slug, {
slug,
slug: slug as CanonicalSlug,
title: fileData.title,
content: fileData.content
})

5
quartz/components/scripts/spa.inline.ts
View File

@ -1,4 +1,5 @@
import micromorph from "micromorph"
import { CanonicalSlug, RelativeURL } from "../../path"
// adapted from `micromorph`
// https://github.com/natemoo-re/micromorph
@ -29,7 +30,7 @@ const getOpts = ({ target }: Event): { url: URL, scroll?: boolean } | undefined
return { url: new URL(href), scroll: 'routerNoscroll' in a.dataset ? false : undefined }
}
function notifyNav(url: string) {
function notifyNav(url: CanonicalSlug) {
const event: CustomEventMap["nav"] = new CustomEvent("nav", { detail: { url } })
document.dispatchEvent(event)
}
@ -100,7 +101,7 @@ function createRouter() {
}
return new class Router {
go(pathname: string) {
go(pathname: RelativeURL) {
const url = new URL(pathname, window.location.toString())
return navigate(url, false)
}

13
quartz/components/scripts/util.ts
View File

@ -18,19 +18,6 @@ export function registerEscapeHandler(outsideContainer: HTMLElement | null, cb:
document.addEventListener('keydown', esc)
}
export function stripIndex(s: string): string {
return s.endsWith("index") ? s.slice(0, -"index".length) : s
}
export function relative(from: string, to: string) {
from = encodeURI(stripIndex(from))
to = encodeURI(stripIndex(to))
const start = [location.protocol, '//', location.host, location.pathname].join('')
const trimEnd = from.length === 0 ? start.length : -from.length
const url = start.slice(0, trimEnd) + to
return url
}
export function removeAllChildren(node: HTMLElement) {
while (node.firstChild) {
node.removeChild(node.firstChild)

2
quartz/components/styles/backlinks.scss
View File

@ -1,4 +1,6 @@
.backlinks {
position: relative;
& > h3 {
font-size: 1rem;
margin: 0;

159
quartz/path.test.ts Normal file
View File

@ -0,0 +1,159 @@
import test, { describe } from 'node:test'
import * as path from './path'
import assert from 'node:assert'
describe('typeguards', () => {
test('isClientSlug', () => {
assert(path.isClientSlug("http://example.com"))
assert(path.isClientSlug("http://example.com/index"))
assert(path.isClientSlug("http://example.com/index.html"))
assert(path.isClientSlug("http://example.com/"))
assert(path.isClientSlug("https://example.com"))
assert(path.isClientSlug("https://example.com/abc/def"))
assert(path.isClientSlug("https://example.com/abc/def/"))
assert(path.isClientSlug("https://example.com/abc/def#cool"))
assert(path.isClientSlug("https://example.com/abc/def?field=1&another=2"))
assert(path.isClientSlug("https://example.com/abc/def?field=1&another=2#cool"))
assert(path.isClientSlug("https://example.com/abc/def.html?field=1&another=2#cool"))
assert(!path.isClientSlug("./"))
assert(!path.isClientSlug(""))
assert(!path.isClientSlug("ipfs://example.com"))
assert(!path.isClientSlug("http"))
assert(!path.isClientSlug("https"))
})
test('isCanonicalSlug', () => {
assert(path.isCanonicalSlug("/"))
assert(path.isCanonicalSlug("/abc"))
assert(path.isCanonicalSlug("/notindex"))
assert(path.isCanonicalSlug("/notindex/def"))
assert(!path.isCanonicalSlug("//"))
assert(!path.isCanonicalSlug("/index"))
assert(!path.isCanonicalSlug(""))
assert(!path.isCanonicalSlug("index"))
assert(!path.isCanonicalSlug("index/abc"))
assert(!path.isCanonicalSlug("https://example.com"))
assert(!path.isCanonicalSlug("/abc/"))
assert(!path.isCanonicalSlug("/abc/index"))
assert(!path.isCanonicalSlug("/abc#anchor"))
assert(!path.isCanonicalSlug("/abc?query=1"))
assert(!path.isCanonicalSlug("/index.md"))
assert(!path.isCanonicalSlug("/index.html"))
})
test('isRelativeURL', () => {
assert(path.isRelativeURL("."))
assert(path.isRelativeURL(".."))
assert(path.isRelativeURL("./abc/def"))
assert(path.isRelativeURL("./abc/def#an-anchor"))
assert(path.isRelativeURL("./abc/def?query=1#an-anchor"))
assert(path.isRelativeURL("../abc/def"))
assert(!path.isRelativeURL("abc"))
assert(!path.isRelativeURL(""))
assert(!path.isRelativeURL("../"))
assert(!path.isRelativeURL("./"))
assert(!path.isRelativeURL("./abc/def.html"))
assert(!path.isRelativeURL("./abc/def.md"))
})
test('isServerSlug', () => {
assert(path.isServerSlug("/index"))
assert(path.isServerSlug("/abc/def"))
assert(!path.isServerSlug("/"))
assert(!path.isServerSlug("."))
assert(!path.isServerSlug("./abc/def"))
assert(!path.isServerSlug("../abc/def"))
assert(!path.isServerSlug("/index.html"))
assert(!path.isServerSlug("/abc/def.html"))
assert(!path.isServerSlug("/abc/def#anchor"))
assert(!path.isServerSlug("/abc/def?query=1"))
assert(!path.isServerSlug("/note with spaces"))
})
test('isFilePath', () => {
assert(path.isFilePath("/content/index.md"))
assert(path.isFilePath("/content/test.png"))
assert(!path.isFilePath("../test.pdf"))
assert(!path.isFilePath("content/test.png"))
assert(!path.isFilePath("content/test"))
assert(!path.isFilePath("./content/test"))
})
})
describe('transforms', () => {
function asserts<Inp, Out>(pairs: [string, string][], transform: (inp: Inp) => Out, checkPre: (x: any) => x is Inp, checkPost: (x: any) => x is Out) {
for (const [inp, expected] of pairs) {
assert(checkPre(inp), `${inp} wasn't the expected input type`)
const actual = transform(inp)
assert.strictEqual(actual, expected, `after transforming ${inp}, ${actual} was not ${expected}`)
assert(checkPost(actual), `${actual} wasn't the expected output type`)
}
}
test('canonicalizeServer', () => {
asserts([
["/index", "/"],
["/abc/def", "/abc/def"],
], path.canonicalizeServer, path.isServerSlug, path.isCanonicalSlug)
})
test('canonicalizeClient', () => {
asserts([
["http://localhost:3000", "/"],
["http://localhost:3000/index", "/"],
["http://localhost:3000/test", "/test"],
["http://example.com", "/"],
["http://example.com/index", "/"],
["http://example.com/index.html", "/"],
["http://example.com/", "/"],
["https://example.com", "/"],
["https://example.com/abc/def", "/abc/def"],
["https://example.com/abc/def/", "/abc/def"],
["https://example.com/abc/def#cool", "/abc/def"],
["https://example.com/abc/def?field=1&another=2", "/abc/def"],
["https://example.com/abc/def?field=1&another=2#cool", "/abc/def"],
["https://example.com/abc/def.html?field=1&another=2#cool", "/abc/def"],
], path.canonicalizeClient, path.isClientSlug, path.isCanonicalSlug)
})
describe('slugifyFilePath', () => {
asserts([
["/content/index.md", "/content/index"],
["/content/cool.png", "/content/cool"],
["/index.md", "/index"],
["/note with spaces.md", "/note-with-spaces"],
], path.slugifyFilePath, path.isFilePath, path.isServerSlug)
})
describe('transformInternalLink', () => {
asserts([
["", "."],
[".", "."],
["./", "."],
["./index", "."],
["./index.html", "."],
["./index.md", "."],
["content", "./content"],
["content/test.md", "./content/test"],
["./content/test.md", "./content/test"],
["../content/test.md", "../content/test"],
["tags/", "./tags"],
["/tags/", "./tags"],
["content/with spaces", "./content/with-spaces"],
["content/with spaces#and Anchor!", "./content/with-spaces#and-anchor"],
], path.transformInternalLink, (x: string): x is string => true, path.isRelativeURL)
})
describe('pathToRoot', () => {
asserts([
["/", "."],
["/abc/def", "../.."],
], path.pathToRoot, path.isCanonicalSlug, path.isRelativeURL)
})
})

212
quartz/path.ts
View File

@ -1,13 +1,154 @@
import path from 'path'
import { slug as slugAnchor } from 'github-slugger'
function slugSegment(s: string): string {
return s.replace(/\s/g, '-')
// Quartz Paths
// Things in boxes are not actual types but rather sources which these types can be acquired from
//
// ┌────────────┐
// ┌───────────┤ Browser ├────────────┐
// │ └────────────┘ │
// │ │
// ▼ ▼
// ┌────────┐ ┌─────────────┐
// ┌───────────────────┤ Window │ │ LinkElement │
// │ └────┬───┘ └──────┬──────┘
// │ │ │
// │ getClientSlug() │ .href │
// │ ▼ ▼
// │
// │ Client Slug Relative URL
// getCanonicalSlug() │ https://test.ca/note/abc#anchor?query=123 ../note/def#anchor
// │
// │ canonicalizeClient() │ ▲
// │ ▼ │
// │ │
// └───────────────► Canonical Slug │
// /note/abc │
// │
// ▲ │
// canonicalizeServer() │ │
// │
// HTML File Server Slug │
// /note/abc/index.html ◄───────────── /note/abc/index │
// │
// ▲ ┌────────┴────────┐
// slugifyFilePath() │ transformInternalLink() │ │
// │ │ │
// ┌─────────┴──────────┐ ┌─────┴─────┐ ┌────────┴──────┐
// │ File Path │ │ Wikilinks │ │ Markdown Link │
// │ /note/abc/index.md │ └───────────┘ └───────────────┘
// └────────────────────┘ ▲ ▲
// ▲ │ │
// │ ┌─────────┐ │ │
// └────────────┤ MD File ├─────┴─────────────────┘
// └─────────┘
/// Utility type to simulate nominal types in TypeScript
type SlugLike<T> = string & { __brand: T }
/** Client-side slug, usually obtained through `window.location` */
export type ClientSlug = SlugLike<"client">
export function isClientSlug(s: string): s is ClientSlug {
return /^https?:\/\/.+/.test(s)
}
// on the client, 'index' isn't ever rendered so we should clean it up
export function clientSideSlug(fp: string): string {
// remove index
/** Canonical slug, should be used whenever you need to refer to the location of a file/note.
* On the client, this is normally stored in `document.body.dataset.slug`
*/
export type CanonicalSlug = SlugLike<"canonical">
export function isCanonicalSlug(s: string): s is CanonicalSlug {
const validStart = s.startsWith("/")
const validEnding = s.length === 1 || (!s.endsWith("/") && !s.endsWith("/index"))
return !_containsForbiddenCharacters(s) && validStart && validEnding && !_hasFileExtension(s)
}
/** A relative link, can be found on `href`s but can also be constructed for
* client-side navigation (e.g. search and graph)
*/
export type RelativeURL = SlugLike<"relative">
export function isRelativeURL(s: string): s is RelativeURL {
const validStart = /^\.{1,2}/.test(s)
const validEnding = !s.endsWith("/") && !s.endsWith("/index")
return validStart && validEnding && !_hasFileExtension(s)
}
/** A server side slug. This is what Quartz uses to emit files so uses index suffixes */
export type ServerSlug = SlugLike<"server">
export function isServerSlug(s: string): s is ServerSlug {
// must start with forward slash
const validStart = s.startsWith("/")
const validEnding = !s.endsWith("/")
return validStart && validEnding && !_containsForbiddenCharacters(s) && !_hasFileExtension(s)
}
/** The real file path to a file on disk */
export type FilePath = SlugLike<"filepath">
export function isFilePath(s: string): s is FilePath {
return s.startsWith("/") && _hasFileExtension(s)
}
export function getClientSlug(window: Window): ClientSlug {
return window.location.href as ClientSlug
}
export function getCanonicalSlug(window: Window): CanonicalSlug {
return window.document.body.dataset.slug! as CanonicalSlug
}
export function canonicalizeClient(slug: ClientSlug): CanonicalSlug {
const { pathname } = new URL(slug)
let fp = pathname
fp = fp.replace(new RegExp(path.extname(fp) + '$'), '')
return _canonicalize(fp) as CanonicalSlug
}
export function canonicalizeServer(slug: ServerSlug): CanonicalSlug {
let fp = slug as string
return _canonicalize(fp) as CanonicalSlug
}
export function slugifyFilePath(fp: FilePath): ServerSlug {
// strip file extension
const withoutFileExt = fp.replace(new RegExp(path.extname(fp) + '$'), '')
const slug = withoutFileExt
.split(path.sep) // fs can have diff interpretations of /
.map((segment) => segment.replace(/\s/g, '-')) // slugify all segments
.join('/') // always use / as sep
.replace(/\/$/, '') // remove trailing slash
return slug as ServerSlug
}
export function transformInternalLink(link: string): RelativeURL {
let [fplike, anchor] = link.split("#", 2)
let segments = fplike.split("/").filter(x => x.length > 0)
let prefix = segments.filter(_isRelativeSegment).join("/")
let fp = "/" + segments.filter(seg => !_isRelativeSegment(seg)).join("/")
fp = canonicalizeServer(slugifyFilePath(fp as FilePath))
if (fp.endsWith("index")) {
fp = fp.slice(0, -"index".length)
}
let joined = [_stripSlashes(prefix), _stripSlashes(fp)].filter(x => x !== "").join("/")
anchor = anchor === undefined ? "" : '#' + slugAnchor(anchor)
return _addRelativeToStart(joined) + anchor as RelativeURL
}
// resolve /a/b/c to ../../
export function pathToRoot(slug: CanonicalSlug): RelativeURL {
let rootPath = slug
.split('/')
.filter(x => x !== '')
.map(_ => '..')
.join('/')
return _addRelativeToStart(rootPath) as RelativeURL
}
export const QUARTZ = "quartz"
function _canonicalize(fp: string): string {
if (fp.endsWith("index")) {
fp = fp.slice(0, -"index".length)
}
@ -17,50 +158,45 @@ export function clientSideSlug(fp: string): string {
fp = fp.slice(0, -1)
}
if (fp.length === 0) {
return "/" as CanonicalSlug
}
return fp
}
export function trimPathSuffix(fp: string): string {
fp = clientSideSlug(fp)
let [cleanPath, anchor] = fp.split("#", 2)
anchor = anchor === undefined ? "" : "#" + anchor
return cleanPath + anchor
function _containsForbiddenCharacters(s: string): boolean {
return s.includes(" ") || s.includes("#") || s.includes("?")
}
export function slugify(s: string): string {
let [fp, anchor] = s.split("#", 2)
const sluggedAnchor = anchor === undefined ? "" : "#" + slugAnchor(anchor)
const withoutFileExt = fp.replace(new RegExp(path.extname(fp) + '$'), '')
const rawSlugSegments = withoutFileExt.split(path.sep)
const slugParts: string = rawSlugSegments
.map((segment) => slugSegment(segment))
.join(path.posix.sep)
.replace(/\/$/, '')
return path.normalize(slugParts) + sluggedAnchor
function _hasFileExtension(s: string): boolean {
return /\.[A-Za-z]+$/.test(s)
}
// resolve /a/b/c to ../../
export function resolveToRoot(slug: string): string {
let fp = trimPathSuffix(slug)
function _isRelativeSegment(s: string): boolean {
return /^\.{0,2}$/.test(s)
}
if (fp === "") {
return "."
function _stripSlashes(s: string): string {
if (s.startsWith("/")) {
s = s.substring(1)
}
return "./" + fp
.split('/')
.filter(x => x !== '')
.map(_ => '..')
.join('/')
if (s.endsWith("/")) {
s = s.slice(0, -1)
}
return s
}
export function relativeToRoot(slug: string, fp: string): string {
return path.join(resolveToRoot(slug), fp)
}
function _addRelativeToStart(s: string): string {
if (s === "") {
s = "."
}
export function relative(src: string, dest: string): string {
return path.relative(src, dest)
}
if (!s.startsWith(".")) {
s = "./" + s
}
export const QUARTZ = "quartz"
return s
}

14
quartz/plugins/emitters/aliases.ts
View File

@ -1,4 +1,4 @@
import { relativeToRoot } from "../../path"
import { CanonicalSlug, FilePath, ServerSlug, relativeToRoot } from "../../path"
import { QuartzEmitterPlugin } from "../types"
import path from 'path'
@ -7,14 +7,14 @@ export const AliasRedirects: QuartzEmitterPlugin = () => ({
getQuartzComponents() {
return []
},
async emit(contentFolder, _cfg, content, _resources, emit): Promise<string[]> {
const fps: string[] = []
async emit(contentFolder, _cfg, content, _resources, emit): Promise<FilePath[]> {
const fps: FilePath[] = []
for (const [_tree, file] of content) {
const ogSlug = file.data.slug!
const dir = path.relative(contentFolder, file.dirname ?? contentFolder)
let aliases: string[] = []
let aliases: CanonicalSlug[] = []
if (file.data.frontmatter?.aliases) {
aliases = file.data.frontmatter?.aliases
} else if (file.data.frontmatter?.alias) {
@ -22,11 +22,11 @@ export const AliasRedirects: QuartzEmitterPlugin = () => ({
}
for (const alias of aliases) {
const slug = alias.startsWith("/")
const slug = (alias.startsWith("/")
? alias
: path.posix.join(dir, alias)
: path.posix.join(dir, alias)) as ServerSlug
const fp = slug + ".html"
const fp = slug + ".html" as FilePath
const redirUrl = relativeToRoot(slug, ogSlug)
await emit({
content: `

15
quartz/plugins/emitters/contentIndex.ts
View File

@ -1,11 +1,12 @@
import { GlobalConfiguration } from "../../cfg"
import { CanonicalSlug, ClientSlug } from "../../path"
import { QuartzEmitterPlugin } from "../types"
import path from "path"
export type ContentIndex = Map<string, ContentDetails>
export type ContentIndex = Map<CanonicalSlug, ContentDetails>
export type ContentDetails = {
title: string,
links: string[],
links: CanonicalSlug[],
tags: string[],
content: string,
date?: Date,
@ -25,8 +26,8 @@ const defaultOptions: Options = {
}
function generateSiteMap(cfg: GlobalConfiguration, idx: ContentIndex): string {
const base = cfg.canonicalUrl ?? ""
const createURLEntry = (slug: string, content: ContentDetails): string => `<url>
const base = cfg.baseUrl ?? ""
const createURLEntry = (slug: CanonicalSlug, content: ContentDetails): string => `<url>
<loc>https://${base}/${slug}</loc>
<lastmod>${content.date?.toISOString()}</lastmod>
</url>`
@ -35,10 +36,10 @@ function generateSiteMap(cfg: GlobalConfiguration, idx: ContentIndex): string {
}
function generateRSSFeed(cfg: GlobalConfiguration, idx: ContentIndex): string {
const base = cfg.canonicalUrl ?? ""
const root = `https://${base}`
const base = cfg.baseUrl ?? ""
const root = `https://${base}` as ClientSlug
const createURLEntry = (slug: string, content: ContentDetails): string => `<items>
const createURLEntry = (slug: CanonicalSlug, content: ContentDetails): string => `<items>
<title>${content.title}</title>
<link>${root}/${slug}</link>
<guid>${root}/${slug}</guid>

7
quartz/plugins/emitters/contentPage.tsx
View File

@ -4,6 +4,7 @@ import HeaderConstructor from "../../components/Header"
import BodyConstructor from "../../components/Body"
import { pageResources, renderPage } from "../../components/renderPage"
import { FullPageLayout } from "../../cfg"
import { FilePath } from "../../path"
export const ContentPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
if (!opts) {
@ -19,8 +20,8 @@ export const ContentPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, Content, ...left, ...right, Footer]
},
async emit(_contentDir, cfg, content, resources, emit): Promise<string[]> {
const fps: string[] = []
async emit(_contentDir, cfg, content, resources, emit): Promise<FilePath[]> {
const fps: FilePath[] = []
const allFiles = content.map(c => c[1].data)
for (const [tree, file] of content) {
const slug = file.data.slug!
@ -41,7 +42,7 @@ export const ContentPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
externalResources
)
const fp = file.data.slug + ".html"
const fp = file.data.slug + ".html" as FilePath
await emit({
content,
slug: file.data.slug!,

6
quartz/plugins/emitters/folderPage.tsx
View File

@ -6,7 +6,7 @@ import { pageResources, renderPage } from "../../components/renderPage"
import { ProcessedContent, defaultProcessedContent } from "../vfile"
import { FullPageLayout } from "../../cfg"
import path from "path"
import { clientSideSlug } from "../../path"
import { FilePath, toServerSlug } from "../../path"
export const FolderPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
if (!opts) {
@ -22,7 +22,7 @@ export const FolderPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, Content, ...left, ...right, Footer]
},
async emit(_contentDir, cfg, content, resources, emit): Promise<string[]> {
async emit(_contentDir, cfg, content, resources, emit): Promise<FilePath[]> {
const fps: string[] = []
const allFiles = content.map(c => c[1].data)
@ -37,7 +37,7 @@ export const FolderPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
])))
for (const [tree, file] of content) {
const slug = clientSideSlug(file.data.slug!)
const slug = toServerSlug(file.data.slug!)
if (folders.has(slug)) {
folderDescriptions[slug] = [tree, file]
}

12
quartz/plugins/emitters/tagPage.tsx
View File

@ -5,7 +5,7 @@ import BodyConstructor from "../../components/Body"
import { pageResources, renderPage } from "../../components/renderPage"
import { ProcessedContent, defaultProcessedContent } from "../vfile"
import { FullPageLayout } from "../../cfg"
import { clientSideSlug } from "../../path"
import { FilePath, ServerSlug, toServerSlug } from "../../path"
export const TagPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
if (!opts) {
@ -21,17 +21,17 @@ export const TagPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, Content, ...left, ...right, Footer]
},
async emit(_contentDir, cfg, content, resources, emit): Promise<string[]> {
const fps: string[] = []
async emit(_contentDir, cfg, content, resources, emit): Promise<FilePath[]> {
const fps: FilePath[] = []
const allFiles = content.map(c => c[1].data)
const tags: Set<string> = new Set(allFiles.flatMap(data => data.frontmatter?.tags ?? []))
const tagDescriptions: Record<string, ProcessedContent> = Object.fromEntries([...tags].map(tag => ([
tag, defaultProcessedContent({ slug: `tags/${tag}`, frontmatter: { title: `Tag: ${tag}`, tags: [] } })
tag, defaultProcessedContent({ slug: `tags/${tag}` as ServerSlug, frontmatter: { title: `Tag: ${tag}`, tags: [] } })
])))
for (const [tree, file] of content) {
const slug = clientSideSlug(file.data.slug!)
const slug = toServerSlug(file.data.slug!)
if (slug.startsWith("tags/")) {
const tag = slug.slice("tags/".length)
if (tags.has(tag)) {
@ -60,7 +60,7 @@ export const TagPage: QuartzEmitterPlugin<FullPageLayout> = (opts) => {
externalResources
)
const fp = file.data.slug + ".html"
const fp = file.data.slug + ".html" as FilePath
await emit({
content,
slug: file.data.slug!,

9
quartz/plugins/index.ts
View File

@ -4,6 +4,7 @@ import { StaticResources } from '../resources'
import { joinStyles } from '../theme'
import { EmitCallback, PluginTypes } from './types'
import styles from '../styles/base.scss'
import { FilePath, ServerSlug } from '../path'
export type ComponentResources = {
css: string[],
@ -51,7 +52,7 @@ function joinScripts(scripts: string[]): string {
return scripts.map(script => `(function () {${script}})();`).join("\n")
}
export async function emitComponentResources(cfg: GlobalConfiguration, res: ComponentResources, emit: EmitCallback): Promise<string[]> {
export async function emitComponentResources(cfg: GlobalConfiguration, res: ComponentResources, emit: EmitCallback): Promise<FilePath[]> {
const fps = await Promise.all([
emit({
slug: "index",
@ -99,8 +100,8 @@ export * from './emitters'
declare module 'vfile' {
// inserted in processors.ts
interface DataMap {
slug: string
allSlugs: string[]
filePath: string
slug: ServerSlug
allSlugs: ServerSlug[]
filePath: FilePath
}
}

14
quartz/plugins/transformers/links.ts
View File

@ -1,5 +1,5 @@
import { QuartzTransformerPlugin } from "../types"
import { clientSideSlug, relative, relativeToRoot, slugify, trimPathSuffix } from "../../path"
import { CanonicalSlug, transformInternalLink } from "../../path"
import path from "path"
import { visit } from 'unist-util-visit'
import isAbsoluteUrl from "is-absolute-url"
@ -27,9 +27,9 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
htmlPlugins() {
return [() => {
return (tree, file) => {
const curSlug = clientSideSlug(file.data.slug!)
const curSlug = file.data.slug!
const transformLink = (target: string) => {
const targetSlug = clientSideSlug(slugify(decodeURI(target).trim()))
const targetSlug = transformInternalLink(target)
if (opts.markdownLinkResolution === 'relative' && !path.isAbsolute(targetSlug)) {
return './' + relative(curSlug, targetSlug)
} else if (opts.markdownLinkResolution === 'shortest') {
@ -38,13 +38,13 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
// if the file name is unique, then it's just the filename
const matchingFileNames = allSlugs.filter(slug => {
const parts = clientSideSlug(slug).split(path.posix.sep)
const parts = toServerSlug(slug).split(path.posix.sep)
const fileName = parts.at(-1)
return targetSlug === fileName
})
if (matchingFileNames.length === 1) {
const targetSlug = clientSideSlug(matchingFileNames[0])
const targetSlug = toServerSlug(matchingFileNames[0])
return './' + relativeToRoot(curSlug, targetSlug)
}
@ -55,7 +55,7 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
return './' + relativeToRoot(curSlug, targetSlug)
}
const outgoing: Set<string> = new Set()
const outgoing: Set<CanonicalSlug> = new Set()
visit(tree, 'element', (node, _index, _parent) => {
// rewrite all links
if (
@ -113,6 +113,6 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
declare module 'vfile' {
interface DataMap {
links: string[]
links: CanonicalSlug[]
}
}

3
quartz/plugins/transformers/toc.ts
View File

@ -3,6 +3,7 @@ import { Root } from "mdast"
import { visit } from "unist-util-visit"
import { toString } from "mdast-util-to-string"
import { slug as slugAnchor } from 'github-slugger'
import { CanonicalSlug } from "../../path"
export interface Options {
maxDepth: 1 | 2 | 3 | 4 | 5 | 6,
@ -19,7 +20,7 @@ const defaultOptions: Options = {
interface TocEntry {
depth: number,
text: string,
slug: string
slug: CanonicalSlug
}
export const TableOfContents: QuartzTransformerPlugin<Partial<Options> | undefined> = (userOpts) => {

7
quartz/plugins/types.ts
View File

@ -3,6 +3,7 @@ import { StaticResources } from "../resources"
import { ProcessedContent } from "./vfile"
import { GlobalConfiguration } from "../cfg"
import { QuartzComponent } from "../components/types"
import { FilePath, ServerSlug } from "../path"
export interface PluginTypes {
transformers: QuartzTransformerPluginInstance[],
@ -29,14 +30,14 @@ export type QuartzFilterPluginInstance = {
export type QuartzEmitterPlugin<Options extends OptionType = undefined> = (opts?: Options) => QuartzEmitterPluginInstance
export type QuartzEmitterPluginInstance = {
name: string
emit(contentDir: string, cfg: GlobalConfiguration, content: ProcessedContent[], resources: StaticResources, emitCallback: EmitCallback): Promise<string[]>
emit(contentDir: string, cfg: GlobalConfiguration, content: ProcessedContent[], resources: StaticResources, emitCallback: EmitCallback): Promise<FilePath[]>
getQuartzComponents(): QuartzComponent[]
}
export interface EmitOptions {
slug: string
slug: ServerSlug
ext: `.${string}` | ""
content: string
}
export type EmitCallback = (data: EmitOptions) => Promise<string>
export type EmitCallback = (data: EmitOptions) => Promise<FilePath>

17
quartz/processors/emit.ts
View File

@ -5,7 +5,7 @@ import { PerfTimer } from "../perf"
import { ComponentResources, emitComponentResources, getComponentResources, getStaticResourcesFromPlugins } from "../plugins"
import { EmitCallback } from "../plugins/types"
import { ProcessedContent } from "../plugins/vfile"
import { QUARTZ, slugify } from "../path"
import { FilePath, QUARTZ, slugifyFilePath } from "../path"
import { globbyStream } from "globby"
import chalk from "chalk"
@ -71,7 +71,7 @@ export async function emitContent(contentFolder: string, output: string, cfg: Qu
log.start(`Emitting output files`)
const emit: EmitCallback = async ({ slug, ext, content }) => {
const pathToPage = path.join(output, slug + ext)
const pathToPage = path.join(output, slug + ext) as FilePath
const dir = path.dirname(pathToPage)
await fs.promises.mkdir(dir, { recursive: true })
await fs.promises.writeFile(pathToPage, content)
@ -123,15 +123,16 @@ export async function emitContent(contentFolder: string, output: string, cfg: Qu
// glob all non MD/MDX/HTML files in content folder and copy it over
const assetsPath = path.join(output, "assets")
for await (const fp of globbyStream("**", {
for await (const rawFp of globbyStream("**", {
ignore: ["**/*.md"],
cwd: contentFolder,
})) {
const ext = path.extname(fp as string)
const src = path.join(contentFolder, fp as string)
const name = slugify(fp as string) + ext
const dest = path.join(assetsPath, name)
const dir = path.dirname(dest)
const fp = rawFp as FilePath
const ext = path.extname(fp)
const src = path.join(contentFolder, fp) as FilePath
const name = slugifyFilePath(fp as FilePath) + ext as FilePath
const dest = path.join(assetsPath, name) as FilePath
const dir = path.dirname(dest) as FilePath
await fs.promises.mkdir(dir, { recursive: true }) // ensure dir exists
await fs.promises.copyFile(src, dest)
emittedFiles += 1

11
quartz/processors/parse.ts
View File

@ -7,7 +7,7 @@ import { Root as HTMLRoot } from 'hast'
import { ProcessedContent } from '../plugins/vfile'
import { PerfTimer } from '../perf'
import { read } from 'to-vfile'
import { slugify } from '../path'
import { FilePath, ServerSlug, slugifyFilePath } from '../path'
import path from 'path'
import os from 'os'
import workerpool, { Promise as WorkerPromise } from 'workerpool'
@ -73,7 +73,7 @@ async function transpileWorkerScript() {
})
}
export function createFileParser(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: string[], allSlugs: string[], verbose: boolean) {
export function createFileParser(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: FilePath[], allSlugs: ServerSlug[], verbose: boolean) {
return async (processor: QuartzProcessor) => {
const res: ProcessedContent[] = []
for (const fp of fps) {
@ -89,7 +89,7 @@ export function createFileParser(transformers: QuartzTransformerPluginInstance[]
}
// base data properties that plugins may use
file.data.slug = slugify(path.relative(baseDir, file.path))
file.data.slug = slugifyFilePath(path.relative(baseDir, file.path) as FilePath)
file.data.allSlugs = allSlugs
file.data.filePath = fp
@ -110,7 +110,7 @@ export function createFileParser(transformers: QuartzTransformerPluginInstance[]
}
}
export async function parseMarkdown(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: string[], verbose: boolean): Promise<ProcessedContent[]> {
export async function parseMarkdown(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: FilePath[], verbose: boolean): Promise<ProcessedContent[]> {
const perf = new PerfTimer()
const log = new QuartzLogger(verbose)
@ -118,8 +118,7 @@ export async function parseMarkdown(transformers: QuartzTransformerPluginInstanc
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
// get all slugs ahead of time as each thread needs a copy
// const slugs: string[] = fps.map(fp => slugify(path))
const allSlugs = fps.map(fp => slugify(path.relative(baseDir, path.resolve(fp))))
const allSlugs = fps.map(fp => slugifyFilePath(path.relative(baseDir, path.resolve(fp)) as FilePath))
let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`)

3
quartz/worker.ts
View File

@ -1,11 +1,12 @@
import config from "../quartz.config"
import { FilePath, ServerSlug } from "./path"
import { createFileParser, createProcessor } from "./processors/parse"
const transformers = config.plugins.transformers
const processor = createProcessor(transformers)
// only called from worker thread
export async function parseFiles(baseDir: string, fps: string[], allSlugs: string[], verbose: boolean) {
export async function parseFiles(baseDir: string, fps: FilePath[], allSlugs: ServerSlug[], verbose: boolean) {
const parse = createFileParser(transformers, baseDir, fps, allSlugs, verbose)
return parse(processor)
}

2
tsconfig.tsbuildinfo
View File

File diff suppressed because one or more lines are too long