build.sh (17885 bytes)
1 #!/bin/sh 2 # build.sh — converts posts/*.md and pages/*.md to site/ using pandoc 3 # Usage: sh build.sh [--no-deploy] 4 5 set -e 6 7 POSTS_DIR="posts" 8 PAGES_DIR="pages" 9 SITE_DIR="site" 10 STATIC_DIR="static" 11 SITE_URL="https://recentlywritten.com" 12 DEPLOY_HOST="root@recentlywritten.com" 13 DEPLOY_PATH="/var/www/recentlywritten/" 14 FEED_SIZE=10 15 DICKT_FEED="https://dickt.store/rss.xml" 16 17 DEPLOY=yes 18 [ "$1" = "--no-deploy" ] && DEPLOY=no 19 20 WORK="${TMPDIR:-/tmp}/rw-build.$$" 21 mkdir -p "$WORK" 22 trap 'rm -rf "$WORK"' EXIT INT TERM 23 24 # ── setup ────────────────────────────────────────────────── 25 rm -rf "$SITE_DIR" 26 mkdir -p "$SITE_DIR" 27 cp style.css "$SITE_DIR/style.css" 28 # Images and downloads the recovered posts link to as static/... 29 [ -d "$STATIC_DIR" ] && cp -R "$STATIC_DIR" "$SITE_DIR/$STATIC_DIR" 30 31 # pandoc HTML template for individual posts. 32 # Kept as a relative path in the working directory on purpose: under cygwin 33 # the pandoc on PATH is the native Windows build, which cannot resolve a 34 # POSIX path like /tmp/..., but does resolve a path relative to the cwd. 35 TEMPLATE=".build-template.html" 36 trap 'rm -rf "$WORK" "$TEMPLATE"' EXIT INT TERM 37 cat > "$TEMPLATE" << 'TMPL' 38 <!DOCTYPE html> 39 <html lang="en"> 40 <head> 41 <meta charset="UTF-8" /> 42 <meta name="viewport" content="width=device-width, initial-scale=1" /> 43 <title>$title$ — Recently Written</title> 44 <meta property="og:type" content="article" /> 45 <meta property="og:site_name" content="Recently Written" /> 46 <meta property="og:title" content="$title$" /> 47 <meta property="og:url" content="$url$" /> 48 <meta property="og:image" content="https://recentlywritten.com/static/og.png" /> 49 <meta name="twitter:card" content="summary_large_image" /> 50 <link rel="stylesheet" href="style.css" /> 51 <link rel="alternate" type="application/rss+xml" title="Recently Written" href="rss.xml" /> 52 $if(feed)$ 53 <link rel="alternate" type="application/atom+xml" title="$title$" href="$feed$" /> 54 $endif$ 55 </head> 56 <body> 57 <div id="container"> 58 59 <div id="top"> 60 <a class="site-title" href="index.html">Recently Written</a> 61 <nav> 62 <a href="index.html">Home</a> 63 <a href="about.html">about</a> 64 <a href="lair.html">code</a> 65 <a href="projects.html">projects</a> 66 <a href="git/index.html">git</a> 67 <a href="links.html">links</a> 68 <a href="youtube.html">youtube</a> 69 <a href="esperanto.html">esperanto</a> 70 <a href="call.html">contact</a> 71 <a href="rss.xml">rss</a> 72 <a href="account/">subscribe</a> 73 </nav> 74 </div> 75 76 <h1 class="post-title">$title$</h1> 77 78 $body$ 79 80 $if(links)$ 81 <p class="links">$for(links)$$links$$sep$ · $endfor$</p> 82 83 $endif$ 84 <div id="footer"> 85 <a href="index.html">← Home</a> · 86 <a href="https://github.com/equwal">Github</a> 87 <p class="more">More sites and apps: 88 <a href="https://subread.space/" title="Read along with an audiobook.">SubRead</a> · 89 <a href="https://booksimulator.com/" title="A reading room for Aozora Bunko and Project Gutenberg books.">Book Simulator</a> · 90 <a href="https://honjimaku.com/" title="Subtitles for Japanese audiobooks.">honjimaku.com</a> · 91 <a href="https://sbmsync.com/" title="Your bookmarks, the same on every device.">sbm Sync</a> · 92 <a href="https://dickt.store/" title="Language-learning tools, flashcards and web toys.">dickt.store</a> · 93 <a href="https://hentaibun.online/" title="Learn kanbun and kobun.">hentaibun.online</a> · 94 <a href="https://mowimypopol.online/" title="Archival study materials for Polish.">mowimypopol.online</a> · 95 <a href="https://alljapanesealltheti.me/" title="All Japanese All The Time.">AJATT mirror</a></p> 96 <p class="more">Apps: 97 <a href="https://github.com/equwal/subread-android/releases/latest">SubRead for Android</a> · 98 <a href="https://github.com/equwal/subread-overlay/releases/latest">SubRead Overlay</a> · 99 <a href="https://github.com/equwal/subread-dictionary/releases/latest">SubRead Dictionary</a> · 100 <a href="https://github.com/equwal/subread-anki">SubRead Anki</a> · 101 <a href="https://github.com/equwal/subread-extension/releases/latest">SubRead for YouTube</a> · 102 <a href="https://github.com/equwal/subread.koplugin">SubRead for KOReader</a> · 103 <a href="https://github.com/equwal/subrep-android/releases/latest">Subrep</a> · 104 <a href="https://github.com/equwal/sbm-android/releases/latest">sbm for Android</a> · 105 <a href="https://github.com/equwal/sbm-extension/releases/latest">sbm add-on</a> · 106 <a href="https://github.com/equwal/rebind/releases">Rebind</a> · 107 <a href="https://github.com/equwal/ink-recents/releases/latest">Ink Recents</a> · 108 <a href="https://github.com/equwal/ink-dim/releases/latest">Ink Dim</a> · 109 <a href="https://github.com/equwal/ink-update/releases/latest">Ink Update</a> · 110 <a href="projects.html">All projects</a></p> 111 </div> 112 113 </div> 114 </body> 115 </html> 116 TMPL 117 118 # Read one `key: value` line out of a file's front matter, dropping the 119 # surrounding quotes that titles containing a colon have to be written with. 120 meta() { 121 sed -n "s/^$2: *//p" "$1" | head -1 | sed -e 's/^"//' -e 's/"$//' -e 's/\\"/"/g' 122 } 123 124 # Escape the characters that must not appear raw in generated HTML. 125 esc() { 126 sed -e 's/&/\&/g' -e 's/</\</g' -e 's/>/\>/g' 127 } 128 129 # render <markdown> <output> <title> 130 render() { 131 pandoc \ 132 --from markdown \ 133 --to html5 \ 134 --template "$TEMPLATE" \ 135 --metadata title="$3" \ 136 --metadata url="$SITE_URL/$(basename "$2")" \ 137 --output "$2" \ 138 "$1" 139 } 140 141 # ── build each post ───────────────────────────────────────── 142 : > "$WORK/posts.tsv" 143 for md in "$POSTS_DIR"/*.md; do 144 [ -e "$md" ] || continue 145 slug=$(basename "$md" .md) 146 title=$(meta "$md" title) 147 date=$(meta "$md" date) 148 # Recovered posts carry `order`: their position on the old front page, 149 # the only surviving record of their sequence. Everything else sorts by 150 # date. The two never interleave, since an order is three digits ("051") 151 # and a date leads with its year ("2026-..."), so new posts land on top. 152 order=$(meta "$md" order) 153 [ -z "$order" ] && order="$date" 154 render "$md" "$SITE_DIR/${slug}.html" "$title" 155 printf '%s\t%s\t%s\t%s\n' "$order" "$date" "$slug" "$title" >> "$WORK/posts.tsv" 156 done 157 158 # Dates order the posts but are never shown on the site: most recovered ones 159 # are approximations taken from archive crawls, good enough to sort by and 160 # not good enough to publish. C collation keeps the sort independent of the 161 # system language. 162 LC_ALL=C sort -r "$WORK/posts.tsv" > "$WORK/sorted.tsv" 163 164 # ── build each standalone page ────────────────────────────── 165 # Pages are dateless and stay out of the chronological list; these are the 166 # hub pages the old site linked from its nav (lair, esperanto, call, ...). 167 : > "$WORK/pages.tsv" 168 for md in "$PAGES_DIR"/*.md; do 169 [ -e "$md" ] || continue 170 slug=$(basename "$md" .md) 171 title=$(meta "$md" title) 172 render "$md" "$SITE_DIR/${slug}.html" "$title" 173 printf '%s\t%s\n' "$slug" "$title" >> "$WORK/pages.tsv" 174 done 175 176 # ── list fragments for the index ──────────────────────────── 177 TAB=$(printf '\t') 178 179 while IFS="$TAB" read -r order date slug title; do 180 [ -z "$slug" ] && continue 181 printf '<li><a href="%s.html">%s</a></li>\n' \ 182 "$slug" "$(printf '%s' "$title" | esc)" 183 done < "$WORK/sorted.tsv" > "$WORK/post-items.html" 184 185 LC_ALL=C sort -f "$WORK/pages.tsv" | while IFS="$TAB" read -r slug title; do 186 [ -z "$slug" ] && continue 187 printf '<li><a href="%s.html">%s</a></li>\n' \ 188 "$slug" "$(printf '%s' "$title" | esc)" 189 done > "$WORK/page-items.html" 190 191 # ── build index ───────────────────────────────────────────── 192 # The fragments are read from disk rather than substituted into an awk 193 # variable, so titles containing & or \ survive intact. 194 awk -v postfile="$WORK/post-items.html" -v pagefile="$WORK/page-items.html" ' 195 /<!-- POST_LIST -->/ { 196 print "<ul class=\"post-list\">" 197 while ((getline line < postfile) > 0) print line 198 close(postfile) 199 print "</ul>" 200 next 201 } 202 /<!-- PAGE_LIST -->/ { 203 print "<ul class=\"page-list\">" 204 while ((getline line < pagefile) > 0) print line 205 close(pagefile) 206 print "</ul>" 207 next 208 } 209 { print } 210 ' index.html > "$SITE_DIR/index.html" 211 212 # ── feed ───────────────────────────────────────────────────── 213 # Only the newest FEED_SIZE posts go in, and that is deliberate as well as 214 # conventional: a feed reader shows each item's date, and the newest posts are 215 # the ones whose dates are exact rather than recovered from a crawl. 216 217 # RFC 822, as RSS requires. The C locale matters: without it the day and 218 # month names come out in the system language. `date -d` is GNU; where it is 219 # missing the item simply goes out without a pubDate. A post has only a 220 # date, so its time is midnight; a podcast episode has an exact time. 221 feed_date() { 222 LC_ALL=C date -u -d "$1" '+%a, %d %b %Y %H:%M:%S +0000' 2>/dev/null 223 } 224 225 # A feed reader shows the HTML away from the site, so links relative to it 226 # need the site put in front of them. 227 absolute() { 228 sed -E \ 229 -e 's@(href|src)="/([^/"][^"]*)"@\1="'"$SITE_URL"'/\2"@g' \ 230 -e 's@(href|src)="([^"/#][^":]*)"@\1="'"$SITE_URL"'/\2"@g' 231 } 232 233 { 234 printf '<?xml version="1.0" encoding="UTF-8"?>\n' 235 printf '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">\n<channel>\n' 236 printf ' <title>Recently Written</title>\n' 237 printf ' <link>%s/</link>\n' "$SITE_URL" 238 printf ' <description>A site full of things which I have recently written.</description>\n' 239 printf ' <language>en-us</language>\n' 240 printf ' <atom:link href="%s/rss.xml" rel="self" type="application/rss+xml" />\n' "$SITE_URL" 241 head -n "$FEED_SIZE" "$WORK/sorted.tsv" | while IFS="$TAB" read -r order date slug title; do 242 url="$SITE_URL/$slug.html" 243 printf ' <item>\n' 244 printf ' <title>%s</title>\n' "$(printf '%s' "$title" | esc)" 245 printf ' <link>%s</link>\n' "$url" 246 printf ' <guid isPermaLink="true">%s</guid>\n' "$url" 247 pub=$(feed_date "$date") && [ -n "$pub" ] && 248 printf ' <pubDate>%s</pubDate>\n' "$pub" 249 printf ' <description><![CDATA[' 250 pandoc --from markdown --to html5 "$POSTS_DIR/$slug.md" | 251 absolute | sed 's/]]>/]]]]><![CDATA[>/g' 252 printf ']]></description>\n' 253 printf ' </item>\n' 254 done 255 # Releases on dickt.store go in the same feed. The build copies each 256 # item from the live dickt.store feed. If the fetch fails, the build 257 # continues without these items, because the posts are more important. 258 if curl -fsS -o "$WORK/dickt.xml" "$DICKT_FEED"; then 259 awk '/<item>/,/<\/item>/' "$WORK/dickt.xml" 260 else 261 echo "warning: no items from $DICKT_FEED" >&2 262 fi 263 printf '</channel>\n</rss>\n' 264 } > "$SITE_DIR/rss.xml" 265 266 # ── podcast ────────────────────────────────────────────────── 267 # podcast.tsv lists the podcast episodes, newest first, one on each line: 268 # date <TAB> file <TAB> bytes <TAB> seconds <TAB> youtube <TAB> title 269 # date is the UTC time of the upload in ISO 8601. file is the name of the 270 # audio in static/language-podcast/. The audio stays out of git (see 271 # .gitignore), so its size and length, which the feed must give, are kept 272 # here. youtube is the ID of the same episode on YouTube, or "-" if none. 273 # Every field needs a value, because read merges empty tab-separated fields. 274 PODCAST_PAGE="polyglot-podcast.html" 275 AUDIO_DIR="static/language-podcast" 276 277 youtube_url() { 278 printf 'https://www.youtube.com/watch?v=%s' "$1" 279 } 280 281 { 282 printf '<?xml version="1.0" encoding="UTF-8"?>\n' 283 printf '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"' 284 printf ' xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"' 285 printf ' xmlns:podcast="https://podcastindex.org/namespace/1.0">\n<channel>\n' 286 printf ' <title>Polyglot Podcast</title>\n' 287 printf ' <link>%s/%s</link>\n' "$SITE_URL" "$PODCAST_PAGE" 288 printf ' <description>Spenser Truex talks about how to learn languages, and about AI.</description>\n' 289 printf ' <language>en-us</language>\n' 290 printf ' <atom:link href="%s/podcast.xml" rel="self" type="application/rss+xml" />\n' "$SITE_URL" 291 printf ' <itunes:author>Spenser Truex</itunes:author>\n' 292 # Podcast directories such as Spotify send the code that proves 293 # ownership of the feed to this email address. 294 printf ' <itunes:owner>\n' 295 printf ' <itunes:name>Spenser Truex</itunes:name>\n' 296 printf ' <itunes:email>truex@equwal.com</itunes:email>\n' 297 printf ' </itunes:owner>\n' 298 printf ' <itunes:image href="%s/static/podcast.png" />\n' "$SITE_URL" 299 printf ' <itunes:category text="Education">\n' 300 printf ' <itunes:category text="Language Learning" />\n' 301 printf ' </itunes:category>\n' 302 printf ' <itunes:explicit>false</itunes:explicit>\n' 303 printf ' <podcast:funding url="https://ko-fi.com/truex">Support the podcast</podcast:funding>\n' 304 while IFS="$TAB" read -r date file bytes seconds youtube title; do 305 [ -z "$date" ] && continue 306 case $file in 307 *.m4a) type=audio/x-m4a ;; 308 *.mp3) type=audio/mpeg ;; 309 *) echo "podcast.tsv: no MIME type for $file" >&2; exit 1 ;; 310 esac 311 url="$SITE_URL/$AUDIO_DIR/$file" 312 printf ' <item>\n' 313 printf ' <title>%s</title>\n' "$(printf '%s' "$title" | esc)" 314 printf ' <enclosure url="%s" length="%s" type="%s" />\n' "$url" "$bytes" "$type" 315 printf ' <guid isPermaLink="false">%s</guid>\n' "$url" 316 pub=$(feed_date "$date") && [ -n "$pub" ] && 317 printf ' <pubDate>%s</pubDate>\n' "$pub" 318 printf ' <itunes:duration>%s</itunes:duration>\n' "$seconds" 319 if [ "$youtube" != - ]; then 320 printf ' <link>%s</link>\n' "$(youtube_url "$youtube")" 321 printf ' <description>Video: %s</description>\n' "$(youtube_url "$youtube")" 322 fi 323 printf ' </item>\n' 324 done < podcast.tsv 325 printf '</channel>\n</rss>\n' 326 } > "$SITE_DIR/podcast.xml" 327 328 # The podcast page lists the same episodes. The list goes in place of the 329 # comment <!-- EPISODE_LIST --> in pages/polyglot-podcast.md. 330 while IFS="$TAB" read -r date file bytes seconds youtube title; do 331 [ -z "$date" ] && continue 332 printf '<li><a href="%s/%s">%s</a>' "$AUDIO_DIR" "$file" \ 333 "$(printf '%s' "$title" | esc)" 334 [ "$youtube" = - ] || 335 printf ' (<a href="%s">video</a>)' "$(youtube_url "$youtube")" 336 printf '</li>\n' 337 done < podcast.tsv > "$WORK/episode-items.html" 338 awk -v listfile="$WORK/episode-items.html" ' 339 /<!-- EPISODE_LIST -->/ { 340 print "<ul>" 341 while ((getline line < listfile) > 0) print line 342 close(listfile) 343 print "</ul>" 344 next 345 } 346 { print } 347 ' "$SITE_DIR/$PODCAST_PAGE" > "$WORK/podcast-page.html" 348 cp "$WORK/podcast-page.html" "$SITE_DIR/$PODCAST_PAGE" 349 350 # ── post data for the accounts service ────────────────────── 351 # accounts/ reads these files to send email about new posts and to make 352 # the private feed of each user. index.tsv has one line for each post, 353 # newest first: date, slug, tags, title. postdata/<slug>.html has the body 354 # of the post, with absolute links. 355 mkdir -p "$SITE_DIR/postdata" 356 while IFS="$TAB" read -r order date slug title; do 357 [ -z "$slug" ] && continue 358 printf '%s\t%s\t%s\t%s\n' "$date" "$slug" "$(meta "$POSTS_DIR/$slug.md" tags)" "$title" 359 pandoc --from markdown --to html5 "$POSTS_DIR/$slug.md" < /dev/null | 360 absolute > "$SITE_DIR/postdata/$slug.html" 361 done < "$WORK/sorted.tsv" > "$SITE_DIR/postdata/index.tsv" 362 363 echo "Built $(ls "$SITE_DIR"/*.html | wc -l | tr -d ' ') pages → $SITE_DIR/" 364 echo " posts: $(wc -l < "$WORK/posts.tsv" | tr -d ' ') pages: $(wc -l < "$WORK/pages.tsv" | tr -d ' ') feed: $(grep -c '<item>' "$SITE_DIR/rss.xml") items podcast: $(grep -c '<item>' "$SITE_DIR/podcast.xml") episodes" 365 366 # ── deploy ─────────────────────────────────────────────────── 367 # git/ and git.html share this web root but are published by deploy-git.sh, 368 # not built here. Excluding them keeps --delete from treating them as stale: 369 # rsync never deletes an excluded path on the receiving side. 370 # static/book/ holds book files that stay out of the repo (see .gitignore). 371 # They exist only on the server, so the same rule protects them. 372 # static/language-podcast/ holds the podcast audio, which also stays out of 373 # the repo. The P (protect) rule stops --delete from removing the server 374 # copy when the working tree has no audio, but the audio that the working 375 # tree has still uploads. 376 # sitemap.xml and robots.txt were written on the server and are not built here. 377 if [ "$DEPLOY" = yes ]; then 378 rsync -avzP --delete --exclude=/git/ --exclude=/git.html \ 379 --exclude=/static/book/ --filter='P /static/language-podcast/***' \ 380 --exclude=/sitemap.xml --exclude=/robots.txt \ 381 "$SITE_DIR/" "$DEPLOY_HOST:$DEPLOY_PATH" 382 ssh "$DEPLOY_HOST" "chmod -R a+rX $DEPLOY_PATH" 383 fi