Recently Written · git

bm-page (7652 bytes)

 #!/bin/sh
 
 # bm-page: print a web page as plain text: its title, a blank line, then its
 # text. When it is installed, the fzf list of bm shows the page of the
 # bookmark under the cursor in the preview, live. SBM_FETCH=0 keeps it off
 # the network.
 #
 #     bm-page https://suckless.org/ | less
 #
 # Only http and https pages are fetched. curl stops after 5 seconds, and dd
 # passes at most 512k of the page. Scripts, styles and comments are left
 # out. Control characters are removed, so that a page cannot send escape
 # sequences to the terminal.
 
 if [ $# -ne 1 ]; then
     printf 'usage: bm-page <url>\n' >&2
     exit 2
 fi
 [ "${SBM_FETCH:-1}" != 0 ] || exit 0
 case $(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') in
     http://*|https://*) ;;
     *) exit 0 ;;
 esac
 
 # The first awk reads the page as one record (RS is a control character),
 # then splits it at each "<". Each part starts with a tag and ends with the
 # text after it, so one pass through the parts reads the page in linear
 # time. The second awk tidies the white space.
 curl -sL --max-time 5 -A 'Mozilla/5.0' -- "$1" 2>/dev/null \
     | dd bs=1024 count=512 2>/dev/null | LC_ALL=C awk '
     function utf8(c) {
         if (c == 160) return " "
         if (c < 128) return sprintf("%c", c)
         if (c < 2048) return sprintf("%c%c", 192 + int(c / 64), 128 + c % 64)
         if (c < 65536)
             return sprintf("%c%c%c", 224 + int(c / 4096), 128 + int(c / 64) % 64, 128 + c % 64)
         if (c < 1114112)
             return sprintf("%c%c%c%c", 240 + int(c / 262144), 128 + int(c / 4096) % 64,
                            128 + int(c / 64) % 64, 128 + c % 64)
         return ""
     }
     function entity(name,    n, i) {
         if (name ~ /^#[0-9]+$/) return utf8(substr(name, 2) + 0)
         if (name ~ /^#[xX][0-9A-Fa-f]+$/) {
             n = 0
             for (i = 3; i <= length(name); i++)
                 n = n * 16 + index("0123456789abcdef", tolower(substr(name, i, 1))) - 1
             return utf8(n)
         }
         if (name in named) return named[name]
         return "&" name ";"
     }
     function decode(s,    out, i, j, name) {
         out = ""
         while ((i = index(s, "&")) > 0) {
             out = out substr(s, 1, i - 1)
             s = substr(s, i + 1)
             j = index(s, ";")
             name = substr(s, 1, j - 1)
             if (j > 1 && j <= 12 && name ~ /^(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z][A-Za-z0-9]*)$/) {
                 out = out entity(name)
                 s = substr(s, j + 1)
             } else {
                 out = out "&"
             }
         }
         return out s
     }
     # text: the text of the page, or of its title. As in a browser, a line
     # break in the text is a space. The line breaks come from the tags: brk
     # tells how many are waiting for the next text.
     function text(s) {
         if (intitle) {
             title = title " " s
             return
         }
         s = decode(s)
         gsub(/[ \t\r\n]+/, " ", s)
         if (s == "" || s == " ") {
             if (s == " " && !brk) printf " "
             return
         }
         if (brk) {
             printf "%s", brk == 2 ? "\n\n" : "\n"
             brk = 0
             sub(/^ /, "", s)
         }
         printf "%s", s
     }
     BEGIN {
         RS = "\001"
         named["amp"] = "&"; named["lt"] = "<"; named["gt"] = ">"; named["quot"] = "\""
         named["apos"] = "\047"; named["nbsp"] = " "
         named["ndash"] = utf8(8211); named["mdash"] = utf8(8212); named["hellip"] = utf8(8230)
         named["lsquo"] = utf8(8216); named["rsquo"] = utf8(8217)
         named["ldquo"] = utf8(8220); named["rdquo"] = utf8(8221)
         named["laquo"] = utf8(171); named["raquo"] = utf8(187); named["middot"] = utf8(183)
         named["bull"] = utf8(8226); named["copy"] = utf8(169); named["reg"] = utf8(174)
         named["trade"] = utf8(8482); named["deg"] = utf8(176); named["times"] = utf8(215)
         named["euro"] = utf8(8364)
         # Tags that start a line, and tags that start a paragraph.
         split("br div li tr dt dd header footer nav main section article aside " \
               "figure figcaption form details summary address hr", b, " ")
         for (i in b) line[b[i]] = 1
         split("p h1 h2 h3 h4 h5 h6 ul ol dl table blockquote pre", b, " ")
         for (i in b) para[b[i]] = 1
         # Tags whose content is no text of the page.
         split("script style noscript template svg iframe object", b, " ")
         for (i in b) hidden[b[i]] = 1
     }
     { page = page $0 }
     END {
         if (substr(page, 1, 5) == "%PDF-") {
             print "(a PDF document)"
             exit
         }
         n = split(page, part, "<")
         text(part[1])
         for (i = 2; i <= n; i++) {
             p = part[i]
             if (comment || substr(p, 1, 3) == "!--") {
                 k = index(p, "-->")
                 comment = !k
                 if (k && skip == "") text(substr(p, k + 3))
                 continue
             }
             k = index(p, ">")
             if (!k) {
                 # A "<" in the text, or the end of a page that dd cut off.
                 if (skip == "" && i < n) text("<" p)
                 continue
             }
             tag = tolower(substr(p, 1, k - 1))
             closing = substr(tag, 1, 1) == "/"
             name = tag
             sub(/^\//, "", name)
             sub(/[^a-z0-9].*/, "", name)
             rest = substr(p, k + 1)
             if (skip != "") {
                 if (closing && name == skip) {
                     skip = ""
                     text(rest)
                 }
                 continue
             }
             if (name in hidden && !closing && substr(tag, length(tag)) != "/") {
                 skip = name
                 continue
             }
             if (name == "title" && !done) {
                 if (closing) {
                     intitle = 0
                     done = 1
                     t = decode(title)
                     gsub(/[ \t\r\n]+/, " ", t)
                     printf "\n\001T%s\n", t
                     brk = 0
                 } else {
                     intitle = 1
                 }
             } else if (name == "main") {
                 # Marks for the second awk: the main part of the page.
                 printf "\n\001%s\n", closing ? ">" : "<"
                 brk = 0
             } else if (name in para) {
                 brk = 2
             } else if (name in line) {
                 if (!brk) brk = 1
             } else if (name == "td" || name == "th") {
                 text(" ")
             }
             text(rest)
         }
         print ""
     }' | LC_ALL=C awk '
     {
         gsub(/\302\240/, " ")
         gsub(/\302[\200-\237]/, "")
         gsub(/[ \t\r]+/, " ")
         sub(/^ /, "")
         sub(/ $/, "")
     }
     /^\001T/ {
         if (title == "") {
             title = substr($0, 3)
             sub(/^ /, "", title)
         }
         next
     }
     /^\001</ { if (!opened) { opened = 1; from = n + 1 }; next }
     /^\001>/ { if (opened && !closed) { closed = 1; to = n }; next }
     { line[++n] = $0 }
     # The title, then the main part of the page when the page marks one, so
     # that menus do not fill the preview. At most one blank line in a row.
     END {
         if (!opened) from = 1
         if (!closed) to = n
         if (title != "") {
             print title
             seen = blank = 1
         }
         for (i = from; i <= to; i++) {
             if (line[i] == "") {
                 blank = seen
                 continue
             }
             if (blank) print ""
             print line[i]
             seen = 1
             blank = 0
         }
     }' | tr -d '\000-\010\013-\037\177'