commit 66861e4e69095a395e33c264051538c46795bd41 equwal <truex@equwal.com> 2026-09-22 18:28:26 -0700 Add the bookmarks of an account to the web site Signed-in users can now use their bookmarks at /bookmarks, in any browser and on a phone. The pages do what bm does: - Search with the search of the add-on, filter by tag, and sort by date, URL, description or tag. Text that no bookmark matches opens as an address or as a web search. - Add, edit and delete a bookmark. A URL that is a bookmark already is refused. - Edit the whole file, as bm -e does. - Import the HTML file that a browser exports, as bm-import and bm --merge do. A file of bookmark lines works too. - Download the file. Each change goes through Store.sync, so the changes that devices made since a page opened stay. The pages obey the rules of sync. The download works also when sync is paused. Sign-in now opens the bookmarks. Sign-up still opens the account page.
README.md | 23 ++ bookmarks.go | 737 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ bookmarks_test.go | 234 +++++++++++++++++ main.go | 26 +- pages.html | 94 ++++++- store.go | 42 +++- web_test.go | 434 ++++++++++++++++++++++++++++++++ 7 files changed, 1571 insertions(+), 19 deletions(-)
diff --git a/README.md b/README.md index 64ec000..bf2dade 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,29 @@ The merge compares whole lines: When the server does not know the last version of a device (for its first sync), nothing is removed: the result is the union of both files. +## Bookmarks on the web + +After you sign in, the page **/bookmarks** shows your bookmarks in any +browser, also on a phone. It does what bm does: + +- Search as fzf does, with the search of the add-on. Pick a tag. Sort by + date, URL, description or tag. Text that no bookmark matches opens as an + address, or as a web search. +- Add a bookmark. A URL that is a bookmark already is refused. +- Edit or delete a bookmark. +- Edit the whole file, as `bm -e` does. +- Import the HTML file that a browser exports, as + `bm-import bookmarks.html | bm --merge` does: folders become tags, and + known URLs are skipped. A file of bookmark lines works too. +- Download the file. + +Each change goes through the same merge as a sync, so the changes that your +devices made since the page opened stay. The pages have no script. + +The pages obey the rules of sync: the email address must be confirmed, and +with billing on, the account needs its trial or a subscription. The +download always works, because the bookmarks belong to their owner. + ## Protocol Plain HTTP, for any client with curl. diff --git a/bookmarks.go b/bookmarks.go new file mode 100644 index 0000000..c5834b9 --- /dev/null +++ b/bookmarks.go @@ -0,0 +1,737 @@ +package main + +// The web pages for the bookmarks of an account. Each change goes through +// Store.sync, as the file of a device does, so the changes that other +// devices made since a page opened stay. + +import ( + "errors" + "fmt" + "html" + "io" + "io/fs" + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +const ( + maxRows = 1000 // bookmarks on one page + maxForm = 64 << 10 // bytes in the form of one bookmark + maxImport = 32 << 20 // bytes in an imported file: browsers put icons in it +) + +var ( + errNoURL = errors.New("type the URL of the bookmark") + errHashURL = errors.New("a URL cannot start with #, because bm reads the line as a comment") + // errChanged tells that a page shows a line that the file does not have + // now. + errChanged = errors.New("the bookmark changed on another device") +) + +// sentence gives the text of an error as a sentence for a page. +func sentence(err error) string { + return strings.ToUpper(err.Error()[:1]) + err.Error()[1:] + "." +} + +// ---- the bookmark file ---- + +// Bookmark is one bookmark line of the file: +// +// URL<tab>description<tab>tag tag tag +type Bookmark struct { + URL, Desc string + Tags []string + N int // the index of the line in the file +} + +// Title gives the text of the link to the bookmark. +func (b Bookmark) Title() string { + if strings.TrimSpace(b.Desc) == "" { + return b.URL + } + return b.Desc +} + +// Host gives the host of the URL, to show under the title. +func (b Bookmark) Host() string { + u := scheme.ReplaceAllString(b.URL, "") + if i := strings.IndexAny(u, "/?#"); i >= 0 { + u = u[:i] + } + return u +} + +// Link gives the address of the link to the bookmark. Only http and https +// links are made, so that a line of the file cannot run script in the page. +// An address without a scheme gets https. Other schemes give "": the page +// shows them as text. +func (b Bookmark) Link() string { + l := strings.ToLower(b.URL) + switch { + case strings.HasPrefix(l, "https://") || strings.HasPrefix(l, "http://"): + return b.URL + case !strings.Contains(b.URL, ":"): + return "https://" + b.URL + } + return "" +} + +// parseLine gives the bookmark of a line, and false for a comment or an +// empty line. A line without a tab has the old format "URL description | +// tags". Until bm-migrate converts it, its first word is the URL, as in the +// add-on. +func parseLine(line string) (Bookmark, bool) { + if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { + return Bookmark{}, false + } + fields := strings.Split(line, "\t") + if len(fields) == 1 { + words := strings.Fields(line) + return Bookmark{URL: words[0], Desc: strings.Join(words[1:], " ")}, true + } + b := Bookmark{URL: strings.TrimSpace(fields[0]), Desc: fields[1]} + if len(fields) > 2 { + b.Tags = strings.FieldsFunc(fields[2], func(r rune) bool { return r == ' ' }) + } + return b, b.URL != "" +} + +// parse gives the bookmarks of a file, in the order of the file. +func parse(text string) []Bookmark { + var out []Bookmark + for i, l := range lines(text) { + if b, ok := parseLine(l); ok { + b.N = i + out = append(out, b) + } + } + return out +} + +// entry is the text in the fields of a bookmark form. +type entry struct{ URL, Desc, Tags string } + +// line gives the line of the bookmark, as bm and the add-on write it: the URL +// without white space, the description without tabs and line breaks, and the +// tags with one space between them. +func (e entry) line() (string, error) { + u := strings.Join(strings.Fields(e.URL), "") + switch { + case u == "": + return "", errNoURL + case strings.HasPrefix(u, "#"): + return "", errHashURL + } + desc := strings.TrimSpace(strings.NewReplacer("\t", " ", "\r", " ", "\n", " ").Replace(e.Desc)) + return u + "\t" + desc + "\t" + strings.Join(strings.Fields(e.Tags), " "), nil +} + +// ---- search ---- + +// score gives the score of query in text, higher for a better match, or -1 +// when a word of the query does not occur in text. This is the search of the +// add-on (lib/fuzzy.js), as in fzf: each word of the query must occur in the +// text, with its letters in order but not always together. Case does not +// matter. +func score(query, text string) int { + hay := strings.ToLower(text) + total := 0 + for _, word := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool { return r == ' ' || r == '\t' }) { + s := scoreWord(word, hay) + if s < 0 { + return -1 + } + total += s + } + return total +} + +// scoreWord gives the score of one word in hay. The whole word in one place +// scores highest, most at the start of a word. +func scoreWord(word, hay string) int { + if at := strings.Index(hay, word); at >= 0 { + s := 100 + utf8.RuneCountInString(word) + if startsWord(hay, at) { + s += 20 + } + return s + } + s, end := 0, -1 // end: the byte after the last letter found + for _, c := range word { + from := max(end, 0) + i := strings.IndexRune(hay[from:], c) + if i < 0 { + return -1 + } + i += from + s++ + if i == end { + s += 5 + } + if startsWord(hay, i) { + s += 3 + } + _, n := utf8.DecodeRuneInString(hay[i:]) + end = i + n + } + return s +} + +// startsWord tells if the letter at byte i of hay starts a word: no letter +// or digit is before it. +func startsWord(hay string, i int) bool { + if i == 0 { + return true + } + r, _ := utf8.DecodeLastRuneInString(hay[:i]) + return !unicode.IsLetter(r) && !unicode.IsNumber(r) +} + +// search gives the bookmarks that match query, best first. Bookmarks with the +// same score keep their order. +func search(bs []Bookmark, query string) []Bookmark { + type hit struct { + b Bookmark + score int + } + var hits []hit + for _, b := range bs { + if s := score(query, b.Desc+" "+strings.Join(b.Tags, " ")+" "+b.URL); s >= 0 { + hits = append(hits, hit{b, s}) + } + } + slices.SortStableFunc(hits, func(x, y hit) int { return y.score - x.score }) + out := make([]Bookmark, len(hits)) + for i, h := range hits { + out[i] = h.b + } + return out +} + +// orders are the sort orders of the list page. bm also has "used", but the +// counts of use stay on each device. +var orders = map[string]func(x, y Bookmark) int{ + "recent": func(x, y Bookmark) int { return y.N - x.N }, + "url": func(x, y Bookmark) int { return strings.Compare(norm(x.URL), norm(y.URL)) }, + "desc": func(x, y Bookmark) int { + return strings.Compare(strings.ToLower(x.Title()), strings.ToLower(y.Title())) + }, + "tag": func(x, y Bookmark) int { return strings.Compare(strings.Join(x.Tags, " "), strings.Join(y.Tags, " ")) }, +} + +// target gives the address for text that no bookmark matches, as bm does: one +// word with "://" or a dot is an address, and other text is a web search. +func target(text string) string { + t := strings.TrimSpace(text) + if !strings.Contains(t, " ") { + if strings.Contains(t, "://") { + return t + } + if strings.Contains(t, ".") { + return "https://" + t + } + } + return "https://duckduckgo.com/?q=" + url.QueryEscape(t) +} + +// ---- import ---- + +var ( + notTag = regexp.MustCompile(`[^a-z0-9]+`) + imported = regexp.MustCompile(`^(https?|ftp|file)://`) +) + +// importRows gives the bookmark lines of an imported file: the Netscape HTML +// that browsers export, or a file of bookmark lines. +func importRows(text string) []string { + text = strings.TrimPrefix(text, string(rune(0xFEFF))) + if strings.HasPrefix(strings.TrimSpace(text), "<") { + return fromHTML(text) + } + return lines(text) +} + +// fromHTML gives the bookmark lines of the Netscape HTML that browsers +// export, as bm-import does: <H3> opens a folder, </DL> closes it, and the +// folders of a bookmark become its tags. Only http, https, ftp and file URLs +// stay. +func fromHTML(text string) []string { + var folders, out []string + for _, line := range lines(text) { + upper := upperASCII(line) + if at := strings.Index(upper, "<H3"); at >= 0 { + name := line[at:] + name = name[strings.IndexByte(name, '>')+1:] + if end := strings.Index(upperASCII(name), "</H3"); end >= 0 { + name = name[:end] + } + folders = append(folders, folderTag(htmlText(name))) + } + if at := strings.Index(upper, `HREF="`); at >= 0 { + u, rest, ok := strings.Cut(line[at+len(`HREF="`):], `"`) + name := rest[strings.IndexByte(rest, '>')+1:] + if end := strings.Index(upperASCII(name), "</A"); end >= 0 { + name = name[:end] + } + var tags []string + for _, f := range folders { + if f != "" && !slices.Contains(tags, f) { + tags = append(tags, f) + } + } + if u = htmlText(u); ok && imported.MatchString(u) { + out = append(out, u+"\t"+htmlText(name)+"\t"+strings.Join(tags, " ")) + } + } + if strings.Contains(upper, "</DL>") && len(folders) > 0 { + folders = folders[:len(folders)-1] + } + } + return out +} + +// upperASCII gives s with the letters a to z in upper case. Each byte keeps +// its index, so an index in the result is an index in s. +func upperASCII(s string) string { + b := []byte(s) + for i, c := range b { + if 'a' <= c && c <= 'z' { + b[i] = c - 'a' + 'A' + } + } + return string(b) +} + +// htmlText gives the text of HTML without its entities, with one space in +// place of each run of white space. +func htmlText(s string) string { + return strings.Join(strings.Fields(html.UnescapeString(s)), " ") +} + +// folderTag gives the tag of a folder name, as bm-import makes it: "Dev +// Tools" is dev-tools. +func folderTag(name string) string { + return strings.Trim(notTag.ReplaceAllString(strings.ToLower(name), "-"), "-") +} + +// mergeRows gives the rows whose URL is not in text and not in a row before +// them, as bm --merge does, with the number of rows that it skipped for that +// reason. Empty rows and comments go. +func mergeRows(text string, rows []string) ([]string, int) { + seen := map[string]bool{} + for _, l := range lines(text) { + seen[key(l)] = true + } + var add []string + skipped := 0 + for _, row := range rows { + k := key(row) + switch { + case k == "": + case seen[k]: + skipped++ + default: + seen[k] = true + f := append(strings.SplitN(row, "\t", 4), "", "") + add = append(add, f[0]+"\t"+f[1]+"\t"+f[2]) + } + } + return add, skipped +} + +// ---- pages ---- + +// bookmarksPage is the data of the pages for the bookmarks of an account. +type bookmarksPage struct { + page + Query, Tag, Sort, Done string + Open string // the form of the list page to open: "add" or "import" + Version string // the version of the file that the page shows + Rows []Bookmark // the bookmarks to show, at most maxRows + Found, Total int // the bookmarks that match, and all bookmarks + Tags []tagCount + Target string // where the query goes when no bookmark matches + Form entry // the fields of the form of a bookmark + N int // the line of the bookmark on the edit page + Text string // the whole file, for the file page +} + +type tagCount struct { + Name string + N int +} + +// done gives the message of the list page after a change. +var done = map[string]string{ + "added": "Bookmark added.", + "saved": "Bookmark saved.", + "deleted": "Bookmark deleted.", + "file": "File saved.", + "merged": "File saved, and merged with the changes of your other devices.", + "changed": "That bookmark changed on another device. Look at it again.", +} + +// owner gives the signed-in account when it can use its bookmarks on this +// site: as for sync, its email is confirmed and sync is not paused. Else +// owner sends the browser to sign in, or to the account page, which tells +// why, and gives nil. +func (s *server) owner(w http.ResponseWriter, r *http.Request) *Account { + u := s.user(r) + if u == nil { + http.Redirect(w, r, "/login", http.StatusSeeOther) + return nil + } + if a := s.store.view(u); !s.verified(a) || !s.active(a) { + http.Redirect(w, r, "/account", http.StatusSeeOther) + return nil + } + return u +} + +// listing gives the list page for the bookmarks in text that match the query +// and the tag, in the order. +func (s *server) listing(text, version, query, tag, order string) bookmarksPage { + p := bookmarksPage{page: s.page("Your bookmarks"), Query: query, Tag: tag, Version: version} + all := parse(text) + count := map[string]int{} + var rows []Bookmark + for _, b := range all { + for _, t := range b.Tags { + count[t]++ + } + if tag == "" || slices.Contains(b.Tags, tag) { + rows = append(rows, b) + } + } + for name, n := range count { + p.Tags = append(p.Tags, tagCount{name, n}) + } + slices.SortFunc(p.Tags, func(x, y tagCount) int { return strings.Compare(x.Name, y.Name) }) + if orders[order] == nil { + order = "recent" + } + p.Sort = order + slices.SortStableFunc(rows, orders[order]) + if strings.TrimSpace(query) != "" { + if rows = search(rows, query); len(rows) == 0 { + p.Target = target(query) + } + } + p.Total, p.Found = len(all), len(rows) + p.Rows = rows[:min(len(rows), maxRows)] + return p +} + +// bookmarks shows the bookmarks that match the query ?q= and the tag ?tag=, +// in the order ?sort=. +func (s *server) bookmarks(w http.ResponseWriter, r *http.Request) { + u := s.owner(w, r) + if u == nil { + return + } + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + q := r.URL.Query() + p := s.listing(text, version, q.Get("q"), q.Get("tag"), q.Get("sort")) + p.Done = done[q.Get("done")] + if q.Get("done") == "imported" { + added, _ := strconv.Atoi(q.Get("added")) + known, _ := strconv.Atoi(q.Get("known")) + p.Done = fmt.Sprintf("Bookmarks imported: %d. Skipped because they are bookmarks already: %d.", added, known) + } + s.render(w, http.StatusOK, "bookmarks", p) +} + +// refuse shows the list page again, with the problem of its add or import +// form. +func (s *server) refuse(w http.ResponseWriter, u *Account, status int, form, problem string, e entry) { + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + p := s.listing(text, version, "", "", "") + p.Error, p.Open, p.Form = problem, form, e + s.render(w, status, "bookmarks", p) +} + +// addBookmark adds a bookmark at the end of the file, where bm adds them. As +// bm does, it refuses a URL that is a bookmark already. +func (s *server) addBookmark(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxForm) + e := entry{URL: r.PostFormValue("url"), Desc: r.PostFormValue("desc"), Tags: r.PostFormValue("tags")} + line, err := e.line() + if err != nil { + s.refuse(w, u, http.StatusBadRequest, "add", sentence(err), e) + return + } + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + for _, b := range parse(text) { + if norm(b.URL) == key(line) { + s.refuse(w, u, http.StatusConflict, "add", "This URL is a bookmark already: "+b.Title()+".", e) + return + } + } + if _, _, err := s.store.sync(u, version, join(append(lines(text), line))); err != nil { + s.fail(w, err) + return + } + http.Redirect(w, r, "/bookmarks?done=added", http.StatusSeeOther) +} + +// lineAt gives line n of version v of the file of the account, and n as a +// number. It gives errChanged when the version is gone, or when the line is +// not a bookmark. +func (s *server) lineAt(u *Account, v, n string) (string, int, error) { + i, err := strconv.Atoi(n) + if err != nil || !hashName.MatchString(v) { + return "", 0, errChanged + } + text, _, err := s.store.read(u, v) + if errors.Is(err, fs.ErrNotExist) { + return "", 0, errChanged + } else if err != nil { + return "", 0, err + } + ls := lines(text) + if i < 0 || i >= len(ls) { + return "", 0, errChanged + } + if _, ok := parseLine(ls[i]); !ok { + return "", 0, errChanged + } + return ls[i], i, nil +} + +// editBookmark shows the form for line ?n= of version ?v= of the file. +func (s *server) editBookmark(w http.ResponseWriter, r *http.Request) { + u := s.owner(w, r) + if u == nil { + return + } + q := r.URL.Query() + line, n, err := s.lineAt(u, q.Get("v"), q.Get("n")) + if errors.Is(err, errChanged) { + http.Redirect(w, r, "/bookmarks?done=changed", http.StatusSeeOther) + return + } else if err != nil { + s.fail(w, err) + return + } + b, _ := parseLine(line) + s.render(w, http.StatusOK, "edit", bookmarksPage{page: s.page("Edit a bookmark"), Version: q.Get("v"), N: n, + Form: entry{URL: b.URL, Desc: b.Desc, Tags: strings.Join(b.Tags, " ")}}) +} + +// saveBookmark changes or deletes the bookmark of the edit page in the +// current file. When the current file does not have the line of the page, +// another device changed it, and nothing changes. +func (s *server) saveBookmark(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxForm) + v := r.PostFormValue("v") + old, n, err := s.lineAt(u, v, r.PostFormValue("n")) + if errors.Is(err, errChanged) { + http.Redirect(w, r, "/bookmarks?done=changed", http.StatusSeeOther) + return + } else if err != nil { + s.fail(w, err) + return + } + e := entry{URL: r.PostFormValue("url"), Desc: r.PostFormValue("desc"), Tags: r.PostFormValue("tags")} + problem := func(status int, text string) { + p := bookmarksPage{page: s.page("Edit a bookmark"), Version: v, N: n, Form: e} + p.Error = text + s.render(w, status, "edit", p) + } + line, msg := "", "deleted" + if r.PostFormValue("do") != "delete" { + if line, err = e.line(); err != nil { + problem(http.StatusBadRequest, sentence(err)) + return + } + msg = "saved" + } + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + if b, _ := parseLine(old); line != "" && key(line) != norm(b.URL) { + for _, other := range parse(text) { + if norm(other.URL) == key(line) { + problem(http.StatusConflict, "Another bookmark has this URL: "+other.Title()+".") + return + } + } + } + var out []string + found := false + for _, l := range lines(text) { + if l != old { + out = append(out, l) + } else if !found { + found = true + if line != "" { + out = append(out, line) + } + } + } + if !found { + http.Redirect(w, r, "/bookmarks?done=changed", http.StatusSeeOther) + return + } + if _, _, err := s.store.sync(u, version, join(out)); err != nil { + s.fail(w, err) + return + } + http.Redirect(w, r, "/bookmarks?done="+msg, http.StatusSeeOther) +} + +// bookmarkFile shows the whole file in a text box, as bm -e opens it in an +// editor. +func (s *server) bookmarkFile(w http.ResponseWriter, r *http.Request) { + u := s.owner(w, r) + if u == nil { + return + } + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + s.render(w, http.StatusOK, "file", bookmarksPage{page: s.page("Edit the file"), Version: version, Text: text}) +} + +// saveFile takes the whole file of the file page. As for the file of a +// device, the server merges the changes since the version that the page +// showed. +func (s *server) saveFile(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + // A form sends a tab as %09, so the form can be three times as large as + // the file. + r.Body = http.MaxBytesReader(w, r.Body, 3*maxFile+maxForm) + if err := r.ParseForm(); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + http.Error(w, "the bookmark file is too large", http.StatusRequestEntityTooLarge) + } else { + http.Error(w, "cannot read the form", http.StatusBadRequest) + } + return + } + base, text := r.PostFormValue("base"), join(lines(r.PostFormValue("text"))) + if base != "" && !hashName.MatchString(base) { + http.Error(w, "base is not a version name", http.StatusBadRequest) + return + } + if len(text) > maxFile { + p := bookmarksPage{page: s.page("Edit the file"), Version: base, Text: text} + p.Error = "The file is larger than 4 MB. Nothing changed." + s.render(w, http.StatusRequestEntityTooLarge, "file", p) + return + } + merged, _, err := s.store.sync(u, base, text) + if err != nil { + s.fail(w, err) + return + } + msg := "file" + if merged != text { + msg = "merged" + } + http.Redirect(w, r, "/bookmarks?done="+msg, http.StatusSeeOther) +} + +// importFile adds the bookmarks of a file, as bm-import and bm --merge do. A +// URL that is a bookmark already is skipped. +func (s *server) importFile(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxImport) + f, _, err := r.FormFile("file") + if err != nil { + s.refuse(w, u, http.StatusBadRequest, "import", "Choose a file of 32 MB or less.", entry{}) + return + } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { + s.fail(w, err) + return + } + text, version, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + add, skipped := mergeRows(text, importRows(string(b))) + merged := join(append(lines(text), add...)) + if len(merged) > maxFile { + s.refuse(w, u, http.StatusRequestEntityTooLarge, "import", + "With these bookmarks, the file would be larger than 4 MB. Nothing changed.", entry{}) + return + } + if len(add) > 0 { + if _, _, err := s.store.sync(u, version, merged); err != nil { + s.fail(w, err) + return + } + } + http.Redirect(w, r, fmt.Sprintf("/bookmarks?done=imported&added=%d&known=%d", len(add), skipped), http.StatusSeeOther) +} + +// download gives the current file. It works also when sync is paused, +// because the bookmarks belong to their owner. +func (s *server) download(w http.ResponseWriter, r *http.Request) { + u := s.user(r) + if u == nil { + http.Redirect(w, r, "/login", http.StatusSeeOther) + return + } + text, _, err := s.store.read(u, "") + if err != nil { + s.fail(w, err) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="bookmarks.txt"`) + io.WriteString(w, text) +} diff --git a/bookmarks_test.go b/bookmarks_test.go new file mode 100644 index 0000000..749dcfa --- /dev/null +++ b/bookmarks_test.go @@ -0,0 +1,234 @@ +package main + +import ( + "slices" + "strings" + "testing" + + "pgregory.net/rapid" +) + +func TestParseLine(t *testing.T) { + for line, want := range map[string]Bookmark{ + "https://a.org\tA b\tx y": {URL: "https://a.org", Desc: "A b", Tags: []string{"x", "y"}}, + " https://a.org \t\t": {URL: "https://a.org"}, + "https://a.org old format | tags": {URL: "https://a.org", Desc: "old format | tags"}, + "https://a.org": {URL: "https://a.org"}, + } { + got, ok := parseLine(line) + if !ok || got.URL != want.URL || got.Desc != want.Desc || !slices.Equal(got.Tags, want.Tags) { + t.Errorf("parseLine(%q) = %+v %v, want %+v", line, got, ok, want) + } + } + for _, line := range []string{"", " ", "# comment", "\tno URL\t"} { + if b, ok := parseLine(line); ok { + t.Errorf("parseLine(%q) = %+v, want no bookmark", line, b) + } + } +} + +// A form gives one line, which holds the URL without white space, the words +// of the description and the tags. +func TestFormGivesOneLine(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + e := entry{rapid.String().Draw(t, "url"), rapid.String().Draw(t, "desc"), rapid.String().Draw(t, "tags")} + u := strings.Join(strings.Fields(e.URL), "") + line, err := e.line() + if u == "" || strings.HasPrefix(u, "#") { + if err == nil { + t.Fatalf("line of URL %q = %q, want an error", e.URL, line) + } + return + } + if err != nil { + t.Fatal(err) + } + if strings.Count(line, "\t") != 2 || strings.ContainsAny(line, "\r\n") { + t.Fatalf("%q is not one line of three fields", line) + } + b, ok := parseLine(line) + if !ok || b.URL != u || !slices.Equal(strings.Fields(b.Desc), strings.Fields(e.Desc)) || + !slices.Equal(b.Tags, strings.Fields(e.Tags)) { + t.Fatalf("parseLine(%q) = %+v %v", line, b, ok) + } + }) +} + +func TestLink(t *testing.T) { + for u, want := range map[string]string{ + "https://a.org": "https://a.org", + "HTTP://A.org/x": "HTTP://A.org/x", + "www.a.org/x": "https://www.a.org/x", + "javascript:alert(1)": "", + "JavaScript:alert(1)": "", + "data:text/html,x": "", + "file:///etc/passwd": "", + } { + if got := (Bookmark{URL: u}).Link(); got != want { + t.Errorf("link of %q = %q, want %q", u, got, want) + } + } +} + +// TestScoreAsTheAddOn compares with the scores of lib/fuzzy.js in the add-on. +func TestScoreAsTheAddOn(t *testing.T) { + for _, c := range []struct { + query, text string + want int + }{ + {"suck", "suckless software c tools https://suckless.org", 124}, + {"sls", "suckless", 6}, + {"jsh", "Jisho: 日本語の辞書 ja https://jisho.org", 11}, + {"日本", "Jisho: 日本語", 122}, + {"go doc", "Go docs dev-tools https://go.dev/doc/", 245}, + {"xyz", "abc", -1}, + {"", "anything", 0}, + {"GITHUB", "https://github.com/equwal/sbm", 126}, + {"eqsbm", "https://github.com/equwal/sbm", 26}, + {"a b", "b a", 242}, + {"tools c", "suckless software c tools", 226}, + {"wiki", "Wikipedia, the free encyclopedia https://en.wikipedia.org", 124}, + {"wkp", "Wikipedia, the free encyclopedia https://en.wikipedia.org", 6}, + {"é", "Café crème", 101}, + {"cr", "Café crème", 122}, + } { + if got := score(c.query, c.text); got != c.want { + t.Errorf("score(%q, %q) = %d, want %d", c.query, c.text, got, c.want) + } + } +} + +// letters draws text from few letters, with case, white space, punctuation +// and letters of more than one byte, so that searches often match. +var letters = rapid.StringOf(rapid.SampledFrom([]rune("abcABC xyz.-/:日本éÉ\t"))) + +// matches is a slow reference for the search: case aside, the letters of +// each word of the query occur in the text in their order. +func matches(query, text string) bool { + hay := []rune(strings.ToLower(text)) + for _, word := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool { return r == ' ' || r == '\t' }) { + w, i := []rune(word), 0 + for _, r := range hay { + if i < len(w) && r == w[i] { + i++ + } + } + if i < len(w) { + return false + } + } + return true +} + +func TestScoreMatchesAsTheReference(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + query, text := letters.Draw(t, "query"), letters.Draw(t, "text") + if s := score(query, text); (s >= 0) != matches(query, text) || s < -1 { + t.Fatalf("score(%q, %q) = %d, reference match %v", query, text, s, matches(query, text)) + } + }) +} + +func TestScoreFindsEachPartOfTheText(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + text := letters.Draw(t, "text") + rs := []rune(strings.ToLower(text)) + var words []string + for range rapid.IntRange(1, 3).Draw(t, "words") { + i := rapid.IntRange(0, len(rs)).Draw(t, "i") + j := rapid.IntRange(i, len(rs)).Draw(t, "j") + words = append(words, string(rs[i:j])) + } + query := strings.Join(words, " ") + n := len(strings.FieldsFunc(query, func(r rune) bool { return r == ' ' || r == '\t' })) + if s := score(query, text); s < 100*n { + t.Fatalf("score(%q, %q) = %d, want at least %d", query, text, s, 100*n) + } + }) +} + +func TestScoreIgnoresCaseAndWordOrder(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + a, b, text := letters.Draw(t, "a"), letters.Draw(t, "b"), letters.Draw(t, "text") + s := score(a+" "+b, text) + if got := score(strings.ToUpper(b+" "+a), text); got != s { + t.Fatalf("score(%q) = %d, but score(%q) = %d", strings.ToUpper(b+" "+a), got, a+" "+b, s) + } + if got := score(a+" "+b, strings.ToUpper(text)); got != s { + t.Fatalf("in %q: %d, in %q: %d", strings.ToUpper(text), got, text, s) + } + }) +} + +func FuzzScore(f *testing.F) { + f.Add("sls", "suckless") + f.Add("é", "Café") + f.Add(string([]byte{0xff}), "a"+string([]byte{0xff, 0xe6, 0x97})+"b") + f.Fuzz(func(t *testing.T, query, text string) { + if s := score(query, text); s < -1 { + t.Fatalf("score(%q, %q) = %d", query, text, s) + } + }) +} + +func TestSearchKeepsTheOrderOfEqualScores(t *testing.T) { + bs := parse("https://a.org\tgo\t\nhttps://b.org\tgolang\t\nhttps://c.org\tgo\t\nhttps://d.org\tpython\t\n") + var got []string + for _, b := range search(bs, "go") { + got = append(got, b.URL) + } + // "go" at the start of a word scores the same in each; b.org has it too. + if want := []string{"https://a.org", "https://b.org", "https://c.org"}; !slices.Equal(got, want) { + t.Errorf("search: %q, want %q", got, want) + } +} + +func TestTarget(t *testing.T) { + for text, want := range map[string]string{ + "suckless.org/dwm": "https://suckless.org/dwm", + "gemini://x.org": "gemini://x.org", + "posix sh printf": "https://duckduckgo.com/?q=posix+sh+printf", + " word ": "https://duckduckgo.com/?q=word", + "c++": "https://duckduckgo.com/?q=c%2B%2B", + } { + if got := target(text); got != want { + t.Errorf("target(%q) = %q, want %q", text, got, want) + } + } +} + +// Each imported row with a new URL comes in once. The other rows are known, +// or are no bookmark. +func TestMergeRowsAddsEachURLOnce(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + text, rows := file(t, "file"), lines(file(t, "rows")) + add, skipped := mergeRows(text, rows) + seen := map[string]bool{} + for _, l := range lines(text) { + seen[key(l)] = true + } + for _, a := range add { + if k := key(a); k == "" || seen[k] { + t.Fatalf("added %q, but its URL is known", a) + } + seen[key(a)] = true + } + empty := 0 + for _, r := range rows { + if key(r) == "" { + empty++ + } + } + if len(add)+skipped+empty != len(rows) { + t.Fatalf("%d added, %d skipped and %d empty of %d rows", len(add), skipped, empty, len(rows)) + } + }) +} + +func TestFolderTag(t *testing.T) { + for name, want := range map[string]string{"Dev Tools": "dev-tools", " -C++ & Go- ": "c-go", "日本語": ""} { + if got := folderTag(name); got != want { + t.Errorf("folderTag(%q) = %q, want %q", name, got, want) + } + } +} diff --git a/main.go b/main.go index 9f4584d..a7b992f 100644 --- a/main.go +++ b/main.go @@ -163,6 +163,14 @@ func (s *server) routes() http.Handler { m.HandleFunc("POST /api/login", s.apiLogin) m.HandleFunc("POST /api/logout", s.apiLogout) m.HandleFunc("POST /api/sync", s.sync) + m.HandleFunc("GET /bookmarks", s.bookmarks) + m.HandleFunc("POST /bookmarks/add", s.addBookmark) + m.HandleFunc("GET /bookmarks/edit", s.editBookmark) + m.HandleFunc("POST /bookmarks/edit", s.saveBookmark) + m.HandleFunc("GET /bookmarks/file", s.bookmarkFile) + m.HandleFunc("POST /bookmarks/file", s.saveFile) + m.HandleFunc("POST /bookmarks/import", s.importFile) + m.HandleFunc("GET /bookmarks.txt", s.download) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h := w.Header() h.Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; "+ @@ -185,6 +193,7 @@ type page struct { Month, Year string TrialDays int Billing, CanSubscribe, Unconfirmed bool + Active bool // the account can sync and use its bookmarks here Customer string } @@ -194,7 +203,7 @@ func (s *server) page(title string) page { Month: s.labels["month"], Year: s.labels["year"], TrialDays: int(s.trial.Hours() / 24)} } -func (s *server) render(w http.ResponseWriter, status int, name string, p page) { +func (s *server) render(w http.ResponseWriter, status int, name string, p any) { w.Header().Set("Content-Type", "text/html; charset=utf-8") w.WriteHeader(status) if err := pages.ExecuteTemplate(w, name, p); err != nil { @@ -204,8 +213,9 @@ func (s *server) render(w http.ResponseWriter, status int, name string, p page) func (s *server) home(w http.ResponseWriter, r *http.Request) { p := s.page("") - if a := s.user(r); a != nil { - p.Email = a.Email + if u := s.user(r); u != nil { + a := s.store.view(u) + p.Email, p.Active = a.Email, s.verified(a) && s.active(a) } s.render(w, http.StatusOK, "home", p) } @@ -277,7 +287,7 @@ func (s *server) signup(w http.ResponseWriter, r *http.Request) { log.Printf("email check for %s: %v", a.ID, err) } } - s.startSession(w, r, a) + s.startSession(w, r, a, "/account") } // verified tells if the account can sync: its email is confirmed, or the @@ -359,17 +369,18 @@ func (s *server) login(w http.ResponseWriter, r *http.Request) { s.render(w, http.StatusUnauthorized, "login", p) return } - s.startSession(w, r, a) + s.startSession(w, r, a, "/bookmarks") } -func (s *server) startSession(w http.ResponseWriter, r *http.Request, a *Account) { +// startSession signs the browser in and sends it to the page at the path to. +func (s *server) startSession(w http.ResponseWriter, r *http.Request, a *Account, to string) { token, err := s.store.signIn(a) if err != nil { s.fail(w, err) return } s.setCookie(w, token, 365*24*3600) - http.Redirect(w, r, "/account", http.StatusSeeOther) + http.Redirect(w, r, to, http.StatusSeeOther) } func (s *server) logout(w http.ResponseWriter, r *http.Request) { @@ -420,6 +431,7 @@ func (s *server) showAccount(w http.ResponseWriter, status int, a Account, probl p := s.page("Your account") p.Email, p.Error, p.Customer = a.Email, problem, a.Customer p.Unconfirmed = !s.verified(a) + p.Active = s.verified(a) && s.active(a) left := s.trialLeft(a) switch { case s.stripe == nil: diff --git a/pages.html b/pages.html index 4f83bd2..bdb8f1f 100644 --- a/pages.html +++ b/pages.html @@ -20,8 +20,12 @@ label { display: block; } input { display: block; width: 100%; box-sizing: border-box; margin: .2em 0 1em; padding: .4em; font: inherit; } button { font: inherit; padding: .4em 1em; margin: .2em .5em .2em 0; } .error { color: #a00; } .note, footer { color: #555; } footer { margin-top: 3em; font-size: .9em; } +select { font: inherit; padding: .3em; margin: .2em 0 1em; } +textarea { display: block; width: 100%; box-sizing: border-box; height: 60vh; margin: .2em 0 1em; font: 13px/1.4 monospace; } +details { margin: 1em 0; } summary { cursor: pointer; } +.bookmarks { list-style: none; padding: 0; } .bookmarks li { margin: .8em 0; overflow-wrap: anywhere; } .bookmarks small { color: #555; } @media (prefers-color-scheme: dark) { - body { background: #111; color: #ddd; } a { color: #4fb3d9; } .note, footer { color: #999; } .error { color: #f77; } + body { background: #111; color: #ddd; } a { color: #4fb3d9; } .note, footer, .bookmarks small { color: #999; } .error { color: #f77; } } </style> </head> @@ -47,9 +51,9 @@ button { font: inherit; padding: .4em 1em; margin: .2em .5em .2em 0; } <p><a href="https://github.com/equwal/sbm">sbm</a> keeps your bookmarks in one plain text file. sbm Sync keeps that file the same on each of your devices: bm on your computers, the sbm app for Android, and the sbm add-on for Firefox and -Chrome.</p> +Chrome. You can also use your bookmarks on this site, in any browser.</p> {{if .Billing}}<p>Free for {{.TrialDays}} days. Then {{.Year}} or {{.Month}}. Cancel at any time.</p>{{end}} -<p>{{if .Email}}<a href="/account">Your account</a>{{else}}<a href="/signup">Create an account</a> or <a href="/login">sign in</a>.{{end}}</p> +<p>{{if .Email}}{{if .Active}}<a href="/bookmarks">Your bookmarks</a> · {{end}}<a href="/account">Your account</a>{{else}}<a href="/signup">Create an account</a> or <a href="/login">sign in</a>.{{end}}</p> {{with .Teams}}<h2>Teams</h2> <p>Coming: bookmarks for teams. The members of a team share one bookmark file, and sync it on each of their devices. Pre-order the founding plan: <a href="{{.}}">for up to 10 people</a>{{with $.TeamsLarge}}, @@ -84,6 +88,7 @@ or <a href="{{.}}">for 11 people or more</a>{{end}}.</p>{{end}} {{define "account"}}{{template "head" .}} <h1>Your account</h1> <p>Signed in as {{.Email}}.</p> +{{if .Active}}<p><a href="/bookmarks"><b>Open your bookmarks</b></a></p>{{end}} {{with .Error}}<p class="error">{{.}}</p>{{end}} {{if .Unconfirmed}}<p><b>Confirm your email address.</b> Open the link in the email that this server sent to {{.Email}}. Sync starts after that.</p> @@ -116,6 +121,89 @@ their bookmark files.</p> </form> {{template "foot" .}}{{end}} +{{define "nav"}}<p class="note"><a href="/bookmarks">Bookmarks</a> · +<a href="/bookmarks/file">Edit the file</a> · +<a href="/bookmarks.txt">Download</a> · +<a href="/account">Account</a></p>{{end}} + +{{define "bookmarks"}}{{template "head" .}} +{{template "nav"}} +<h1>Your bookmarks</h1> +{{with .Error}}<p class="error">{{.}}</p>{{end}} +{{with .Done}}<p><b>{{.}}</b></p>{{end}} +<form method="get" action="/bookmarks"> +<input type="search" name="q" value="{{.Query}}" placeholder="Search, or type an address" aria-label="Search"> +{{with .Tag}}<input type="hidden" name="tag" value="{{.}}">{{end}} +<label>Order <select name="sort"> +<option value="recent"{{if eq .Sort "recent"}} selected{{end}}>newest first</option> +<option value="url"{{if eq .Sort "url"}} selected{{end}}>URL</option> +<option value="desc"{{if eq .Sort "desc"}} selected{{end}}>description</option> +<option value="tag"{{if eq .Sort "tag"}} selected{{end}}>tag</option> +</select></label> +<button>Search</button> +</form> +<details{{if eq .Open "add"}} open{{end}}><summary>Add a bookmark</summary> +<form method="post" action="/bookmarks/add"> +<label>URL <input name="url" value="{{.Form.URL}}" required></label> +<label>Description <input name="desc" value="{{.Form.Desc}}"></label> +<label>Tags, with spaces between them <input name="tags" value="{{.Form.Tags}}" autocomplete="off"></label> +<button>Add</button> +</form></details> +<details{{if eq .Open "import"}} open{{end}}><summary>Import bookmarks</summary> +<form method="post" action="/bookmarks/import" enctype="multipart/form-data"> +<p class="note">Export the bookmarks of your browser to an HTML file, then +import that file here. Folders become tags. A file of sbm bookmark lines +works too. URLs that are bookmarks already are skipped.</p> +<label>File <input type="file" name="file" required></label> +<button>Import</button> +</form></details> +<p class="note">{{.Found}} of {{.Total}} bookmarks{{with .Tag}} with the tag <b>{{.}}</b>{{end}}.{{if gt .Found (len .Rows)}} +This page shows the first {{len .Rows}}.{{end}}{{if or .Query .Tag}} <a href="/bookmarks">Show all</a>{{end}}</p> +{{with .Target}}<p>No bookmark matches. Go to <a href="{{.}}">{{.}}</a></p>{{end}} +<ul class="bookmarks"> +{{range .Rows}}<li>{{if .Link}}<a href="{{.Link}}">{{.Title}}</a>{{else}}{{.Title}}{{end}}<br> +<small>{{.Host}}{{range .Tags}} · <a href="/bookmarks?tag={{.}}">{{.}}</a>{{end}} · +<a href="/bookmarks/edit?v={{$.Version}}&n={{.N}}">edit</a></small></li> +{{end}}</ul> +{{with .Tags}}<h2>Tags</h2> +<p>{{range .}}<a href="/bookmarks?tag={{.Name}}">{{.Name}}</a> ({{.N}}) {{end}}</p>{{end}} +{{template "foot" .}}{{end}} + +{{define "edit"}}{{template "head" .}} +{{template "nav"}} +<h1>Edit a bookmark</h1> +{{with .Error}}<p class="error">{{.}}</p>{{end}} +<form method="post" action="/bookmarks/edit"> +<input type="hidden" name="v" value="{{.Version}}"> +<input type="hidden" name="n" value="{{.N}}"> +<label>URL <input name="url" value="{{.Form.URL}}" required></label> +<label>Description <input name="desc" value="{{.Form.Desc}}"></label> +<label>Tags, with spaces between them <input name="tags" value="{{.Form.Tags}}" autocomplete="off"></label> +<button name="do" value="save">Save</button> +<button name="do" value="delete" formnovalidate>Delete</button> +<a href="/bookmarks">Cancel</a> +</form> +{{template "foot" .}}{{end}} + +{{define "file"}}{{template "head" .}} +{{template "nav"}} +<h1>Edit the file</h1> +{{with .Error}}<p class="error">{{.}}</p>{{end}} +<p class="note">One bookmark on each line: the URL, a tab, the description, a +tab, and the tags with spaces between them. Lines that start with # are +comments. A browser does not type a tab in this box: paste one, or use +<a href="/bookmarks">Add a bookmark</a>.</p> +<p class="note">When you save, the server merges your changes with the +changes of your other devices, as a sync does. Lines that you remove go, new +lines come at the end, and the other lines keep their order.</p> +<form method="post" action="/bookmarks/file"> +<input type="hidden" name="base" value="{{.Version}}"> +<textarea name="text" spellcheck="false" aria-label="The bookmark file"> +{{.Text}}</textarea> +<button>Save</button> <a href="/bookmarks">Cancel</a> +</form> +{{template "foot" .}}{{end}} + {{define "confirmed"}}{{template "head" .}} <h1>{{.Title}}</h1> {{with .Error}}<p class="error">{{.}}</p>{{else}}<p>Sync is on for {{.Email}}.</p>{{end}} diff --git a/store.go b/store.go index d391200..e66122d 100644 --- a/store.go +++ b/store.go @@ -370,15 +370,8 @@ func (s *Store) sync(a *Account, base, local string) (string, string, error) { if err := os.MkdirAll(dir, 0o700); err != nil { return "", "", err } - remote := "" - head, err := os.ReadFile(filepath.Join(dir, "HEAD")) - if err == nil { - b, err := os.ReadFile(filepath.Join(dir, strings.TrimSpace(string(head)))) - if err != nil { - return "", "", err - } - remote = string(b) - } else if !errors.Is(err, fs.ErrNotExist) { + remote, _, err := head(dir) + if err != nil { return "", "", err } old := "" @@ -405,6 +398,37 @@ func (s *Store) sync(a *Account, base, local string) (string, string, error) { return merged, name, prune(dir, name) } +// head gives the current version of the file in dir and its name. Without a +// version, both are "". The caller holds the lock of the account. +func head(dir string) (string, string, error) { + b, err := os.ReadFile(filepath.Join(dir, "HEAD")) + if errors.Is(err, fs.ErrNotExist) { + return "", "", nil + } else if err != nil { + return "", "", err + } + name := strings.TrimSpace(string(b)) + text, err := os.ReadFile(filepath.Join(dir, name)) + return string(text), name, err +} + +// read gives the version of the file of the account with the name, or the +// current version when name is "", and the name of the version. +func (s *Store) read(a *Account, name string) (string, string, error) { + m := s.lock(a.ID) + m.Lock() + defer m.Unlock() + dir := filepath.Join(s.dir, "files", a.ID) + if name == "" { + return head(dir) + } + if !hashName.MatchString(name) { + return "", "", fs.ErrNotExist + } + b, err := os.ReadFile(filepath.Join(dir, name)) + return string(b), name, err +} + // prune deletes all but the newest versions in dir, and never keep. func prune(dir, keep string) error { entries, err := os.ReadDir(dir) diff --git a/web_test.go b/web_test.go new file mode 100644 index 0000000..c8b588f --- /dev/null +++ b/web_test.go @@ -0,0 +1,434 @@ +package main + +import ( + "bytes" + "io" + "mime/multipart" + "net/http" + "net/url" + "regexp" + "slices" + "strings" + "testing" + "time" +) + +// get gives the status, the Location header and the body of a page. +func (e *harness) get(c *http.Client, path string) (int, string, string) { + resp, err := c.Get(e.web.URL + path) + if err != nil { + e.t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, resp.Header.Get("Location"), string(b) +} + +// send posts a form, and gives the status, the Location header and the body. +func (e *harness) send(c *http.Client, path string, form url.Values) (int, string, string) { + resp := e.post(c, path, form) + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, resp.Header.Get("Location"), string(b) +} + +// upload sends a file with the import form, as a browser does. +func (e *harness) upload(c *http.Client, content string) (int, string, string) { + var body bytes.Buffer + m := multipart.NewWriter(&body) + f, _ := m.CreateFormFile("file", "bookmarks.html") + io.WriteString(f, content) + m.Close() + req, _ := http.NewRequest("POST", e.web.URL+"/bookmarks/import", &body) + req.Header.Set("Content-Type", m.FormDataContentType()) + req.Header.Set("Origin", e.web.URL) + resp, err := c.Do(req) + if err != nil { + e.t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, resp.Header.Get("Location"), string(b) +} + +// account makes an account whose file is text. It gives a signed-in browser, +// the token of a device and the version of the file. +func (e *harness) account(text string) (*http.Client, string, string) { + c := e.signup("me@example.org") + tok := e.token("me@example.org") + code, _, v := e.sync(tok, "", text) + if code != http.StatusOK { + e.t.Fatalf("first sync: %d", code) + } + return c, tok, v +} + +// fileOf gives the file on the server, as a device without a file gets it. +func (e *harness) fileOf(tok string) string { + _, text, _ := e.sync(tok, "", "") + return text +} + +var editLink = regexp.MustCompile(`href="/bookmarks/edit\?v=([0-9a-f]{64})&n=([0-9]+)"`) + +// editForm gives the fields v and n of the edit link of the bookmark with the +// title on the page. +func editForm(t *testing.T, page, title string) url.Values { + t.Helper() + for _, li := range strings.Split(page, "<li>") { + if strings.Contains(li, ">"+title+"</a>") { + if m := editLink.FindStringSubmatch(li); m != nil { + return url.Values{"v": {m[1]}, "n": {m[2]}} + } + } + } + t.Fatalf("no edit link for %q on the page:\n%s", title, page) + return nil +} + +var rowTitle = regexp.MustCompile(`<li>(?:<a href="[^"]*">)?([^<]*)`) + +// titles gives the titles of the bookmarks on the page, in their order. +func titles(page string) []string { + var out []string + for _, m := range rowTitle.FindAllStringSubmatch(page, -1) { + out = append(out, m[1]) + } + return out +} + +func TestBookmarkPagesNeedSignIn(t *testing.T) { + e := newEnv(t, false) + c := e.browser() + for _, path := range []string{"/bookmarks", "/bookmarks/file", "/bookmarks.txt", + "/bookmarks/edit?v=" + strings.Repeat("0", 64) + "&n=0"} { + if code, to, _ := e.get(c, path); code != http.StatusSeeOther || to != "/login" { + t.Errorf("GET %s: %d to %q", path, code, to) + } + } + for _, path := range []string{"/bookmarks/add", "/bookmarks/edit", "/bookmarks/file", "/bookmarks/import"} { + if code, to, _ := e.send(c, path, url.Values{"url": {"https://a.org"}}); code != http.StatusSeeOther || to != "/login" { + t.Errorf("POST %s: %d to %q", path, code, to) + } + } +} + +func TestSignInOpensTheBookmarks(t *testing.T) { + e := newEnv(t, false) + form := url.Values{"email": {"me@example.org"}, "password": {"password1"}} + if code, to, _ := e.send(e.browser(), "/signup", form); code != http.StatusSeeOther || to != "/account" { + t.Errorf("sign up: %d to %q", code, to) + } + c := e.browser() + if code, to, _ := e.send(c, "/login", form); code != http.StatusSeeOther || to != "/bookmarks" { + t.Errorf("sign in: %d to %q", code, to) + } + if code, _, page := e.get(c, "/bookmarks"); code != http.StatusOK || !strings.Contains(page, "0 of 0 bookmarks.") { + t.Errorf("bookmarks of a new account: %d\n%s", code, page) + } + for _, path := range []string{"/", "/account"} { + if !strings.Contains(e.body(c, path), `<a href="/bookmarks">`) { + t.Errorf("%s has no link to the bookmarks", path) + } + } +} + +func TestWebChangesReachTheDevices(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("https://a.org\tA\tx\nhttps://b.org\tB\t\n") + if got := titles(e.body(c, "/bookmarks")); !slices.Equal(got, []string{"B", "A"}) { + t.Fatalf("titles: %q", got) + } + code, to, _ := e.send(c, "/bookmarks/add", url.Values{"url": {" https://c.org "}, "desc": {"C\tc"}, "tags": {" y z "}}) + if code != http.StatusSeeOther || to != "/bookmarks?done=added" { + t.Fatalf("add: %d to %q", code, to) + } + if got, want := e.fileOf(tok), "https://a.org\tA\tx\nhttps://b.org\tB\t\nhttps://c.org\tC c\ty z\n"; got != want { + t.Fatalf("after the add: %q, want %q", got, want) + } + if !strings.Contains(e.body(c, to), "Bookmark added.") { + t.Error("the page does not tell that the bookmark is added") + } + + form := editForm(t, e.body(c, "/bookmarks"), "A") + page := e.body(c, "/bookmarks/edit?v="+form.Get("v")+"&n="+form.Get("n")) + if !strings.Contains(page, `name="url" value="https://a.org"`) || !strings.Contains(page, `name="tags" value="x"`) { + t.Fatalf("edit page:\n%s", page) + } + form.Set("url", "https://a.org") + form.Set("desc", "A, edited") + form.Set("tags", "x w") + if code, to, _ := e.send(c, "/bookmarks/edit", form); code != http.StatusSeeOther || to != "/bookmarks?done=saved" { + t.Fatalf("save: %d to %q", code, to) + } + form = editForm(t, e.body(c, "/bookmarks"), "B") + form.Set("do", "delete") + if code, to, _ := e.send(c, "/bookmarks/edit", form); code != http.StatusSeeOther || to != "/bookmarks?done=deleted" { + t.Fatalf("delete: %d to %q", code, to) + } + if got, want := e.fileOf(tok), "https://a.org\tA, edited\tx w\nhttps://c.org\tC c\ty z\n"; got != want { + t.Fatalf("after the edit and the delete: %q, want %q", got, want) + } +} + +func TestWebEditsKeepTheChangesOfDevices(t *testing.T) { + e := newEnv(t, false) + c, tok, v := e.account("https://a.org\tA\t\nhttps://b.org\tB\t\n") + form := editForm(t, e.body(c, "/bookmarks"), "A") + // A device adds a bookmark while the page is open. + e.sync(tok, v, "https://a.org\tA\t\nhttps://b.org\tB\t\nhttps://d.org\tD\t\n") + form.Set("do", "delete") + if code, to, _ := e.send(c, "/bookmarks/edit", form); code != http.StatusSeeOther || to != "/bookmarks?done=deleted" { + t.Fatalf("delete: %d to %q", code, to) + } + if got, want := e.fileOf(tok), "https://b.org\tB\t\nhttps://d.org\tD\t\n"; got != want { + t.Fatalf("after the delete: %q, want %q", got, want) + } + + // A device edits B while the page is open. The page does not change B. + form = editForm(t, e.body(c, "/bookmarks"), "B") + _, text, v := e.sync(tok, "", "") + e.sync(tok, v, strings.Replace(text, "\tB\t", "\tB on a device\t", 1)) + form.Set("url", "https://b.org") + form.Set("desc", "B on the web") + if code, to, _ := e.send(c, "/bookmarks/edit", form); code != http.StatusSeeOther || to != "/bookmarks?done=changed" { + t.Fatalf("save of a bookmark that a device changed: %d to %q", code, to) + } + if got, want := e.fileOf(tok), "https://b.org\tB on a device\t\nhttps://d.org\tD\t\n"; got != want { + t.Fatalf("after the refused save: %q, want %q", got, want) + } +} + +func TestAddAndEditRefusals(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("http://a.org/\tA\t\nhttps://b.org\tB\t\n") + file := e.fileOf(tok) + for _, tc := range []struct { + form url.Values + code int + text string + }{ + {url.Values{"url": {"https://www.A.org"}, "desc": {"typed"}}, http.StatusConflict, "This URL is a bookmark already: A."}, + {url.Values{"url": {" "}}, http.StatusBadRequest, "Type the URL of the bookmark."}, + {url.Values{"url": {"#x"}}, http.StatusBadRequest, "A URL cannot start with #"}, + } { + code, _, page := e.send(c, "/bookmarks/add", tc.form) + if code != tc.code || !strings.Contains(page, tc.text) || !strings.Contains(page, "<details open>") { + t.Errorf("add %v: %d, want %d with %q in an open form", tc.form, code, tc.code, tc.text) + } + } + if _, _, page := e.send(c, "/bookmarks/add", url.Values{"url": {"https://a.org"}, "desc": {"typed"}}); !strings.Contains(page, `value="typed"`) { + t.Error("the add form lost the text that was typed") + } + + form := editForm(t, e.body(c, "/bookmarks"), "A") + form.Set("url", "https://b.org/") + if code, _, page := e.send(c, "/bookmarks/edit", form); code != http.StatusConflict || !strings.Contains(page, "Another bookmark has this URL: B.") { + t.Errorf("edit to the URL of B: %d", code) + } + form.Set("url", "") + if code, _, _ := e.send(c, "/bookmarks/edit", form); code != http.StatusBadRequest { + t.Errorf("edit without a URL: %d", code) + } + // A version that is gone, or a line that the version does not have. + if code, to, _ := e.get(c, "/bookmarks/edit?v="+strings.Repeat("0", 64)+"&n=0"); code != http.StatusSeeOther || to != "/bookmarks?done=changed" { + t.Errorf("edit page of an unknown version: %d to %q", code, to) + } + if code, to, _ := e.send(c, "/bookmarks/edit", url.Values{"v": {form.Get("v")}, "n": {"9"}, "url": {"https://x.org"}}); code != http.StatusSeeOther || to != "/bookmarks?done=changed" { + t.Errorf("save of an unknown line: %d to %q", code, to) + } + if got := e.fileOf(tok); got != file { + t.Errorf("a refusal changed the file: %q, want %q", got, file) + } +} + +func TestSearchTagsAndOrders(t *testing.T) { + e := newEnv(t, false) + c, _, _ := e.account("https://b.org\tBee\tx\nhttps://c.org\tAnt\ty\nhttps://a.org\tCat\tz w\n# a comment\nhttps://jisho.org\tJisho: 日本語\tja\n") + for _, tc := range []struct { + path string + want []string + }{ + {"/bookmarks", []string{"Jisho: 日本語", "Cat", "Ant", "Bee"}}, + {"/bookmarks?sort=url", []string{"Cat", "Bee", "Ant", "Jisho: 日本語"}}, + {"/bookmarks?sort=desc", []string{"Ant", "Bee", "Cat", "Jisho: 日本語"}}, + {"/bookmarks?sort=tag", []string{"Jisho: 日本語", "Bee", "Ant", "Cat"}}, + {"/bookmarks?sort=nonsense", []string{"Jisho: 日本語", "Cat", "Ant", "Bee"}}, + {"/bookmarks?q=jsh", []string{"Jisho: 日本語"}}, + {"/bookmarks?q=" + url.QueryEscape("日本"), []string{"Jisho: 日本語"}}, + {"/bookmarks?tag=z", []string{"Cat"}}, + {"/bookmarks?tag=z&q=bee", nil}, + } { + if got := titles(e.body(c, tc.path)); !slices.Equal(got, tc.want) { + t.Errorf("%s: %q, want %q", tc.path, got, tc.want) + } + } + page := e.body(c, "/bookmarks?q=jsh") + if !strings.Contains(page, "1 of 4 bookmarks.") || !strings.Contains(page, `<a href="/bookmarks?tag=ja">ja</a> (1)`) { + t.Errorf("the count or the tags are wrong:\n%s", page) + } + // Text that no bookmark matches goes to the web, as in bm. + if page := e.body(c, "/bookmarks?q=posix+sh"); !strings.Contains(page, "No bookmark matches.") || !strings.Contains(page, "https://duckduckgo.com/?q=posix") { + t.Errorf("no web search for text that no bookmark matches:\n%s", page) + } + if page := e.body(c, "/bookmarks?q=example.org/x"); !strings.Contains(page, `Go to <a href="https://example.org/x">`) { + t.Errorf("no link for an address that no bookmark matches:\n%s", page) + } +} + +func TestBookmarkPagesEscapeTheFile(t *testing.T) { + e := newEnv(t, false) + c, _, _ := e.account("javascript:alert(1)\t<script>alert(2)</script>\t<marquee>\nwww.example.org\tNo scheme\t\n") + page := e.body(c, "/bookmarks") + for _, bad := range []string{`href="javascript:`, "<script>", "<marquee>"} { + if strings.Contains(page, bad) { + t.Errorf("the page has %q", bad) + } + } + if !strings.Contains(page, "<script>alert(2)</script>") { + t.Error("the page does not show the description as text") + } + if !strings.Contains(page, `<a href="https://www.example.org">No scheme</a>`) { + t.Error("an address without a scheme does not get https") + } +} + +func TestBookmarksWaitForTheEmailCheck(t *testing.T) { + e := newEnv(t, false) + e.withMail() + c := e.signup("me@example.org") + if code, to, _ := e.get(c, "/bookmarks"); code != http.StatusSeeOther || to != "/account" { + t.Errorf("bookmarks before the check: %d to %q", code, to) + } + if code, to, _ := e.send(c, "/bookmarks/add", url.Values{"url": {"https://a.org"}}); code != http.StatusSeeOther || to != "/account" { + t.Errorf("add before the check: %d to %q", code, to) + } + if strings.Contains(e.body(c, "/account"), `href="/bookmarks"`) { + t.Error("the account page links to the bookmarks before the check") + } + if code := e.open(e.link()); code != http.StatusOK { + t.Fatalf("the link of the email: %d", code) + } + if code, _, _ := e.get(c, "/bookmarks"); code != http.StatusOK { + t.Errorf("bookmarks after the check: %d", code) + } +} + +func TestBookmarksPauseWithSync(t *testing.T) { + e := newEnv(t, true) + c, tok, _ := e.account("https://a.org\tA\t\n") + e.s.store.byTok(tok).Created = time.Now().Add(-31 * 24 * time.Hour) + if code, to, _ := e.get(c, "/bookmarks"); code != http.StatusSeeOther || to != "/account" { + t.Errorf("bookmarks after the trial: %d to %q", code, to) + } + // The download works: the bookmarks belong to their owner. + resp, err := c.Get(e.web.URL + "/bookmarks.txt") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || string(b) != "https://a.org\tA\t\n" || + resp.Header.Get("Content-Disposition") != `attachment; filename="bookmarks.txt"` { + t.Errorf("download after the trial: %s %q %q", resp.Status, b, resp.Header.Get("Content-Disposition")) + } +} + +func TestEditTheWholeFile(t *testing.T) { + e := newEnv(t, false) + c, tok, v := e.account("https://a.org\tA\t\nhttps://b.org\tB\t\n") + page := e.body(c, "/bookmarks/file") + if !strings.Contains(page, `name="base" value="`+v+`"`) || !strings.Contains(page, ">\nhttps://a.org\tA\t\nhttps://b.org\tB\t\n</textarea>") { + t.Fatalf("file page:\n%s", page) + } + // A device adds D while the page is open. The browser sends CRLF, and a + // file without A and with C. + e.sync(tok, v, "https://a.org\tA\t\nhttps://b.org\tB\t\nhttps://d.org\tD\t\n") + code, to, _ := e.send(c, "/bookmarks/file", url.Values{"base": {v}, "text": {"https://b.org\tB\t\r\nhttps://c.org\tC\t\r\n"}}) + if code != http.StatusSeeOther || to != "/bookmarks?done=merged" { + t.Fatalf("save: %d to %q", code, to) + } + want := "https://b.org\tB\t\nhttps://d.org\tD\t\nhttps://c.org\tC\t\n" + if got := e.fileOf(tok); got != want { + t.Fatalf("after the save: %q, want %q", got, want) + } + // Without changes of other devices, the file is as the page sent it. + _, _, v = e.sync(tok, "", "") + if code, to, _ := e.send(c, "/bookmarks/file", url.Values{"base": {v}, "text": {want + "https://e.org\tE\t\n"}}); code != http.StatusSeeOther || to != "/bookmarks?done=file" { + t.Errorf("save without other changes: %d to %q", code, to) + } + if code, _, _ := e.send(c, "/bookmarks/file", url.Values{"base": {"../accounts.json"}, "text": {"x"}}); code != http.StatusBadRequest { + t.Errorf("a base that is not a version name: %d", code) + } +} + +// browserExport is a file as the browsers export it. +const browserExport = `<!DOCTYPE NETSCAPE-Bookmark-file-1> +<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8"> +<TITLE>Bookmarks</TITLE> +<H1>Bookmarks</H1> +<DL><p> + <DT><A HREF="https://www.a.org/" ADD_DATE="1" ICON="data:image/png;base64,AAAA">A & more</A> + <DT><H3 ADD_DATE="1">Dev Tools</H3> + <DL><p> + <DT><A HREF="https://go.dev/doc/">Go docs</A> + <DT><H3>Go & Co</H3> + <DL><p> + <DT><A HREF="https://pkg.go.dev/">Packages</A> + <DT><A HREF="https://go.dev/doc">Go docs again</A> + </DL><p> + <DT><A HREF="javascript:alert(1)">Bookmarklet</A> + </DL><p> + <DT><A HREF="https://news.example/">News</A> +</DL> +` + +func TestImportFiles(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("http://a.org\tA\t\n") + code, to, _ := e.upload(c, browserExport) + if code != http.StatusSeeOther || to != "/bookmarks?done=imported&added=3&known=2" { + t.Fatalf("import of a browser file: %d to %q", code, to) + } + want := "http://a.org\tA\t\n" + + "https://go.dev/doc/\tGo docs\tdev-tools\n" + + "https://pkg.go.dev/\tPackages\tdev-tools go-co\n" + + "https://news.example/\tNews\t\n" + if got := e.fileOf(tok); got != want { + t.Fatalf("after the import: %q, want %q", got, want) + } + if !strings.Contains(e.body(c, to), "Bookmarks imported: 3. Skipped because they are bookmarks already: 2.") { + t.Error("the page does not tell what the import did") + } + // A file of bookmark lines, as bm --merge takes them. + if code, to, _ := e.upload(c, "https://b.org\tB\tx\n# comment\nhttps://pkg.go.dev\tknown\t\nhttps://c.org\n"); code != http.StatusSeeOther || to != "/bookmarks?done=imported&added=2&known=1" { + t.Fatalf("import of bookmark lines: %d to %q", code, to) + } + if got := e.fileOf(tok); got != want+"https://b.org\tB\tx\nhttps://c.org\t\t\n" { + t.Fatalf("after the import of lines: %q", got) + } + if code, _, page := e.send(c, "/bookmarks/import", url.Values{}); code != http.StatusBadRequest || !strings.Contains(page, "Choose a file") { + t.Errorf("import without a file: %d", code) + } +} + +func TestOtherSiteCannotChangeBookmarks(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("https://a.org\tA\t\n") + for _, path := range []string{"/bookmarks/add", "/bookmarks/edit", "/bookmarks/file", "/bookmarks/import"} { + req, _ := http.NewRequest("POST", e.web.URL+path, strings.NewReader("url=https%3A%2F%2Fevil.example&text=")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Origin", "https://evil.example") + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("POST %s from another site: %s", path, resp.Status) + } + } + if got := e.fileOf(tok); got != "https://a.org\tA\t\n" { + t.Errorf("another site changed the file: %q", got) + } +}