commit dc26d4e99c675e56200e7edf28dc918b8e341ea0 equwal <truex@equwal.com> 2026-09-25 02:15:58 -0700 Follow RSS and Atom feeds on /feed, with sfeed An account can follow feeds. The list is one plain file, feeds.txt, one feed on each line: URL, tab, name. The command `sbm-sync feed` writes an sfeedrc from it and runs sfeed_update, which keeps the items of each feed in one sfeed(5) file. The page /feed shows the items newest first, can open links in a new tab, and has the settings. GET /api/feed gives the items in the sfeed format, so sfeed_plain and the other sfeed tools read them. GET /api/feeds gives feeds.txt. Explore: when the account asks for it, the job looks through the bookmarked pages for feeds with sfeed_web, 20 pages in each run, and the page offers what it found. Digest: when the account asks for it, the job sends the new items by email once a day, in the form of sfeed_plain. The first run after the opt-in only notes the items of that day. The job fetches only public hosts, so a feed URL cannot reach the services of the machine or its private networks. A feed URL is printable ASCII without single quotes, so it holds between the quotes of the sfeedrc. The job writes only its own files under feeds/<account>/, and the web server writes only feeds.txt and accounts.json.
feed.go | 566 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ feed_test.go | 378 +++++++++++++++++++++++++++++++++++++ feedjob.go | 372 +++++++++++++++++++++++++++++++++++++ feedjob_test.go | 326 ++++++++++++++++++++++++++++++++ main.go | 27 ++- pages.html | 60 +++++- store.go | 26 +-- 7 files changed, 1732 insertions(+), 23 deletions(-)
diff --git a/feed.go b/feed.go new file mode 100644 index 0000000..3d8d537 --- /dev/null +++ b/feed.go @@ -0,0 +1,566 @@ +package main + +// Feeds. An account can follow RSS and Atom feeds. The list of an account +// is one plain file, feeds.txt, with one feed on each line: +// +// URL<tab>name +// +// The command `sbm-sync feed` (feedjob.go) fetches the feeds with +// sfeed_update into one sfeed(5) file for each feed, looks through the +// bookmarks of the account for feeds with sfeed_web when the account asked +// for it, and sends a daily digest of the new items by email. The page /feed +// shows the items, the feeds and the settings. GET /api/feed gives the items +// in the sfeed format, for sfeed_plain and the other sfeed tools. +// +// The files of an account are under feeds/<account ID>/: +// +// feeds.txt the feeds, written by the web pages +// sfeedrc the sfeed_update settings, made from feeds.txt by the job +// items/<file> the items of one feed, written by sfeed_update +// explore the feeds found on the bookmarked pages, written by the job +// mailed the keys of the items that went out in a digest +// digest the time of the last digest + +import ( + "errors" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "time" +) + +const ( + maxFeeds = 200 // feeds that an account can follow + maxFeedURL = 2048 // bytes in the URL of a feed + maxFeedName = 100 // characters in the name of a feed + maxFileName = 40 // characters in the file name of a feed, before a number + itemsOnPage = 100 +) + +var ( + errFeedURL = errors.New("the URL must start with http:// or https:// and have no spaces or quotes") + errFeedLong = errors.New("the URL is longer than 2048 characters") + errFeedKnown = errors.New("you follow this feed already") + errFeedMany = errors.New("you follow 200 feeds already, the most that this server keeps") + notFile = regexp.MustCompile(`[^A-Za-z0-9._-]+`) +) + +// feedSettings are the feed choices of an account. +type feedSettings struct { + NewTab bool `json:"newtab,omitempty"` // the feed page opens links in a new tab + Mail bool `json:"mail,omitempty"` // an email with the new items, once a day + Explore bool `json:"explore,omitempty"` // the job looks through the bookmarks for feeds +} + +// Feed is one line of feeds.txt. File is the name of the item file of the +// feed under items/, and the name that sfeed_update knows the feed by. +type Feed struct { + URL, Name, File string +} + +// feedURL checks the URL of a feed. It takes only http and https addresses +// of printable ASCII without quotes, so that the URL can go into the +// sfeedrc, a shell script, between single quotes. +func feedURL(raw string) (string, error) { + u := strings.TrimSpace(raw) + if u == "" { + return "", errNoURL + } + if len(u) > maxFeedURL { + return "", errFeedLong + } + for i := 0; i < len(u); i++ { + if c := u[i]; c <= ' ' || c > '~' || c == '\'' { + return "", errFeedURL + } + } + p, err := url.Parse(u) + if err != nil || p.Host == "" || (p.Scheme != "http" && p.Scheme != "https") { + return "", errFeedURL + } + return u, nil +} + +// feedName gives the name of a feed for feeds.txt: one line of at most +// maxFeedName characters. Without a name, the host of the URL is the name. +func feedName(name, u string) string { + n := strings.Join(strings.Fields(name), " ") + if n == "" { + n = Bookmark{URL: u}.Host() + } + if r := []rune(n); len(r) > maxFeedName { + n = string(r[:maxFeedName]) + } + return n +} + +// fileName gives the base of the file name of a feed: the letters, digits, +// dots, dashes and underscores of its name, or "feed". +func fileName(name string) string { + f := strings.Trim(notFile.ReplaceAllString(name, "-"), "-.") + if len(f) > maxFileName { + f = strings.Trim(f[:maxFileName], "-.") + } + if f == "" { + return "feed" + } + return f +} + +// parseFeeds gives the feeds of feeds.txt, in the order of the file. Each +// feed gets a file name that no other feed of the file has. +func parseFeeds(text string) []Feed { + var out []Feed + used := map[string]bool{} + for _, l := range lines(text) { + u, name, _ := strings.Cut(l, "\t") + if u == "" { + continue + } + f := Feed{URL: u, Name: name, File: fileName(name)} + for i := 2; used[f.File]; i++ { + f.File = fileName(name) + "-" + strconv.Itoa(i) + } + used[f.File] = true + out = append(out, f) + } + return out +} + +// Item is one line of an sfeed(5) file. +type Item struct { + Time time.Time // zero when the feed gives none + Title string + Link string // the address, or "" when it is not http or https + Author string + Feed string // the name of the feed + File string // the item file of the feed + Key string // "file<tab>id" names the item in mailed; the link or the title stand in for a missing id + Raw string // the line as sfeed wrote it +} + +// unescape undoes the escapes of the sfeed format: \t, \n and \\. +func unescape(s string) string { + if !strings.Contains(s, `\`) { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+1 < len(s) { + switch s[i+1] { + case 't': + b.WriteByte('\t') + i++ + continue + case 'n': + b.WriteByte('\n') + i++ + continue + case '\\': + b.WriteByte('\\') + i++ + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +// parseItems gives the items of the sfeed file of a feed, in the order of +// the file. +func parseItems(text string, f Feed) []Item { + var out []Item + for _, l := range lines(text) { + c := strings.Split(l, "\t") + if len(c) < 3 { + continue + } + for len(c) < 9 { + c = append(c, "") + } + it := Item{Title: unescape(c[1]), Author: unescape(c[6]), Feed: f.Name, File: f.File, Raw: l} + if t, err := strconv.ParseInt(c[0], 10, 64); err == nil { + it.Time = time.Unix(t, 0).UTC() + } + link := unescape(c[2]) + if link != "" { + it.Link = Bookmark{URL: link}.Link() + } + id := c[5] + if id == "" { + id = c[2] + } + if id == "" { + id = c[1] + } + it.Key = f.File + "\t" + id + if it.Title == "" { + it.Title = link + } + if it.Title == "" { + it.Title = "(no title)" + } + out = append(out, it) + } + return out +} + +// ---- the files of an account ---- + +func (s *Store) feedDir(id string) string { + return filepath.Join(s.dir, "feeds", id) +} + +// readFeeds gives feeds.txt of the account. A missing file is empty. +func (s *Store) readFeeds(a *Account) (string, error) { + m := s.lock(a.ID) + m.Lock() + defer m.Unlock() + b, err := os.ReadFile(filepath.Join(s.feedDir(a.ID), "feeds.txt")) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + return string(b), err +} + +// changeFeeds gives feeds.txt to f and writes what f gives back. No other +// change to the file comes between the two. +func (s *Store) changeFeeds(a *Account, f func(text string) (string, error)) error { + m := s.lock(a.ID) + m.Lock() + defer m.Unlock() + dir := s.feedDir(a.ID) + b, err := os.ReadFile(filepath.Join(dir, "feeds.txt")) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + text, err := f(string(b)) + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return writeFile(filepath.Join(dir, "feeds.txt"), []byte(text)) +} + +func (s *Store) setFeed(a *Account, f feedSettings) error { + s.mu.Lock() + defer s.mu.Unlock() + a.Feed = f + return s.save() +} + +// all gives every account, oldest first. +func (s *Store) all() []*Account { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]*Account, 0, len(s.accounts)) + for _, a := range s.accounts { + out = append(out, a) + } + slices.SortFunc(out, func(x, y *Account) int { return x.Created.Compare(y.Created) }) + return out +} + +// feedItems gives the items of the feeds of the account, newest first. +// Items without a time come last. +func (s *server) feedItems(a *Account, feeds []Feed) []Item { + var all []Item + for _, f := range feeds { + b, err := os.ReadFile(filepath.Join(s.store.feedDir(a.ID), "items", f.File)) + if err != nil { + continue // not fetched yet + } + all = append(all, parseItems(string(b), f)...) + } + slices.SortStableFunc(all, func(x, y Item) int { return y.Time.Compare(x.Time) }) + return all +} + +// suggestion is a feed that the job found on a bookmarked page. +type suggestion struct { + URL, Page, Name string +} + +// suggestions gives the feeds of the explore file that the account does not +// follow, the number of bookmarked pages that the job looked at, and the +// number of bookmarks with a web address. +func (s *server) suggestions(a *Account, feeds []Feed) ([]suggestion, int, int, error) { + text, _, err := s.store.read(a, "") + if err != nil { + return nil, 0, 0, err + } + names := map[string]string{} + total := 0 + for _, b := range parse(text) { + if l := b.Link(); l != "" { + names[l] = b.Desc + total++ + } + } + b, err := os.ReadFile(filepath.Join(s.store.feedDir(a.ID), "explore")) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, 0, 0, err + } + known := map[string]bool{} + for _, f := range feeds { + known[norm(f.URL)] = true + } + pages := map[string]bool{} + var out []suggestion + for _, l := range lines(string(b)) { + f := strings.Split(l, "\t") + if len(f) < 2 { + continue + } + if _, ok := names[f[0]]; ok { + pages[f[0]] = true // a page that is a bookmark still + } + if f[1] == "-" || known[norm(f[1])] { + continue + } + known[norm(f[1])] = true + out = append(out, suggestion{URL: f[1], Page: f[0], Name: feedName(names[f[0]], f[1])}) + } + return out, len(pages), total, nil +} + +// ---- pages ---- + +// feedPage is the data of the feed page. +type feedPage struct { + page + Feeds []Feed + Items []Item + Start, Next int // the first item on the page, and the start of the next page, or 0 + Total int // items that match + Filter string // the file name of the one feed that the page shows, or "" + FilterName string + Settings feedSettings + HasMail bool // the server sends email, so the digest is a choice + Open string // "add" when the form to follow a feed must be open + Form struct{ URL, Name string } + Done string + Suggestions []suggestion + Checked int // bookmarked pages that the job looked at + Bookmarks int // bookmarks with a web address +} + +var feedDone = map[string]string{ + "added": "You follow the feed. Its items come with the next update, within 30 minutes.", + "deleted": "You do not follow the feed any more.", + "settings": "Settings saved.", +} + +// feedData gives the feed page of the account: the items from start, or +// only the items of the feed with the file name filter. +func (s *server) feedData(a *Account, filter string, start int) (feedPage, error) { + p := feedPage{page: s.page("Your feed"), Filter: filter, Start: start, HasMail: s.mail != nil, + Settings: s.store.view(a).Feed} + text, err := s.store.readFeeds(a) + if err != nil { + return p, err + } + p.Feeds = parseFeeds(text) + items := s.feedItems(a, p.Feeds) + if filter != "" { + items = slices.DeleteFunc(items, func(it Item) bool { return it.File != filter }) + for _, f := range p.Feeds { + if f.File == filter { + p.FilterName = f.Name + } + } + } + p.Total = len(items) + if start > len(items) { + start = len(items) + } + p.Items = items[start:min(start+itemsOnPage, len(items))] + if start+itemsOnPage < len(items) { + p.Next = start + itemsOnPage + } + if p.Settings.Explore { + p.Suggestions, p.Checked, p.Bookmarks, err = s.suggestions(a, p.Feeds) + } + return p, err +} + +// feed shows the items of the feeds of the account, newest first: from item +// ?start=, and only the feed ?feed= when that is given. +func (s *server) feed(w http.ResponseWriter, r *http.Request) { + u := s.owner(w, r) + if u == nil { + return + } + q := r.URL.Query() + start, _ := strconv.Atoi(q.Get("start")) + p, err := s.feedData(u, q.Get("feed"), max(start, 0)) + if err != nil { + s.fail(w, err) + return + } + p.Done = feedDone[q.Get("done")] + s.render(w, http.StatusOK, "feed", p) +} + +// followFeed adds the feed of the form to feeds.txt. +func (s *server) followFeed(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxForm) + raw, name := r.PostFormValue("url"), r.PostFormValue("name") + addr, err := feedURL(raw) + if err == nil { + err = s.store.changeFeeds(u, func(text string) (string, error) { + feeds := parseFeeds(text) + if len(feeds) >= maxFeeds { + return "", errFeedMany + } + for _, f := range feeds { + if norm(f.URL) == norm(addr) { + return "", errFeedKnown + } + } + return join(append(lines(text), addr+"\t"+feedName(name, addr))), nil + }) + } + status := http.StatusBadRequest + switch { + case err == nil: + http.Redirect(w, r, "/feed?done=added", http.StatusSeeOther) + return + case errors.Is(err, errFeedKnown): + status = http.StatusConflict + case errors.Is(err, errNoURL), errors.Is(err, errFeedURL), errors.Is(err, errFeedLong), errors.Is(err, errFeedMany): + default: + s.fail(w, err) + return + } + p, perr := s.feedData(u, "", 0) + if perr != nil { + s.fail(w, perr) + return + } + p.Error, p.Open, p.Form.URL, p.Form.Name = sentence(err), "add", raw, name + s.render(w, status, "feed", p) +} + +// unfollowFeed removes the feed with the URL of the form from feeds.txt. +func (s *server) unfollowFeed(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxForm) + k := norm(strings.TrimSpace(r.PostFormValue("url"))) + err := s.store.changeFeeds(u, func(text string) (string, error) { + var out []string + for _, l := range lines(text) { + if k == "" || key(l) != k { + out = append(out, l) + } + } + return join(out), nil + }) + if err != nil { + s.fail(w, err) + return + } + http.Redirect(w, r, "/feed?done=deleted", http.StatusSeeOther) +} + +// saveFeedSettings takes the check boxes of the settings form. +func (s *server) saveFeedSettings(w http.ResponseWriter, r *http.Request) { + if !s.sameSite(w, r) { + return + } + u := s.owner(w, r) + if u == nil { + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxForm) + f := feedSettings{ + NewTab: r.PostFormValue("newtab") != "", + Mail: s.mail != nil && r.PostFormValue("mail") != "", + Explore: r.PostFormValue("explore") != "", + } + if err := s.store.setFeed(u, f); err != nil { + s.fail(w, err) + return + } + http.Redirect(w, r, "/feed?done=settings", http.StatusSeeOther) +} + +// ---- API ---- + +// apiOwner gives the account of the token when it can use its bookmarks +// and feeds: as for sync, its email is confirmed and sync is not paused. +// Else apiOwner answers the request and gives nil. +func (s *server) apiOwner(w http.ResponseWriter, r *http.Request) *Account { + u := s.apiUser(r) + if u == nil { + http.Error(w, "not signed in: sign in again", http.StatusUnauthorized) + return nil + } + if !s.verified(s.store.view(u)) { + http.Error(w, "confirm your email address first: open the link in the email from sbm Sync, or see "+s.site+"/account", http.StatusForbidden) + return nil + } + if !s.active(s.store.view(u)) { + http.Error(w, "sbm Sync is paused for this account: see "+s.site+"/account", http.StatusPaymentRequired) + return nil + } + return u +} + +// apiFeed gives the items of all feeds of the account in the sfeed format, +// newest first, for the sfeed tools: +// +// curl -H "Authorization: Bearer $token" https://sbmsync.com/api/feed | sfeed_plain +func (s *server) apiFeed(w http.ResponseWriter, r *http.Request) { + u := s.apiOwner(w, r) + if u == nil { + return + } + text, err := s.store.readFeeds(u) + if err != nil { + s.fail(w, err) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + for _, it := range s.feedItems(u, parseFeeds(text)) { + io.WriteString(w, it.Raw+"\n") + } +} + +// apiFeeds gives feeds.txt: the feeds that the account follows. +func (s *server) apiFeeds(w http.ResponseWriter, r *http.Request) { + u := s.apiOwner(w, r) + if u == nil { + return + } + text, err := s.store.readFeeds(u) + if err != nil { + s.fail(w, err) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + io.WriteString(w, text) +} diff --git a/feed_test.go b/feed_test.go new file mode 100644 index 0000000..f28caf5 --- /dev/null +++ b/feed_test.go @@ -0,0 +1,378 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "pgregory.net/rapid" +) + +// setFeeds writes text as feeds.txt of the account of the token. +func (e *harness) setFeeds(tok, text string) *Account { + a := e.s.store.byTok(tok) + if err := e.s.store.changeFeeds(a, func(string) (string, error) { return text, nil }); err != nil { + e.t.Fatal(err) + } + return a +} + +// setItems writes an sfeed file for the feed with the file name. +func (e *harness) setItems(a *Account, file, text string) { + dir := filepath.Join(e.s.store.feedDir(a.ID), "items") + if err := os.MkdirAll(dir, 0o700); err != nil { + e.t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, file), []byte(text), 0o600); err != nil { + e.t.Fatal(err) + } +} + +// itemTitles gives the titles of the items on the feed page, without the +// lists of feeds and suggestions under them. +func itemTitles(page string) []string { + head, _, _ := strings.Cut(page, "<h2>") + return titles(head) +} + +// feedsOf gives feeds.txt as the API gives it. +func (e *harness) feedsOf(tok string) string { + code, body := e.api("GET", tok, "/api/feeds", nil) + if code != http.StatusOK { + e.t.Fatalf("GET /api/feeds: %d %s", code, body) + } + return body +} + +func TestFeedPagesNeedSignIn(t *testing.T) { + e := newEnv(t, false) + c := e.browser() + if code, to, _ := e.get(c, "/feed"); code != http.StatusSeeOther || to != "/login" { + t.Errorf("GET /feed: %d to %q", code, to) + } + for _, path := range []string{"/feed/add", "/feed/delete", "/feed/settings"} { + if code, to, _ := e.send(c, path, url.Values{"url": {"https://a.org/rss.xml"}}); code != http.StatusSeeOther || to != "/login" { + t.Errorf("POST %s: %d to %q", path, code, to) + } + } + for _, path := range []string{"/api/feed", "/api/feeds"} { + if code, _ := e.api("GET", "wrong", path, nil); code != http.StatusUnauthorized { + t.Errorf("GET %s with a bad token: %d", path, code) + } + } +} + +func TestFeedWaitsForTheEmailCheck(t *testing.T) { + e := newEnv(t, false) + e.withMail() + c := e.signup("me@example.org") + tok := e.token("me@example.org") + if code, to, _ := e.get(c, "/feed"); code != http.StatusSeeOther || to != "/account" { + t.Errorf("feed before the check: %d to %q", code, to) + } + if code, _ := e.api("GET", tok, "/api/feed", nil); code != http.StatusForbidden { + t.Errorf("API before the check: %d", code) + } + e.open(e.link()) + if code, _, _ := e.get(c, "/feed"); code != http.StatusOK { + t.Errorf("feed after the check: %d", code) + } + if code, _ := e.api("GET", tok, "/api/feed", nil); code != http.StatusOK { + t.Errorf("API after the check: %d", code) + } +} + +func TestFollowAndUnfollow(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + code, to, _ := e.send(c, "/feed/add", url.Values{"url": {" https://a.org/rss.xml "}, "name": {" A\tfeed "}}) + if code != http.StatusSeeOther || to != "/feed?done=added" { + t.Fatalf("follow: %d to %q", code, to) + } + if got := e.feedsOf(tok); got != "https://a.org/rss.xml\tA feed\n" { + t.Fatalf("feeds.txt: %q", got) + } + if !strings.Contains(e.body(c, to), "You follow the feed.") { + t.Error("the page does not tell that the feed is followed") + } + code, _, page := e.send(c, "/feed/add", url.Values{"url": {"http://www.A.org/rss.xml/"}, "name": {"again"}}) + if code != http.StatusConflict || !strings.Contains(page, "You follow this feed already.") || !strings.Contains(page, "<details open>") || !strings.Contains(page, `value="again"`) { + t.Errorf("follow the same feed: %d\n%s", code, page) + } + // Without a name, the host is the name. + e.send(c, "/feed/add", url.Values{"url": {"https://b.org/atom"}}) + want := "https://a.org/rss.xml\tA feed\nhttps://b.org/atom\tb.org\n" + if got := e.feedsOf(tok); got != want { + t.Fatalf("feeds.txt: %q, want %q", got, want) + } + for _, bad := range []string{"javascript:alert(1)", "https://a.org/it's", "https://a.org/x y", "ftp://a.org/f", "a.org/rss", "", "https://" + strings.Repeat("a", 2100)} { + if code, _, _ := e.send(c, "/feed/add", url.Values{"url": {bad}}); code != http.StatusBadRequest { + t.Errorf("follow %q: %d", bad, code) + } + } + if got := e.feedsOf(tok); got != want { + t.Fatalf("a refusal changed feeds.txt: %q", got) + } + if code, to, _ := e.send(c, "/feed/delete", url.Values{"url": {"https://A.org/rss.xml/"}}); code != http.StatusSeeOther || to != "/feed?done=deleted" { + t.Fatalf("unfollow: %d to %q", code, to) + } + if got := e.feedsOf(tok); got != "https://b.org/atom\tb.org\n" { + t.Fatalf("feeds.txt after the unfollow: %q", got) + } +} + +func TestFeedLimit(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + var b strings.Builder + for i := range maxFeeds { + fmt.Fprintf(&b, "https://f%d.org/rss\tF%d\n", i, i) + } + e.setFeeds(tok, b.String()) + if code, _, page := e.send(c, "/feed/add", url.Values{"url": {"https://one-more.org/rss"}}); code != http.StatusBadRequest || !strings.Contains(page, "200 feeds already") { + t.Errorf("feed 201: %d", code) + } +} + +func TestFeedTimeline(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\nhttps://b.org/feed\tBeta & Co\n") + e.setItems(a, "Alpha", "1700000000\tOld \\t tab\thttps://a.org/1\t\t\tid1\tAnn\t\t\n1700003600\tNewer\thttps://a.org/2\t\t\t\t\t\t\n") + e.setItems(a, "Beta-Co", "1700001800\tScript <b>\tjavascript:alert(1)\t\t\tid3\t\t\t\n\tNo time\thttps://b.org/4\t\t\tid4\t\t\t\n") + page := e.body(c, "/feed") + if got := itemTitles(page); !slices.Equal(got, []string{"Newer", "Script <b>", "Old \t tab", "No time"}) { + t.Fatalf("titles: %q\n%s", got, page) + } + for _, want := range []string{`<a href="https://a.org/2">Newer</a>`, "Beta & Co", " · Ann", "2023-11-14 23:13", "4 items."} { + if !strings.Contains(page, want) { + t.Errorf("the page lacks %q", want) + } + } + for _, bad := range []string{`href="javascript:`, `target="_blank"`} { + if strings.Contains(page, bad) { + t.Errorf("the page has %q", bad) + } + } + page = e.body(c, "/feed?feed=Beta-Co") + if got := itemTitles(page); !slices.Equal(got, []string{"Script <b>", "No time"}) || !strings.Contains(page, "from <b>Beta & Co</b>") { + t.Errorf("one feed: %q", got) + } + if page := e.body(c, "/feed?feed=nothing"); !strings.Contains(page, "0 items.") { + t.Error("an unknown feed shows items") + } + + if code, to, _ := e.send(c, "/feed/settings", url.Values{"newtab": {"on"}}); code != http.StatusSeeOther || to != "/feed?done=settings" { + t.Fatalf("settings: %d to %q", code, to) + } + if page := e.body(c, "/feed"); !strings.Contains(page, `<a href="https://a.org/2" target="_blank" rel="noopener">Newer</a>`) { + t.Error("links do not open in a new tab") + } + if !e.s.store.view(a).Feed.NewTab { + t.Error("the setting is not kept") + } + e.send(c, "/feed/settings", url.Values{}) + if page := e.body(c, "/feed"); strings.Contains(page, `target="_blank"`) || e.s.store.view(a).Feed.NewTab { + t.Error("links still open in a new tab") + } + + code, body := e.api("GET", tok, "/api/feed", nil) + want := "1700003600\tNewer\thttps://a.org/2\t\t\t\t\t\t\n" + + "1700001800\tScript <b>\tjavascript:alert(1)\t\t\tid3\t\t\t\n" + + "1700000000\tOld \\t tab\thttps://a.org/1\t\t\tid1\tAnn\t\t\n" + + "\tNo time\thttps://b.org/4\t\t\tid4\t\t\t\n" + if code != http.StatusOK || body != want { + t.Errorf("GET /api/feed: %d\n%q\nwant\n%q", code, body, want) + } +} + +func TestFeedPages(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + var b strings.Builder + for i := range itemsOnPage + 5 { + fmt.Fprintf(&b, "%d\tn%d\thttps://a.org/%d\t\t\t\t\t\t\n", 1700000000+i, i, i) + } + e.setItems(a, "Alpha", b.String()) + page := e.body(c, "/feed") + if got := itemTitles(page); len(got) != itemsOnPage || got[0] != "n104" || !strings.Contains(page, `<a href="/feed?start=100">Older items</a>`) { + t.Errorf("page 1: %d titles, first %q", len(got), got[0]) + } + page = e.body(c, "/feed?start=100&feed=Alpha") + if got := itemTitles(page); !slices.Equal(got, []string{"n4", "n3", "n2", "n1", "n0"}) || strings.Contains(page, "Older items") { + t.Errorf("page 2: %q", got) + } +} + +func TestFeedMailSettingNeedsAnSMTPServer(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + a := e.s.store.byTok(tok) + if strings.Contains(e.body(c, "/feed"), `name="mail"`) { + t.Error("the page offers the digest without an SMTP server") + } + e.send(c, "/feed/settings", url.Values{"mail": {"on"}, "explore": {"on"}}) + if f := e.s.store.view(a).Feed; f.Mail || !f.Explore { + t.Errorf("settings without an SMTP server: %+v", f) + } + e.withMail() + if !strings.Contains(e.body(c, "/feed"), `name="mail"`) { + t.Error("the page does not offer the digest") + } + e.send(c, "/feed/settings", url.Values{"mail": {"on"}}) + if f := e.s.store.view(a).Feed; !f.Mail || f.Explore { + t.Errorf("settings with an SMTP server: %+v", f) + } +} + +func TestOtherSiteCannotChangeFeeds(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + for _, path := range []string{"/feed/add", "/feed/delete", "/feed/settings"} { + req, _ := http.NewRequest("POST", e.web.URL+path, strings.NewReader("url=https%3A%2F%2Fa.org%2Frss.xml&newtab=on")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Origin", "https://evil.example") + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("POST %s from another site: %s", path, resp.Status) + } + } + if got := e.feedsOf(tok); got != "https://a.org/rss.xml\tAlpha\n" { + t.Errorf("another site changed the feeds: %q", got) + } +} + +func TestDeleteAccountRemovesTheFeeds(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + if resp := e.post(c, "/account/delete", url.Values{"password": {"password1"}}); resp.StatusCode != http.StatusSeeOther { + t.Fatalf("delete: %s", resp.Status) + } + if _, err := os.Stat(e.s.store.feedDir(a.ID)); !os.IsNotExist(err) { + t.Errorf("the feeds of the account are still there: %v", err) + } +} + +// feedURLs draws strings that are often feed URLs and often not. +var feedURLs = rapid.Custom(func(t *rapid.T) string { + head := rapid.SampledFrom([]string{"https://a.org/", "http://b.org", "HTTPS://c.org/x", "https://", "ftp://c.org/", "javascript:", "", " https://d.org/"}).Draw(t, "head") + tail := rapid.StringOf(rapid.SampledFrom([]rune("abcz09:/.-_?=&%#~'\" \t\n\\日é\x01"))).Draw(t, "tail") + return head + tail +}) + +// A URL that feedURL takes has only printable ASCII without single quotes, +// so that it holds between single quotes in the sfeedrc. Each such http or +// https URL with a host is taken. +func TestFeedURLIsShellSafe(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + raw := feedURLs.Draw(t, "url") + u := strings.TrimSpace(raw) + safe := u != "" && len(u) <= maxFeedURL + for i := 0; i < len(u); i++ { + if c := u[i]; c <= ' ' || c > '~' || c == '\'' { + safe = false + } + } + p, err := url.Parse(u) + want := safe && err == nil && p.Host != "" && (p.Scheme == "http" || p.Scheme == "https") + got, err := feedURL(raw) + if (err == nil) != want { + t.Fatalf("feedURL(%q) = %q, %v; want ok %v", raw, got, err, want) + } + if err != nil { + return + } + if got != u { + t.Fatalf("feedURL(%q) = %q, want %q", raw, got, u) + } + rc := sfeedrc("/data/items", []Feed{{URL: got, Name: "x", File: "x"}}) + line := "\tfeed 'x' '" + got + "'\n" + if !strings.Contains(rc, line) || strings.Count(line, "'") != 4 { + t.Fatalf("sfeedrc for %q:\n%s", got, rc) + } + }) +} + +var fileNames = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9._-]*$`) + +// Each line of feeds.txt parses back, and each feed gets a safe file name +// that no other feed of the file has. +func TestFeedsParseBackWithUniqueFiles(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + var b strings.Builder + var want []Feed + for i := range rapid.IntRange(0, 6).Draw(t, "feeds") { + u := fmt.Sprintf("https://f%d.org/rss", i) + name := feedName(rapid.StringOf(rapid.SampledFrom([]rune("aA0 -._/\t\n日'"))).Draw(t, "name"), u) + if strings.ContainsAny(name, "\t\n") || len([]rune(name)) > maxFeedName { + t.Fatalf("feedName gave %q", name) + } + b.WriteString(u + "\t" + name + "\n") + want = append(want, Feed{URL: u, Name: name}) + } + got := parseFeeds(b.String()) + if len(got) != len(want) { + t.Fatalf("%d feeds parsed, want %d", len(got), len(want)) + } + files := map[string]bool{} + for i, f := range got { + if f.URL != want[i].URL || f.Name != want[i].Name { + t.Fatalf("feed %d is %+v, want %+v", i, f, want[i]) + } + if !fileNames.MatchString(f.File) || files[f.File] || len(f.File) > maxFileName+4 { + t.Fatalf("feed %d has the file name %q; files %v", i, f.File, files) + } + files[f.File] = true + } + }) +} + +// escape is the escaping of the sfeed format, as a reference. +func escape(s string) string { + return strings.NewReplacer(`\`, `\\`, "\t", `\t`, "\n", `\n`).Replace(s) +} + +func TestUnescapeUndoesEscape(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + s := rapid.StringOf(rapid.SampledFrom([]rune("ab\\tn\t\n日"))).Draw(t, "text") + if got := unescape(escape(s)); got != s { + t.Fatalf("unescape(%q) = %q, want %q", escape(s), got, s) + } + }) +} + +func TestParseItems(t *testing.T) { + f := Feed{URL: "https://a.org/rss", Name: "A", File: "A"} + items := parseItems("1700000000\tT\\ta\\\\b\\n\thttps://a.org/1\tbody\thtml\tid1\tAnn\t\tcat\n"+ + "x\t\thttps://a.org/2\t\t\t\t\t\n"+ + "short\n"+ + "1\t\t\t\t\t\t\t\t\n", f) + if len(items) != 3 { + t.Fatalf("%d items", len(items)) + } + a, b, c := items[0], items[1], items[2] + if a.Title != "T\ta\\b\n" || a.Link != "https://a.org/1" || a.Author != "Ann" || a.Key != "A\tid1" || a.Time.Unix() != 1700000000 || a.Feed != "A" { + t.Errorf("item 1: %+v", a) + } + // Without a time, an id or a title: the time is zero, the link stands in + // for the id, and the title comes from the link. + if !b.Time.IsZero() || b.Key != "A\thttps://a.org/2" || b.Title != "https://a.org/2" { + t.Errorf("item 2: %+v", b) + } + if c.Title != "(no title)" || c.Key != "A\t" || c.Link != "" { + t.Errorf("item 3: %+v", c) + } +} diff --git a/feedjob.go b/feedjob.go new file mode 100644 index 0000000..cc56e78 --- /dev/null +++ b/feedjob.go @@ -0,0 +1,372 @@ +package main + +// The feed job: `sbm-sync feed`. A systemd timer runs it every 30 minutes +// (contrib/sbm-feed.timer). For each account it: +// +// 1. writes an sfeedrc from feeds.txt and runs sfeed_update, which fetches +// each feed and merges the new items into items/<file>; +// 2. when the account asked for it, looks through the bookmarks of the +// account for feeds with sfeed_web, and keeps what it found in explore; +// 3. when the account asked for it, sends the items that are new since the +// last digest by email, once a day, in the form of sfeed_plain. +// +// The job writes only its own files under feeds/<account>/. The web server +// writes only feeds.txt and accounts.json. So the two never write the same +// file. + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log" + "net" + "net/url" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +const ( + digestEvery = 24 * time.Hour + crawlPerRun = 20 // bookmarked pages that explore looks at in one run + feedsPerPage = 5 // feeds that explore keeps from one page + updateTime = 10 * time.Minute +) + +// feedTools are the programs and the clock that the job uses. Tests +// replace them. +type feedTools struct { + update func(sfeedrc string) error // runs sfeed_update + plain func(files []string) (string, error) // runs sfeed_plain on sfeed files + discover func(page string) ([]string, error) // gives "URL<tab>type" for each feed of a page, as curl | sfeed_web + lookup func(host string) ([]net.IP, error) // resolves a host name + now func() time.Time +} + +// execTools gives the tools that run the programs. +func execTools() feedTools { + return feedTools{ + update: func(sfeedrc string) error { + ctx, cancel := context.WithTimeout(context.Background(), updateTime) + defer cancel() + out, err := exec.CommandContext(ctx, "sfeed_update", sfeedrc).CombinedOutput() + for _, l := range lines(string(out)) { + log.Print(l) + } + return err + }, + plain: func(files []string) (string, error) { + out, err := exec.Command("sfeed_plain", files...).Output() + return string(out), err + }, + discover: func(page string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + curl := exec.CommandContext(ctx, "curl", "-sfL", "--proto", "=http,https", "--max-redirs", "3", + "-m", "10", "--max-filesize", "1048576", "-A", "sbm-sync (+https://github.com/equwal/sbm-sync)", page) + web := exec.CommandContext(ctx, "sfeed_web", page) + pipe, err := curl.StdoutPipe() + if err != nil { + return nil, err + } + web.Stdin = pipe + if err := curl.Start(); err != nil { + return nil, err + } + out, err := web.Output() + curl.Wait() // a page that does not load gives no feeds + if err != nil { + return nil, err + } + return lines(string(out)), nil + }, + lookup: net.LookupIP, + now: time.Now, + } +} + +// runFeeds does the work of the job for every account. +func (s *server) runFeeds() { + accounts := s.store.all() + for _, a := range accounts { + if err := s.feedAccount(a); err != nil { + log.Printf("account %s: %v", a.ID, err) + } + } + log.Printf("feeds of %d accounts done", len(accounts)) +} + +// sfeedrc gives the sfeed_update settings for the feeds: the item files go +// to itemsDir. Each URL passed feedURL, and each file name has only safe +// characters, so the single quotes hold. +func sfeedrc(itemsDir string, feeds []Feed) string { + var b strings.Builder + b.WriteString("# sbm-sync makes this file from feeds.txt. Do not edit it.\n") + b.WriteString("sfeedpath='" + itemsDir + "'\n") + b.WriteString("maxjobs=4\n") + b.WriteString("fetch() {\n\tcurl --proto '=http,https' -L --max-redirs 3 -A 'sbm-sync (+https://github.com/equwal/sbm-sync)' -f -s -m 20 --max-filesize 5242880 -z \"$3\" \"$2\" 2>/dev/null\n}\n") + b.WriteString("feeds() {\n") + for _, f := range feeds { + b.WriteString("\tfeed '" + f.File + "' '" + f.URL + "'\n") + } + b.WriteString("}\n") + return b.String() +} + +// publicIP tells if an address is on the public internet. +func publicIP(ip net.IP) bool { + if !ip.IsGlobalUnicast() || ip.IsPrivate() { + return false + } + // 100.64.0.0/10 is for carrier networks, as private as 10.0.0.0/8. + if v4 := ip.To4(); v4 != nil && v4[0] == 100 && v4[1]&0xc0 == 64 { + return false + } + return true +} + +// public tells if the host of the URL is on the public internet. The job +// fetches only such hosts, so that a feed URL cannot reach the services on +// this machine or in its private networks. cache keeps the answers of one +// run. +func (s *server) public(rawURL string, cache map[string]bool) bool { + p, err := url.Parse(rawURL) + if err != nil || p.Hostname() == "" { + return false + } + host := p.Hostname() + if ok, seen := cache[host]; seen { + return ok + } + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else if r, err := s.feeds.lookup(host); err == nil { + ips = r + } + ok := len(ips) > 0 + for _, ip := range ips { + if !publicIP(ip) { + ok = false + } + } + cache[host] = ok + return ok +} + +// feedAccount does the work of the job for one account. +func (s *server) feedAccount(p *Account) error { + a := s.store.view(p) + dir := s.store.feedDir(a.ID) + itemsDir := filepath.Join(dir, "items") + text, err := s.store.readFeeds(p) + if err != nil { + return err + } + cache := map[string]bool{} + var feeds []Feed + for _, f := range parseFeeds(text) { + if _, err := feedURL(f.URL); err != nil || !s.public(f.URL, cache) { + log.Printf("account %s: feed %s: not a public http address, skipped", a.ID, f.URL) + continue + } + feeds = append(feeds, f) + } + if len(feeds) == 0 && !a.Feed.Explore && !a.Feed.Mail { + if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) { + return nil // the account never used feeds + } + } + if err := os.MkdirAll(itemsDir, 0o700); err != nil { + return err + } + rcPath := filepath.Join(dir, "sfeedrc") + if len(feeds) > 0 { + rc := sfeedrc(itemsDir, feeds) + if old, _ := os.ReadFile(rcPath); string(old) != rc { + if err := writeFile(rcPath, []byte(rc)); err != nil { + return err + } + } + if err := s.feeds.update(rcPath); err != nil { + log.Printf("account %s: sfeed_update: %v", a.ID, err) + } + } else { + os.Remove(rcPath) + } + // The items of feeds that the account left go. + keep := map[string]bool{} + for _, f := range feeds { + keep[f.File] = true + } + entries, err := os.ReadDir(itemsDir) + if err != nil { + return err + } + for _, e := range entries { + if !keep[e.Name()] { + os.Remove(filepath.Join(itemsDir, e.Name())) + } + } + if a.Feed.Explore { + if err := s.explore(p, dir, cache); err != nil { + log.Printf("account %s: explore: %v", a.ID, err) + } + } + if !a.Feed.Mail { + os.Remove(filepath.Join(dir, "mailed")) + os.Remove(filepath.Join(dir, "digest")) + } else if s.mail != nil && s.verified(a) && s.active(a) { + if err := s.digest(a, dir, feeds); err != nil { + log.Printf("account %s: digest: %v", a.ID, err) + } + } + return nil +} + +// explore looks at up to crawlPerRun bookmarked pages that it did not look +// at before, newest first, and adds the feeds that it finds to the explore +// file. A page without a feed, or one that the job cannot fetch, gets a +// line with "-", so that the job does not look at it again. +func (s *server) explore(p *Account, dir string, cache map[string]bool) error { + text, _, err := s.store.read(p, "") + if err != nil { + return err + } + path := filepath.Join(dir, "explore") + old, err := os.ReadFile(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + seen := map[string]bool{} + for _, l := range lines(string(old)) { + page, _, _ := strings.Cut(l, "\t") + seen[page] = true + } + var add []string + n := 0 + for _, b := range slices.Backward(parse(text)) { + if n >= crawlPerRun { + break + } + link := b.Link() + if link == "" || seen[link] { + continue + } + seen[link] = true + if _, err := feedURL(link); err != nil || !s.public(link, cache) { + add = append(add, link+"\t-\t") + continue + } + n++ + found, err := s.feeds.discover(link) + if err != nil { + log.Printf("account %s: explore %s: %v", p.ID, link, err) + } + kept := 0 + for _, f := range found { + u, typ, _ := strings.Cut(f, "\t") + if _, err := feedURL(u); err != nil || kept >= feedsPerPage { + continue + } + add = append(add, link+"\t"+u+"\t"+typ) + kept++ + } + if kept == 0 { + add = append(add, link+"\t-\t") + } + } + if len(add) == 0 { + return nil + } + return writeFile(path, []byte(string(old)+join(add))) +} + +// digest sends the items that came in since the last digest, once a day. +// The first run after the opt-in sends nothing: it only notes the items of +// that day, so that nobody gets the whole history. +func (s *server) digest(a Account, dir string, feeds []Feed) error { + mailedPath, digestPath := filepath.Join(dir, "mailed"), filepath.Join(dir, "digest") + now := s.feeds.now() + stamp := []byte(strconv.FormatInt(now.Unix(), 10) + "\n") + type feedItems struct { + f Feed + items []Item + } + var all []feedItems + var keys []string + for _, f := range feeds { + b, err := os.ReadFile(filepath.Join(dir, "items", f.File)) + if err != nil { + continue + } + items := parseItems(string(b), f) + all = append(all, feedItems{f, items}) + for _, it := range items { + keys = append(keys, it.Key) + } + } + old, err := os.ReadFile(mailedPath) + if errors.Is(err, fs.ErrNotExist) { + if err := writeFile(mailedPath, []byte(join(keys))); err != nil { + return err + } + return writeFile(digestPath, stamp) + } else if err != nil { + return err + } + b, _ := os.ReadFile(digestPath) + last, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + if now.Sub(time.Unix(last, 0)) < digestEvery { + return nil + } + mailed := set(lines(string(old))) + tmp, err := os.MkdirTemp("", "sbm-digest-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + var files, newKeys []string + for _, fi := range all { + var ls []string + for _, it := range fi.items { + if !mailed[it.Key] { + ls = append(ls, it.Raw) + newKeys = append(newKeys, it.Key) + } + } + if len(ls) == 0 { + continue + } + path := filepath.Join(tmp, fi.f.File) + if err := os.WriteFile(path, []byte(join(ls)), 0o600); err != nil { + return err + } + files = append(files, path) + } + if len(newKeys) == 0 { + return nil + } + body, err := s.feeds.plain(files) + if err != nil { + return err + } + subject := fmt.Sprintf("sbm feed: %d new items", len(newKeys)) + if len(newKeys) == 1 { + subject = "sbm feed: 1 new item" + } + body = strings.TrimRight(body, "\n") + "\n\nYour feed: " + s.site + "/feed\n" + + "To stop these emails, turn off the daily digest in the settings of that page.\n" + if err := s.mail(a.Email, subject, body); err != nil { + return err + } + if err := writeFile(mailedPath, []byte(string(old)+join(newKeys))); err != nil { + return err + } + return writeFile(digestPath, stamp) +} diff --git a/feedjob_test.go b/feedjob_test.go new file mode 100644 index 0000000..c5bcf22 --- /dev/null +++ b/feedjob_test.go @@ -0,0 +1,326 @@ +package main + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// fakeTools are feed tools that need no sfeed: "sfeed_update" writes the +// items of a table, and "sfeed_web" answers from a table. +type fakeTools struct { + items map[string]string // feed URL: the sfeed lines of the feed + pages map[string][]string // page URL: the "URL<tab>type" lines that sfeed_web finds + updates []string // the sfeedrc of each update + crawled []string // the pages given to discover + plain int // calls of plain + now time.Time +} + +func (f *fakeTools) tools() feedTools { + return feedTools{ + update: func(rc string) error { + f.updates = append(f.updates, rc) + b, err := os.ReadFile(rc) + if err != nil { + return err + } + dir := "" + for _, l := range lines(string(b)) { + if v, ok := strings.CutPrefix(l, "sfeedpath='"); ok { + dir = strings.TrimSuffix(v, "'") + } + if v, ok := strings.CutPrefix(l, "\tfeed '"); ok { + file, u, _ := strings.Cut(strings.TrimSuffix(v, "'"), "' '") + if err := os.WriteFile(filepath.Join(dir, file), []byte(f.items[u]), 0o600); err != nil { + return err + } + } + } + return nil + }, + plain: func(files []string) (string, error) { + f.plain++ + var b strings.Builder + for _, path := range files { + text, err := os.ReadFile(path) + if err != nil { + return "", err + } + b.WriteString(filepath.Base(path) + ":\n") + for _, l := range lines(string(text)) { + c := strings.Split(l, "\t") + b.WriteString(" " + c[1] + " " + c[2] + "\n") + } + } + return b.String(), nil + }, + discover: func(page string) ([]string, error) { + f.crawled = append(f.crawled, page) + return f.pages[page], nil + }, + lookup: func(host string) ([]net.IP, error) { + switch host { + case "internal.test": + return []net.IP{net.ParseIP("10.66.0.4")}, nil + case "nowhere.test": + return nil, errors.New("no such host") + } + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, + now: func() time.Time { return f.now }, + } +} + +// fileOf gives the text of a file of the feed directory of the account, or +// "" when the file is missing. +func (e *harness) feedFile(a *Account, name string) string { + b, err := os.ReadFile(filepath.Join(e.s.store.feedDir(a.ID), name)) + if err != nil { + return "" + } + return string(b) +} + +const oneItem = "1700000000\tOne\thttps://a.org/1\t\t\tid1\t\t\t\n" + +func TestJobFetchesTheFeeds(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\nhttp://internal.test/rss\tInside\nhttps://nowhere.test/x\tGone\nhttps://10.0.0.7/rss\tPrivate\n") + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: time.Unix(1_800_000_000, 0)} + e.s.feeds = f.tools() + e.s.runFeeds() + rc := e.feedFile(a, "sfeedrc") + itemsDir := filepath.Join(e.s.store.feedDir(a.ID), "items") + for _, want := range []string{"sfeedpath='" + itemsDir + "'\n", "\tfeed 'Alpha' 'https://a.org/rss.xml'\n", "curl --proto '=http,https'"} { + if !strings.Contains(rc, want) { + t.Errorf("the sfeedrc lacks %q:\n%s", want, rc) + } + } + // A host on a private network, an unknown host and an address on a + // private network stay out of the sfeedrc. + for _, bad := range []string{"internal.test", "nowhere.test", "10.0.0.7"} { + if strings.Contains(rc, bad) { + t.Errorf("the sfeedrc has %q", bad) + } + } + if len(f.updates) != 1 || f.updates[0] != filepath.Join(e.s.store.feedDir(a.ID), "sfeedrc") { + t.Errorf("updates: %q", f.updates) + } + if got := e.feedFile(a, filepath.Join("items", "Alpha")); got != oneItem { + t.Errorf("items of Alpha: %q", got) + } + if page := e.body(c, "/feed"); !strings.Contains(page, `<a href="https://a.org/1">One</a>`) { + t.Error("the page does not show the item") + } + // The items of a feed that the account left go. + e.setFeeds(tok, "") + e.s.runFeeds() + if e.feedFile(a, filepath.Join("items", "Alpha")) != "" || e.feedFile(a, "sfeedrc") != "" || len(f.updates) != 1 { + t.Error("the items or the sfeedrc of the left feed stay") + } +} + +func TestJobLeavesAccountsWithoutFeedsAlone(t *testing.T) { + e := newEnv(t, false) + _, tok, _ := e.account("") + a := e.s.store.byTok(tok) + f := &fakeTools{now: time.Now()} + e.s.feeds = f.tools() + e.s.runFeeds() + if _, err := os.Stat(e.s.store.feedDir(a.ID)); !os.IsNotExist(err) { + t.Errorf("the job made a feed directory: %v", err) + } +} + +func TestJobSendsADailyDigest(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + e.withMail() + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + start := time.Unix(1_800_000_000, 0) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: start} + e.s.feeds = f.tools() + + e.s.runFeeds() + if e.sent() != 0 || e.feedFile(a, "mailed") != "" { + t.Fatal("the job mails without the opt-in") + } + e.send(c, "/feed/settings", url.Values{"mail": {"on"}}) + // The first run after the opt-in notes the items and sends nothing. + e.s.runFeeds() + if e.sent() != 0 || e.feedFile(a, "mailed") != "Alpha\tid1\n" || strings.TrimSpace(e.feedFile(a, "digest")) != "1800000000" { + t.Fatalf("first run: %d emails, mailed %q, digest %q", e.sent(), e.feedFile(a, "mailed"), e.feedFile(a, "digest")) + } + f.items["https://a.org/rss.xml"] = "1700003600\tTwo\thttps://a.org/2\t\t\tid2\t\t\t\n" + oneItem + f.now = start.Add(time.Hour) + e.s.runFeeds() + if e.sent() != 0 { + t.Fatal("a digest went out within a day") + } + f.now = start.Add(25 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 { + t.Fatalf("%d emails after a day", e.sent()) + } + mail := e.mails[0] + for _, want := range []string{"me@example.org\nsbm feed: 1 new item\n", "Alpha:\n Two https://a.org/2\n", "Your feed: " + e.s.site + "/feed\n"} { + if !strings.Contains(mail, want) { + t.Errorf("the digest lacks %q:\n%s", want, mail) + } + } + if strings.Contains(mail, "One") { + t.Error("the digest has the old item") + } + if e.feedFile(a, "mailed") != "Alpha\tid1\nAlpha\tid2\n" || strings.TrimSpace(e.feedFile(a, "digest")) != fmt.Sprint(f.now.Unix()) { + t.Errorf("after the digest: mailed %q, digest %q", e.feedFile(a, "mailed"), e.feedFile(a, "digest")) + } + f.now = start.Add(50 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 { + t.Error("a digest went out without new items") + } + f.items["https://a.org/rss.xml"] = "1700007200\tThree\thttps://a.org/3\t\t\tid3\t\t\t\n" + f.items["https://a.org/rss.xml"] + f.now = start.Add(51 * time.Hour) + e.s.runFeeds() + if e.sent() != 2 || !strings.Contains(e.mails[1], "sbm feed: 1 new item\n") || !strings.Contains(e.mails[1], "Three") { + t.Errorf("second digest: %d emails, %q", e.sent(), e.mails[len(e.mails)-1]) + } + // The opt-out forgets the digests, so that a later opt-in starts fresh. + e.send(c, "/feed/settings", url.Values{}) + e.s.runFeeds() + if e.feedFile(a, "mailed") != "" || e.feedFile(a, "digest") != "" { + t.Error("the opt-out kept the digest files") + } + if e.sent() != 2 { + t.Error("a digest went out after the opt-out") + } +} + +// A digest that does not go out is sent again with the next run. +func TestJobKeepsTheItemsOfAFailedDigest(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + e.withMail() + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + start := time.Unix(1_800_000_000, 0) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": ""}, now: start} + e.s.feeds = f.tools() + e.send(c, "/feed/settings", url.Values{"mail": {"on"}}) + e.s.runFeeds() + f.items["https://a.org/rss.xml"] = oneItem + f.now = start.Add(25 * time.Hour) + e.s.mail = func(string, string, string) error { return errors.New("no SMTP server") } + e.s.runFeeds() + if e.feedFile(a, "mailed") != "" || strings.TrimSpace(e.feedFile(a, "digest")) != "1800000000" { + t.Errorf("a failed digest changed the files: mailed %q, digest %q", e.feedFile(a, "mailed"), e.feedFile(a, "digest")) + } + e.withMail() + e.s.runFeeds() + if e.sent() != 1 || !strings.Contains(e.mails[0], "One") { + t.Errorf("the next run: %d emails", e.sent()) + } +} + +func TestJobMailsOnlyConfirmedAccounts(t *testing.T) { + e := newEnv(t, false) + e.withMail() + e.signup("me@example.org") + tok := e.token("me@example.org") + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + e.s.store.setFeed(a, feedSettings{Mail: true}) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: time.Now()} + e.s.feeds = f.tools() + e.s.runFeeds() + // The one email is the email check of the sign-up. + if e.sent() != 1 || e.feedFile(a, "mailed") != "" { + t.Errorf("an unconfirmed account got a digest: %d emails, mailed %q", e.sent(), e.feedFile(a, "mailed")) + } + if e.feedFile(a, filepath.Join("items", "Alpha")) != oneItem { + t.Error("the feed of an unconfirmed account is not fetched") + } +} + +func TestJobExploresTheBookmarks(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("https://old.org\tOld\t\nhttps://blog.org\tBlog\t\nhttp://internal.test/\tInside\t\njavascript:x\tBad\t\n") + a := e.s.store.byTok(tok) + f := &fakeTools{now: time.Now(), pages: map[string][]string{ + "https://blog.org": {"https://blog.org/feed.xml\tapplication/rss+xml", "https://blog.org/it's\tapplication/atom+xml"}, + }} + e.s.feeds = f.tools() + e.s.runFeeds() + if len(f.crawled) != 0 || e.feedFile(a, "explore") != "" { + t.Fatal("the job looked at the bookmarks without the opt-in") + } + e.send(c, "/feed/settings", url.Values{"explore": {"on"}}) + e.s.runFeeds() + // Newest first. The page on a private network is not fetched. + if want := []string{"https://blog.org", "https://old.org"}; strings.Join(f.crawled, " ") != strings.Join(want, " ") { + t.Errorf("crawled %q, want %q", f.crawled, want) + } + want := "http://internal.test/\t-\t\nhttps://blog.org\thttps://blog.org/feed.xml\tapplication/rss+xml\nhttps://old.org\t-\t\n" + if got := e.feedFile(a, "explore"); got != want { + t.Errorf("explore:\n%q\nwant\n%q", got, want) + } + page := e.body(c, "/feed") + for _, want := range []string{"looked at 3 of your 3 bookmarked pages", `value="https://blog.org/feed.xml"`, `value="Blog"`, "on https://blog.org"} { + if !strings.Contains(page, want) { + t.Errorf("the page lacks %q", want) + } + } + // The next run looks at no page twice. + e.s.runFeeds() + if len(f.crawled) != 2 { + t.Errorf("crawled %q", f.crawled) + } + // A feed that the account follows is no suggestion any more. + e.send(c, "/feed/add", url.Values{"url": {"https://blog.org/feed.xml"}, "name": {"Blog"}}) + _, explore, _ := strings.Cut(e.body(c, "/feed"), "<h2>Explore</h2>") + if strings.Contains(explore, "feed.xml") || !strings.Contains(explore, "No new feed to follow yet.") { + t.Errorf("the page still suggests the followed feed:\n%s", explore) + } +} + +func TestJobExploresTwentyPagesInOneRun(t *testing.T) { + e := newEnv(t, false) + var b strings.Builder + for i := range crawlPerRun + 5 { + fmt.Fprintf(&b, "https://p%d.org\tP%d\t\n", i, i) + } + c, tok, _ := e.account(b.String()) + f := &fakeTools{now: time.Now()} + e.s.feeds = f.tools() + e.send(c, "/feed/settings", url.Values{"explore": {"on"}}) + e.s.runFeeds() + if len(f.crawled) != crawlPerRun || f.crawled[0] != "https://p24.org" { + t.Errorf("first run: %d pages, first %q", len(f.crawled), f.crawled[0]) + } + if !strings.Contains(e.body(c, "/feed"), "looked at 20 of your 25 bookmarked pages") { + t.Error("the page does not show the count") + } + e.s.runFeeds() + if len(f.crawled) != crawlPerRun+5 { + t.Errorf("second run: %d pages", len(f.crawled)) + } + _ = tok +} + +func TestPublicIP(t *testing.T) { + for ip, want := range map[string]bool{ + "93.184.216.34": true, "2606:2800:220:1:248:1893:25c8:1946": true, + "127.0.0.1": false, "10.66.0.4": false, "192.168.1.1": false, "172.16.0.9": false, + "169.254.169.254": false, "100.64.0.1": false, "0.0.0.0": false, "::1": false, "fd00::1": false, "fe80::1": false, + } { + if got := publicIP(net.ParseIP(ip)); got != want { + t.Errorf("publicIP(%s) = %v, want %v", ip, got, want) + } + } +} diff --git a/main.go b/main.go index 261ec2d..a0bc4ab 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,8 @@ type server struct { // mail sends an email. It is nil when the server has no SMTP server: // then accounts need no email check. mail func(to, subject, body string) error + // feeds are the programs of the feed job, see feedjob.go. + feeds feedTools } func main() { @@ -115,6 +117,14 @@ func main() { From: from, Hello: u.Hostname()} s.mail = m.send } + if len(os.Args) > 1 { + if os.Args[1] != "feed" { + log.Fatalf("unknown command %q: the one command is feed", os.Args[1]) + } + s.feeds = execTools() + s.runFeeds() + return + } addr := env("SBM_ADDR", "127.0.0.1:8750") log.Printf("sbm-sync on %s for %s, billing %v, email check %v", addr, site, s.stripe != nil, s.mail != nil) hs := &http.Server{ @@ -180,6 +190,12 @@ func (s *server) routes() http.Handler { m.HandleFunc("GET /bookmarks.txt", s.download) m.HandleFunc("GET /search.js", s.script) m.HandleFunc("GET /opensearch.xml", s.openSearch) + m.HandleFunc("GET /feed", s.feed) + m.HandleFunc("POST /feed/add", s.followFeed) + m.HandleFunc("POST /feed/delete", s.unfollowFeed) + m.HandleFunc("POST /feed/settings", s.saveFeedSettings) + m.HandleFunc("GET /api/feed", s.apiFeed) + m.HandleFunc("GET /api/feeds", s.apiFeeds) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h := w.Header() // Scripts only from this server: the live search of the bookmark @@ -664,17 +680,8 @@ func (s *server) apiLogout(w http.ResponseWriter, r *http.Request) { // name of its version in the Sbm-Version header. The device writes the file // and sends that name as base next time. func (s *server) sync(w http.ResponseWriter, r *http.Request) { - u := s.apiUser(r) + u := s.apiOwner(w, r) if u == nil { - http.Error(w, "not signed in: sign in again", http.StatusUnauthorized) - return - } - if !s.verified(s.store.view(u)) { - http.Error(w, "confirm your email address first: open the link in the email from sbm Sync, or see "+s.site+"/account", http.StatusForbidden) - return - } - if !s.active(s.store.view(u)) { - http.Error(w, "sbm Sync is paused for this account: see "+s.site+"/account", http.StatusPaymentRequired) return } base := r.URL.Query().Get("base") diff --git a/pages.html b/pages.html index 9e4d4d4..3288acd 100644 --- a/pages.html +++ b/pages.html @@ -25,6 +25,8 @@ select { font: inherit; padding: .3em; margin: .2em 0 1em; } textarea { display: block; width: 100%; box-sizing: border-box; height: 60vh; margin: .2em 0 1em; font: 13px/1.4 monospace; } details { margin: 1em 0; } summary { cursor: pointer; } .bookmarks { list-style: none; padding: 0; } .bookmarks li { margin: .8em 0; overflow-wrap: anywhere; } .bookmarks small { color: #555; } +.bookmarks form { display: inline; } .bookmarks form button { padding: .1em .5em; margin: 0; font-size: .9em; } +.check { display: inline; width: auto; margin: 0 .4em 0 0; } pre { overflow-x: auto; } @media (prefers-color-scheme: dark) { body { background: #111; color: #ddd; } a { color: #4fb3d9; } .note, footer, .bookmarks small { color: #999; } .error { color: #f77; } } @@ -61,7 +63,10 @@ text file. sbm Sync keeps that file the same on each of your devices: bm on your computers, the sbm app for Android, and the sbm add-on for Firefox and Chrome. You can also use your bookmarks on this site, in any browser.</p> {{if .Billing}}<p>Free for {{.TrialDays}} days. Then {{.Year}} or {{.Month}}. Cancel at any time.</p>{{end}} -<p>{{if .Email}}{{if .Active}}<a href="/bookmarks">Your bookmarks</a> · {{end}}<a href="/account">Your account</a>{{else}}<a href="/signup">Create an account</a> or <a href="/login">sign in</a>.{{end}}</p> +<p>{{if .Email}}{{if .Active}}<a href="/bookmarks">Your bookmarks</a> · <a href="/feed">Your feed</a> · {{end}}<a href="/account">Your account</a>{{else}}<a href="/signup">Create an account</a> or <a href="/login">sign in</a>.{{end}}</p> +<p>The <a href="/feed">feed page</a> follows RSS and Atom feeds for you, and can +look through your bookmarks for feeds. Read it here, or with the sfeed tools +in a terminal.</p> {{with .Teams}}<h2>Teams</h2> <p>Coming: bookmarks for teams. The members of a team share one bookmark file, and sync it on each of their devices. Pre-order the founding plan: <a href="{{.}}">for up to 10 people</a>{{with $.TeamsLarge}}, @@ -106,7 +111,7 @@ your own server</a>. See the <a href="/privacy">privacy policy</a>.</p> {{define "account"}}{{template "head" .}} <h1>Your account</h1> <p>Signed in as {{.Email}}.</p> -{{if .Active}}<p><a href="/bookmarks"><b>Open your bookmarks</b></a></p>{{end}} +{{if .Active}}<p><a href="/bookmarks"><b>Open your bookmarks</b></a> · <a href="/feed">Your feed</a></p>{{end}} {{with .Error}}<p class="error">{{.}}</p>{{end}} {{if .Unconfirmed}}<p><b>Confirm your email address.</b> Open the link in the email that this server sent to {{.Email}}. Sync starts after that.</p> @@ -142,8 +147,56 @@ their bookmark files.</p> {{define "nav"}}<p class="note"><a href="/bookmarks">Bookmarks</a> · <a href="/bookmarks/file">Edit the file</a> · <a href="/bookmarks.txt">Download</a> · +<a href="/feed">Feed</a> · <a href="/account">Account</a></p>{{end}} +{{define "feed"}}{{template "head" .}} +{{template "nav"}} +<h1>Your feed</h1> +{{with .Error}}<p class="error">{{.}}</p>{{end}} +{{with .Done}}<p><b>{{.}}</b></p>{{end}} +{{if not .Feeds}}<p>Follow a feed: type the address of an RSS or Atom feed +below, or turn on Explore in the settings, and the server looks through your +bookmarks for feeds. The server fetches new items every 30 minutes.</p>{{end}} +<details{{if eq .Open "add"}} open{{end}}><summary>Follow a feed</summary> +<form method="post" action="/feed/add"> +<label>Feed URL <input name="url" value="{{.Form.URL}}" placeholder="https://example.org/rss.xml" required></label> +<label>Name <input name="name" value="{{.Form.Name}}"></label> +<button>Follow</button> +</form></details> +<details><summary>Settings</summary> +<form method="post" action="/feed/settings"> +<label><input type="checkbox" class="check" name="newtab"{{if .Settings.NewTab}} checked{{end}}>Open links in a new tab</label> +{{if .HasMail}}<label><input type="checkbox" class="check" name="mail"{{if .Settings.Mail}} checked{{end}}>Send me the new items by email, once a day</label>{{end}} +<label><input type="checkbox" class="check" name="explore"{{if .Settings.Explore}} checked{{end}}>Explore: look through my bookmarks for feeds</label> +<button>Save</button> +</form></details> +<details><summary>Read the feed in a terminal</summary> +<p class="note">The items come in the <a href="https://codemadness.org/sfeed.html">sfeed</a> +format, newest first, with the token of a device (<code>bm-sync login</code> +keeps one in <code>~/.config/sbm/sync</code>):</p> +<pre>curl -H "Authorization: Bearer $token" {{.URL}}/api/feed | sfeed_plain</pre> +<p class="note"><code>{{.URL}}/api/feeds</code> gives the feeds that you follow, one on each line: the URL, a tab and the name.</p> +</details> +<p class="note">{{.Total}} items{{with .FilterName}} from <b>{{.}}</b>{{end}}.{{if .Filter}} <a href="/feed">All feeds</a>{{end}}</p> +<ul class="bookmarks"> +{{range .Items}}<li>{{if .Link}}<a href="{{.Link}}"{{if $.Settings.NewTab}} target="_blank" rel="noopener"{{end}}>{{.Title}}</a>{{else}}{{.Title}}{{end}}<br> +<small><a href="/feed?feed={{.File}}">{{.Feed}}</a>{{if not .Time.IsZero}} · {{.Time.Format "2006-01-02 15:04"}}{{end}}{{with .Author}} · {{.}}{{end}}</small></li> +{{end}}</ul> +{{if .Next}}<p><a href="/feed?start={{.Next}}{{with .Filter}}&feed={{.}}{{end}}">Older items</a></p>{{end}} +{{with .Feeds}}<h2>Feeds you follow</h2> +<ul class="bookmarks"> +{{range .}}<li><a href="/feed?feed={{.File}}">{{.Name}}</a> <small>{{.URL}}</small> +<form method="post" action="/feed/delete"><input type="hidden" name="url" value="{{.URL}}"><button>Unfollow</button></form></li> +{{end}}</ul>{{end}} +{{if .Settings.Explore}}<h2>Explore</h2> +<p class="note">The server looked at {{.Checked}} of your {{.Bookmarks}} bookmarked pages for feeds, 20 more with each update.{{if not .Suggestions}} No new feed to follow yet.{{end}}</p> +{{with .Suggestions}}<ul class="bookmarks"> +{{range .}}<li>{{.URL}}<br><small>on {{.Page}}</small> +<form method="post" action="/feed/add"><input type="hidden" name="url" value="{{.URL}}"><input type="hidden" name="name" value="{{.Name}}"><button>Follow</button></form></li> +{{end}}</ul>{{end}}{{end}} +{{template "foot" .}}{{end}} + {{define "bookmarks"}}{{template "head" .}} {{template "nav"}} <h1>Your bookmarks</h1> @@ -283,6 +336,9 @@ When this server checks email addresses, it also sends you an email with a link <li>Your bookmark file, the last 50 versions that your devices sent: to sync your devices.</li> <li>A hash of the sign-in token of each device: to know your devices.</li> <li>When you subscribe: your Stripe customer ID and the state of your subscription: to bill you.</li> +<li>When you follow feeds: the addresses of the feeds, the items that the server fetched from them, and +the addresses of the feeds that the server found on your bookmarked pages when you asked it to look: +to show you the items, and to send them to you by email once a day when you asked for that.</li> </ul> <p>The server does not sell or share this data, and does not use it for ads. There are no analytics and no trackers. The only cookie keeps you signed in.</p> diff --git a/store.go b/store.go index e66122d..478b3ac 100644 --- a/store.go +++ b/store.go @@ -36,15 +36,16 @@ var ( // Account is one user. The file of the user is in files/<ID>. type Account struct { - ID string `json:"id"` - Email string `json:"email"` - Hash string `json:"hash"` // bcrypt hash of the password - Created time.Time `json:"created"` - Tokens []string `json:"tokens"` // SHA-256 of each sign-in token - Customer string `json:"customer,omitempty"` - Status string `json:"status,omitempty"` // of the Stripe subscription - StatusAt int64 `json:"status_at,omitempty"` // time of the Stripe event that set Status - Verify string `json:"verify,omitempty"` // SHA-256 of the code that confirms the email; empty when confirmed + ID string `json:"id"` + Email string `json:"email"` + Hash string `json:"hash"` // bcrypt hash of the password + Created time.Time `json:"created"` + Tokens []string `json:"tokens"` // SHA-256 of each sign-in token + Customer string `json:"customer,omitempty"` + Status string `json:"status,omitempty"` // of the Stripe subscription + StatusAt int64 `json:"status_at,omitempty"` // time of the Stripe event that set Status + Verify string `json:"verify,omitempty"` // SHA-256 of the code that confirms the email; empty when confirmed + Feed feedSettings `json:"feed,omitzero"` // the feed choices, see feed.go } // Store keeps the accounts in one JSON file, accounts.json, and the versions @@ -337,7 +338,7 @@ func (s *Store) lock(id string) *sync.Mutex { return m } -// remove deletes the account and all versions of its file. +// remove deletes the account, all versions of its file and its feeds. func (s *Store) remove(a *Account) error { m := s.lock(a.ID) m.Lock() @@ -353,7 +354,10 @@ func (s *Store) remove(a *Account) error { if err != nil { return err } - return os.RemoveAll(filepath.Join(s.dir, "files", a.ID)) + if err := os.RemoveAll(filepath.Join(s.dir, "files", a.ID)); err != nil { + return err + } + return os.RemoveAll(s.feedDir(a.ID)) } // sync merges the file that a device sends into the current version, keeps