commit 80c3de324d8fd025e60b3ec61aa45c25fec2bb5e equwal <truex@equwal.com> 2026-09-25 14:58:53 -0700 Digest: send no old items of a feed that is followed later The digest noted the items of the feeds only at its first run. A feed that the account followed after that brought all its old items into the next digest. Example: 112 feeds followed at one time would give a digest of more than 1,500 items. Now the job notes the first items that it finds of a new feed, and sends none of them. The new file known has a line for each noted feed: the URL, a tab and the item file. A feed that the account leaves, or that gets another item file, is a new feed again. Without that file, as after this update, all followed feeds are known, so no waiting item is lost.
README.md | 4 +- feedjob.go | 66 ++++++++++++++++++++++++++++--- feedjob_test.go | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 184 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md index e6b328a..18509a1 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,9 @@ It uses [sfeed](https://codemadness.org/sfeed.html) on the server: - Digest: the server sends you the new items by email once a day, in the form of `sfeed_plain`, unless you turn that off in the settings of the page. The first day sends nothing, so that you do not get the whole - history. A server without an SMTP server sends no digest. + history. A feed that you follow later is the same: the digest sends only + the items that come after its first fetch. A server without an SMTP + server sends no digest. Read the feed in a terminal with the token of a device (`bm-sync login` keeps one in `~/.config/sbm/sync`): diff --git a/feedjob.go b/feedjob.go index 61b4fdf..075ad30 100644 --- a/feedjob.go +++ b/feedjob.go @@ -175,8 +175,9 @@ func (s *server) feedAccount(p *Account) error { return err } cache := map[string]bool{} + followed := parseFeeds(text) var feeds []Feed - for _, f := range parseFeeds(text) { + for _, f := range followed { if _, err := feedURL(f.URL); err != nil || !s.public(f.URL, cache) { log.Printf("account %s: feed %s: not a public http address, skipped", a.ID, f.URL) continue @@ -227,8 +228,11 @@ func (s *server) feedAccount(p *Account) error { if a.Feed.NoMail { os.Remove(filepath.Join(dir, "mailed")) os.Remove(filepath.Join(dir, "digest")) + os.Remove(filepath.Join(dir, "known")) } else if s.mail != nil && s.verified(a) && s.active(a) { - if err := s.digest(a, dir, feeds); err != nil { + // The digest gets all followed feeds, also a feed that this run + // skipped because its host did not resolve. So that feed stays known. + if err := s.digest(a, dir, followed); err != nil { log.Printf("account %s: digest: %v", a.ID, err) } } @@ -295,18 +299,29 @@ func (s *server) explore(p *Account, dir string, cache map[string]bool) error { // digest sends the items that came in since the last digest, once a day. // The first run for an account sends nothing: it only notes the items of -// that day, so that nobody gets the whole history. +// that day, so that nobody gets the whole history. A feed that the account +// follows later is the same: the first run that finds items of the feed +// notes them and sends none of them. +// +// The file known has a line for each feed that the digest noted: the URL, +// a tab and the item file. A feed that gets another item file, for example +// under a new name, is a new feed, because the keys of its items change. A +// new feed without items does not become known, because its first fetch +// can have failed. So the first items of a feed that starts empty are +// noted too, and not sent. func (s *server) digest(a Account, dir string, feeds []Feed) error { - mailedPath, digestPath := filepath.Join(dir, "mailed"), filepath.Join(dir, "digest") + mailedPath, digestPath, knownPath := filepath.Join(dir, "mailed"), filepath.Join(dir, "digest"), filepath.Join(dir, "known") now := s.feeds.now() stamp := []byte(strconv.FormatInt(now.Unix(), 10) + "\n") + id := func(f Feed) string { return f.URL + "\t" + f.File } type feedItems struct { f Feed items []Item } var all []feedItems - var keys []string + var keys, ids []string for _, f := range feeds { + ids = append(ids, id(f)) b, err := os.ReadFile(filepath.Join(dir, "items", f.File)) if err != nil { continue @@ -322,10 +337,51 @@ func (s *server) digest(a Account, dir string, feeds []Feed) error { if err := writeFile(mailedPath, []byte(join(keys))); err != nil { return err } + if err := writeFile(knownPath, []byte(join(ids))); err != nil { + return err + } return writeFile(digestPath, stamp) } else if err != nil { return err } + // Without the file known, as after an update from a version without + // it, the digest knows all feeds that the account follows now. + kb, err := os.ReadFile(knownPath) + noKnown := errors.Is(err, fs.ErrNotExist) + if noKnown { + kb = []byte(join(ids)) + } else if err != nil { + return err + } + known := set(lines(string(kb))) + var noted []string + for _, fi := range all { + if !known[id(fi.f)] && len(fi.items) > 0 { + known[id(fi.f)] = true + for _, it := range fi.items { + noted = append(noted, it.Key) + } + } + } + if len(noted) > 0 { + old = append(old, join(noted)...) + if err := writeFile(mailedPath, old); err != nil { + return err + } + } + // A feed that the account left is not known any more. When the account + // follows it again, it is a new feed. + var keep []string + for _, k := range ids { + if known[k] { + keep = append(keep, k) + } + } + if k := join(keep); noKnown || k != string(kb) { + if err := writeFile(knownPath, []byte(k)); err != nil { + return err + } + } b, _ := os.ReadFile(digestPath) last, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) if now.Sub(time.Unix(last, 0)) < digestEvery { diff --git a/feedjob_test.go b/feedjob_test.go index ee645df..558380c 100644 --- a/feedjob_test.go +++ b/feedjob_test.go @@ -194,7 +194,7 @@ func TestJobSendsADailyDigest(t *testing.T) { // The opt-out forgets the digests, so that a later opt-in starts fresh. e.send(c, "/feed/settings", url.Values{}) e.s.runFeeds() - if e.feedFile(a, "mailed") != "" || e.feedFile(a, "digest") != "" { + if e.feedFile(a, "mailed") != "" || e.feedFile(a, "digest") != "" || e.feedFile(a, "known") != "" { t.Error("the opt-out kept the digest files") } f.items["https://a.org/rss.xml"] = "1700010800\tFour\thttps://a.org/4\t\t\tid4\t\t\t\n" + f.items["https://a.org/rss.xml"] @@ -235,6 +235,125 @@ func TestJobKeepsTheItemsOfAFailedDigest(t *testing.T) { } } +// A feed that the account follows after the first digest run does not +// bring its old items into the digest: the first run that finds items of +// the feed notes them. The items that come after that go out. +func TestJobKeepsTheOldItemsOfANewFeedOutOfTheDigest(t *testing.T) { + e := newEnv(t, false) + c, tok, _ := e.account("") + e.withMail() + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + start := time.Unix(1_800_000_000, 0) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: start} + e.s.feeds = f.tools() + e.s.runFeeds() + + // The first fetch of Beta gives its old items. The first fetch of Gamma + // gives nothing, as a fetch that failed. + old := "1600000000\tOld 1\thttps://b.org/1\t\t\tb1\t\t\t\n1600000001\tOld 2\thttps://b.org/2\t\t\tb2\t\t\t\n" + f.items["https://b.org/rss.xml"] = old + f.items["https://c.org/rss.xml"] = "" + e.send(c, "/feed/add", url.Values{"url": {"https://b.org/rss.xml"}, "name": {"Beta"}}) + e.send(c, "/feed/add", url.Values{"url": {"https://c.org/rss.xml"}, "name": {"Gamma"}}) + f.now = start.Add(time.Hour) + e.s.runFeeds() + // Now the fetch of Gamma works, and Beta has a new item. + f.items["https://c.org/rss.xml"] = "1600000002\tOld 3\thttps://c.org/3\t\t\tc3\t\t\t\n" + f.items["https://b.org/rss.xml"] = "1800003600\tNew\thttps://b.org/new\t\t\tbnew\t\t\t\n" + old + f.now = start.Add(2 * time.Hour) + e.s.runFeeds() + f.now = start.Add(25 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 { + t.Fatalf("%d emails after a day", e.sent()) + } + mail := e.mails[0] + if !strings.Contains(mail, "sbm feed: 1 new item\n") || !strings.Contains(mail, "New") { + t.Errorf("the digest lacks the new item:\n%s", mail) + } + for _, item := range []string{"Old 1", "Old 2", "Old 3"} { + if strings.Contains(mail, item) { + t.Errorf("the digest has the old item %q:\n%s", item, mail) + } + } + if got := e.feedFile(a, "known"); got != "https://a.org/rss.xml\tAlpha\nhttps://b.org/rss.xml\tBeta\nhttps://c.org/rss.xml\tGamma\n" { + t.Errorf("known: %q", got) + } + + // A feed that the account left and follows again is a new feed: the + // item that came while the account did not follow it is old. + e.send(c, "/feed/delete", url.Values{"url": {"https://b.org/rss.xml"}}) + f.now = start.Add(26 * time.Hour) + e.s.runFeeds() + f.items["https://b.org/rss.xml"] = "1800090000\tWhile away\thttps://b.org/away\t\t\tbaway\t\t\t\n" + f.items["https://b.org/rss.xml"] + e.send(c, "/feed/add", url.Values{"url": {"https://b.org/rss.xml"}, "name": {"Beta"}}) + f.now = start.Add(27 * time.Hour) + e.s.runFeeds() + // Under another name, the items of the feed get other keys. So the + // feed is new, also without a run between the two changes. + e.send(c, "/feed/delete", url.Values{"url": {"https://b.org/rss.xml"}}) + e.send(c, "/feed/add", url.Values{"url": {"https://b.org/rss.xml"}, "name": {"Bee"}}) + f.now = start.Add(28 * time.Hour) + e.s.runFeeds() + f.now = start.Add(50 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 { + t.Errorf("old items of the feed that came back went out:\n%s", e.mails[len(e.mails)-1]) + } + if got := e.feedFile(a, "known"); got != "https://a.org/rss.xml\tAlpha\nhttps://c.org/rss.xml\tGamma\nhttps://b.org/rss.xml\tBee\n" { + t.Errorf("known at the end: %q", got) + } +} + +// A feed whose host does not resolve in one run stays known. So its new +// items still go out. +func TestJobKeepsAFeedKnownWhenItsHostFailsOnce(t *testing.T) { + e := newEnv(t, false) + _, tok, _ := e.account("") + e.withMail() + e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + start := time.Unix(1_800_000_000, 0) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: start} + e.s.feeds = f.tools() + e.s.runFeeds() + e.s.feeds.lookup = func(string) ([]net.IP, error) { return nil, errors.New("no such host") } + f.now = start.Add(time.Hour) + e.s.runFeeds() + e.s.feeds = f.tools() + f.items["https://a.org/rss.xml"] = "1700003600\tTwo\thttps://a.org/2\t\t\tid2\t\t\t\n" + oneItem + f.now = start.Add(25 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 || !strings.Contains(e.mails[0], "Two") { + t.Errorf("the new item did not go out: %d emails", e.sent()) + } +} + +// After an update from a version without the file known, the digest knows +// all feeds that the account follows. So no item that waits for the next +// digest is lost. +func TestJobKnowsTheFeedsOfAnOlderDigest(t *testing.T) { + e := newEnv(t, false) + _, tok, _ := e.account("") + e.withMail() + a := e.setFeeds(tok, "https://a.org/rss.xml\tAlpha\n") + start := time.Unix(1_800_000_000, 0) + f := &fakeTools{items: map[string]string{"https://a.org/rss.xml": oneItem}, now: start} + e.s.feeds = f.tools() + e.s.runFeeds() + if err := os.Remove(filepath.Join(e.s.store.feedDir(a.ID), "known")); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + f.items["https://a.org/rss.xml"] = "1700003600\tTwo\thttps://a.org/2\t\t\tid2\t\t\t\n" + oneItem + f.now = start.Add(25 * time.Hour) + e.s.runFeeds() + if e.sent() != 1 || !strings.Contains(e.mails[0], "Two") { + t.Errorf("the update lost the new item: %d emails", e.sent()) + } + if got := e.feedFile(a, "known"); got != "https://a.org/rss.xml\tAlpha\n" { + t.Errorf("known: %q", got) + } +} + func TestJobMailsOnlyConfirmedAccounts(t *testing.T) { e := newEnv(t, false) e.withMail()