From cb00405041ee4714b6e817e9570cfa10ae972840 Mon Sep 17 00:00:00 2001 From: nsfisis Date: Mon, 27 Apr 2026 21:20:10 +0900 Subject: feat(backend): adapt feed fetch interval to update frequency Add per-feed fetch_interval_seconds (clamped to [1h, 24h]) that halves on new articles and grows 1.5x when a fetch yields nothing, replacing the fixed 1h schedule with the 10min cooldown filter. Scheduler tick shortened to 30min so the 1h floor is honored with reasonable precision. --- backend/api/handler_feeds.go | 2 +- backend/cmd/serve.go | 68 +++++++++++++--------- backend/cmd/serve_test.go | 30 ++++++++++ backend/db/feeds.sql.go | 41 ++++++++++--- backend/db/migrations.go | 2 +- .../007_add_feeds_fetch_interval_seconds.sql | 1 + backend/db/models.go | 13 +++-- backend/db/queries/feeds.sql | 16 +++-- backend/db/schema.sql | 13 +++-- backend/feed/feed.go | 19 +++--- 10 files changed, 144 insertions(+), 61 deletions(-) create mode 100644 backend/cmd/serve_test.go create mode 100644 backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql (limited to 'backend') diff --git a/backend/api/handler_feeds.go b/backend/api/handler_feeds.go index 4d16e4b..936153e 100644 --- a/backend/api/handler_feeds.go +++ b/backend/api/handler_feeds.go @@ -61,7 +61,7 @@ func (h *Handler) FeedsAddFeed(ctx context.Context, request FeedsAddFeedRequestO return FeedsAddFeed400JSONResponse{Message: fmt.Sprintf("failed to insert feed: %v", err)}, nil } - if err := feed.Sync(ctx, h.Queries, dbFeed.ID, result.Feed); err != nil { + if _, err := feed.Sync(ctx, h.Queries, dbFeed.ID, result.Feed); err != nil { return FeedsAddFeed400JSONResponse{Message: fmt.Sprintf("failed to sync articles: %v", err)}, nil } diff --git a/backend/cmd/serve.go b/backend/cmd/serve.go index 4b32868..75bcb53 100644 --- a/backend/cmd/serve.go +++ b/backend/cmd/serve.go @@ -23,46 +23,62 @@ import ( "undef.ninja/x/feedaka/feed" ) -func fetchOneFeed(feedID int64, url string, ctx context.Context, queries *db.Queries) error { - log.Printf("Fetching %s...\n", url) - result, err := feed.Fetch(ctx, url) - if err != nil { - return err +const ( + minFetchIntervalSeconds = 60 * 60 // 1 hour + maxFetchIntervalSeconds = 24 * 60 * 60 // 1 day +) + +// nextFetchInterval adapts the per-feed poll interval based on whether the +// last fetch yielded new articles. Active feeds converge toward the floor +// quickly (halving) while quiet feeds back off gently (1.5x) so a brief lull +// doesn't push the interval to the ceiling. +func nextFetchInterval(current int64, hadNewArticles bool) int64 { + var next int64 + if hadNewArticles { + next = current / 2 + } else { + next = current * 3 / 2 } - return feed.Sync(ctx, queries, feedID, result.Feed) + if next < minFetchIntervalSeconds { + next = minFetchIntervalSeconds + } + if next > maxFetchIntervalSeconds { + next = maxFetchIntervalSeconds + } + return next } -func listFeedsToBeFetched(ctx context.Context, queries *db.Queries) (map[int64]string, error) { - feeds, err := queries.GetFeedsToFetch(ctx) +func fetchOneFeed(ctx context.Context, queries *db.Queries, f db.GetFeedsToFetchRow) error { + log.Printf("Fetching %s...\n", f.Url) + result, err := feed.Fetch(ctx, f.Url) + if err != nil { + return err + } + newCount, err := feed.Sync(ctx, queries, f.ID, result.Feed) if err != nil { - return nil, err + return err } - - result := make(map[int64]string) - for _, feed := range feeds { - fetchedAtTime, err := time.Parse(time.RFC3339, feed.FetchedAt) - if err != nil { - log.Fatal(err) - } - now := time.Now().UTC() - if now.Sub(fetchedAtTime).Minutes() <= 10 { - continue + next := nextFetchInterval(f.FetchIntervalSeconds, newCount > 0) + if next != f.FetchIntervalSeconds { + if err := queries.UpdateFeedFetchInterval(ctx, db.UpdateFeedFetchIntervalParams{ + FetchIntervalSeconds: next, + ID: f.ID, + }); err != nil { + return err } - result[feed.ID] = feed.Url } - return result, nil + return nil } func fetchAllFeeds(ctx context.Context, queries *db.Queries) error { - feeds, err := listFeedsToBeFetched(ctx, queries) + feeds, err := queries.GetFeedsToFetch(ctx) if err != nil { return err } var result *multierror.Error - for feedID, url := range feeds { - err := fetchOneFeed(feedID, url, ctx, queries) - if err != nil { + for _, f := range feeds { + if err := fetchOneFeed(ctx, queries, f); err != nil { result = multierror.Append(result, err) } time.Sleep(5 * time.Second) @@ -121,7 +137,7 @@ func RunServe(database *sql.DB, cfg *config.Config, publicFS embed.FS) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - scheduled(ctx, 1*time.Hour, func() { + scheduled(ctx, 30*time.Minute, func() { err := fetchAllFeeds(ctx, queries) if err != nil { log.Printf("Failed to fetch feeds: %v\n", err) diff --git a/backend/cmd/serve_test.go b/backend/cmd/serve_test.go new file mode 100644 index 0000000..82b7d53 --- /dev/null +++ b/backend/cmd/serve_test.go @@ -0,0 +1,30 @@ +package cmd + +import "testing" + +func TestNextFetchInterval(t *testing.T) { + tests := []struct { + name string + current int64 + hadNewArticles bool + want int64 + }{ + {"new articles halve from mid-range", 14400, true, 7200}, + {"new articles halve from max", maxFetchIntervalSeconds, true, 43200}, + {"new articles clamp at min when halving below", 3600, true, minFetchIntervalSeconds}, + {"new articles clamp at min from slightly above", 4000, true, minFetchIntervalSeconds}, + {"no articles grow by 1.5x from min", minFetchIntervalSeconds, false, 5400}, + {"no articles grow by 1.5x from mid-range", 14400, false, 21600}, + {"no articles clamp at max", maxFetchIntervalSeconds, false, maxFetchIntervalSeconds}, + {"no articles clamp at max from slightly below", 60000, false, maxFetchIntervalSeconds}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := nextFetchInterval(tt.current, tt.hadNewArticles) + if got != tt.want { + t.Errorf("nextFetchInterval(%d, %v) = %d, want %d", tt.current, tt.hadNewArticles, got, tt.want) + } + }) + } +} diff --git a/backend/db/feeds.sql.go b/backend/db/feeds.sql.go index 0226a7d..9fcc43e 100644 --- a/backend/db/feeds.sql.go +++ b/backend/db/feeds.sql.go @@ -12,7 +12,7 @@ import ( const createFeed = `-- name: CreateFeed :one INSERT INTO feeds (url, title, fetched_at, user_id) VALUES (?, ?, ?, ?) -RETURNING id, url, title, fetched_at, is_subscribed, user_id +RETURNING id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds ` type CreateFeedParams struct { @@ -37,6 +37,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e &i.FetchedAt, &i.IsSubscribed, &i.UserID, + &i.FetchIntervalSeconds, ) return i, err } @@ -52,7 +53,7 @@ func (q *Queries) DeleteFeed(ctx context.Context, id int64) error { } const getFeed = `-- name: GetFeed :one -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE id = ? ` @@ -67,12 +68,13 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) { &i.FetchedAt, &i.IsSubscribed, &i.UserID, + &i.FetchIntervalSeconds, ) return i, err } const getFeedByURL = `-- name: GetFeedByURL :one -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE url = ? AND user_id = ? ` @@ -92,6 +94,7 @@ func (q *Queries) GetFeedByURL(ctx context.Context, arg GetFeedByURLParams) (Fee &i.FetchedAt, &i.IsSubscribed, &i.UserID, + &i.FetchIntervalSeconds, ) return i, err } @@ -133,7 +136,7 @@ func (q *Queries) GetFeedUnreadCounts(ctx context.Context, userID int64) ([]GetF } const getFeeds = `-- name: GetFeeds :many -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE is_subscribed = 1 AND user_id = ? ORDER BY id @@ -155,6 +158,7 @@ func (q *Queries) GetFeeds(ctx context.Context, userID int64) ([]Feed, error) { &i.FetchedAt, &i.IsSubscribed, &i.UserID, + &i.FetchIntervalSeconds, ); err != nil { return nil, err } @@ -170,16 +174,18 @@ func (q *Queries) GetFeeds(ctx context.Context, userID int64) ([]Feed, error) { } const getFeedsToFetch = `-- name: GetFeedsToFetch :many -SELECT id, url, fetched_at, user_id +SELECT id, url, fetched_at, user_id, fetch_interval_seconds FROM feeds WHERE is_subscribed = 1 + AND datetime(fetched_at, '+' || fetch_interval_seconds || ' seconds') <= datetime('now') ` type GetFeedsToFetchRow struct { - ID int64 - Url string - FetchedAt string - UserID int64 + ID int64 + Url string + FetchedAt string + UserID int64 + FetchIntervalSeconds int64 } func (q *Queries) GetFeedsToFetch(ctx context.Context) ([]GetFeedsToFetchRow, error) { @@ -196,6 +202,7 @@ func (q *Queries) GetFeedsToFetch(ctx context.Context) ([]GetFeedsToFetchRow, er &i.Url, &i.FetchedAt, &i.UserID, + &i.FetchIntervalSeconds, ); err != nil { return nil, err } @@ -221,6 +228,22 @@ func (q *Queries) UnsubscribeFeed(ctx context.Context, id int64) error { return err } +const updateFeedFetchInterval = `-- name: UpdateFeedFetchInterval :exec +UPDATE feeds +SET fetch_interval_seconds = ? +WHERE id = ? +` + +type UpdateFeedFetchIntervalParams struct { + FetchIntervalSeconds int64 + ID int64 +} + +func (q *Queries) UpdateFeedFetchInterval(ctx context.Context, arg UpdateFeedFetchIntervalParams) error { + _, err := q.db.ExecContext(ctx, updateFeedFetchInterval, arg.FetchIntervalSeconds, arg.ID) + return err +} + const updateFeedMetadata = `-- name: UpdateFeedMetadata :exec UPDATE feeds SET title = ?, fetched_at = ? diff --git a/backend/db/migrations.go b/backend/db/migrations.go index 4ccfba9..28a608f 100644 --- a/backend/db/migrations.go +++ b/backend/db/migrations.go @@ -14,7 +14,7 @@ import ( //go:embed migrations/*.sql var migrationsFS embed.FS -const EXPECTED_SCHEMA_VERSION = 4 +const EXPECTED_SCHEMA_VERSION = 7 type Migration struct { Version int diff --git a/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql b/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql new file mode 100644 index 0000000..102a555 --- /dev/null +++ b/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql @@ -0,0 +1 @@ +ALTER TABLE feeds ADD COLUMN fetch_interval_seconds INTEGER NOT NULL DEFAULT 3600; diff --git a/backend/db/models.go b/backend/db/models.go index e397102..342d6a0 100644 --- a/backend/db/models.go +++ b/backend/db/models.go @@ -14,12 +14,13 @@ type Article struct { } type Feed struct { - ID int64 - Url string - Title string - FetchedAt string - IsSubscribed int64 - UserID int64 + ID int64 + Url string + Title string + FetchedAt string + IsSubscribed int64 + UserID int64 + FetchIntervalSeconds int64 } type User struct { diff --git a/backend/db/queries/feeds.sql b/backend/db/queries/feeds.sql index 094a0f8..678f7cc 100644 --- a/backend/db/queries/feeds.sql +++ b/backend/db/queries/feeds.sql @@ -1,10 +1,10 @@ -- name: GetFeed :one -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE id = ?; -- name: GetFeeds :many -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE is_subscribed = 1 AND user_id = ? ORDER BY id; @@ -24,14 +24,20 @@ DELETE FROM feeds WHERE id = ?; -- name: GetFeedByURL :one -SELECT id, url, title, fetched_at, is_subscribed, user_id +SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds FROM feeds WHERE url = ? AND user_id = ?; -- name: GetFeedsToFetch :many -SELECT id, url, fetched_at, user_id +SELECT id, url, fetched_at, user_id, fetch_interval_seconds FROM feeds -WHERE is_subscribed = 1; +WHERE is_subscribed = 1 + AND datetime(fetched_at, '+' || fetch_interval_seconds || ' seconds') <= datetime('now'); + +-- name: UpdateFeedFetchInterval :exec +UPDATE feeds +SET fetch_interval_seconds = ? +WHERE id = ?; -- name: UnsubscribeFeed :exec UPDATE feeds diff --git a/backend/db/schema.sql b/backend/db/schema.sql index 2596c84..85f39d1 100644 --- a/backend/db/schema.sql +++ b/backend/db/schema.sql @@ -8,12 +8,13 @@ CREATE TABLE IF NOT EXISTS users ( -- Feeds CREATE TABLE IF NOT EXISTS feeds ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT NOT NULL, - title TEXT NOT NULL, - fetched_at TEXT NOT NULL, - is_subscribed INTEGER NOT NULL DEFAULT 1, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL, + title TEXT NOT NULL, + fetched_at TEXT NOT NULL, + is_subscribed INTEGER NOT NULL DEFAULT 1, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + fetch_interval_seconds INTEGER NOT NULL DEFAULT 3600 ); -- Articles diff --git a/backend/feed/feed.go b/backend/feed/feed.go index 2d84798..e37a883 100644 --- a/backend/feed/feed.go +++ b/backend/feed/feed.go @@ -76,25 +76,29 @@ func discoverFeedURL(ctx context.Context, rawURL string) (string, error) { return feedURL, nil } -func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed) error { +// Sync upserts the parsed feed's items into the database and returns the +// number of newly inserted articles. The caller uses that count to drive the +// adaptive fetch interval. +func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed) (int, error) { err := queries.UpdateFeedMetadata(ctx, db.UpdateFeedMetadataParams{ Title: f.Title, FetchedAt: time.Now().UTC().Format(time.RFC3339), ID: feedID, }) if err != nil { - return err + return 0, err } guids, err := queries.GetArticleGUIDsByFeed(ctx, feedID) if err != nil { - return err + return 0, err } existingFeedGUIDs := make(map[string]bool, len(guids)) for _, guid := range guids { existingFeedGUIDs[guid] = true } + newCount := 0 for _, item := range f.Items { if existingFeedGUIDs[item.GUID] { err := queries.UpdateArticle(ctx, db.UpdateArticleParams{ @@ -104,12 +108,12 @@ func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed Guid: item.GUID, }) if err != nil { - return err + return newCount, err } } else { exists, err := queries.CheckArticleExistsByGUID(ctx, item.GUID) if err != nil { - return err + return newCount, err } if exists == 1 { continue @@ -122,9 +126,10 @@ func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed IsRead: 0, }) if err != nil { - return err + return newCount, err } + newCount++ } } - return nil + return newCount, nil } -- cgit v1.3.1