aboutsummaryrefslogtreecommitdiffhomepage
path: root/backend
diff options
context:
space:
mode:
Diffstat (limited to 'backend')
-rw-r--r--backend/api/handler_feeds.go2
-rw-r--r--backend/cmd/serve.go68
-rw-r--r--backend/cmd/serve_test.go30
-rw-r--r--backend/db/feeds.sql.go41
-rw-r--r--backend/db/migrations.go2
-rw-r--r--backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql1
-rw-r--r--backend/db/models.go13
-rw-r--r--backend/db/queries/feeds.sql16
-rw-r--r--backend/db/schema.sql13
-rw-r--r--backend/feed/feed.go19
10 files changed, 144 insertions, 61 deletions
diff --git a/backend/api/handler_feeds.go b/backend/api/handler_feeds.go
index 4d16e4b..936153e 100644
--- a/backend/api/handler_feeds.go
+++ b/backend/api/handler_feeds.go
@@ -61,7 +61,7 @@ func (h *Handler) FeedsAddFeed(ctx context.Context, request FeedsAddFeedRequestO
return FeedsAddFeed400JSONResponse{Message: fmt.Sprintf("failed to insert feed: %v", err)}, nil
}
- if err := feed.Sync(ctx, h.Queries, dbFeed.ID, result.Feed); err != nil {
+ if _, err := feed.Sync(ctx, h.Queries, dbFeed.ID, result.Feed); err != nil {
return FeedsAddFeed400JSONResponse{Message: fmt.Sprintf("failed to sync articles: %v", err)}, nil
}
diff --git a/backend/cmd/serve.go b/backend/cmd/serve.go
index 4b32868..75bcb53 100644
--- a/backend/cmd/serve.go
+++ b/backend/cmd/serve.go
@@ -23,46 +23,62 @@ import (
"undef.ninja/x/feedaka/feed"
)
-func fetchOneFeed(feedID int64, url string, ctx context.Context, queries *db.Queries) error {
- log.Printf("Fetching %s...\n", url)
- result, err := feed.Fetch(ctx, url)
- if err != nil {
- return err
+const (
+ minFetchIntervalSeconds = 60 * 60 // 1 hour
+ maxFetchIntervalSeconds = 24 * 60 * 60 // 1 day
+)
+
+// nextFetchInterval adapts the per-feed poll interval based on whether the
+// last fetch yielded new articles. Active feeds converge toward the floor
+// quickly (halving) while quiet feeds back off gently (1.5x) so a brief lull
+// doesn't push the interval to the ceiling.
+func nextFetchInterval(current int64, hadNewArticles bool) int64 {
+ var next int64
+ if hadNewArticles {
+ next = current / 2
+ } else {
+ next = current * 3 / 2
}
- return feed.Sync(ctx, queries, feedID, result.Feed)
+ if next < minFetchIntervalSeconds {
+ next = minFetchIntervalSeconds
+ }
+ if next > maxFetchIntervalSeconds {
+ next = maxFetchIntervalSeconds
+ }
+ return next
}
-func listFeedsToBeFetched(ctx context.Context, queries *db.Queries) (map[int64]string, error) {
- feeds, err := queries.GetFeedsToFetch(ctx)
+func fetchOneFeed(ctx context.Context, queries *db.Queries, f db.GetFeedsToFetchRow) error {
+ log.Printf("Fetching %s...\n", f.Url)
+ result, err := feed.Fetch(ctx, f.Url)
+ if err != nil {
+ return err
+ }
+ newCount, err := feed.Sync(ctx, queries, f.ID, result.Feed)
if err != nil {
- return nil, err
+ return err
}
-
- result := make(map[int64]string)
- for _, feed := range feeds {
- fetchedAtTime, err := time.Parse(time.RFC3339, feed.FetchedAt)
- if err != nil {
- log.Fatal(err)
- }
- now := time.Now().UTC()
- if now.Sub(fetchedAtTime).Minutes() <= 10 {
- continue
+ next := nextFetchInterval(f.FetchIntervalSeconds, newCount > 0)
+ if next != f.FetchIntervalSeconds {
+ if err := queries.UpdateFeedFetchInterval(ctx, db.UpdateFeedFetchIntervalParams{
+ FetchIntervalSeconds: next,
+ ID: f.ID,
+ }); err != nil {
+ return err
}
- result[feed.ID] = feed.Url
}
- return result, nil
+ return nil
}
func fetchAllFeeds(ctx context.Context, queries *db.Queries) error {
- feeds, err := listFeedsToBeFetched(ctx, queries)
+ feeds, err := queries.GetFeedsToFetch(ctx)
if err != nil {
return err
}
var result *multierror.Error
- for feedID, url := range feeds {
- err := fetchOneFeed(feedID, url, ctx, queries)
- if err != nil {
+ for _, f := range feeds {
+ if err := fetchOneFeed(ctx, queries, f); err != nil {
result = multierror.Append(result, err)
}
time.Sleep(5 * time.Second)
@@ -121,7 +137,7 @@ func RunServe(database *sql.DB, cfg *config.Config, publicFS embed.FS) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- scheduled(ctx, 1*time.Hour, func() {
+ scheduled(ctx, 30*time.Minute, func() {
err := fetchAllFeeds(ctx, queries)
if err != nil {
log.Printf("Failed to fetch feeds: %v\n", err)
diff --git a/backend/cmd/serve_test.go b/backend/cmd/serve_test.go
new file mode 100644
index 0000000..82b7d53
--- /dev/null
+++ b/backend/cmd/serve_test.go
@@ -0,0 +1,30 @@
+package cmd
+
+import "testing"
+
+func TestNextFetchInterval(t *testing.T) {
+ tests := []struct {
+ name string
+ current int64
+ hadNewArticles bool
+ want int64
+ }{
+ {"new articles halve from mid-range", 14400, true, 7200},
+ {"new articles halve from max", maxFetchIntervalSeconds, true, 43200},
+ {"new articles clamp at min when halving below", 3600, true, minFetchIntervalSeconds},
+ {"new articles clamp at min from slightly above", 4000, true, minFetchIntervalSeconds},
+ {"no articles grow by 1.5x from min", minFetchIntervalSeconds, false, 5400},
+ {"no articles grow by 1.5x from mid-range", 14400, false, 21600},
+ {"no articles clamp at max", maxFetchIntervalSeconds, false, maxFetchIntervalSeconds},
+ {"no articles clamp at max from slightly below", 60000, false, maxFetchIntervalSeconds},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := nextFetchInterval(tt.current, tt.hadNewArticles)
+ if got != tt.want {
+ t.Errorf("nextFetchInterval(%d, %v) = %d, want %d", tt.current, tt.hadNewArticles, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/db/feeds.sql.go b/backend/db/feeds.sql.go
index 0226a7d..9fcc43e 100644
--- a/backend/db/feeds.sql.go
+++ b/backend/db/feeds.sql.go
@@ -12,7 +12,7 @@ import (
const createFeed = `-- name: CreateFeed :one
INSERT INTO feeds (url, title, fetched_at, user_id)
VALUES (?, ?, ?, ?)
-RETURNING id, url, title, fetched_at, is_subscribed, user_id
+RETURNING id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
`
type CreateFeedParams struct {
@@ -37,6 +37,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
&i.FetchedAt,
&i.IsSubscribed,
&i.UserID,
+ &i.FetchIntervalSeconds,
)
return i, err
}
@@ -52,7 +53,7 @@ func (q *Queries) DeleteFeed(ctx context.Context, id int64) error {
}
const getFeed = `-- name: GetFeed :one
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE id = ?
`
@@ -67,12 +68,13 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) {
&i.FetchedAt,
&i.IsSubscribed,
&i.UserID,
+ &i.FetchIntervalSeconds,
)
return i, err
}
const getFeedByURL = `-- name: GetFeedByURL :one
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE url = ? AND user_id = ?
`
@@ -92,6 +94,7 @@ func (q *Queries) GetFeedByURL(ctx context.Context, arg GetFeedByURLParams) (Fee
&i.FetchedAt,
&i.IsSubscribed,
&i.UserID,
+ &i.FetchIntervalSeconds,
)
return i, err
}
@@ -133,7 +136,7 @@ func (q *Queries) GetFeedUnreadCounts(ctx context.Context, userID int64) ([]GetF
}
const getFeeds = `-- name: GetFeeds :many
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE is_subscribed = 1 AND user_id = ?
ORDER BY id
@@ -155,6 +158,7 @@ func (q *Queries) GetFeeds(ctx context.Context, userID int64) ([]Feed, error) {
&i.FetchedAt,
&i.IsSubscribed,
&i.UserID,
+ &i.FetchIntervalSeconds,
); err != nil {
return nil, err
}
@@ -170,16 +174,18 @@ func (q *Queries) GetFeeds(ctx context.Context, userID int64) ([]Feed, error) {
}
const getFeedsToFetch = `-- name: GetFeedsToFetch :many
-SELECT id, url, fetched_at, user_id
+SELECT id, url, fetched_at, user_id, fetch_interval_seconds
FROM feeds
WHERE is_subscribed = 1
+ AND datetime(fetched_at, '+' || fetch_interval_seconds || ' seconds') <= datetime('now')
`
type GetFeedsToFetchRow struct {
- ID int64
- Url string
- FetchedAt string
- UserID int64
+ ID int64
+ Url string
+ FetchedAt string
+ UserID int64
+ FetchIntervalSeconds int64
}
func (q *Queries) GetFeedsToFetch(ctx context.Context) ([]GetFeedsToFetchRow, error) {
@@ -196,6 +202,7 @@ func (q *Queries) GetFeedsToFetch(ctx context.Context) ([]GetFeedsToFetchRow, er
&i.Url,
&i.FetchedAt,
&i.UserID,
+ &i.FetchIntervalSeconds,
); err != nil {
return nil, err
}
@@ -221,6 +228,22 @@ func (q *Queries) UnsubscribeFeed(ctx context.Context, id int64) error {
return err
}
+const updateFeedFetchInterval = `-- name: UpdateFeedFetchInterval :exec
+UPDATE feeds
+SET fetch_interval_seconds = ?
+WHERE id = ?
+`
+
+type UpdateFeedFetchIntervalParams struct {
+ FetchIntervalSeconds int64
+ ID int64
+}
+
+func (q *Queries) UpdateFeedFetchInterval(ctx context.Context, arg UpdateFeedFetchIntervalParams) error {
+ _, err := q.db.ExecContext(ctx, updateFeedFetchInterval, arg.FetchIntervalSeconds, arg.ID)
+ return err
+}
+
const updateFeedMetadata = `-- name: UpdateFeedMetadata :exec
UPDATE feeds
SET title = ?, fetched_at = ?
diff --git a/backend/db/migrations.go b/backend/db/migrations.go
index 4ccfba9..28a608f 100644
--- a/backend/db/migrations.go
+++ b/backend/db/migrations.go
@@ -14,7 +14,7 @@ import (
//go:embed migrations/*.sql
var migrationsFS embed.FS
-const EXPECTED_SCHEMA_VERSION = 4
+const EXPECTED_SCHEMA_VERSION = 7
type Migration struct {
Version int
diff --git a/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql b/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql
new file mode 100644
index 0000000..102a555
--- /dev/null
+++ b/backend/db/migrations/007_add_feeds_fetch_interval_seconds.sql
@@ -0,0 +1 @@
+ALTER TABLE feeds ADD COLUMN fetch_interval_seconds INTEGER NOT NULL DEFAULT 3600;
diff --git a/backend/db/models.go b/backend/db/models.go
index e397102..342d6a0 100644
--- a/backend/db/models.go
+++ b/backend/db/models.go
@@ -14,12 +14,13 @@ type Article struct {
}
type Feed struct {
- ID int64
- Url string
- Title string
- FetchedAt string
- IsSubscribed int64
- UserID int64
+ ID int64
+ Url string
+ Title string
+ FetchedAt string
+ IsSubscribed int64
+ UserID int64
+ FetchIntervalSeconds int64
}
type User struct {
diff --git a/backend/db/queries/feeds.sql b/backend/db/queries/feeds.sql
index 094a0f8..678f7cc 100644
--- a/backend/db/queries/feeds.sql
+++ b/backend/db/queries/feeds.sql
@@ -1,10 +1,10 @@
-- name: GetFeed :one
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE id = ?;
-- name: GetFeeds :many
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE is_subscribed = 1 AND user_id = ?
ORDER BY id;
@@ -24,14 +24,20 @@ DELETE FROM feeds
WHERE id = ?;
-- name: GetFeedByURL :one
-SELECT id, url, title, fetched_at, is_subscribed, user_id
+SELECT id, url, title, fetched_at, is_subscribed, user_id, fetch_interval_seconds
FROM feeds
WHERE url = ? AND user_id = ?;
-- name: GetFeedsToFetch :many
-SELECT id, url, fetched_at, user_id
+SELECT id, url, fetched_at, user_id, fetch_interval_seconds
FROM feeds
-WHERE is_subscribed = 1;
+WHERE is_subscribed = 1
+ AND datetime(fetched_at, '+' || fetch_interval_seconds || ' seconds') <= datetime('now');
+
+-- name: UpdateFeedFetchInterval :exec
+UPDATE feeds
+SET fetch_interval_seconds = ?
+WHERE id = ?;
-- name: UnsubscribeFeed :exec
UPDATE feeds
diff --git a/backend/db/schema.sql b/backend/db/schema.sql
index 2596c84..85f39d1 100644
--- a/backend/db/schema.sql
+++ b/backend/db/schema.sql
@@ -8,12 +8,13 @@ CREATE TABLE IF NOT EXISTS users (
-- Feeds
CREATE TABLE IF NOT EXISTS feeds (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- url TEXT NOT NULL,
- title TEXT NOT NULL,
- fetched_at TEXT NOT NULL,
- is_subscribed INTEGER NOT NULL DEFAULT 1,
- user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ url TEXT NOT NULL,
+ title TEXT NOT NULL,
+ fetched_at TEXT NOT NULL,
+ is_subscribed INTEGER NOT NULL DEFAULT 1,
+ user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+ fetch_interval_seconds INTEGER NOT NULL DEFAULT 3600
);
-- Articles
diff --git a/backend/feed/feed.go b/backend/feed/feed.go
index 2d84798..e37a883 100644
--- a/backend/feed/feed.go
+++ b/backend/feed/feed.go
@@ -76,25 +76,29 @@ func discoverFeedURL(ctx context.Context, rawURL string) (string, error) {
return feedURL, nil
}
-func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed) error {
+// Sync upserts the parsed feed's items into the database and returns the
+// number of newly inserted articles. The caller uses that count to drive the
+// adaptive fetch interval.
+func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed) (int, error) {
err := queries.UpdateFeedMetadata(ctx, db.UpdateFeedMetadataParams{
Title: f.Title,
FetchedAt: time.Now().UTC().Format(time.RFC3339),
ID: feedID,
})
if err != nil {
- return err
+ return 0, err
}
guids, err := queries.GetArticleGUIDsByFeed(ctx, feedID)
if err != nil {
- return err
+ return 0, err
}
existingFeedGUIDs := make(map[string]bool, len(guids))
for _, guid := range guids {
existingFeedGUIDs[guid] = true
}
+ newCount := 0
for _, item := range f.Items {
if existingFeedGUIDs[item.GUID] {
err := queries.UpdateArticle(ctx, db.UpdateArticleParams{
@@ -104,12 +108,12 @@ func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed
Guid: item.GUID,
})
if err != nil {
- return err
+ return newCount, err
}
} else {
exists, err := queries.CheckArticleExistsByGUID(ctx, item.GUID)
if err != nil {
- return err
+ return newCount, err
}
if exists == 1 {
continue
@@ -122,9 +126,10 @@ func Sync(ctx context.Context, queries *db.Queries, feedID int64, f *gofeed.Feed
IsRead: 0,
})
if err != nil {
- return err
+ return newCount, err
}
+ newCount++
}
}
- return nil
+ return newCount, nil
}
generated by cgit v1.3.1 (git 2.54.0) at 2026-06-01 18:53:54 +0000