From 97d50db15e86386f8bbbc36f4fcab42edaea3db0 Mon Sep 17 00:00:00 2001 From: lemon24 Date: Sat, 11 May 2024 00:01:50 +0300 Subject: [PATCH] Expose Feed.update_after and Feed.last_retrieved. #332 --- CHANGES.rst | 7 ++ src/reader/_storage/_entries.py | 6 +- src/reader/_storage/_feeds.py | 8 +- src/reader/types.py | 14 ++- tests/test_reader.py | 141 +++++++++++++++++++++++++++---- tests/test_reader_integration.py | 4 +- 6 files changed, 157 insertions(+), 23 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 25b8f212..622802b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,13 @@ Unreleased * Update entries whenever their :attr:`~Entry.updated` changes, don't compare the values. (:issue:`332`) +* Add :attr:`Feed.update_after` and :attr:`Feed.last_retrieved`. (:issue:`332`) +* The ``new`` filter of :meth:`~Reader.update_feeds()` etc. considers + a feed as new if it was never retrieved (:attr:`~Feed.last_retrieved`), + not if it was never updated successfully (:attr:`~Feed.last_updated`). (:issue:`332`) + + .. FIXME: versionchanged on update_feeds() etc. + * Group mutually-exclusive attributes of :class:`~.FeedUpdateIntent` into its :attr:`~.FeedUpdateIntent.value` union attribute. (:issue:`332`) diff --git a/src/reader/_storage/_entries.py b/src/reader/_storage/_entries.py index 727dbd58..fa3ae759 100644 --- a/src/reader/_storage/_entries.py +++ b/src/reader/_storage/_entries.py @@ -415,6 +415,8 @@ def get_entries_query( feeds.last_updated feeds.last_exception feeds.updates_enabled + feeds.update_after + feeds.last_retrieved entries.id entries.updated entries.title @@ -444,7 +446,7 @@ def get_entries_query( def entry_factory(row: tuple[Any, ...]) -> Entry: - feed = feed_factory(row[0:12]) + feed = feed_factory(row[0:14]) ( id, updated, @@ -464,7 +466,7 @@ def entry_factory(row: tuple[Any, ...]) -> Entry: last_updated, original_feed, sequence, - ) = row[12:30] + ) = row[14:32] return Entry( id, convert_timestamp(updated) if updated else None, diff --git a/src/reader/_storage/_feeds.py b/src/reader/_storage/_feeds.py index 3bc128e6..f540e9d0 100644 --- a/src/reader/_storage/_feeds.py +++ b/src/reader/_storage/_feeds.py @@ -285,6 +285,8 @@ def get_feeds_query(filter: FeedFilter, sort: FeedSort) -> tuple[Query, dict[str 'last_updated', 'last_exception', 'updates_enabled', + 'update_after', + 'last_retrieved', ) .FROM("feeds") .scrolling_window_sort_key(FEED_SORT_KEYS[sort]) @@ -307,7 +309,9 @@ def feed_factory(row: tuple[Any, ...]) -> Feed: last_updated, last_exception, updates_enabled, - ) = row[:12] + update_after, + last_retrieved, + ) = row[:14] return Feed( url, convert_timestamp(updated) if updated else None, @@ -321,6 +325,8 @@ def feed_factory(row: tuple[Any, ...]) -> Feed: convert_timestamp(last_updated) if last_updated else None, ExceptionInfo(**json.loads(last_exception)) if last_exception else None, updates_enabled == 1, + convert_timestamp(update_after) if update_after else None, + convert_timestamp(last_retrieved) if last_retrieved else None, ) diff --git a/src/reader/types.py b/src/reader/types.py index 0e9cd933..4ed49892 100644 --- a/src/reader/types.py +++ b/src/reader/types.py @@ -119,7 +119,7 @@ class Feed(_namedtuple_compat): #: .. versionadded:: 1.3 added: datetime = cast(datetime, None) - #: The date when the feed was last retrieved by reader. + #: The date when the feed was last (successfully) updated by reader. #: #: .. versionadded:: 1.3 last_updated: datetime | None = None @@ -139,6 +139,18 @@ class Feed(_namedtuple_compat): #: .. versionadded:: 1.11 updates_enabled: bool = True + #: The earliest time the feed will next be updated + #: (when using scheduled updates). + #: + #: .. versionadded:: 3.13 + update_after: datetime | None = None + + #: The date when the feed was last retrieved by reader, + #: regardless of the outcome. + #: + #: .. versionadded:: 3.13 + last_retrieved: datetime | None = None + @property def resource_id(self) -> tuple[str]: """Alias for (:attr:`~url`,). diff --git a/tests/test_reader.py b/tests/test_reader.py index 7f286de8..17d4d33f 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -80,7 +80,9 @@ def test_update_feed_updated(reader, update_feed, caplog): update_feed(reader, old_feed.url) feed = old_feed.as_feed( - added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 2) + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 2), + last_retrieved=datetime(2010, 1, 2), ) assert set(reader.get_entries()) == { entry_one.as_entry( @@ -100,7 +102,9 @@ def test_update_feed_updated(reader, update_feed, caplog): update_feed(reader, old_feed.url) feed = old_feed.as_feed( - added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 3) + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), ) assert set(reader.get_entries()) == { entry_one.as_entry( @@ -125,7 +129,9 @@ def test_update_feed_updated(reader, update_feed, caplog): update_feed(reader, old_feed.url) feed = old_feed.as_feed( - added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 3, 12) + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 3, 12), + last_retrieved=datetime(2010, 1, 3, 12), ) assert reader.get_feed(feed) == feed assert "feed hash changed, treating as updated" in caplog.text @@ -144,6 +150,7 @@ def test_update_feed_updated(reader, update_feed, caplog): added=datetime(2010, 1, 1), updated=datetime(2009, 1, 1), last_updated=datetime(2010, 1, 4), + last_retrieved=datetime(2010, 1, 4), ) assert set(reader.get_entries()) == { entry_one.as_entry( @@ -178,6 +185,8 @@ def test_update_feed_updated(reader, update_feed, caplog): updated=datetime(2009, 1, 1), # doesn't change because nothing changed last_updated=datetime(2010, 1, 4), + # changes always + last_retrieved=datetime(2010, 1, 4, 12), ) assert set(reader.get_entries()) == { entry_one.as_entry( @@ -205,7 +214,9 @@ def test_update_feed_updated(reader, update_feed, caplog): entry_four = parser.entry(1, 4, datetime(2010, 2, 1)) reader._now = lambda: datetime(2010, 1, 5) feed = new_feed.as_feed( - added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 5) + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 5), + last_retrieved=datetime(2010, 1, 5), ) with caplog.at_level(logging.DEBUG, logger='reader'): @@ -256,7 +267,11 @@ def test_update_entry_updated(reader, update_feed, caplog, monkeypatch): with caplog.at_level(logging.DEBUG, logger='reader'): update_feed(reader, feed.url) - feed = feed.as_feed(added=datetime(2010, 2, 1), last_updated=datetime(2010, 2, 2)) + feed = feed.as_feed( + added=datetime(2010, 2, 1), + last_updated=datetime(2010, 2, 2), + last_retrieved=datetime(2010, 2, 2), + ) assert set(reader.get_entries()) == { old_entry.as_entry( @@ -279,6 +294,7 @@ def test_update_entry_updated(reader, update_feed, caplog, monkeypatch): added=datetime(2010, 2, 1), updated=datetime(2010, 1, 1), last_updated=datetime(2010, 2, 2), + last_retrieved=datetime(2010, 2, 3), ) assert set(reader.get_entries()) == { old_entry.as_entry( @@ -301,7 +317,9 @@ def test_update_entry_updated(reader, update_feed, caplog, monkeypatch): update_feed(reader, feed.url) feed = feed.as_feed( - added=datetime(2010, 2, 1), last_updated=datetime(2010, 2, 3, 12) + added=datetime(2010, 2, 1), + last_updated=datetime(2010, 2, 3, 12), + last_retrieved=datetime(2010, 2, 3, 12), ) assert set(reader.get_entries()) == { new_entry.as_entry( @@ -322,7 +340,11 @@ def test_update_entry_updated(reader, update_feed, caplog, monkeypatch): with caplog.at_level(logging.DEBUG, logger='reader'): update_feed(reader, feed.url) - feed = feed.as_feed(added=datetime(2010, 2, 1), last_updated=datetime(2010, 2, 4)) + feed = feed.as_feed( + added=datetime(2010, 2, 1), + last_updated=datetime(2010, 2, 4), + last_retrieved=datetime(2010, 2, 4), + ) assert set(reader.get_entries()) == { new_entry.as_entry( feed=feed, @@ -366,7 +388,11 @@ def test_update_no_updated(reader, chunk_size, update_feed): reader._now = lambda: datetime(2010, 1, 1) reader.add_feed(feed.url) update_feed(reader, feed) - feed = feed.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 1)) + feed = feed.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 1), + last_retrieved=datetime(2010, 1, 1), + ) assert set(reader.get_feeds()) == {feed} assert set(reader.get_entries()) == { @@ -383,7 +409,11 @@ def test_update_no_updated(reader, chunk_size, update_feed): entry_two = parser.entry(1, 2, None) reader._now = lambda: datetime(2010, 1, 2) update_feed(reader, feed) - feed = feed.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 2)) + feed = feed.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 2), + last_retrieved=datetime(2010, 1, 2), + ) assert set(reader.get_feeds()) == {feed} assert set(reader.get_entries()) == { @@ -491,7 +521,11 @@ def test_update_new(reader): entry_one = parser.entry(1, 1, datetime(2010, 1, 1)) reader.update_feeds(new=True) - two = two.as_feed(added=datetime(2010, 1, 1, 12), last_updated=datetime(2010, 1, 2)) + two = two.as_feed( + added=datetime(2010, 1, 1, 12), + last_updated=datetime(2010, 1, 2), + last_retrieved=datetime(2010, 1, 2), + ) assert len(set(reader.get_feeds())) == 2 assert set(reader.get_entries()) == { entry_two.as_entry( @@ -504,7 +538,11 @@ def test_update_new(reader): reader._now = lambda: datetime(2010, 1, 3) reader.update_feeds() - one = one.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 3)) + one = one.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), + ) assert len(set(reader.get_feeds())) == 2 assert set(reader.get_entries()) == { entry_one.as_entry( @@ -513,7 +551,7 @@ def test_update_new(reader): last_updated=datetime(2010, 1, 3), ), entry_two.as_entry( - feed=two, + feed=two._replace(last_retrieved=datetime(2010, 1, 3)), added=datetime(2010, 1, 2), last_updated=datetime(2010, 1, 2), ), @@ -802,6 +840,49 @@ def _update_feed(*_, **__): assert excinfo.value is exc +def test_last_retrieved(reader): + reader._parser = parser = Parser() + feed = parser.feed(1) + reader.add_feed(feed.url) + + reader._now = lambda: datetime(2010, 1, 1) + reader.update_feeds() + feed = reader.get_feed(feed) + + assert feed.last_retrieved == datetime(2010, 1, 1) + assert feed.last_updated == datetime(2010, 1, 1) + + +def test_last_retrieved_not_modified(reader): + reader._parser = parser = Parser() + feed = parser.feed(1) + reader.add_feed(feed.url) + + parser.not_modified() + + reader._now = lambda: datetime(2010, 1, 1) + reader.update_feeds() + feed = reader.get_feed(feed) + + assert feed.last_retrieved == datetime(2010, 1, 1) + assert feed.last_updated == None + + +def test_last_retrieved_error(reader): + reader._parser = parser = Parser() + feed = parser.feed(1) + reader.add_feed(feed.url) + + parser.raise_exc() + + reader._now = lambda: datetime(2010, 1, 1) + reader.update_feeds() + feed = reader.get_feed(feed) + + assert feed.last_retrieved == datetime(2010, 1, 1) + assert feed.last_updated == None + + class FeedAction(Enum): none = object() update = object() @@ -904,8 +985,16 @@ def test_update_feed(reader, feed_arg): reader.update_feed(feed_arg(one)) reader._now = lambda: datetime(2010, 1, 1) - one = one.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 1)) - two = two.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 1)) + one = one.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 1), + last_retrieved=datetime(2010, 1, 1), + ) + two = two.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 1), + last_retrieved=datetime(2010, 1, 1), + ) reader.add_feed(one.url) reader.add_feed(two.url) @@ -1223,8 +1312,16 @@ def test_add_remove_get_feeds(reader, feed_arg): reader._now = lambda: datetime(2010, 1, 2) reader.update_feeds() - one = one.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 2)) - two = two.as_feed(added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 2)) + one = one.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 2), + last_retrieved=datetime(2010, 1, 2), + ) + two = two.as_feed( + added=datetime(2010, 1, 1), + last_updated=datetime(2010, 1, 2), + last_retrieved=datetime(2010, 1, 2), + ) entry_one = entry_one.as_entry( feed=one, added=datetime(2010, 1, 2), last_updated=datetime(2010, 1, 2) ) @@ -1401,7 +1498,9 @@ def test_data_roundtrip(reader): assert list(reader.get_entries()) == [ entry.as_entry( feed=feed.as_feed( - added=datetime(2010, 1, 2), last_updated=datetime(2010, 1, 3) + added=datetime(2010, 1, 2), + last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), ), added=datetime(2010, 1, 3), last_updated=datetime(2010, 1, 3), @@ -1474,7 +1573,9 @@ def test_get_entry(reader, entry_arg): entry = entry.as_entry( feed=feed.as_feed( - added=datetime(2010, 1, 2), last_updated=datetime(2010, 1, 3) + added=datetime(2010, 1, 2), + last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), ), added=datetime(2010, 1, 3), last_updated=datetime(2010, 1, 3), @@ -1647,6 +1748,7 @@ def test_change_feed_url_feed(reader): updated=None, last_updated=None, last_exception=None, + last_retrieved=None, ) @@ -1727,6 +1829,7 @@ def test_change_feed_url_second_update(reader, new_feed_url): url=new_feed_url, updated=None, last_updated=None, + last_retrieved=None, ) reader._parser.feed( @@ -1746,6 +1849,7 @@ def test_change_feed_url_second_update(reader, new_feed_url): url=new_feed_url, updated=datetime(2010, 1, 2), last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), title='new title', author='new author', link='new link', @@ -2387,6 +2491,7 @@ def test_add_entry(reader): feed=feed.as_feed( added=datetime(2010, 1, 1), last_updated=datetime(2010, 1, 3), + last_retrieved=datetime(2010, 1, 3), ), ) diff --git a/tests/test_reader_integration.py b/tests/test_reader_integration.py index c1a8d3a7..7dc056ed 100644 --- a/tests/test_reader_integration.py +++ b/tests/test_reader_integration.py @@ -132,7 +132,9 @@ class datetime_mock(datetime): exec(data_dir.joinpath(feed_filename + '.py').read_text(), expected) expected_feed = expected['feed'].as_feed( - added=utc_datetime(2010, 1, 1), last_updated=utc_datetime(2010, 1, 2) + added=utc_datetime(2010, 1, 1), + last_updated=utc_datetime(2010, 1, 2), + last_retrieved=utc_datetime(2010, 1, 2), ) assert feed == expected_feed