diff --git a/Gemfile b/Gemfile index 5fed213..5df134c 100644 --- a/Gemfile +++ b/Gemfile @@ -42,9 +42,10 @@ group :plugins, :store, optional: true do end # An HTML parser, for the plugins that read HTML: FilterFullFeed, -# FilterImageSource, FilterDescriptionLink, and FeedParser.parse_html for -# SubscriptionLink and SubscriptionTumblr. PublishMarkdown uses it when it is -# installed and reduces a body to text without it. +# FilterImageSource, FilterDescriptionLink, CustomFeedWeb, and +# FeedParser.parse_html for SubscriptionLink and SubscriptionTumblr. +# PublishMarkdown uses it when it is installed and reduces a body to text +# without it. group :plugins, :html, optional: true do gem 'nokogiri', '>= 1.15', '< 2.0' end diff --git a/README.md b/README.md index f3c4c12..419bb2d 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ a plugin set every part of which still has somewhere to talk to. See - **Recipes in YAML.** A job is a file, not a program. No Ruby is written to wire a pipeline together. -- **34 plugins** across seven categories: subscribe, custom feed, filter, +- **35 plugins** across seven categories: subscribe, custom feed, filter, store, provide, notify, publish — and every one of them has a current use. - **Markdown out of the box.** `PublishMarkdown` writes the result as a plain Markdown document, to a file or to standard output, with no service and no @@ -369,13 +369,13 @@ like a shipped plugin replaces it. ### Which plugins still work -34 plugins ship with the gem. Every one is classified in +35 plugins ship with the gem. Every one is classified in [`doc/PLUGINS.md`](doc/PLUGINS.md) section 6, with its settings and the reason for its status: | Status | Count | Meaning | | --- | --- | --- | -| **Supported** | 23 | Works on the supported Rubies with current dependencies | +| **Supported** | 24 | Works on the supported Rubies with current dependencies | | **Supported (external)** | 10 | Works, but needs something you provide: a service, a command, a credential, a data file | | **Needs rework** | 1 | The service exists; this plugin speaks a replaced interface | diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index 3c5d191..e04c800 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -456,7 +456,7 @@ bundle. `plugins` is every group in the first block at once. | Plugin | Needs | Installed gem | Checkout group | Status | | --- | --- | --- | --- | --- | | `StorePermalink`, `StoreFullText` | `activerecord`, `sqlite3` | `gem install activerecord sqlite3` | `store` | Supported | -| `FilterImageSource`, `FilterDescriptionLink`, `SubscriptionLink`, `SubscriptionTumblr` | `nokogiri` | `gem install nokogiri` | `html` | Supported (`SubscriptionTumblr` external) | +| `FilterImageSource`, `FilterDescriptionLink`, `SubscriptionLink`, `SubscriptionTumblr`, `CustomFeedWeb` | `nokogiri` | `gem install nokogiri` | `html` | Supported (`SubscriptionTumblr` external) | | `PublishMarkdown` | `nokogiri`, for HTML bodies only | `gem install nokogiri` | `html` | Supported; runs without it | | `FilterSanitize` | `sanitize` | `gem install sanitize` | `sanitize` | Supported | | `autodiscovery` and `inspect` subcommands | `feedbag` | `gem install feedbag` | `autodiscovery` | Supported | diff --git a/doc/PLUGINS.md b/doc/PLUGINS.md index b2db031..73d41e5 100644 --- a/doc/PLUGINS.md +++ b/doc/PLUGINS.md @@ -688,6 +688,130 @@ to follow before putting it in `cron`, and set `interval`. ### 6.2 CustomFeed +#### CustomFeedWeb — **Supported** + +`custom_feed/web.rb`. Fetches HTML index pages and builds one feed per page +from the article links it lists. For a site that publishes no feed and whose +list page has more structure than `SubscriptionLink` reads: CSS selectors say +where an article is and what belongs to it, and the links are resolved, +filtered and deduplicated on the way into the feed. + +```yaml + - module: CustomFeedWeb + config: + retry: 2 + interval: 1 + sites: + - url: https://example.com/news/ + name: Example News + item_selector: article + link_selector: h2 a + title_selector: h2 + description_selector: .summary + date_selector: time + same_host: true + include: + - ^https://example\.com/news/ + exclude: + - /category/ + fetch_items: 50 +``` + +| Key | Type | Meaning | +| --- | --- | --- | +| `sites` | sequence | Page mappings, fetched in order. Required. | +| `retry` | integer | Attempts after the first, per page. Default `0`. | +| `interval` | integer | Seconds between requests. Default `0`. | + +Each element of `sites` is a mapping. A bare `- https://example.com/news/` is +not accepted: a page's settings are what this plugin is for, and one shorthand +kept working forever is a second format to support. + +| Key | Type | Meaning | +| --- | --- | --- | +| `url` | string | The page to fetch. Required. | +| `name` | string | Channel title. Default the page's ``, then its host. | +| `item_selector` | string | The node one article occupies. | +| `link_selector` | string | The permalink, evaluated inside the article where there is one. Default `a[href]`. | +| `title_selector` | string | The title, inside the article. Default the link's own text. | +| `description_selector` | string | The summary the page prints, taken as text. | +| `date_selector` | string | The publication date, inside the article. | +| `same_host` | boolean | Drop a URL whose host is not the page's. Default `true`. | +| `include` | sequence | Regular expressions; a URL matching none of them is dropped. | +| `exclude` | sequence | Regular expressions; a URL matching one of them is dropped. | +| `fetch_items` | integer | Items per page, from the top. Default `100`; `0`, a negative value and an absent one all mean the default. | + +There are three ways a page is read, and which one applies follows from the +selectors given: + +- **Neither `item_selector` nor `link_selector`.** Every `a[href]` on the page + is a candidate and its text is the title. This is the mode to start with. +- **`link_selector` only.** Each node it selects is a candidate, and its text + is the title. `main h2 a` is the usual shape of it. +- **`item_selector`.** Each node it selects is one article, and + `link_selector`, `title_selector`, `description_selector` and + `date_selector` are evaluated inside that node. Without `link_selector` the + article's first `a[href]` is the permalink; without `title_selector` the + link's own text is the title. + +`title_selector`, `description_selector` and `date_selector` are read inside an +article, so giving one without `item_selector` names no article to read it in +and is refused as a settings error. + +A candidate URL is resolved against the page it was found on — `/articles/42`, +`../42` and `//example.com/42` all become the URL a reader would follow — and +then judged in this order: HTTP or HTTPS, not the page itself, `same_host`, +`include`, `exclude`, already seen, and finally `fetch_items`. The fragment is +removed, because two links differing only in their anchor are one article. The +query string is kept, because `?id=42` is frequently the whole of what +identifies one; no canonical form is guessed. `same_host` is an exact host +match, so `blog.example.com` is not `www.example.com`. + +The page's own order is kept. A list page's order is the only ordering +information it carries, and nothing here sorts by date; `FilterSort` is where +a Recipe asks for that. + +`date_selector` prefers the `datetime` attribute of a `<time>` element and +otherwise parses the node's text. A date that cannot be read is logged and the +item keeps its place without one — the time the page was fetched is not the +time the article was published, and is never substituted for it. + +The plugin keeps no state: it fetches the page, and what the page lists now is +what it returns. Whether an item has been published before is the record +`StorePermalink` keeps, which is what the usual Recipe puts after it: + +```yaml +plugins: + - module: CustomFeedWeb + config: + sites: + - url: https://example.com/news/ + link_selector: main h2 a + + - module: StorePermalink + config: + db: web-watch.db + + - module: PublishMarkdown + config: + file: ~/.automatic/markdown/web-watch.md + mode: append +``` + +A page that could not be fetched is retried, then logged and skipped, and the +other pages still produce their feeds. Settings that cannot be carried out — +a site that is not a mapping, a missing or unfetchable `url`, an `include` or +`exclude` that is not a regular expression, a selector combination that names +no article — are refused before anything is fetched, because a second attempt +would fail identically. + +Nothing else is fetched: no article body, no next page, no sitemap, no feed +autodiscovery, and no link found on the page is followed. One run makes one +request per site. Set `interval` when several sites are on one host. + +Needs `nokogiri`, which it reads the page with: `gem install nokogiri`, or the +`html` group in a checkout. + #### CustomFeedSVNLog — **Supported (external)** `custom_feed/svn_log.rb`. Runs `svn log --xml` against a repository and makes a @@ -1310,11 +1434,11 @@ is a claim that the plugin works. | Status | Count | Plugins | | --- | --- | --- | -| Supported | 23 | `SubscriptionFeed`, `SubscriptionLink`, `SubscriptionXml`, `SubscriptionText`, `FilterIgnore`, `FilterAccept`, `FilterSort`, `FilterOne`, `FilterRand`, `FilterClear`, `FilterImage`, `FilterImageSource`, `FilterAbsoluteURI`, `FilterSanitize`, `FilterTumblrResize`, `FilterDescriptionLink`, `FilterGithubFeed`, `StorePermalink`, `StoreFullText`, `StoreFile`, `PublishMarkdown`, `PublishConsole`, `PublishConsoleLink` | +| Supported | 24 | `SubscriptionFeed`, `SubscriptionLink`, `SubscriptionXml`, `SubscriptionText`, `CustomFeedWeb`, `FilterIgnore`, `FilterAccept`, `FilterSort`, `FilterOne`, `FilterRand`, `FilterClear`, `FilterImage`, `FilterImageSource`, `FilterAbsoluteURI`, `FilterSanitize`, `FilterTumblrResize`, `FilterDescriptionLink`, `FilterGithubFeed`, `StorePermalink`, `StoreFullText`, `StoreFile`, `PublishMarkdown`, `PublishConsole`, `PublishConsoleLink` | | Supported (external) | 10 | `SubscriptionTumblr`, `CustomFeedSVNLog`, `FilterFullFeed`, `ProvideFluentd`, `NotifyIkachan`, `PublishEject`, `PublishMemcached`, `PublishFluentd`, `PublishInstapaper`, `PublishAmazonS3` | | Needs rework | 1 | `PublishHatenaBookmark` | -Thirty-four plugins. Every one of them either runs, or names the one thing it +Thirty-five plugins. Every one of them either runs, or names the one thing it needs from the operator; the single exception says what is wrong with it and what fixing it would take. diff --git a/doc/VERSIONS b/doc/VERSIONS index 4e8c559..c1fa7fe 100644 --- a/doc/VERSIONS +++ b/doc/VERSIONS @@ -11,6 +11,7 @@ v26.08 (Release Date: TBD) - Classify every shipped plugin by its current support status rather than simulating obsolete services in tests. - Modernize the bundled plugins for current Ruby, libraries and services, and remove the integrations that are no longer viable. - Remove the plugins for Twitter, Pocket, HipChat, Google Calendar, livedoor Weather, So-net G-Guide and Chan-Toru, and Google News link rewriting; their services or APIs no longer exist, and a Recipe naming one now fails at load. +- Add the plugin CustomFeedWeb, which builds one feed per HTML index page from the article links it lists, selected with CSS selectors and filtered by host, pattern and count. - Rebuild the test and CI strategy for current RSpec and Ruby, with deterministic isolation from user data and external services. - Add Markdown as the primary service-independent publication format, with a documented and tested first-run workflow. - Rebuild the maintained documentation around current usage, architecture, policy, plugins and deployment, and remove superseded historical documents. diff --git a/plugins/custom_feed/web.rb b/plugins/custom_feed/web.rb new file mode 100644 index 0000000..3395fdc --- /dev/null +++ b/plugins/custom_feed/web.rb @@ -0,0 +1,348 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::CustomFeed::Web +# Description:: Build a feed from the article links of an HTML index page. +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Aug 17, 2026 +# Updated:: Aug 17, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +# +# One page in, one feed out. The pages named in the Recipe are fetched, their +# article links are taken with CSS selectors, and each page becomes a feed of +# what it currently lists. Nothing else is fetched: the links are not +# followed, no article body is read, and no page beyond the ones named is +# visited, so a run costs one request per site. +# +# It keeps no state of its own, deliberately. What was published last time is +# StorePermalink's question, and the Recipe that answers it puts StorePermalink +# after this plugin. See doc/PLUGINS.md section 6.2. + +require 'rss/maker' +require 'set' +require 'time' +require 'uri' + +module Automatic::Plugin + class CustomFeedWeb + Automatic.require_optional('nokogiri', needed_by: 'CustomFeedWeb') + + DEFAULT_FETCH_ITEMS = 100 + + # What an article link is, where the Recipe does not say. Every mode ends + # at an `<a href>`, because a permalink is what a feed item needs. + DEFAULT_LINK_SELECTOR = 'a[href]' + + # A link in a page is written by whoever wrote the page. `mailto:`, + # `javascript:` and `file:` links are ordinary in a navigation bar and are + # not articles. + SCHEMES = %w[http https].freeze + + # One article as the page presents it, before it becomes an item. Internal + # to this plugin: what leaves here is the pipeline value of section 3.4. + Candidate = Struct.new(:title, :url, :description, :date, keyword_init: true) + + def initialize(config, pipeline = []) + @config = config || {} + @pipeline = pipeline + @seen = Set.new + end + + def run + sites.each do |site| + feed = feed_for(site) + @pipeline << feed unless feed.nil? + end + @pipeline + end + + private + + # Every site, checked before anything is fetched. + # + # A Recipe this plugin cannot carry out -- a site that is not a mapping, a + # URL that cannot be fetched, a pattern that is not a regular expression, + # a selector combination that names no article -- is the operator's + # mistake, and it will be the same mistake after a retry and a wait. It is + # refused here, where no request has been made yet, rather than inside the + # retry loop, which is for a host that did not answer. + def sites + @sites ||= Array(@config['sites']).each { |site| validate(site) } + end + + def validate(site) + unless site.is_a?(Hash) + raise ArgumentError, + 'CustomFeedWeb takes a mapping with a url key for each site, ' \ + "not #{site.inspect}" + end + + source(site) + patterns(site, 'include') + patterns(site, 'exclude') + article_unit(site) + end + + # The page URL as this plugin will fetch and resolve against it. + # Automatic::Http is what says which URLs those are. + def source(site) + Automatic::Http.uri(site['url']) + rescue ArgumentError, URI::InvalidURIError => e + raise ArgumentError, "CustomFeedWeb needs a url for each site: #{e.message}" + end + + def patterns(site, key) + Array(site[key]).map { |pattern| Regexp.new(pattern.to_s) } + rescue RegexpError => e + raise ArgumentError, + "CustomFeedWeb was given an invalid #{key} pattern for " \ + "#{site['url']}: #{e.message}" + end + + # `title_selector`, `description_selector` and `date_selector` are read + # inside one article's node, and `item_selector` is what says where an + # article begins and ends. Without it there is no node to read them in, + # and a page of anchors cannot be divided into articles by guessing. + def article_unit(site) + return if presence(site['item_selector']) + + named = %w[title_selector description_selector date_selector]. + select { |key| presence(site[key]) } + return if named.empty? + + raise ArgumentError, + "CustomFeedWeb needs item_selector to use #{named.join(', ')}: " \ + "#{site['url']}" + end + + def feed_for(site) + base = source(site) + document = fetch(site) + return nil if document.nil? + + articles = entries(document, site, base) + if articles.empty? + # A list page with nothing on it that this Recipe recognises is an + # ordinary answer rather than a failure, and an empty feed would only + # give the rest of the pipeline something to skip. + Automatic::Log.puts('warn', "No article links found on #{base}") + return nil + end + + Automatic::Log.puts('info', "Web feed: #{articles.size} items from #{base}") + feed(site, document, base, articles) + end + + # The page, or nil where it could not be read after its retries. One site + # that is down does not take the rest of the Recipe with it. + # + # The parser is given the stream rather than a decoded string, so that it + # reads the document's own meta charset instead of the encoding open-uri + # settled on. See doc/PLUGINS.md section 3.8.1. + def fetch(site) + url = site['url'] + retries = 0 + retry_max = @config['retry'].to_i + begin + Automatic::Log.puts('info', "Parsing Web page: #{url}") + document = Automatic::Http.open(url) { |io| Nokogiri::HTML(io) } + sleep(@config['interval'].to_i) + document + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in fetching: #{url}, #{e.message}") + if retries > retry_max + Automatic::Log.puts('warn', "Skipping #{url}") + return nil + end + + sleep(@config['interval'].to_i) + retry + end + end + + # The candidates that survive, in the order the page lists them, up to + # `fetch_items`. A list page's own order is the only ordering information + # it carries, so nothing here sorts. + def entries(document, site, base) + includes = patterns(site, 'include') + excludes = patterns(site, 'exclude') + limit = fetch_items(site) + + candidates(document, site).each_with_object([]) do |candidate, kept| + next if candidate.title.empty? + + url = permalink(candidate.url, base, site, includes, excludes) + next if url.nil? + + candidate.url = url + kept << candidate + break kept if kept.size >= limit + end + end + + # The three ways a page is read, and the only three: every anchor, the + # anchors a selector names, or an article node with its parts inside it. + def candidates(document, site) + item_selector = presence(site['item_selector']) + return anchors(document, site) if item_selector.nil? + + document.css(item_selector).filter_map { |node| article(node, site) } + end + + def anchors(document, site) + selector = presence(site['link_selector']) || DEFAULT_LINK_SELECTOR + document.css(selector).filter_map { |node| + next if node['href'].nil? + + Candidate.new(title: normalize(node.text), url: node['href'], description: '') + } + end + + def article(node, site) + link = node.at_css(presence(site['link_selector']) || DEFAULT_LINK_SELECTOR) + return nil if link.nil? || link['href'].nil? + + Candidate.new( + title: title(node, site, link), + url: link['href'], + description: description(node, site), + date: date(node, site) + ) + end + + # A `title_selector` that selects nothing in this particular article falls + # back to the link's own text rather than dropping the article: one entry + # laid out differently from the rest of a list is ordinary. + def title(node, site, link) + selector = presence(site['title_selector']) + found = selector.nil? ? nil : node.at_css(selector) + normalize((found || link).text) + end + + # The summary the list page prints, as text. Not the article: this plugin + # does not fetch one. + def description(node, site) + selector = presence(site['description_selector']) + return '' if selector.nil? + + found = node.at_css(selector) + found.nil? ? '' : normalize(found.text) + end + + # `<time datetime="2026-08-17T09:00:00+09:00">yesterday</time>` is the + # reason the attribute is preferred: it is written for a machine, and the + # text beside it is written for a reader. + def date(node, site) + selector = presence(site['date_selector']) + return nil if selector.nil? + + found = node.at_css(selector) + return nil if found.nil? + + published(found.name == 'time' && found['datetime'] ? found['datetime'] : found.text) + end + + # A date that cannot be read costs the article its date and not its place + # in the feed. Nothing is substituted for it: the time this ran is when + # the page was fetched, which is not when the article was published. + def published(value) + Time.parse(value.to_s) + rescue ArgumentError => e + Automatic::Log.puts('warn', "Unreadable date #{value.to_s.strip.inspect}: #{e.message}") + nil + end + + # The judgements a candidate URL passes, in this order: it is resolved + # against the page it was found on, then it is one this framework fetches, + # then it is not the page itself, then the host, then include, then + # exclude, and only what has survived all of them is recorded as seen. + def permalink(href, base, site, includes, excludes) + uri = absolute(href, base) + return nil if uri.nil? || !SCHEMES.include?(uri.scheme) + + url = uri.to_s + return nil if url == base.to_s + return nil if same_host?(site) && !same_host_as?(uri, base) + return nil if includes.any? && includes.none? { |pattern| pattern.match?(url) } + return nil if excludes.any? { |pattern| pattern.match?(url) } + + # Within one run only. Whether a URL was published last week is + # StorePermalink's record, not this plugin's. + @seen.add?(url).nil? ? nil : url + end + + # Resolved against the page it was found on, so that `/articles/42`, + # `../42` and `//example.com/42` all become the URL a reader would follow. + # The fragment goes, because two links differing only in their anchor are + # one article. The query string stays, because `?id=42` is frequently the + # whole of what identifies one, and no canonical form is guessed. + def absolute(href, base) + uri = base.merge(href.to_s.strip) + uri.fragment = nil + uri + rescue URI::Error, ArgumentError + nil + end + + def same_host?(site) + value = site['same_host'] + value.nil? ? true : value + end + + # An exact host match, and nothing cleverer. `www.example.com` and + # `blog.example.com` are one organisation and are not one site, and which + # of them a Recipe wants is the Recipe's to say. + def same_host_as?(uri, base) + uri.host.to_s.downcase == base.host.to_s.downcase + end + + def fetch_items(site) + value = site['fetch_items'].to_i + value.positive? ? value : DEFAULT_FETCH_ITEMS + end + + def feed(site, document, base, entries) + RSS::Maker.make('2.0') do |maker| + maker.channel.title = channel_title(site, document, base) + maker.channel.link = base.to_s + maker.channel.description = "Web page items from #{base}" + # The page's order is kept. RSS::Maker sorts its items by date when it + # is asked to, and an article the page listed first is first for a + # reason a date does not carry. + maker.items.do_sort = false + + entries.each do |entry| + item = maker.items.new_item + item.title = entry.title + item.link = entry.url + item.description = entry.description + item.date = entry.date unless entry.date.nil? + end + end + end + + # `name`, then what the page calls itself, then the host it came from. + def channel_title(site, document, base) + name = presence(site['name']) + return name unless name.nil? + + title = normalize(document.title.to_s) + title.empty? ? base.host.to_s : title + end + + # " Ruby\n 4.0\t released " is "Ruby 4.0 released". A title in a list + # page is laid out for a browser, and the line breaks and indentation of + # the markup are not part of it. + def normalize(text) + text.to_s.gsub(/[[:space:]]+/, ' ').strip + end + + def presence(value) + string = value.to_s.strip + string.empty? ? nil : string + end + end +end diff --git a/spec/plugins/custom_feed/web_spec.rb b/spec/plugins/custom_feed/web_spec.rb new file mode 100644 index 0000000..8b0cb36 --- /dev/null +++ b/spec/plugins/custom_feed/web_spec.rb @@ -0,0 +1,529 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::CustomFeed::Web +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Aug 17, 2026 +# Updated:: Aug 17, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +# CustomFeedWeb reads HTML with nokogiri, which the Gemfile declares in its +# optional :plugins group. The default suite and CI do not install it, so this +# spec runs only where the operator has. See doc/POLICY.md section 5. +return unless AutomaticSpec.optional_dependency?('nokogiri') + +require 'custom_feed/web' +require 'stringio' + +# Nothing here reaches a network: Automatic::Http.open yields the page the +# example wrote, and everything this plugin decides -- which links are +# articles, what they resolve to, what their titles are -- it decides between +# that page and the feed it returns, which is what a local double can hold +# still. +module WebSpec + module_function + + # What open-uri hands a caller: the bytes of the page, tagged with the + # encoding open-uri settled on. The plugin gives the stream to the parser + # rather than a decoded string, so the double is a stream. + def response(body, content_type = 'text/html; charset=UTF-8') + io = StringIO.new(body.dup.force_encoding('UTF-8')) + io.define_singleton_method(:meta) { { 'content-type' => content_type } } + io + end + + def page(body, head = '<title>Example News') + "#{head}#{body}" + end +end + +describe Automatic::Plugin::CustomFeedWeb do + let(:source) { 'https://example.com/news/' } + let(:body) { 'First article' } + let(:head) { 'Example News' } + let(:pages) { { source => WebSpec.page(body, head) } } + let(:incoming) { [] } + + before do + Automatic::Http.stub(:open) { |url, &block| + html = pages[url.to_s] + raise "the spec did not expect a request for #{url}" if html.nil? + + block.call(WebSpec.response(html)) + } + end + + # One site, whose settings the example adds to, and the pipeline it returns. + def run(site = {}, config = {}, pipeline = incoming) + described_class.new( + { 'sites' => [{ 'url' => source }.merge(site)] }.merge(config), pipeline + ).run + end + + def items(site = {}, config = {}) + run(site, config)[0].items + end + + def links(site = {}, config = {}) + items(site, config).map(&:link) + end + + describe 'the generic mode' do + let(:body) { + '

intro

' \ + 'First article' \ + 'Second article' + } + + it 'makes an item of every a[href]' do + run.should have(1).feed + items.should have(2).items + end + + it 'takes the anchor text as the title' do + items.map(&:title).should == ['First article', 'Second article'] + end + + it 'keeps the order the page lists them in' do + links.should == ['https://example.com/articles/42', + 'https://example.com/articles/43'] + end + end + + describe 'a candidate URL' do + it 'resolves a relative URL against the page' do + links.should == ['https://example.com/articles/42'] + end + + context 'written with ../' do + let(:body) { 'Older' } + + it 'resolves it' do + links.should == ['https://example.com/archive/7'] + end + end + + context 'written scheme-relative' do + let(:body) { 'Same host' } + + it 'takes the scheme from the page' do + links.should == ['https://example.com/articles/42'] + end + end + + context 'carrying a fragment' do + let(:body) { 'With an anchor' } + + it 'drops the fragment' do + links.should == ['https://example.com/articles/42'] + end + end + + context 'carrying a query string' do + let(:body) { 'Query' } + + it 'keeps the query string' do + links.should == ['https://example.com/article?id=42&page=2'] + end + end + + context 'that is not HTTP or HTTPS' do + let(:body) { + 'Mail' \ + 'Menu' \ + 'Archive' \ + 'Article' + } + + it 'is dropped' do + links.should == ['https://example.com/articles/42'] + end + end + + context 'that is the page itself' do + let(:body) { + 'This page' \ + 'This page again' \ + 'Article' + } + + it 'is dropped' do + links.should == ['https://example.com/articles/42'] + end + end + + context 'appearing twice' do + let(:body) { + 'First' \ + 'First again' \ + 'First, with an anchor' + } + + it 'becomes one item' do + links.should == ['https://example.com/articles/42'] + end + end + end + + describe 'a title' do + let(:body) { " Ruby\n 4.0\t released " } + + it 'has its whitespace normalized' do + items.map(&:title).should == ['Ruby 4.0 released'] + end + + context 'that is empty' do + let(:body) { + '' \ + 'Second article' + } + + it 'costs the candidate its place' do + links.should == ['https://example.com/articles/43'] + end + end + end + + describe 'same_host' do + let(:body) { + 'Ours' \ + 'A subdomain' \ + 'Another site' + } + + it 'drops another host by default, subdomains included' do + links.should == ['https://example.com/articles/42'] + end + + it 'keeps another host when it is false' do + links('same_host' => false).should == ['https://example.com/articles/42', + 'https://blog.example.com/x', + 'https://example.org/y'] + end + end + + describe 'include and exclude' do + let(:body) { + 'An article' \ + 'A category' \ + 'Not news' + } + + it 'keeps only what include matches' do + links('include' => ['^https://example\.com/news/']). + should == ['https://example.com/news/42', + 'https://example.com/news/category/ruby'] + end + + it 'drops what exclude matches' do + links('exclude' => ['/category/']).should == ['https://example.com/news/42', + 'https://example.com/about'] + end + + it 'applies exclude to what include kept' do + links('include' => ['^https://example\.com/news/'], + 'exclude' => ['/category/']).should == ['https://example.com/news/42'] + end + + it 'keeps everything when neither is given' do + links.should have(3).links + end + end + + describe 'fetch_items' do + let(:body) { + (1..120).map { |number| "Article #{number}" }.join + } + + it 'limits the feed to the first N the page lists' do + links('fetch_items' => 3).should == ['https://example.com/articles/1', + 'https://example.com/articles/2', + 'https://example.com/articles/3'] + end + + it 'defaults to 100' do + items.should have(100).items + end + + it 'takes 0 as the default' do + items('fetch_items' => 0).should have(100).items + end + + it 'takes a negative value as the default' do + items('fetch_items' => -5).should have(100).items + end + end + + describe 'link_selector' do + let(:body) { + '' \ + '

An article

' \ + 'A bare link
' + } + + it 'takes the anchors it names and no others' do + links('link_selector' => 'main h2 a').should == ['https://example.com/articles/42'] + end + end + + describe 'item_selector' do + let(:body) { + '' \ + '
' \ + '

First article

' \ + '

What it is about.

' \ + '' \ + 'Comments' \ + '
' \ + '
' \ + '

Second article

' \ + '

Something else.

' \ + '' \ + '
' + } + let(:item_site) { + { 'item_selector' => 'article', 'link_selector' => 'h2 a', + 'title_selector' => 'h2', 'description_selector' => '.summary', + 'date_selector' => 'time' } + } + + it 'makes one item per article node' do + items(item_site).should have(2).items + end + + it 'evaluates link_selector inside the article' do + links(item_site).should == ['https://example.com/articles/42', + 'https://example.com/articles/43'] + end + + it 'takes the first a[href] in the article when link_selector is absent' do + links('item_selector' => 'article').should == ['https://example.com/articles/42', + 'https://example.com/articles/43'] + end + + it 'takes the title from title_selector' do + items(item_site).map(&:title).should == ['First article', 'Second article'] + end + + it 'takes the title from the link when title_selector is absent' do + items('item_selector' => 'article', 'link_selector' => 'h2 a'). + map(&:title).should == ['First article', 'Second article'] + end + + it 'takes the description from description_selector' do + items(item_site).map(&:description).should == ['What it is about.', 'Something else.'] + end + + it 'leaves the description empty when description_selector is absent' do + items('item_selector' => 'article').map(&:description).should == ['', ''] + end + + describe 'a date' do + it 'is taken from the datetime attribute of a time element' do + items(item_site)[0].date.should == Time.parse('2026-08-01T09:15:22+09:00') + end + + it 'is parsed from the node text where there is no datetime attribute' do + items(item_site)[1].date.should == Time.parse('30 July 2026') + end + + it 'is absent when date_selector is not given' do + items('item_selector' => 'article')[0].date.should be_nil + end + + context 'that cannot be read' do + let(:body) { + '

An article

' \ + '
' + } + + it 'costs the item its date and not its place' do + feed = items('item_selector' => 'article', 'date_selector' => 'time') + feed.should have(1).item + feed[0].date.should be_nil + end + end + end + end + + describe 'the channel' do + it 'takes its title from name' do + run('name' => 'Example News')[0].channel.title.should == 'Example News' + end + + it 'takes the page title where name is absent' do + run[0].channel.title.should == 'Example News' + end + + context 'on a page with no title element' do + let(:head) { '' } + + it 'takes the host' do + run[0].channel.title.should == 'example.com' + end + end + + it 'links to the page it was built from' do + run[0].channel.link.should == source + end + end + + describe 'a site that cannot be fetched' do + let(:other) { 'https://example.org/news/' } + let(:pages) { { other => WebSpec.page('Theirs') } } + let(:plugin) { + described_class.new( + 'retry' => 0, 'sites' => [{ 'url' => source }, { 'url' => other }] + ) + } + + before do + Automatic::Http.stub(:open) { |url, &block| + raise Errno::ECONNREFUSED if url.to_s == source + + block.call(WebSpec.response(pages[url.to_s])) + } + end + + it 'is skipped, and the sites after it are still fetched' do + feeds = plugin.run + feeds.should have(1).feed + feeds[0].items[0].link.should == 'https://example.org/articles/7' + end + + it 'is attempted once more per retry' do + Automatic::Http.should_receive(:open).exactly(3).times.and_raise(Errno::ECONNREFUSED) + described_class.new('retry' => 2, 'interval' => 0, + 'sites' => [{ 'url' => source }]).run.should be_empty + end + end + + # A Recipe this plugin cannot carry out will not be carried out by a second + # attempt either. It is refused before anything is fetched. + describe 'a settings error' do + it 'is raised for an invalid include pattern, without fetching' do + Automatic::Http.should_not_receive(:open) + lambda { run('include' => ['[']) }.should raise_error(ArgumentError) + end + + it 'is raised for an invalid exclude pattern, without fetching' do + Automatic::Http.should_not_receive(:open) + lambda { run('exclude' => ['(']) }.should raise_error(ArgumentError) + end + + it 'is raised for a site with no url, without fetching' do + Automatic::Http.should_not_receive(:open) + lambda { described_class.new('sites' => [{ 'name' => 'No URL' }]).run }. + should raise_error(ArgumentError) + end + + it 'is raised for a url this framework does not fetch' do + Automatic::Http.should_not_receive(:open) + lambda { described_class.new('sites' => [{ 'url' => 'ftp://example.com/' }]).run }. + should raise_error(ArgumentError) + end + + it 'is raised for the sites shorthand, which is not accepted' do + Automatic::Http.should_not_receive(:open) + lambda { described_class.new('sites' => ['https://example.com/news/']).run }. + should raise_error(ArgumentError, /mapping/) + end + + it 'is raised where an article unit cannot be decided' do + Automatic::Http.should_not_receive(:open) + lambda { run('title_selector' => 'h2') }.should raise_error(ArgumentError, /item_selector/) + end + + # An invalid selector is only found by the parser, which is on the far + # side of the request. What matters is that the request is not repeated. + it 'is not retried when the parser rejects a selector' do + fetched = 0 + Automatic::Http.stub(:open) { |_url, &block| + fetched += 1 + block.call(WebSpec.response(WebSpec.page(body))) + } + lambda { run({ 'link_selector' => 'h2 >>' }, 'retry' => 2) }. + should raise_error(Nokogiri::CSS::SyntaxError) + fetched.should == 1 + end + end + + describe 'the pipeline' do + let(:incoming) { + AutomaticSpec.generate_pipeline { + feed { item 'https://example.net/earlier', 'An earlier feed' } + } + } + + it 'keeps what was already in it and appends the new feed' do + feeds = run + feeds.should have(2).feeds + feeds[0].items[0].link.should == 'https://example.net/earlier' + feeds[1].items[0].link.should == 'https://example.com/articles/42' + end + + context 'when a page yields no candidate' do + let(:body) { '

Nothing to link to.

' } + + it 'is returned unchanged' do + run.should == incoming + end + end + + it 'is returned unchanged when there are no sites' do + described_class.new({}, incoming).run.should == incoming + end + end + + # nokogiri is this plugin's own dependency and not the framework's: a Recipe + # that does not name CustomFeedWeb runs without it. See doc/POLICY.md + # section 9.1. + describe 'its dependency on nokogiri' do + let(:source_file) { + File.read(File.join(APP_ROOT, 'plugins', 'custom_feed', 'web.rb'), encoding: 'UTF-8') + } + + it 'is required through Automatic.require_optional' do + source_file.should include( + "Automatic.require_optional('nokogiri', needed_by: 'CustomFeedWeb')" + ) + source_file.should_not match(/^\s*require 'nokogiri'/) + end + + it 'is declared as an optional plugin gem rather than a runtime one' do + AutomaticSpec::OPTIONAL_PLUGIN_GEMS.should include('nokogiri') + File.read(File.join(APP_ROOT, 'automatic.gemspec'), encoding: 'UTF-8'). + should_not match(/add_dependency\s+'nokogiri'/) + end + end +end + +# Excluded from the default suite, which reaches no network. The examples +# above are what says this plugin works; this one says the page it was written +# for is still a page. Run it deliberately with AUTOMATIC_NETWORK_SPECS=1. +# +# It selects by URL rather than by selector on purpose: a pattern outlives a +# redesign, and a shipped test that depends on somebody else's markup is a +# test that breaks without anything here changing. +describe Automatic::Plugin::CustomFeedWeb, 'against a real page', :network do + subject { + Automatic::Plugin::CustomFeedWeb.new( + 'sites' => [{ 'url' => 'https://www.ruby-lang.org/en/news/', + 'name' => 'Ruby News', + 'include' => ['/en/news/20'], + 'fetch_items' => 3 }] + ) + } + + its(:run) { should have(1).feed } + + it 'takes the release announcements the page lists' do + feed = subject.run[0] + feed.channel.title.should == 'Ruby News' + feed.items.should have(3).items + feed.items.each { |item| item.link.should include('/en/news/20') } + end +end