Allow reposting feeds on schedule.

If a feed has an interval set to a positive number of seconds it may be
reposted after that many seconds.

diff --git a/plugin/feeds/feeds.py b/plugin/feeds/feeds.py
index 0400754..2d1d429 100644 (file)
--- a/plugin/feeds/feeds.py
+++ b/plugin/feeds/feeds.py
@@ -3,6 +3,7 @@ import bs4
 import discord
 import feedparser
 import json
+import math
 import multiprocessing
 import re
 import shlex
@@ -25,7 +26,7 @@ class Feeds(DBConnection):
 
   def create_tables(self):
     self.open_db()
-    cursor = self.query('create table if not exists feeds (id char(36) not null, bot_id varchar(32) not null, server_id varchar(32) not null, channel_id varchar(32) not null, description varchar(64), url varchar(128) not null, date boolean not null default true, summary boolean not null default false, show_link boolean not null default true, link_json varchar(256), permissions boolean not null default true, enabled boolean not null default true)')
+    cursor = self.query('create table if not exists feeds (id char(36) not null, bot_id varchar(32) not null, server_id varchar(32) not null, channel_id varchar(32) not null, description varchar(64), url varchar(128) not null, date boolean not null default true, summary boolean not null default false, show_link boolean not null default true, link_json varchar(256), permissions boolean not null default true, interval int, enabled boolean not null default true, last_spoke datetime)')
     cursor = self.query('create unique index if not exists feed_id on feeds (id)')
     cursor = self.query('create index if not exists feed_bot_id on feeds (bot_id)')
     self.dbh.commit()
@@ -35,7 +36,7 @@ class Feeds(DBConnection):
 
   def get_feeds(self, client):
     now = self.now()
-    cursor = self.query("select id, server_id, channel_id, url, date, summary, show_link, link_json, permissions, enabled from feeds where bot_id=?", [client.user.id])
+    cursor = self.query("select id, server_id, channel_id, url, date, summary, show_link, link_json, permissions, interval, enabled, last_spoke from feeds where bot_id=? and interval is null or (last_spoke is null or last_spoke < ? - interval)", [client.user.id, now])
     for row in cursor.fetchall():
       yield dict(row)
     self.close_db()
@@ -57,6 +58,7 @@ class Feeds(DBConnection):
     d = {
       'channel_id': 'in',
       'description': 'title',
+      'interval': 'every',
       'show_link': 'link'
     }
     if k in d:
@@ -100,8 +102,9 @@ class Feeds(DBConnection):
       return
     log.debug('{} feeds to get for {}.'.format(len(feeds), channel.name))
 
+    updates = {}
     all_entries = []
-    now = time.time()
+    now = self.now()
     for feed in feeds:
       # Date can be trusted.
       date = bot.parse_boolean(feed['date']) if 'date' in feed else True
@@ -109,6 +112,30 @@ class Feeds(DBConnection):
       summary = bot.parse_boolean(feed['summary']) if 'summary' in feed else False
       # Post link
       show_link = bot.parse_boolean(feed['show_link']) if 'show_link' in feed else True
+      # Post on schedule.
+      interval = feed['interval'] if 'interval' in feed else 0
+      last_spoke = feed['last_spoke'] if 'last_spoke' in feed else None
+      updates[feed['id']] = { 'last_spoke': now }
+      if interval and last_spoke:
+        update = updates[feed['id']]
+        start = last_spoke
+        intervals = math.floor((now - start) / interval)
+        if intervals:
+          # We missed a schedule.
+          scheduled = start + intervals * interval
+          if now - scheduled > max(60, interval / 2):
+            log.info('Feed {} should have been posted at {}'.format(feed['id'], bot.iso8601(scheduled)))
+            update['last_spoke'] = scheduled
+            continue
+        # Post the feed now but set the original schedule.
+        last_spoke = start + (intervals + 1) * interval
+        if last_spoke < now:
+          update['last_spoke'] = last_spoke
+          log.info('Forcing last_spoke for feed {} to {} in line with schedule'.format(feed['id'], bot.iso8601(update['last_spoke'])))
+        dedupe = False
+      else:
+        dedupe = True
+
       url = feed['url']
       d = feedparser.parse(url)
       for entry in d.entries:
@@ -117,63 +144,66 @@ class Feeds(DBConnection):
         else:
           order = now
           now += 0.001
-        all_entries.append({ 'order': order, 'date': date, 'summary': summary, 'show_link': show_link, 'url': feed['url'], 'link_json': feed['link_json'], 'entry': entry })
+        all_entries.append({ 'order': order, 'dedupe': dedupe, 'date': date, 'summary': summary, 'show_link': show_link, 'url': feed['url'], 'link_json': feed['link_json'], 'entry': entry })
     if not len(all_entries):
       log.debug('No messages to post to {}'.format(channel.name))
-      return
-    all_entries.sort(key = lambda entry: entry['order'])
-
-    limit = len(all_entries) * 2
-    if not limit:
-      limit = bot.get('message_limit')
-
-    digests = []
-    logs = yield from client.logs_from(channel, limit = limit)
-    for message in logs:
-      digest = bot.digest(message.content)
-      log.debug('Saw previously posted RSS with digest {}'.format(digest))
-      if digest not in digests:
-        digests.append(digest)
-
-    for feed in all_entries:
-      entry = feed['entry']
-      formatted = '*{}*\n'.format(entry['published']) if feed['date'] else ''
-      link = None
-      if not feed['show_link']:
-        log.debug('Entry {} link is hidden.'.format(feed['url']))
-      elif 'links' in entry:
-        link = entry['links'][0]['href']
-      elif 'link_json' in feed:
+    else:
+      all_entries.sort(key = lambda entry: entry['order'])
+
+      limit = min(len(all_entries) * 2, int(bot.get('message_limit') * 1.5))
+
+      digests = []
+      logs = yield from client.logs_from(channel, limit = limit)
+      for message in logs:
+        digest = bot.digest(message.content)
+        log.debug('Saw previously posted RSS with digest {}'.format(digest))
+        if digest not in digests:
+          digests.append(digest)
+
+      for feed in all_entries:
+        entry = feed['entry']
+        formatted = '*{}*\n'.format(entry['published']) if feed['date'] else ''
+        link = None
+        if not feed['show_link']:
+          log.debug('Entry {} link is hidden.'.format(feed['url']))
+        elif 'links' in entry:
+          link = entry['links'][0]['href']
+        elif 'link_json' in feed:
+          try:
+            l = json.loads(feed['link_json'])
+            args = [entry[k] for k in l[1:]]
+            link = l[0].format(*args)
+          except ValueError:
+            log.error('Invalid link_json in feed {}'.format(feed['url']))
+          except:
+            log.exception('do_rss:link_json')
+        if feed['show_link'] and not link:
+          log.warning('No link for entry {}'.format(entry['title']))
+          continue
+        formatted += '**{}**{}'.format(entry['title'], '\n{}'.format(link) if feed['show_link'] else '')
+        if feed['summary']:
+          formatted += '\n{}'.format(''.join(bs4.BeautifulSoup(entry['summary'], 'html.parser').findAll(text = True)))
+        formatted = formatted.strip()
+        digest = bot.digest(formatted)
+        if digest in digests:
+          if feed['dedupe']:
+            log.debug("Already posted RSS with digest {}".format(digest))
+            continue
         try:
-          l = json.loads(feed['link_json'])
-          args = [entry[k] for k in l[1:]]
-          link = l[0].format(*args)
-        except ValueError:
-          log.error('Invalid link_json in feed {}'.format(feed['url']))
+          if bot.get('dryrun'):
+            log.info('Dryrun: Not posting to {}: {}'.format(channel, formatted))
+          else:
+            log.info('Posting to {}: {}'.format(channel, formatted))
+            message = yield from bot.say(channel, formatted)
+            digest = bot.digest(message.content)
+            digests.append(digest)
+            log.debug('Posted RSS with digest {}'.format(digest))
         except:
-          log.exception('do_rss:link_json')
-      if feed['show_link'] and not link:
-        log.warning('No link for entry {}'.format(entry['title']))
-        continue
-      formatted += '**{}**{}'.format(entry['title'], '\n{}'.format(link) if feed['show_link'] else '')
-      if feed['summary']:
-        formatted += '\n{}'.format(''.join(bs4.BeautifulSoup(entry['summary'], 'html.parser').findAll(text = True)))
-      formatted = formatted.strip()
-      digest = bot.digest(formatted)
-      if digest in digests:
-        log.debug("Already posted RSS with digest {}".format(digest))
-        continue
-      try:
-        if bot.get('dryrun'):
-          log.info('Dryrun: Not posting to {}: {}'.format(channel, formatted))
-        else:
-          log.info('Posting to {}: {}'.format(channel, formatted))
-          message = yield from bot.say(channel, formatted)
-          digest = bot.digest(message.content)
-          digests.append(digest)
-          log.debug('Posted RSS with digest {}'.format(digest))
-      except:
-        log.exception("do_rss")
+          log.exception("do_rss")
+
+    if not bot.get('dryrun'):
+      for id, update in updates.items():
+        self.update_feed(client, id, **update)
 
   @asyncio.coroutine
   def on_ready(self):
@@ -314,6 +344,8 @@ class Feeds(DBConnection):
     text += ' title "{}"'.format(feed['description'])
     text += ' url {}'.format(feed['url'])
     text += ' in <#{}>'.format(feed['channel_id'])
+    if feed['interval']:
+      text += ' every {}'.format(bot.unparse_seconds(int(feed['interval'])))
     text += ' date {}'.format(feed['date'])
     text += ' summary {}'.format(feed['summary'])
     text += ' link {}'.format(feed['show_link'])
@@ -363,6 +395,17 @@ class Feeds(DBConnection):
         k = 'description'
         parsed[k] = param
         ok = True
+      elif arg == 'every':
+        seconds = bot.parse_seconds(param)
+        if seconds is not None:
+          parsed['interval'] = seconds
+          ok = True
+        else:
+          break
+      elif arg == 'once':
+        parsed['interval'] = 0
+        i -= 1
+        ok = True
       elif arg == 'in':
         k = 'channel_id'
         if param:
@@ -559,6 +602,14 @@ class Feeds(DBConnection):
         '```url URL```',
         'The URL to retrieve the feed in RSS format.',
         '',
+        '```every INTERVAL```',
+        'Specify the interval to repeat the feed.  You can include **d**ays, **h**ours, **m**inutes or **s**econds.  `every 6h` means every six hours.  `every 10m30s` means every ten minutes and thirty seconds.',
+        '',
+        '```',
+        'once',
+        '```',
+        'Equivalent to `interval 0` and means no reposting of a feed.',
+        '',
         '```date false```',
         "Use this if the dates returned by the RSS URL aren't valid.  For instance the official GalNet feed always returns the date its cache was updated NOT the date the story was posted.",
         '',