From: CMDR furrycat Date: Sun, 4 Dec 2016 15:25:55 +0000 (+0000) Subject: Allow reposting feeds on schedule. X-Git-Url: http://git.furryclan.net/?a=commitdiff_plain;h=ba275c9b4e363ebd1cc8762fdfc2517d0672ff96;p=furrycat%2Fcatbot.git Allow reposting feeds on schedule. If a feed has an interval set to a positive number of seconds it may be reposted after that many seconds. --- diff --git a/plugin/feeds/feeds.py b/plugin/feeds/feeds.py index 0400754..2d1d429 100644 --- a/plugin/feeds/feeds.py +++ b/plugin/feeds/feeds.py @@ -3,6 +3,7 @@ import bs4 import discord import feedparser import json +import math import multiprocessing import re import shlex @@ -25,7 +26,7 @@ class Feeds(DBConnection): def create_tables(self): self.open_db() - cursor = self.query('create table if not exists feeds (id char(36) not null, bot_id varchar(32) not null, server_id varchar(32) not null, channel_id varchar(32) not null, description varchar(64), url varchar(128) not null, date boolean not null default true, summary boolean not null default false, show_link boolean not null default true, link_json varchar(256), permissions boolean not null default true, enabled boolean not null default true)') + cursor = self.query('create table if not exists feeds (id char(36) not null, bot_id varchar(32) not null, server_id varchar(32) not null, channel_id varchar(32) not null, description varchar(64), url varchar(128) not null, date boolean not null default true, summary boolean not null default false, show_link boolean not null default true, link_json varchar(256), permissions boolean not null default true, interval int, enabled boolean not null default true, last_spoke datetime)') cursor = self.query('create unique index if not exists feed_id on feeds (id)') cursor = self.query('create index if not exists feed_bot_id on feeds (bot_id)') self.dbh.commit() @@ -35,7 +36,7 @@ class Feeds(DBConnection): def get_feeds(self, client): now = self.now() - cursor = self.query("select id, server_id, channel_id, url, date, summary, show_link, link_json, permissions, enabled from feeds where bot_id=?", [client.user.id]) + cursor = self.query("select id, server_id, channel_id, url, date, summary, show_link, link_json, permissions, interval, enabled, last_spoke from feeds where bot_id=? and interval is null or (last_spoke is null or last_spoke < ? - interval)", [client.user.id, now]) for row in cursor.fetchall(): yield dict(row) self.close_db() @@ -57,6 +58,7 @@ class Feeds(DBConnection): d = { 'channel_id': 'in', 'description': 'title', + 'interval': 'every', 'show_link': 'link' } if k in d: @@ -100,8 +102,9 @@ class Feeds(DBConnection): return log.debug('{} feeds to get for {}.'.format(len(feeds), channel.name)) + updates = {} all_entries = [] - now = time.time() + now = self.now() for feed in feeds: # Date can be trusted. date = bot.parse_boolean(feed['date']) if 'date' in feed else True @@ -109,6 +112,30 @@ class Feeds(DBConnection): summary = bot.parse_boolean(feed['summary']) if 'summary' in feed else False # Post link show_link = bot.parse_boolean(feed['show_link']) if 'show_link' in feed else True + # Post on schedule. + interval = feed['interval'] if 'interval' in feed else 0 + last_spoke = feed['last_spoke'] if 'last_spoke' in feed else None + updates[feed['id']] = { 'last_spoke': now } + if interval and last_spoke: + update = updates[feed['id']] + start = last_spoke + intervals = math.floor((now - start) / interval) + if intervals: + # We missed a schedule. + scheduled = start + intervals * interval + if now - scheduled > max(60, interval / 2): + log.info('Feed {} should have been posted at {}'.format(feed['id'], bot.iso8601(scheduled))) + update['last_spoke'] = scheduled + continue + # Post the feed now but set the original schedule. + last_spoke = start + (intervals + 1) * interval + if last_spoke < now: + update['last_spoke'] = last_spoke + log.info('Forcing last_spoke for feed {} to {} in line with schedule'.format(feed['id'], bot.iso8601(update['last_spoke']))) + dedupe = False + else: + dedupe = True + url = feed['url'] d = feedparser.parse(url) for entry in d.entries: @@ -117,63 +144,66 @@ class Feeds(DBConnection): else: order = now now += 0.001 - all_entries.append({ 'order': order, 'date': date, 'summary': summary, 'show_link': show_link, 'url': feed['url'], 'link_json': feed['link_json'], 'entry': entry }) + all_entries.append({ 'order': order, 'dedupe': dedupe, 'date': date, 'summary': summary, 'show_link': show_link, 'url': feed['url'], 'link_json': feed['link_json'], 'entry': entry }) if not len(all_entries): log.debug('No messages to post to {}'.format(channel.name)) - return - all_entries.sort(key = lambda entry: entry['order']) - - limit = len(all_entries) * 2 - if not limit: - limit = bot.get('message_limit') - - digests = [] - logs = yield from client.logs_from(channel, limit = limit) - for message in logs: - digest = bot.digest(message.content) - log.debug('Saw previously posted RSS with digest {}'.format(digest)) - if digest not in digests: - digests.append(digest) - - for feed in all_entries: - entry = feed['entry'] - formatted = '*{}*\n'.format(entry['published']) if feed['date'] else '' - link = None - if not feed['show_link']: - log.debug('Entry {} link is hidden.'.format(feed['url'])) - elif 'links' in entry: - link = entry['links'][0]['href'] - elif 'link_json' in feed: + else: + all_entries.sort(key = lambda entry: entry['order']) + + limit = min(len(all_entries) * 2, int(bot.get('message_limit') * 1.5)) + + digests = [] + logs = yield from client.logs_from(channel, limit = limit) + for message in logs: + digest = bot.digest(message.content) + log.debug('Saw previously posted RSS with digest {}'.format(digest)) + if digest not in digests: + digests.append(digest) + + for feed in all_entries: + entry = feed['entry'] + formatted = '*{}*\n'.format(entry['published']) if feed['date'] else '' + link = None + if not feed['show_link']: + log.debug('Entry {} link is hidden.'.format(feed['url'])) + elif 'links' in entry: + link = entry['links'][0]['href'] + elif 'link_json' in feed: + try: + l = json.loads(feed['link_json']) + args = [entry[k] for k in l[1:]] + link = l[0].format(*args) + except ValueError: + log.error('Invalid link_json in feed {}'.format(feed['url'])) + except: + log.exception('do_rss:link_json') + if feed['show_link'] and not link: + log.warning('No link for entry {}'.format(entry['title'])) + continue + formatted += '**{}**{}'.format(entry['title'], '\n{}'.format(link) if feed['show_link'] else '') + if feed['summary']: + formatted += '\n{}'.format(''.join(bs4.BeautifulSoup(entry['summary'], 'html.parser').findAll(text = True))) + formatted = formatted.strip() + digest = bot.digest(formatted) + if digest in digests: + if feed['dedupe']: + log.debug("Already posted RSS with digest {}".format(digest)) + continue try: - l = json.loads(feed['link_json']) - args = [entry[k] for k in l[1:]] - link = l[0].format(*args) - except ValueError: - log.error('Invalid link_json in feed {}'.format(feed['url'])) + if bot.get('dryrun'): + log.info('Dryrun: Not posting to {}: {}'.format(channel, formatted)) + else: + log.info('Posting to {}: {}'.format(channel, formatted)) + message = yield from bot.say(channel, formatted) + digest = bot.digest(message.content) + digests.append(digest) + log.debug('Posted RSS with digest {}'.format(digest)) except: - log.exception('do_rss:link_json') - if feed['show_link'] and not link: - log.warning('No link for entry {}'.format(entry['title'])) - continue - formatted += '**{}**{}'.format(entry['title'], '\n{}'.format(link) if feed['show_link'] else '') - if feed['summary']: - formatted += '\n{}'.format(''.join(bs4.BeautifulSoup(entry['summary'], 'html.parser').findAll(text = True))) - formatted = formatted.strip() - digest = bot.digest(formatted) - if digest in digests: - log.debug("Already posted RSS with digest {}".format(digest)) - continue - try: - if bot.get('dryrun'): - log.info('Dryrun: Not posting to {}: {}'.format(channel, formatted)) - else: - log.info('Posting to {}: {}'.format(channel, formatted)) - message = yield from bot.say(channel, formatted) - digest = bot.digest(message.content) - digests.append(digest) - log.debug('Posted RSS with digest {}'.format(digest)) - except: - log.exception("do_rss") + log.exception("do_rss") + + if not bot.get('dryrun'): + for id, update in updates.items(): + self.update_feed(client, id, **update) @asyncio.coroutine def on_ready(self): @@ -314,6 +344,8 @@ class Feeds(DBConnection): text += ' title "{}"'.format(feed['description']) text += ' url {}'.format(feed['url']) text += ' in <#{}>'.format(feed['channel_id']) + if feed['interval']: + text += ' every {}'.format(bot.unparse_seconds(int(feed['interval']))) text += ' date {}'.format(feed['date']) text += ' summary {}'.format(feed['summary']) text += ' link {}'.format(feed['show_link']) @@ -363,6 +395,17 @@ class Feeds(DBConnection): k = 'description' parsed[k] = param ok = True + elif arg == 'every': + seconds = bot.parse_seconds(param) + if seconds is not None: + parsed['interval'] = seconds + ok = True + else: + break + elif arg == 'once': + parsed['interval'] = 0 + i -= 1 + ok = True elif arg == 'in': k = 'channel_id' if param: @@ -559,6 +602,14 @@ class Feeds(DBConnection): '```url URL```', 'The URL to retrieve the feed in RSS format.', '', + '```every INTERVAL```', + 'Specify the interval to repeat the feed. You can include **d**ays, **h**ours, **m**inutes or **s**econds. `every 6h` means every six hours. `every 10m30s` means every ten minutes and thirty seconds.', + '', + '```', + 'once', + '```', + 'Equivalent to `interval 0` and means no reposting of a feed.', + '', '```date false```', "Use this if the dates returned by the RSS URL aren't valid. For instance the official GalNet feed always returns the date its cache was updated NOT the date the story was posted.", '',