Merge pull request #1227 from garybuhrmaster/blocklist

Example blocklist application

etc/exabgp/api-blocklist.conf

@@ -0,0 +1,45 @@
+
+process blocklist-10.0.0.2 {
+    run ./run/api-blocklist.run;
+    encoder text;
+}
+
+process blocklist-10.0.0.3 {
+    run ./run/api-blocklist.run;
+    encoder text;
+}
+
+template {
+    neighbor blocklist {
+        local-as 64512;
+        peer-as 64512;
+        router-id 10.0.0.17;
+        local-address 10.0.0.17;
+        group-updates true;
+        hold-time 180;
+        capability {
+            graceful-restart 1200;
+            route-refresh enable;
+            operational enable;
+        }
+        family {
+            ipv4 unicast;
+            ipv6 unicast;
+        }
+    }
+}
+
+neighbor 10.0.0.2 {
+    inherit blocklist;
+    api {
+        processes [ blocklist-10.0.0.2 ];
+    }
+}
+
+neighbor 10.0.0.3 {
+    inherit blocklist;
+    api {
+        processes [ blocklist-10.0.0.3 ];
+    }
+}
+

etc/exabgp/run/api-blocklist.run

@@ -0,0 +1,227 @@
+#!/usr/bin/python3
+# encoding utf-8
+
+"""
+"""
+
+import os
+import sys
+import errno
+import threading
+import time
+import json
+import ipaddress
+import traceback
+import requests
+import requests_file
+import random
+import urllib.parse
+
+#
+# Adjustable values.
+#
+#    The next-hop addresses are typically null routed in the router along with uRPF
+#       (the next-hop may also be set in the router route-map (belt and suspenders))
+#    The community 65535:666 can be used for additional matching checks
+#       (no-advertise may also be set in the router route-map (belt and suspenders))
+#
+
+delay        = 600
+specs4       = ' next-hop 192.0.2.1 community [65535:666 no-advertise]'
+specs6       = ' next-hop 100::1 community [65535:666 no-advertise]'
+
+#
+# Blocklists mostly currated from:
+#    https://docs.danami.com/juggernaut/user-guide/ip-block-lists
+#
+# The blocklist lines supported by this script consist of an ip address,
+#    an optional addr mask, and various end of data markers (space, ';', '#').
+#
+# If one has a local source of bad IP's in a file, one can use a
+#    url of the form 'file:///var/tmp/badips.txt'
+#
+
+blocklists = [
+                { 'url': 'https://www.spamhaus.org/drop/drop.txt', 'refresh': 7200 },
+                { 'url': 'https://www.spamhaus.org/drop/edrop.txt', 'refresh': 7200 },
+                { 'url': 'https://www.spamhaus.org/drop/dropv6.txt', 'refresh': 7200 },
+                { 'url': 'https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt', 'refresh': 7200 },
+                { 'url': 'https://blocklist.greensnow.co/greensnow.txt', 'refresh': 7200 },
+                { 'url': 'https://www.darklist.de/raw.php', 'refresh': 7200 },
+                { 'url': 'https://sigs.interserver.net/ipslim.txt', 'refresh': 7200 },
+                { 'url': 'https://api.blocklist.de/getlast.php?time=3600', 'refresh': 3600 }
+             ]
+
+def requestsGet(url):
+    r_session = requests.session()
+    r_session.mount('file://', requests_file.FileAdapter())
+    r = r_session.get(url, stream=True)
+    r.raise_for_status()
+    return r
+
+def lineFilter(line):
+    if not line:
+        return None
+    l = line.strip()
+    if l.startswith(';'):
+        return None
+    if l.startswith('#'):
+        return None
+    return (l.split(' ')[0].split(';')[0].split('#')[0].strip())
+
+class blocklistThread(object):
+
+    def __init__(self, url=None, refresh=86400):
+        try:
+             refresh = int(refresh)
+        except ValueError:
+             raise ValueError('{} is not a valid refresh time interval'.format(refresh))
+        if refresh < 60:
+             raise ValueError('{} is not a valid refresh interval of at least 60 seconds'.format(refresh))
+        try:
+            result = urllib.parse.urlparse(url)
+        except ValueError:
+             raise ValueError('{} is not a valid url'.format(url))
+        if not all ([result.scheme, result.netloc]):
+             raise ValueError('{} is not a valid url'.format(url))
+        self._prefixes = []
+        self._valid = False
+        self._url = url
+        self._refresh = refresh
+        thread = threading.Thread(target=self.run, args=())
+        thread.daemon = True
+        thread.start()
+
+    def getUrl(self):
+        return self._url
+
+    def getRefresh(self):
+        return self._refresh
+
+    def getPrefixes(self):
+        # Give our thread a chance to get data once
+        count = 0
+        while ((not self._valid) and (count < 12)):
+            count = count + 1
+            time.sleep(5)
+        self._valid = True
+        return self._prefixes
+
+    def run(self):
+        backoff = 0
+        while True:
+            newPrefixesList = []
+            refresh = self._refresh
+            try:
+                r = requestsGet(self._url)
+            except Exception:
+                traceback.print_exc(file=sys.stderr)
+                sys.stderr.flush()
+                backoff = backoff + 1
+                refresh = min(self._refresh, backoff * 600)
+            else:
+                backoff = 0
+                for line in r.iter_lines():
+                    try:
+                        linePrefix = lineFilter(line.decode('utf-8'))
+                        if linePrefix:
+                            netPrefix = ipaddress.ip_network(linePrefix, strict=False)
+                            newPrefixesList.append(netPrefix.compressed)
+                    except Exception:
+                        traceback.print_exc(file=sys.stderr)
+                        sys.stderr.flush()
+            self._prefixes = newPrefixesList.copy()
+            self._valid = True
+            newPrefixesList = None
+            r = None
+            # We add in a little jitter to assist source site load
+            time.sleep(refresh + random.randint(-300, 300))
+
+class responseThread(object):
+
+    def __init__(self):
+        thread = threading.Thread(target=self.run, args=())
+        thread.daemon = True
+        thread.start()
+
+    def run(self):
+        while True:
+            try:
+                line = sys.stdin.readline().strip()
+            except KeyboardInterrupt:
+                pass
+            except IOError as e:
+                if e.errno == errno.EPIPE:
+                    sys.stderr.write('broken pipe, terminating process.\n')
+                    sys.stderr.flush()
+                    os._exit(1)
+                else:
+                    sys.stderr.write('error {} reading from stdin.\n'.format(e.errno))
+                    sys.stderr.flush()
+            else:
+                if (line == 'shutdown'):
+                    sys.stderr.write('shutdown request received, terminating process.\n')
+                    sys.stderr.flush()
+                    os._exit(1)
+                if (line != 'done'):
+                    sys.stderr.write('unexpected response {} received.\n'.format(line))
+                    sys.stderr.flush()
+
+#
+# Start at the start
+#
+
+if __name__ == '__main__':
+
+    # Start our blocklist retrival threads
+
+    blocklistThreads = []
+
+    for bl in blocklists:
+        if bl['url'] is None or bl['refresh'] is None:
+            pass
+        blocklistThreads.append(blocklistThread(bl['url'], bl['refresh']))
+
+    # Start our exabgp response thread
+
+    rt = responseThread()
+
+    #
+    # Process the blocklist prefixes returned from the threads
+    #
+
+    currentBlocklist = dict()
+
+    while True:
+
+        newBlocklist = dict()
+        for blt in blocklistThreads:
+            for prefix in blt.getPrefixes():
+                newBlocklist[prefix] = None
+
+        for prefix in currentBlocklist:
+            if not prefix in newBlocklist:
+                specs = specs4
+                if ipaddress.ip_network(prefix).version == 6:
+                    specs = specs6
+                sys.stdout.write('withdraw route ' + str(prefix) + specs + '\n')
+                sys.stdout.flush()
+
+        for prefix in newBlocklist:
+            if not prefix in currentBlocklist:
+                specs = specs4
+                if ipaddress.ip_network(prefix).version == 6:
+                    specs = specs6
+                sys.stdout.write('announce route ' + str(prefix) + specs + '\n')
+                sys.stdout.flush()
+
+        currentBlocklist = newBlocklist.copy()
+        newBlocklist = None
+
+        try:
+            time.sleep(delay)
+        except KeyboardInterrupt:
+            sys.stderr.write('\nshutting down due to user request\n')
+            sys.stderr.flush()
+            os._exit(1) 
+