livebox now longer answer to ping6 on LAN :(

* zone.list moved to zone.json
* entries are deleted from zone only after a grace period
* preparation for firewall support

.gitignore

@@ -1,6 +1,4 @@
-cron.log
+*.log
 home.conf
 ovh.conf
-ovh.conf.old
+zone.json
-zone.list
-zone.log

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "sysbus"]
+	path = sysbus
+	url = https://github.com/rene-d/sysbus.git
+[submodule "ovh"]
+	path = ovh
+	url = https://github.com/ovh/python-ovh.git

README

@@ -143,6 +143,10 @@ Setup - Réseau local
 
         Chaque mise à jour génerera un mail qui sera envoyé à cette adresse.
 
+* configuration [MailIgnore]:
+
+	Si les mises à jour ne concernent que des entrées dans cette liste
+	le mail ne sera pas envoyé
 
 Utilisation
 -----------

dyndomain

@@ -1,25 +1,29 @@
 #!/usr/bin/python3
 
-import sys
-import os
 import configparser
-import urllib.request
+import copy
-import urllib.parse
+import json
-import socket
+import os
+import smtplib
 import subprocess
+import sys
 import time
+from email.message import EmailMessage
+from pprint import pprint
 
 rundir=os.path.realpath(os.path.dirname(sys.argv[0]))
 os.chdir(rundir)
 
+sys.path.append(os.path.join(rundir,'sysbus/src'))
+from sysbus import sysbus
 from ovh import ovh
 
 
 def load_conf():
     global zone_filename, log_filename
-    global wan_hostname
+    global wan_hostname, zone_domain, zone_subdomain, zone_timeout
-    global hosts6_list, hosts4_list
+    global hosts_list, nat_list, pinhole_list
-    global zone_domain, zone_subdomain
+    global mail_from, mail_to, mail_ignore_list
 
     conf = configparser.ConfigParser(allow_no_value=True)
     conf.read('home.conf')
@@ -27,119 +31,383 @@ def load_conf():
     zone_filename = conf['Files']['zonefile']
     log_filename = conf['Files']['logfile']
 
-    wan_hostname = conf['Wan']['hostname']
+    wan_hostname = conf['Wan']['hostname'].lower()
+    zone_domain = conf['Zone']['domain'].lower()
+    zone_subdomain = conf['Zone']['subdomain'].lower()
+    mail_from = conf['Mail']['from']
+    mail_to   = conf['Mail']['to']
+    mail_ignore_list = [ host.lower() for host in conf['MailIgnore'] ]
+    mail_ignore_list = [ '.'.join([h, zone_subdomain]) if not h.endswith('.'+zone_subdomain) else h for h in mail_ignore_list ]
 
-    hosts6_list = [ host for host in conf['Hosts'] ]
+    hosts_list = [ host.lower() for host in conf['Hosts'] ]
-    hosts4_list = [ host for host in conf['NatHosts'] ]
 
-    zone_domain = conf['Zone']['domain']
+    nat_list = {}
-    zone_subdomain = conf['Zone']['subdomain']
+    for e in conf.items('PortsNat'):
+        if e[1]:
+            nat_list[e[0]] = e[1].split(',')
+        else:
+            nat_list[e[0]] = []
+
+    pinhole_list = {}
+    for e in conf.items('Firewall'):
+        if e[1]:
+            pinhole_list[e[0]] = e[1].split(',')
+        else:
+            pinhole_list[e[0]] = []
+    zone_timeout = conf['Zone']['keep']
+    if zone_timeout[-1] == 's':
+        zone_timeout = int(zone_timeout[:-1])
+    elif zone_timeout[-1] == 'm':
+        zone_timeout = int(zone_timeout[:-1]) * 60
+    elif zone_timeout[-1] == 'h':
+        zone_timeout = int(zone_timeout[:-1]) * 3600
+    elif zone_timeout[-1] == 'd':
+        zone_timeout = int(zone_timeout[:-1]) * 86400
+    else:
+        zone_timeout = int(zone_timeout)
 
 
-def get_host_ipv6(name):
+def ping(hostname):
+    cmd = "ping -4 -c1 -w3 %s" % hostname
+    ret = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    return 0 if ret.returncode else 1
+
+
+def get_hosts():
+    r = sysbus.requete('Devices:get', silent=True)
+    hosts = { 'A': {}, 'AAAA': {} }
+    if not r:
+        return hosts
+
+    for h in r['status']:
+        ns = {}
+        for n in h['Names']:
+            ns[n['Source']] = n['Name']
+        hostname = h['Name']
+        hostname = ns.get('mdns', hostname)
+        hostname = ns.get('dhcp', hostname)
+        hostname = ns.get('webui', hostname)
+        hostname = hostname.lower()
+        for a in h.get('IPv6Address', []):
+            if a['Scope'] != 'global' or a['Status'] != 'reachable':
+                continue
+            if not hostname in hosts['AAAA']:
+                hosts['AAAA'][hostname] = []
+            hosts['AAAA'][hostname].append(a['Address'])
+        for a in h.get('IPv4Address', []):
+            if a['Scope'] != 'global' or a['Status'] != 'reachable':
+                continue
+            if not hostname in hosts['A']:
+                hosts['A'][hostname] = []
+            hosts['A'][hostname].append(a['Address'])
+    return hosts
+
+
+def get_wan_addr():
+    r = sysbus.requete('NMC:getWANStatus', silent=True)
+    wan = dict()
+    if not r or not r.get('data') or not r['data'].get('IPAddress') or not r['data'].get('IPv6Address'):
+        log('get_wan_addr: {}'.format(r))
+        return None
+    wan['ipv4'] = r['data']['IPAddress']
+    wan['ipv6'] = r['data']['IPv6Address']
+    if wan['ipv4'] == '0.0.0.0':
+        wan['ipv4'] = ''
+    return wan
+
+
+def full_name(host, domain):
+    host = host.replace('_', '-')
+    return '.'.join([host, domain])
+
+
+def zone_add_entry(zone, prot, key, value, stamp):
+    #print('zone_add_entry: {} {} {} {}'.format(prot, key, value, stamp))
+    if not zone.get(prot):
+        zone[prot] = {}
+    if not zone[prot].get(key):
+        zone[prot][key] = {}
+    if not zone[prot][key].get(value):
+        zone[prot][key][value] = { 'first': stamp }
+    zone[prot][key][value]['last'] = stamp
+
+
+def populate_zone(zone, wan_hostname, wan_addr, hosts, hosts_list, hosts_nat, pinhole_list, domain, stamp):
+    wan_hostname = full_name(wan_hostname, domain)
+
+    if not zone.get('A'):
+        zone['A'] = {}
+    if not zone.get('AAAA'):
+        zone['AAAA'] = {}
+
+    if wan_addr['ipv4'] != '':
+        zone_add_entry(zone, 'A', wan_hostname, wan_addr['ipv4'], stamp)
+        for host in hosts_nat:
+            if hosts['A'].get(host):
+                zone_add_entry(zone, 'A', full_name(host, domain), wan_addr['ipv4'], stamp)
+                # we can only add PortNat entry for on Address,
+                # so let's arbitrarely take the 1st one
+                for port in hosts_nat[host]:
+                    zone_add_entry(zone, 'nat', hosts['A'][host][0], port, stamp)
+
+    if wan_addr['ipv6'] != '':
+        zone_add_entry(zone, 'AAAA', wan_hostname, wan_addr['ipv6'], stamp)
+        for host in hosts_list:
+            for addr in hosts['AAAA'].get(host, []):
+                zone_add_entry(zone, 'AAAA', full_name(host, domain), addr, stamp)
+                for port in pinhole_list.get(host, []):
+                    zone_add_entry(zone, 'pin', addr, port, stamp)
+
+
+
+def process_zone(zone, stamp, grace_period, sync_zone):
+    update = { 'add': [], 'delete': [] }
+    for prot in zone:
+        for name in zone[prot]:
+            active = False
+            for addr in zone[prot][name]:
+                e = zone[prot][name][addr]
+                if e['first'] == stamp:
+                    update['add'].append([prot, name, addr])
+                if e['last'] == stamp:
+                    active = True
+            for addr in zone[prot][name]:
+                e = zone[prot][name][addr]
+                if active and e['last'] < stamp:
+                    update['delete'].append([prot, name, addr])
+                elif not active and stamp - e['last'] > grace_period:
+                    update['delete'].append([prot, name, addr])
+                elif sync_zone:
+                    update['add'].append([prot, name, addr])
+
+    if not update['add'] and not update['delete']:
+        update = None
+    return update
+
+
+def read_zone_list(zone_filename):
     try:
-        res = socket.getaddrinfo(name, None, socket.AF_INET6, socket.SOCK_STREAM)
+        with open(zone_filename) as jsonfile:
-    except socket.gaierror:
+            zone = json.load(jsonfile)
-        return []
+        return zone
-    ips = [ r[4][0] for r in res ]
-    return ips
-
-
-def get_wan_ipv4():
-    try:
-        f = urllib.request.urlopen('https://ipv4.wtfismyip.com/text')
-        return f.read().decode('utf-8').split('\n')[0]
-    except urllib.error.URLError as e:
-        print(e)
-        sys.exit(0)
-
-
-def get_wan_ipv6():
-    return get_host_ipv6(wan_hostname)[0]
-
-
-def ping_alive(ip):
-    cmd = ['fping6', '-q', '-B1', ip]
-    ret = subprocess.call(cmd)
-    if not ret:
-        return True
-    return False
-
-
-def make_zone_list(wan_ipv4, hosts4, hosts6, subdomain):
-    zone = []
-    for host in hosts4:
-        entry = [ host+'.'+subdomain, 'A', wan_ipv4 ]
-        zone.append(entry)
-    for host in hosts6:
-        for ip in get_host_ipv6(host):
-            if ping_alive(ip):
-                entry = [ host+'.'+subdomain, 'AAAA', ip ]
-                zone.append(entry)
-    zone.sort()
-    return zone
-
-
-def read_zone_list_from_file(zone_filename):
-    zone = []
-    try:
-        with open(zone_filename, 'r') as zf:
-            for line in zf:
-                line = line.rstrip('\n')
-                entry = line.split(' ')
-                if entry[1] not in ['A', 'AAAA']:
-                    continue
-                zone.append(entry)
     except FileNotFoundError:
-        pass
+        return {}
-    return zone
 
 
-def write_zone_list_to_file(zone_filename, zone_list):
+def write_zone_list(zone_filename, zone, update):
-    with open(zone_filename, 'w') as zf:
+    zone = copy.deepcopy(zone)
-        for entry in zone_list:
+    if update:
-            line = ' '.join(entry)+'\n'
+        for prot, name, addr in update['delete']:
-            zf.write(line)
+            del zone[prot][name][addr]
+            if not zone[prot][name]:
+                del zone[prot][name]
+    with open(zone_filename, 'w') as jsonfile:
+        json.dump(zone, jsonfile, indent=2, sort_keys=True)
 
-def ovh_update_zone(domain, prev_zone_list, zone_list):
+
-    if not len(zone_list) and not len(remove_zone_list):
+def log(msg):
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    if sys.stdout.isatty():
+        for line in msg.split('\n'):
+            print("%s - %s" % (stamp, msg))
+    else:
+        with open(log_filename, 'a') as logfile:
+            for line in msg.split('\n'):
+                logfile.write("%s - %s\n" % (stamp, msg))
+
+
+def log_update_zone(update):
+    op_str = { 'add': '', 'delete': '[DEL]' }
+
+    for op in update:
+        for prot, name, addr in update[op]:
+            op = op_str.get(op, op)
+            log('%5s %-20s %-6s %s' % (op, name, prot, addr))
+
+
+def ovh_update_zone(domain, zone, update, sync_zone):
+    try:
+        client = ovh.Client()
+
+        if sync_zone:
+            for prot in ['A', 'AAAA']:
+                for name in zone.get(prot, []):
+
+                    result = client.get('/domain/zone/{}/record'.format(domain),
+                                        fieldType=prot, subDomain=name)
+                    for id in result:
+                        log('OVH: delete entry #{} {} {}'.format(id, name, prot))
+                        client.delete('/domain/zone/{}/record/{}'.format(domain, id))
+
+        for prot, name, addr in update['delete']:
+            if not prot in ['A', 'AAAA']:
+                continue
+            result = client.get('/domain/zone/%s/record' % domain,
+                                fieldType=prot,
+                                subDomain=name)
+            for id in result:
+                r = client.get('/domain/zone/%s/record/%d' % (domain, id))
+                if r['fieldType'] == prot and r['target'] == addr:
+                    log("OVH: delete entry for %s %s %s (#%s)" % (name, prot, addr, id))
+                    client.delete('/domain/zone/%s/record/%d' % (domain, id))
+
+        for prot, name, addr in update['add']:
+            if not prot in ['A', 'AAAA']:
+                continue
+            log("OVH: create entry for %s %s %s" % (name, prot, addr))
+            client.post('/domain/zone/%s/record' % domain,
+                    fieldType=prot,
+                    subDomain=name,
+                    target=addr,
+                    ttl=60)
+
+        log("OVH: Refresh zone %s" % domain)
+        client.post('/domain/zone/%s/refresh' % domain)
+        return True
+    except:
+        log('OVH update error\n')
         return False
 
-    client = ovh.Client()
 
-    print("\ndate:", time.strftime("%c"))
+def livebox_rule_id(words):
-    for host, fieldtype, target in prev_zone_list:
+    id='dyndomain_{}'.format('_'.join(words))
-        result = client.get('/domain/zone/%s/record' % domain,
+    id = id.replace('.', '_')
-                            fieldType=fieldtype,
+    return id
-                            subDomain=host)
-        for id in result:
-            print("Deleting old entry for %s %s" % (host, fieldtype))
-            client.delete('/domain/zone/%s/record/%ld' % (domain, id))
 
 
-    for host, fieldtype, target in zone_list:
+def livebox_delete_port_nat(port, addr):
-        print("Create new entry for %s %s %s" % (host, fieldtype, target))
+    log('livebox: delete PortNat {} to {}'.format(port, addr))
-        client.post('/domain/zone/%s/record' % domain,
+    id = livebox_rule_id([port])
-                fieldType=fieldtype,
+    r = sysbus.requete('Firewall:deletePortForwarding', { 'id': id, 'origin': 'webui' }, silent=True)
-                subDomain=host,
-                target=target,
-                ttl=60)
 
-    print("Refresh zone %s" % domain)
+def livebox_delete_pinhole(port, addr):
-    client.post('/domain/zone/%s/refresh' % domain)
+    log('livebox: delete pinhole {} to {}'.format(port, addr))
-    return True
+    id = livebox_rule_id([port, addr])
+    r = sysbus.requete('Firewall:deletePinhole', { 'id': id, 'origin': 'webui' }, silent=True)
+
+def livebox_add_port_nat(port, addr):
+    log('livebox: add PortNat {} to {}'.format(port, addr))
+    id = livebox_rule_id([port])
+    a = {
+        'id': id,
+        'origin': 'webui',
+        'sourceInterface': 'data',
+        'destinationIPAddress': addr,
+        'protocol': '6,17',
+        'internalPort': port,
+        'enable': True,
+    }
+    r = sysbus.requete('Firewall:setPortForwarding', a, silent=True)
+
+def livebox_add_pinhole(port, addr):
+    log('livebox: add pinhole {} to {}'.format(port, addr))
+    id = livebox_rule_id([port, addr])
+    a = {
+        'id': id,
+        'origin': 'webui',
+        'sourceInterface': 'data',
+        'destinationPort': port,
+        'destinationIPAddress': addr,
+        'protocol': '6,17',
+        'ipversion': '6',
+        'enable': True,
+    }
+    r = sysbus.requete('Firewall:setPinhole', a, silent=True)
+
+def livebox_update_fw(zone, update, sync_zone):
+    for prot, addr, port in update['delete']:
+        if prot == 'nat':
+            livebox_delete_port_nat(port, addr)
+        elif prot == 'pin':
+            livebox_delete_pinhole(port, addr)
+    for prot, addr, port in update['add']:
+        if prot == 'nat':
+            livebox_add_port_nat(port, addr)
+        elif prot == 'pin':
+            livebox_add_pinhole(port, addr)
+
+
+
+def send_update_mail(mail_to, mail_from, zone_domain, update, mail_ignore_list, wan):
+    okmail=False
+    for op in update:
+        for prot,name,addr in update[op]:
+            if not name in mail_ignore_list:
+                okmail = True
+                break
+
+    if not okmail:
+        return
+
+    log('Send email to %s' % mail_to)
+    msg = EmailMessage()
+
+    msg['Subject'] = "Livebox update in %s" % zone_domain
+    msg['From'] = mail_from
+    msg['To' ] = mail_to
+
+    txt = "Livebox update\n\n"
+
+
+    if updated['add']:
+        txt = txt + "\nUpdated entries:\n"
+        for prot,name,addr in update['add']:
+            txt = txt + "  %-20s %-4s %s\n" % (name,prot,addr)
+
+    if update['delete']:
+        txt = txt + "\nRemoved entries:\n"
+        for prot,name,addr in update['delete']:
+            txt = txt + "  %-20s %-4s %s\n" % (name,prot,addr)
+
+    txt = txt + '\n'
+    txt = txt + "WAN IPv4 : %s\n" % wan['ipv4']
+    txt = txt + "WAN IPv6 : %s\n" % wan['ipv6']
+
+    msg.set_content(txt)
+
+    s = smtplib.SMTP('localhost')
+    s.send_message(msg)
+    s.quit()
 
 
 load_conf()
-hosts4_list.append(wan_hostname)
-hosts6_list.append(wan_hostname)
 
-wan_ipv4 = get_wan_ipv4()
+if not ping(wan_hostname):
-wan_ipv6 = get_wan_ipv6()
+    log("%s is down" % wan_hostname)
+    sys.exit(0)
 
-zone_list = make_zone_list(wan_ipv4, hosts4_list, hosts6_list, zone_subdomain)
+sysbus.load_conf()
-prev_zone_list = read_zone_list_from_file(zone_filename)
+try:
+    r = sysbus.auth(False)
+    if not r:
+        log('Error: cannot authenticate on livebox')
+        sys.exit(1)
+except Exception as e:
+    log('Error: %s'.format(e))
+    sys.exit(1)
 
-if zone_list != prev_zone_list:
+hosts = get_hosts()
-    success = ovh_update_zone(zone_domain, prev_zone_list, zone_list)
+wan = get_wan_addr()
-    write_zone_list_to_file(zone_filename, zone_list)
+if not wan:
+    sys.exit(0)
+
+zone = read_zone_list(zone_filename)
+sync_zone = False
+if not zone:
+    sync_zone = True
+
+stamp = int(time.time())
+populate_zone(zone, wan_hostname, wan, hosts, hosts_list, nat_list, pinhole_list, zone_subdomain, stamp)
+update = process_zone(zone, stamp, zone_timeout, sync_zone)
+if update:
+    log_update_zone(update)
+
+    success = ovh_update_zone(zone_domain, zone, update, sync_zone)
+    livebox_update_fw(zone, update, sync_zone)
+    if success:
+        try:
+            send_update_mail(mail_to, mail_from, zone_domain, update, mail_ignore_list, wan)
+        except:
+            pass
+
+write_zone_list(zone_filename, zone, update)

home-sample.conf

@@ -2,7 +2,7 @@
 # edit it and save it as home.conf
 
 [Files]
-zonefile = zone.list
+zonefile = zone.json
 logfile = zone.log
 
 [Wan]
@@ -13,15 +13,22 @@ mypc1
 mypc2
 mypc3
 
-[NatHosts]
+[PortsNat]
-mypc2
+mypc2 = 22,443
-mypc4
+mypc3 = 10022
+
+[Firewall]
+mypc2 = 22,443
+mypc3 = 10022
 
 [Zone]
 domain = example.com
 subdomain = home
+keep = 3d
 
 [Mail]
 from = me@example.com
 to = you@example.com
 
+[MailIgnore]
+mypc2

notes.txt

@@ -1,40 +0,0 @@
-get ipv4 wan
-------------
-
-$ curl -4 http://ifconfig.co
-90.116.135.67
-[please limits to 1 req/min]
-
-$ curl https://ipinfo.io/ip
-90.116.135.67
-
-$ curl https://ipv4.wtfismyip.com/text
-90.116.135.67
-[1 request per minute]
-
-
-get ipv6 wan
-------------
-
-$ host livebox.home
-livebox.home has address 192.168.1.1
-livebox.home has IPv6 address 2a01:cb1d:164:f500:327c:b2ff:fe9e:cf16
-
-get ipv6 host
--------------
-
-$ host arwen.home
-arwen.home has address 192.168.1.113
-arwen.home has IPv6 address 2a01:cb1d:164:f500:8ff:3eea:a3b2:714e
-
-$ host arwen.home
-arwen.home has address 192.168.1.113
-arwen.home has IPv6 address 2a01:cb1d:164:f500:3df0:c245:7d7a:26e2
-arwen.home has IPv6 address 2a01:cb1d:164:f500:8ff:3eea:a3b2:714e
-arwen.home has IPv6 address 2a01:cb1d:164:f500:28af:1644:6308:f13
-
-$ fping6 2a01:cb1d:164:f500:8ff:3eea:a3b2:714e 2a01:cb1d:164:f500:3df0:c245:7d7a:26e2 2a01:cb1d:164:f500:28af:1644:6308:f13
-2a01:cb1d:164:f500:8ff:3eea:a3b2:714e is alive
-2a01:cb1d:164:f500:3df0:c245:7d7a:26e2 is unreachable
-2a01:cb1d:164:f500:28af:1644:6308:f13 is unreachable
-

ovh

@@ -1 +1 @@
-Subproject commit 526c9e0fab750f40dde235a127befd88df5086e3
+Subproject commit 11d03357a58986a582803457238b33ad60342a4c

register_ovh_account

@@ -1,14 +1,19 @@
 #!/usr/bin/python3
 
+import sys
 from ovh import ovh
 
-domain = 'example.com'
+if len(sys.argv) != 2:
+    print('usage: {} <domain>'.format(sys.argv[0]))
+    sys.exit(1)
+
+domain = sys.argv[1]
 
 client = ovh.Client()
 
 ck = client.new_consumer_key_request()
 ck.add_rules(ovh.API_READ_WRITE_SAFE, '/domain/zone/%s/record' % domain)
-ck.add_rules(ovh.API_READ_WRITE,      '/domain/zone/%s/record/*' % domain)
+ck.add_rules(ovh.API_READ_WRITE_SAFE, '/domain/zone/%s/record/*' % domain)
 ck.add_rules(ovh.API_READ_WRITE_SAFE, '/domain/zone/%s/refresh' % domain)
 
 validation = ck.request()

sysbus

@@ -0,0 +1 @@
+Subproject commit 87040a0906df80ade21ea863452838a8a574bdbd