big update

* zone.list moved to zone.json
* entries are deleted from zone only after a grace period
* preparation for firewall support

.gitignore

@@ -1,4 +1,4 @@
+*.log
 home.conf
 ovh.conf
-zone.list
+zone.json
-*.log

dyndomain

@@ -1,10 +1,12 @@
 #!/usr/bin/python3
 
-import sys
-import os
 import configparser
-import subprocess
+import copy
+import json
+import os
 import smtplib
+import subprocess
+import sys
 import time
 from email.message import EmailMessage
 from pprint import pprint
@@ -19,8 +21,8 @@ from ovh import ovh
 
 def load_conf():
     global zone_filename, log_filename
-    global wan_hostname, zone_domain, zone_subdomain
+    global wan_hostname, zone_domain, zone_subdomain, zone_timeout
-    global hosts_list, hosts_ipv4_nat_list, hosts_alias_list
+    global hosts_list, nat_list, pinhole_list
     global mail_from, mail_to, mail_ignore_list
 
     conf = configparser.ConfigParser(allow_no_value=True)
@@ -38,8 +40,31 @@ def load_conf():
     mail_ignore_list = [ '.'.join([h, zone_subdomain]) if not h.endswith('.'+zone_subdomain) else h for h in mail_ignore_list ]
 
     hosts_list = [ host.lower() for host in conf['Hosts'] ]
-    hosts_ipv4_nat_list = [ host.lower() for host in conf['NatHosts'] ]
+
-    hosts_alias_list = conf['Aliases']
+    nat_list = {}
+    for e in conf.items('PortsNat'):
+        if e[1]:
+            nat_list[e[0]] = e[1].split(',')
+        else:
+            nat_list[e[0]] = []
+
+    pinhole_list = {}
+    for e in conf.items('Firewall'):
+        if e[1]:
+            pinhole_list[e[0]] = e[1].split(',')
+        else:
+            pinhole_list[e[0]] = []
+    zone_timeout = conf['Zone']['keep']
+    if zone_timeout[-1] == 's':
+        zone_timeout = int(zone_timeout[:-1])
+    elif zone_timeout[-1] == 'm':
+        zone_timeout = int(zone_timeout[:-1]) * 60
+    elif zone_timeout[-1] == 'h':
+        zone_timeout = int(zone_timeout[:-1]) * 3600
+    elif zone_timeout[-1] == 'd':
+        zone_timeout = int(zone_timeout[:-1]) * 86400
+    else:
+        zone_timeout = int(zone_timeout)
 
 
 def ping(hostname):
@@ -55,9 +80,14 @@ def get_ipv6_hosts():
     for h in r['status']:
         if not 'IPv6Address' in h or not h['IPv6Address']:
             continue
-        hostname = h['Name'].lower()
+        ns = {}
-        if hostname in hosts_alias_list:
+        for n in h['Names']:
-            hostname = hosts_alias_list[hostname]
+            ns[n['Source']] = n['Name']
+        hostname = h['Name']
+        hostname = ns.get('mdns', hostname)
+        hostname = ns.get('dhcp', hostname)
+        hostname = ns.get('webui', hostname)
+        hostname = hostname.lower()
         for a in h['IPv6Address']:
             if a['Scope'] != 'global' or a['Status'] != 'reachable':
                 continue
@@ -82,60 +112,101 @@ def full_name(host, domain):
     return '.'.join([host, domain])
 
 
-def make_zone_list(wan_hostname, wan_addr, hosts, hosts_nat, domain):
+def zone_add_entry(zone, prot, name, addr, stamp):
-    zone = []
+    #print('zone_add_entry: {} {} {} {}'.format(prot, name, addr, stamp))
+    if not zone.get(prot):
+        zone[prot] = {}
+    if not zone[prot].get(name):
+        zone[prot][name] = {}
+    if not zone[prot][name].get(addr):
+        zone[prot][name][addr] = { 'first': stamp }
+    zone[prot][name][addr]['last'] = stamp
+
+
+def populate_zone(zone, wan_hostname, wan_addr, hosts, hosts_list, hosts_nat, domain, stamp):
 
     wan_hostname = full_name(wan_hostname, domain)
 
+    if not zone.get('A'):
+        zone['A'] = {}
+    if not zone.get('AAAA'):
+        zone['AAAA'] = {}
+
     if wan_addr['ipv4'] != '':
-        zone.append([wan_hostname, 'A', wan_addr['ipv4']])
+        zone_add_entry(zone, 'A', wan_hostname, wan_addr['ipv4'], stamp)
         for host in hosts_nat:
-            zone.append([full_name(host, domain), 'A', wan_addr['ipv4']])
+            if hosts.get(host):
+                zone_add_entry(zone, 'A', full_name(host, domain), wan_addr['ipv4'], stamp)
 
     if wan_addr['ipv6'] != '':
-        zone.append([wan_hostname, 'AAAA', wan_addr['ipv6']])
+        zone_add_entry(zone, 'AAAA', wan_hostname, wan_addr['ipv6'], stamp)
-        for host in hosts:
+        for host in hosts_list:
-            for addr in hosts[host]:
+            for addr in hosts.get(host, []):
-                if host in hosts_list:
+                zone_add_entry(zone, 'AAAA', full_name(host, domain), addr, stamp)
-                    zone.append([full_name(host, domain), 'AAAA', addr])
 
+def process_zone(zone, stamp, grace_period):
+    update = { 'add': [], 'delete': [] }
+    for prot in zone:
+        for name in zone[prot]:
+            add = False
+            for addr in zone[prot][name]:
+                e = zone[prot][name][addr]
+                if e['first'] == stamp:
+                    update['add'].append([prot, name, addr])
+                    add = True
+            for addr in zone[prot][name]:
+                e = zone[prot][name][addr]
+                if add and e['last'] < stamp:
+                    update['delete'].append([prot, name, addr])
+                elif not add and stamp - e['last'] > grace_period:
+                    update['delete'].append([prot, name, addr])
+
+    if not update['add'] and not update['delete']:
+        update = None
+    return update
+
+
+def read_zone_list(zone_filename):
+    try:
+        with open(zone_filename) as jsonfile:
+            zone = json.load(jsonfile)
         return zone
+    except FileNotFoundError:
+        return {}
 
 
-def read_zone_list_from_file(zone_filename):
+def write_zone_list(zone_filename, zone, update):
-    zone_list = []
+    zone = copy.deepcopy(zone)
-    config = configparser.ConfigParser()
+    if update:
-    config.read(zone_filename)
+        for prot, name, addr in update['delete']:
-    for host in config:
+            del zone[prot][name][addr]
-        for p in config[host]:
+            if not zone[prot][name]:
-            for target in config[host][p].split('\n'):
+                del zone[prot][name]
-                zone_list.append([host.lower(), p.upper(), target])
+    with open(zone_filename, 'w') as jsonfile:
-    return zone_list
+        json.dump(zone, jsonfile, indent=2, sort_keys=True)
 
 
-def make_delete_zone_list(zone_list, prev_zone_list):
+#def make_delete_zone_list(zone_list, prev_zone_list):
-    return make_update_zone_list(prev_zone_list, zone_list)
+#    return make_update_zone_list(prev_zone_list, zone_list)
-
+#
-
+#
-def make_update_zone_list(zone_list, prev_zone_list):
+#def make_update_zone_list(zone_list, prev_zone_list):
-    update_zone_list = []
+#    update_zone_list = []
-    for entry in zone_list:
+#    for entry in zone_list:
-        if not any(entry == x for x in prev_zone_list):
+#        if not any(entry == x for x in prev_zone_list):
-            update_zone_list.append(entry)
+#            update_zone_list.append(entry)
-    return update_zone_list
+#    return update_zone_list
-
+#
-
+#
-def write_zone_list_to_file(zone_filename, zone_list):
+#    for host,typefield,target in zone_list:
-    config = configparser.ConfigParser()
+#        if not host in config:
-    for host,typefield,target in zone_list:
+#            config[host] = {}
-        if not host in config:
+#        if typefield in config[host]:
-            config[host] = {}
+#            config[host][typefield] += '\n'+target
-        if typefield in config[host]:
+#        else:
-            config[host][typefield] += '\n'+target
+#            config[host][typefield] = target
-        else:
+#    with open(zone_filename, 'w') as configfile:
-            config[host][typefield] = target
+#        config.write(configfile)
-    with open(zone_filename, 'w') as configfile:
-        config.write(configfile)
 
 
 def log(msg):
@@ -145,49 +216,38 @@ def log(msg):
             logfile.write("%s - %s\n" % (stamp, msg))
 
 
-def log_update_zone(zone_list, delete_zone_list):
+def log_update_zone(update):
-    for host,typea,addr in delete_zone_list:
+    op_str = { 'add': '', 'delete': '[DEL]' }
-        log("[DEL] %-20s %-6s %s" % (host, typea, addr))
+
-    for host,typea,addr in zone_list:
+    for op in update:
-        log("      %-20s %-6s %s" % (host, typea, addr))
+        for prot, name, addr in update[op]:
+            op = op_str.get(op, op)
+            log('%5s %-20s %-6s %s' % (op, name, prot, addr))
 
 
-def ovh_update_zone(domain, update_zone_list, delete_zone_list):
+def ovh_update_zone(domain, update):
-    if not len(update_zone_list) and not len(delete_zone_list):
-        return False
-
     try:
         client = ovh.Client()
 
-        for host, fieldtype, target in delete_zone_list:
+        for prot, name, addr in update['delete']:
             result = client.get('/domain/zone/%s/record' % domain,
-                                fieldType=fieldtype,
+                                fieldType=prot,
-                                subDomain=host)
+                                subDomain=name)
             for id in result:
                 r = client.get('/domain/zone/%s/record/%d' % (domain, id))
-                if r['fieldType'] == fieldtype and r['target'] == target:
+                if r['fieldType'] == prot and r['target'] == addr:
-                    #print("Delete entry for %s %s %s" % (host, fieldtype, target))
+                    print("OVH: delete entry for %s %s %s (#%s)" % (name, prot, addr, id))
                     client.delete('/domain/zone/%s/record/%d' % (domain, id))
 
-        for host, fieldtype, target in update_zone_list:
+        for prot, name, addr in update['add']:
-            result = client.get('/domain/zone/%s/record' % domain,
+            print("OVH: create entry for %s %s %s" % (name, prot, addr))
-                                fieldType=fieldtype,
-                                subDomain=host)
-            skip = False
-            for id in result:
-                r = client.get('/domain/zone/%s/record/%d' % (domain, id))
-                if r['fieldType'] == fieldtype and r['target'] == target:
-                    skip = True
-            if skip:
-                continue
-            #print("Create new entry for %s %s %s" % (host, fieldtype, target))
             client.post('/domain/zone/%s/record' % domain,
-                    fieldType=fieldtype,
+                    fieldType=prot,
-                    subDomain=host,
+                    subDomain=name,
-                    target=target,
+                    target=addr,
                     ttl=60)
 
-        #print("Refresh zone %s" % domain)
+        print("OVH: Refresh zone %s" % domain)
         client.post('/domain/zone/%s/refresh' % domain)
         return True
     except:
@@ -195,8 +255,18 @@ def ovh_update_zone(domain, update_zone_list, delete_zone_list):
         return False
 
 
-def send_update_mail(mail_to, mail_from, zone_domain, update_zone_list, delete_zone_list, mail_ignore_list, wan):
+def send_update_mail(mail_to, mail_from, zone_domain, update, mail_ignore_list, wan):
-    #print('Send email to %s' % mail_to)
+    okmail=False
+    for op in update:
+        for prot,name,addr in update[op]:
+            if not name in mail_ignore_list:
+                okmail = True
+                break
+
+    if not okmail:
+        return
+
+    print('Send email to %s' % mail_to)
     msg = EmailMessage()
 
     msg['Subject'] = "Livebox update in %s" % zone_domain
@@ -209,17 +279,13 @@ def send_update_mail(mail_to, mail_from, zone_domain, update_zone_list, delete_z
 
     txt = txt + "\nZone %s has been updated:\n" % zone_domain
 
-    filtered_update_list = [ h for h in update_zone_list if not h[0] in mail_ignore_list ]
+    for prot,name,addr in update['add']:
-    filtered_delete_list = [ h for h in delete_zone_list if not h[0] in mail_ignore_list ]
+        txt = txt + "  %-20s %-4s %s\n" % (name,prot,addr)
-
-    if len(filtered_update_list) or len(filtered_delete_list):
-        for host,tp,addr in update_zone_list:
-            txt = txt + "  %-20s %-4s %s\n" % (host,tp,addr)
 
     txt = txt + "\nRemoved entries:\n"
 
-        for host,tp,addr in delete_zone_list:
+    for prot,name,addr in update['delete']:
-            txt = txt + "  %-20s %-4s %s\n" % (host,tp,addr)
+        txt = txt + "  %-20s %-4s %s\n" % (name,prot,addr)
 
     txt = txt + '\n'
 
@@ -228,6 +294,7 @@ def send_update_mail(mail_to, mail_from, zone_domain, update_zone_list, delete_z
     s = smtplib.SMTP('localhost')
     s.send_message(msg)
     s.quit()
+    #print(txt)
 
 
 load_conf()
@@ -245,27 +312,15 @@ if not r:
 hosts = get_ipv6_hosts()
 wan = get_wan_addr()
 
-zone_list = make_zone_list(wan_hostname, wan, hosts, hosts_ipv4_nat_list, zone_subdomain)
+zone = read_zone_list(zone_filename)
-prev_zone_list = read_zone_list_from_file(zone_filename)
+stamp = int(time.time())
-delete_zone_list = make_delete_zone_list(zone_list, prev_zone_list)
+populate_zone(zone, wan_hostname, wan, hosts, hosts_list, nat_list, zone_subdomain, stamp)
-update_zone_list = make_update_zone_list(zone_list, prev_zone_list)
+update = process_zone(zone, stamp, zone_timeout)
-
+if update:
-#if update_zone_list or delete_zone_list:
+    log_update_zone(update)
-#    print('zone_list:')
-#    pprint(zone_list)
-#    print('prev_zone_list:')
-#    pprint(prev_zone_list)
-#    print('update_zone_list:')
-#    pprint(update_zone_list)
-#    print('delete_zone_list:')
-#    pprint(delete_zone_list)
-
-log_update_zone(update_zone_list, delete_zone_list)
-
-success = ovh_update_zone(zone_domain, update_zone_list, delete_zone_list)
-
-if success:
-    new_zone_list = write_zone_list_to_file(zone_filename, zone_list)
-    send_update_mail(mail_to, mail_from, zone_domain, update_zone_list, delete_zone_list, mail_ignore_list, wan)
 
+    success = ovh_update_zone(zone_domain, update)
+    if success:
+        send_update_mail(mail_to, mail_from, zone_domain, update, mail_ignore_list, wan)
 
+write_zone_list(zone_filename, zone, update)

home-sample.conf

@@ -2,7 +2,7 @@
 # edit it and save it as home.conf
 
 [Files]
-zonefile = zone.list
+zonefile = zone.json
 logfile = zone.log
 
 [Wan]
@@ -13,20 +13,22 @@ mypc1
 mypc2
 mypc3
 
-[Aliases]
+[PortsNat]
-mypc1-wifi = mypc1
+mypc2 = 22,443
+mypc3 = 10022
 
-[NatHosts]
+[Firewall]
-mypc2
+mypc2 = 22,443
-mypc4
+mypc3 = 10022
 
 [Zone]
 domain = example.com
 subdomain = home
+keep = 3d
 
 [Mail]
 from = me@example.com
 to = you@example.com
 
 [MailIgnore]
-mypc2.home
+mypc2