rdiff-manager: first iteration

rdiff-manager.py

+#! /usr/bin/python3
+
+# rdiff-manager - a tool to run more than one rdiff-backup process concurrently
+# and report their output in one hit
+
+configs_path = '/backups/conf'
+backups_base = '/backups'
+# number of jobs to run concurrently (0 for all)
+concurrent_num = 0
+duration_to_keep = '4W'
+
+from multiprocessing import Pool
+import subprocess, os, glob
+
+run_rdiff = lambda *args: subprocess.check_output(('/usr/bin/rdiff-backup',) + args, stderr=subprocess.STDOUT).decode('utf-8')
+
+class Host(object):
+    def __init__(self, config):
+        self.hostname = config['hostname']
+        self.duration_to_keep =  config['duration_to_keep']
+        self.run_backup_flags = config ['backup_flags']
+        self.destination = config['backup_destination']
+        self.include_file = config['include_file']
+
+    def test_server(self):
+        return run_rdiff('--test-server', '[email protected]%s::/' % self.hostname)
+
+    def test_directory(self):
+        try:
+            os.makedirs(self.destination)
+        except OSError as e:
+            if os.access(self.destination, os.F_OK):
+                pass
+            else:
+                raise e
+
+    def remove_old(self):
+        return run_rdiff('--remove-older-than', self.duration_to_keep, self.destination)
+
+    def run_backup(self):
+        return run_rdiff(*(self.run_backup_flags.split(' ') + [ '--include-globbing-filelist',
+            self.include_file, '[email protected]%s::/' % (self.hostname), self.destination ] ) )
+
+    def run_all(self):
+        result = ''
+        try:
+            # only add output if error occurs
+            self.test_server()
+            self.test_directory()
+            # always print output
+            result += self.remove_old()
+            result += self.run_backup()
+        except subprocess.CalledProcessError as e:
+            result += e.output.decode('utf-8')
+            result += 'Backup failed with error %d' % e.returncode
+        except OSError as e:
+            result += 'Backup failed: %s (%d)' % (e.strerror, e.errno)
+        return result
+
+def rdiff_backup(config):
+    target = Host(config)
+    return target.run_all()
+
+if __name__ == '__main__':
+    config_files = glob.iglob(configs_path + '/*.conf')
+    filenames = (os.path.basename(x) for x in config_files)
+    hostnames = (x.replace('.conf', '') for x in filenames)
+
+    hosts = []
+
+    # TODO: add hosts in a sensible order
+    # e.g. alternating slowest & fastest
+    for host in hostnames:
+        hosts.append( { 'hostname': host,
+            'include_file': '%s/%s.conf' % (configs_path, host),
+            'duration_to_keep': duration_to_keep,
+            'backup_flags': '--print-statistics --ssh-no-compression --exclude-sockets --exclude-device-files --exclude-fifos',
+            'backup_destination': '%s/%s' % (backups_base, host),
+            } )
+
+    if concurrent_num == 0:
+        concurrent_num = len(hosts)
+    pool = Pool(concurrent_num)
+
+    results = [ (host, pool.apply_async(rdiff_backup, (host, )) ) for host in hosts ]
+
+    for host, r in results:
+        print("Backup results for", host['hostname'])
+        print(r.get())
+        print('-' * 40)
+        print()