#!/usr/bin/python import os import sys import getopt import subprocess import socket import tempfile import yaml config_path = os.path.expanduser("~")+"/.config/data-sync.yaml" text_red = '\033[31m' text_blue = '\033[34m' text_green = '\033[32m' text_cyan = '\033[36m' text_bold = '\033[1m' text_reset = '\033[0m' class data_sync(): hosts = [] diff = False merge = False verbose = False # config def config_load(self): s = open(config_path, "r") y = yaml.load(s, Loader = yaml.SafeLoader) s.close() self.hostname = socket.gethostname() self.hosts = y["hosts"] # hosts def host_find(self, name): for host in self.hosts: if host["name"] == name: return host return None def host_target_check(self, host, target): if "targets" not in host: return False for entry in host["targets"]: if entry["label"] == target: return True return False # excludes def excludes_prepare(self, source_path, excludes): fd, path = tempfile.mkstemp() s = os.fdopen(fd, 'w') for exclude in excludes: s.write(source_path+exclude+"\n") return path def excludes_cleanup(self, excludes_path): os.unlink(excludes_path) # rsync def rsync_address(self, host, path): if host["name"] == self.hostname or ("location" in host and host["location"] == "local"): return path else: address = "" if "user" in host: address += host["user"]+"@" if "address" in host: address += host["address"] else: address += host["name"] address += ":" + path return address def rsync_command(self, source, source_path, sink, sink_path, excludes_path): rsync_command = [ "rsync", "-a", "-s", "--progress" ] if not self.merge: rsync_command += [ "--delete" ] if self.diff: rsync_command += [ "--dry-run" ] if excludes_path: rsync_command += [ "--exclude-from="+excludes_path ] rsync_command += [ self.rsync_address(source, source_path) ] rsync_command += [ self.rsync_address(sink, sink_path) ] if self.verbose: print("rsync command: " + str(rsync_command)) return rsync_command # sync def sync_target(self, source, sink, target): source_path = None sink_path = None print("Sync target "+text_green+text_bold+target+text_reset+" from "+text_cyan+text_bold+source["name"]+text_reset+" to "+text_cyan+text_bold+sink["name"]+text_reset) excludes_path = "" excludes = [] for entry in source["targets"]: if entry["label"] == target: source_path = os.path.join(source["base"], entry["path"]) excludes_path = entry["path"].split("/")[-1] # meh if "excludes" in entry: excludes += entry["excludes"] break for entry in sink["targets"]: if entry["label"] == target: sink_path = os.path.join(sink["base"], entry["path"]) if "excludes" in entry: excludes += entry["excludes"] break if not source_path or not sink_path: return 1 if len(excludes) > 1: excludes_path = self.excludes_prepare(excludes_path, excludes) else: excludes_path = None command = self.rsync_command(source, source_path, sink, sink_path, excludes_path) subprocess.call(command) if excludes_path: self.excludes_cleanup(excludes_path) def sync_targets(self, source, sink, targets): targets_list = [] for target in targets: if self.host_target_check(source, target) and self.host_target_check(sink, target): targets_list.append(target) if len(targets_list) == 0: return 1 for target in targets_list: self.sync_target(source, sink, target) return 0 def sync_data_entry(self, source, sink, sync_host, entry): print("\nSync "+text_green+text_bold+entry["path"]+text_reset+" from "+text_cyan+source["name"]+text_reset+" to "+text_cyan+sink["name"]+text_reset+" started") if "base" in entry: base = entry["base"] else: base = source["base"] source_path = os.path.join(base, entry["path"]) sink_path = os.path.join(sink["base"], sync_host["path"]) + "/" if "excludes" in entry: excludes_path = self.excludes_prepare(entry["path"], entry["excludes"]) else: excludes_path = None command = self.rsync_command(source, source_path, sink, sink_path, excludes_path) subprocess.call(command) if excludes_path: self.excludes_cleanup(excludes_path) print("\nSync "+text_green+text_bold+entry["path"]+text_reset+" from "+text_cyan+source["name"]+text_reset+" to "+text_cyan+sink["name"]+text_reset+" finished") def sync_data(self, source, sink, targets): found = False print("Sync data from "+text_cyan+text_bold+source["name"]+text_reset+" to "+text_cyan+text_bold+sink["name"]+text_reset) sync_host = None if "sync-hosts" not in sink: return 1 for entry in sink["sync-hosts"]: if entry["name"] == source["name"]: sync_host = entry if sync_host is None: print(sink["name"]+" is not a sync host for "+source["name"]) if "data" not in source: return 1 for entry in source["data"]: if len(targets) > 0 and entry["path"] not in targets: continue self.sync_data_entry(source, sink, sync_host, entry) found = True if not found: return 1 return 0 def main(self): global config_path opts, args = getopt.getopt(sys.argv[1:], "vmdc:") for key, value in opts: if key == "-m": self.merge = True elif key == "-v": self.verbose = True elif key == "-d": self.diff = True elif key == "-c": config_path = value if len(args) < 2: return 1 source_name = args[0] sink_name = args[1] targets = args[2:] self.config_load() source = self.host_find(source_name) if source is None: print("Unknown source host: "+source_name) return 1 sink = self.host_find(sink_name) if sink is None: print("Unknown sink host: "+sink_name) return 1 if len(targets) == 0: return self.sync_data(source, sink, targets) ret = self.sync_targets(source, sink, targets) if ret != 0: ret = self.sync_data(source, sink, targets) if ret != 0: print("No common target or data to sync") return ret if __name__ == "__main__": sys.exit(data_sync().main())