diff options
| -rwxr-xr-x | data-sync | 218 |
1 files changed, 218 insertions, 0 deletions
diff --git a/data-sync b/data-sync new file mode 100755 index 0000000..a505a38 --- /dev/null +++ b/data-sync @@ -0,0 +1,218 @@ +#!/usr/bin/python + +import os +import sys +import getopt +import subprocess +import socket +import tempfile +import yaml + +config_path = os.path.expanduser("~")+"/.config/data-sync.yaml" + +text_red = '\033[31m' +text_blue = '\033[34m' +text_green = '\033[32m' +text_cyan = '\033[36m' +text_bold = '\033[1m' +text_reset = '\033[0m' + +class data_sync(): + hosts = [] + diff = False + merge = False + + # config + + def config_load(self): + s = open(config_path, "r") + y = yaml.load(s, Loader = yaml.SafeLoader) + s.close() + + self.hostname = socket.gethostname() + self.hosts = y["hosts"] + + # hosts + + def host_find(self, name): + for host in self.hosts: + if host["name"] == name: + return host + + return None + + def host_target_check(self, host, target): + if "targets" not in host: + return False + + for entry in host["targets"]: + if entry["label"] == target: + return True + + return False + + # excludes + + def excludes_prepare(self, entry): + if "excludes" not in entry: + return None + + fd, path = tempfile.mkstemp() + + s = os.fdopen(fd, 'w') + + for exclude in entry["excludes"]: + s.write(entry["path"]+exclude+"\n") + + return path + + def excludes_cleanup(self, excludes_path): + os.unlink(excludes_path) + + # rsync + + def rsync_address(self, host, path): + if host["name"] == self.hostname or ("location" in host and host["location"] == "local"): + return path + else: + address = "" + + if "user" in host: + address += host["user"]+"@" + + address += host["name"]+":"+path + + return address + + def rsync_command(self, source, source_path, sink, sink_path, excludes_path): + rsync_command = [ "rsync", "-a", "-s", "--progress" ] + + if not self.merge: + rsync_command += [ "--delete" ] + + if self.diff: + rsync_command += [ "--dry-run" ] + + if excludes_path: + rsync_command += [ "--exclude-from="+excludes_path ] + + rsync_command += [ self.rsync_address(source, source_path) ] + rsync_command += [ self.rsync_address(sink, sink_path) ] + + return rsync_command + + # sync + + def sync_target(self, source, sink, target): + print("Sync target "+target+" between "+source["name"]+" and "+sink["name"]) + + def sync_targets(self, source, sink, targets): + targets_list = [] + + for target in targets: + if self.host_target_check(source, target) and self.host_target_check(sink, target): + targets_list.append(target) + + if len(targets_list) == 0: + return 1 + + for target in targets_list: + self.sync_target(source, sink, target) + + return 0 + + def sync_data_entry(self, source, sink, sync_host, entry): + print("\nSync "+text_green+text_bold+entry["path"]+text_reset+" from "+text_cyan+source["name"]+text_reset+" to "+text_cyan+sink["name"]+text_reset+" started") + + source_path = os.path.join(source["base"], entry["path"]) + sink_path = os.path.join(sink["base"], sync_host["path"]) + "/" + + excludes_path = self.excludes_prepare(entry) + + command = self.rsync_command(source, source_path, sink, sink_path, excludes_path) + subprocess.call(command) + + if excludes_path: + self.excludes_cleanup(excludes_path) + + print("\nSync "+text_green+text_bold+entry["path"]+text_reset+" from "+text_cyan+source["name"]+text_reset+" to "+text_cyan+sink["name"]+text_reset+" finished") + + def sync_data(self, source, sink, targets): + found = False + + print("Sync data from "+text_cyan+text_bold+source["name"]+text_reset+" to "+text_cyan+text_bold+sink["name"]+text_reset) + + sync_host = None + + if "sync-hosts" not in sink: + return 1 + + for entry in sink["sync-hosts"]: + if entry["name"] == source["name"]: + sync_host = entry + + if sync_host is None: + print(sink["name"]+" is not a sync host for "+source["name"]) + + if "data" not in source: + return 1 + + for entry in source["data"]: + if len(targets) > 0 and entry["path"] not in targets: + continue + + self.sync_data_entry(source, sink, sync_host, entry) + found = True + + if not found: + return 1 + + return 0 + + def main(self): + global config_path + + opts, args = getopt.getopt(sys.argv[1:], "mdc:") + + for key, value in opts: + if key == "-m": + self.merge = True + elif key == "-d": + self.diff = True + elif key == "-c": + config_path = value + + if len(args) < 2: + return 1 + + source_name = args[0] + sink_name = args[1] + + targets = args[2:] + + self.config_load() + + source = self.host_find(source_name) + if source is None: + print("Unknown source host: "+source_name) + return 1 + + sink = self.host_find(sink_name) + if sink is None: + print("Unknown sink host: "+sink_name) + return 1 + + if len(targets) == 0: + return self.sync_data(source, sink, targets) + + ret = self.sync_targets(source, sink, targets) + if ret != 0: + ret = self.sync_data(source, sink, targets) + + if ret != 0: + print("No common target or data to sync") + + return ret + +if __name__ == "__main__": + data_sync().main() |