#!/usr/bin/env python3 # -*- coding: utf-8 -*- # BAREOS - Backup Archiving REcovery Open Sourced # # Copyright (C) 2014-2024 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public # License as published by the Free Software Foundation, which is # listed in the file LICENSE. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. # # Author: Stephan Duehr """ Python program to generate configuration files for Bareos VMware plugin jobs """ import argparse import atexit import getpass import sys import logging import os import re import hashlib import pprint from collections import defaultdict from enum import Enum import jinja2.exceptions from jinja2 import Template, StrictUndefined from pyVim.connect import SmartConnect, Disconnect from pyVmomi import vim from pyVmomi import vmodl import bareos.bsock import bareos.exceptions pp = pprint.PrettyPrinter(indent=4) slashes_rex = re.compile(r"\/+") logger = logging.getLogger(os.path.basename(sys.argv[0])) def get_args(): """ Supports the command-line arguments listed below. """ parser = argparse.ArgumentParser( description="Process args to generate configuration files for Bareos VMware plugin jobs" ) # global args parser.add_argument("-s", "--host", required=True, help="Remote host to connect to") parser.add_argument( "-o", "--port", type=int, default=443, help="Port to connect on" ) parser.add_argument( "-u", "--user", required=True, help="User name to use when connecting to vCenter", ) parser.add_argument( "-p", "--password", required=False, help="Password to use when connecting to vCenter", ) parser.add_argument( "--sslverify", action="store_true", default=False, help="Force SSL certificate verification", ) parser.add_argument( "--config-path", required=False, default="/etc/bareos", help="Path to Bareos config, default: /etc/bareos", ) log_group = parser.add_mutually_exclusive_group() log_group.add_argument( "-v", "--verbose", action="store_true", default=False, help="Verbose output", ) log_group.add_argument( "--debug", action="store_true", default=False, help="Debug output", ) parser.add_argument( "--folder", required=False, default="/", help="VM Folder, must start with /, default: /", ) parser.add_argument("-d", "--datacenter", required=True, help="DataCenter Name") parser.add_argument( "--list-all-vms", action="store_true", default=False, help="List all VMs in the given datacenter with UUID and containing folder", ) parser.add_argument( "--template-path", required=False, default="/usr/local/etc/bareos-vmware-tool/templates", help="Path to directory with template files", ) subparsers = parser.add_subparsers(dest="subcommand") # subcommand genconfig args genconfig_parser = subparsers.add_parser("genconfig") genconfig_parser.add_argument( "--exclude-attr-name", required=False, help="Attribute Name to check for exclude", ) genconfig_parser.add_argument( "--exclude-attr-value", required=False, help="Attribute Value to check for exclude", ) genconfig_parser.add_argument( "--module-path", required=False, default=None, help="Bareos Python plugin module path, default: None", ) genconfig_parser.add_argument( "--jobdefs", required=False, default="DefaultJob", help="Bareos JobDefs that generated job configs will refer to, default: DefaultJob", ) # subcommand schedule args schedule_parser = subparsers.add_parser("schedule") schedule_parser.add_argument( "--client-template", required=True, help="Template for ESX specific client config", ) args = parser.parse_args() if not args.subcommand: parser.print_help() sys.exit(1) if args.subcommand == "genconfig": if [args.exclude_attr_name, args.exclude_attr_value].count(None) not in [0, 2]: parser.error( "Requires either both --exclude-attr-name and --exclude-attr-value or none." ) if args.folder and args.folder != Util.proper_path(args.folder): logger.warning( "Improper folder %s, adjusting to %s", args.folder, Util.proper_path(args.folder), ) args.folder = Util.proper_path(args.folder) return args def main(): """ Python program to generate configuration files for Bareos VMware plugin jobs """ setup_logging(verbose=False, debug=False) args = get_args() setup_logging(verbose=args.verbose, debug=args.debug) if args.password: password = args.password else: password = getpass.getpass( prompt=f"Enter password for host {args.host} and user {args.user}: " ) args.password = password vapi = VMwareApi() vapi.connect_vmware(args) dcftree = {} vapi.get_dc_foldertree(dcftree) if args.list_all_vms: print_dcftree(dcftree) sys.exit(0) if args.subcommand == "genconfig": generate_config_files(args, dcftree) if args.subcommand == "schedule": bareos_config = BareosConfig() bareos_config.get_vmware_jobs() print("=== VMware Jobs ===") # pp.pprint(bareos_config.vmware_jobs) schedule_jobs(vapi, bareos_config) def setup_logging(*, verbose, debug): """ Setup logging """ logging.basicConfig( format="%(levelname)s: %(name)s: %(message)s", stream=sys.stderr ) # As %(name)s is "root" the output is useless and confusing, better omit it: # logging.basicConfig(format="%(levelname)s: %(message)s", stream=sys.stderr) if verbose: logger.setLevel(logging.INFO) if debug: logger.setLevel(logging.DEBUG) def find_vm_folder(si, dc, folder_path): """ Find a folder for a given folder path """ current_folder = dc.vmFolder folder_path = Util.proper_path(folder_path) if folder_path == "/": return current_folder folder_path_parts = folder_path.split("/")[1:] current_level = 0 max_level = len(folder_path_parts) - 1 while True: for folder in get_subfolders(si, current_folder): if folder.name == folder_path_parts[current_level]: if current_level == max_level: return folder current_level += 1 current_folder = folder break else: break return False def get_subfolders(si, folder): """ Get subfolders of a given folder """ folder_view = si.content.viewManager.CreateContainerView( folder, type=[vim.Folder], recursive=False ) folders = folder_view.view folder_view.Destroy() return folders def generate_config_files(args, dcftree): """ Check if VMs are specified by UUID or folder + VM names and return list of VMs to work with """ vm_list = [] config_generator = BareosGenVMwarePluginConfig( config_path=args.config_path, jobdefs=args.jobdefs, vcserver=args.host, vcuser=args.user, vcpass=args.password, module_path=args.module_path, template_path=args.template_path, ) if config_generator.generate_common_ini() is ConfigFileState.ERROR: sys.exit(1) for dc in dcftree: for vm_path in dcftree[dc]: vm = dcftree[dc][vm_path] if is_excluded_by_custom_attribute(dcftree[dc][vm_path], args): continue if os.path.dirname(vm_path).startswith(args.folder): vm_list.append(dcftree[dc][vm_path]) logger.info("VM: dc: %s, vm_path: %s", dc, vm_path) try: vm_path.encode("ascii") except UnicodeEncodeError: logger.warning( ( "VM %s contains non-ascii characters, this is currently " " invalid, skipping" ), repr(vm_path), ) continue if not config_generator.generate_vm_backup_config(vm, dc, vm_path): sys.exit(1) if not vm_list: logger.error("No VMs found, probably all excluded or empty folder.") sys.exit(1) return vm_list def is_excluded_by_custom_attribute(vm, args): """ Check for exclude by custom attribute """ if args.exclude_attr_name and args.exclude_attr_value: if get_custom_value(vm, args.exclude_attr_name) == args.exclude_attr_value: logger.info("Excluding VM %s (exclude attribute match)", vm.name) return True return False def get_custom_value(vm, attr_name): """ Get custom value for given custum attribue name """ attr_key = None attr_value = None for available_field in vm.availableField: if available_field.name == attr_name: attr_key = available_field.key break if not attr_key: logger.warning( "VM %s does not have custom attribute %s, ignoring", vm.name, attr_name ) return "" for custom_field in vm.customValue: if custom_field.key == attr_key: attr_value = custom_field.value break if not attr_key: logger.warning( "VM %s could not find custom value for key %s, ignoring", vm.name, attr_key ) return "" return attr_value def print_dcftree(dcftree): """ Print the Datacenter Folder Tree """ for dc in dcftree: dcftree_by_folder = defaultdict(dict) for vm_path in dcftree[dc]: dcftree_by_folder[os.path.dirname(vm_path)][os.path.basename(vm_path)] = ( dcftree[dc][vm_path].config.instanceUuid ) print(f"DataCenter: {dc}") print(f"{'VM-Instance-UUID':37} {'VM-Name':50} VM-Folder and/or vApp") for vm_folder in sorted(dcftree_by_folder): for vm_name in sorted(dcftree_by_folder[vm_folder]): print( f"{dcftree_by_folder[vm_folder][vm_name]:37} {vm_name:50} {vm_folder}" ) def schedule_jobs(vapi, bareos_config): """ Schedule VMware jobs by ESX host """ jobs_to_run = {} config_generator = BareosGenVMwarePluginConfig( config_path=vapi.args.config_path, jobdefs=None, vcserver=vapi.args.host, vcuser=vapi.args.user, vcpass=vapi.args.password, template_path=vapi.args.template_path, ) vmware_jobs = bareos_config.vmware_jobs for vmware_job in vmware_jobs: if vmware_jobs[vmware_job]["plugin_options"]["folder"] == vapi.args.folder: logger.info("Scheduling Job %s", vmware_job) logger.debug(vmware_jobs[vmware_job]) plugin_options = vmware_jobs[vmware_job]["plugin_options"] vm = vapi.get_vm( dc=plugin_options["dc"], folder=plugin_options["folder"], vmname=plugin_options["vmname"], ) if vm is None: logger.error( "Skipping Job %s, DC %s folder %s VM %s not found", vmware_job, plugin_options["dc"], plugin_options["folder"], plugin_options["vmname"], ) continue logger.info("VM %s is on host %s", vm.name, vm.runtime.host.name) config_generator.generate_esx_host_client_config( esxhost=vm.runtime.host.name, template=vapi.args.client_template ) jobs_to_run[vmware_job] = {"client": "esx-" + vm.runtime.host.name} if ConfigFileState.ERROR in config_generator.esx_client_config.values(): logger.fatal("Error occured during client config generation, aborting.") sys.exit(1) if ConfigFileState.CHANGED in config_generator.esx_client_config.values(): logger.info("Client config changed, reloading Bareos config") if not bareos_config.reload(): sys.exit(1) all_job_run_success = True for job_name, job_data in jobs_to_run.items(): logger.info("enqueuing job %s on client %s", job_name, job_data["client"]) if not bareos_config.run_job(job=job_name, client=job_data["client"]): all_job_run_success = False return all_job_run_success class ConfigFileState(Enum): """ Result of generating config files can have three states: - already exists and won't change - created or changed - error on write or template processing """ UNCHANGED = 1 CHANGED = 2 ERROR = 3 class BareosGenVMwarePluginConfig: """ A Class that generates Bareos config files for VMWare plugin jobs """ def __init__( self, config_path="/etc/bareos", jobdefs="DefaultJob", vcserver=None, vcuser=None, vcpass=None, module_path=None, template_path=None, ): self.config_path = config_path self.client_config_path = Util.proper_path(f"{config_path}/bareos-dir.d/client") self.jobdefs = jobdefs self.vcserver = vcserver self.vcuser = vcuser self.vcpass = vcpass self.module_path = module_path self.template_path = template_path self.fileset_path = None self.job_path = None self.common_ini_name = None self.common_ini_template = "common.ini.j2" self.fileset_template = "fileset.conf.j2" self.job_template = "job.conf.j2" self.esx_client_config = {} Util.mkdir(self.config_path) Util.mkdir(self.client_config_path) def _set_common_ini_name(self): config_path = self.config_path + "/vmware.d" vc_user = self.vcuser.replace("@", "_at_") self.common_ini_name = f"{config_path}/VMware-{self.vcserver}-{vc_user}.ini" def generate_common_ini(self): """ Generate common ini file """ self._set_common_ini_name() Util.mkdir(self.config_path + "/vmware.d") template_vars = { "vcserver": self.vcserver, "vcuser": self.vcuser, "vcpass": self.vcpass, } return self._template_to_config( template=self.common_ini_template, target=self.common_ini_name, data=template_vars, ) def generate_vm_backup_config(self, vm, dc, vm_path): """ Generate Bareos config for backup of the given VM """ self._create_dir(dc, vm_path) fileset_name = self._generate_fileset(vm, dc, vm_path) if not fileset_name: return False if self._generate_job(dc, vm_path, fileset_name) is ConfigFileState.ERROR: return False return True def _generate_fileset(self, vm, dc, vm_folder_path): """ Generate Bareos FileSet config for a given VM """ fileset_name = ( dc + "_" + vm_folder_path.lstrip("/").replace("/", ".") + "_FileSet" ) fileset_config_filename = self.fileset_path + "/" + fileset_name + ".conf" logger.info("Creating %s", fileset_config_filename) template_vars = { "fileset_name": fileset_name, "module_path": self.module_path, "compression_type": None, "common_ini_name": self.common_ini_name, "dc": dc, "folder": os.path.dirname(vm_folder_path), "vm_name": vm.name, } if ( self._template_to_config( template=self.fileset_template, target=fileset_config_filename, data=template_vars, ) is not ConfigFileState.ERROR ): return fileset_name return False def _generate_job(self, dc, vm_folder_path, fileset_name): """ Generate Bareos Job config for a given VM """ job_name = dc + "_" + vm_folder_path.lstrip("/").replace("/", ".") + "_Job" job_config_filename = self.job_path + "/" + job_name + ".conf" logger.info("Creating %s", job_config_filename) template_vars = { "job_name": job_name, "jobdefs": self.jobdefs, "fileset": fileset_name, } return self._template_to_config( template=self.job_template, target=job_config_filename, data=template_vars ) def generate_esx_host_client_config(self, esxhost=None, template=None): """ Generate per ESX host client config if not yet exists """ if esxhost in self.esx_client_config: # already seen or created return True client_config_filename = f"{self.client_config_path}/esx-{esxhost}.conf" template_vars = {"esx_host_name": esxhost} self.esx_client_config[esxhost] = self._template_to_config( template=template, target=client_config_filename, data=template_vars ) return True def _template_to_config(self, template, target, data): """ Write config file from template """ template_filename = f"{self.template_path}/{template}" try: with open(template_filename, "r", encoding="utf-8") as template_fh: template = Template( template_fh.read(), undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, ) except OSError as exc: logger.error("Could not open template file: %s", exc) return ConfigFileState.ERROR try: rendered_template = template.render(data) except jinja2.exceptions.UndefinedError as exc: logger.error("Bad template %s: %s", template_filename, exc) return ConfigFileState.ERROR if not Util.needs_write(rendered_template, target): logger.debug("No change of file %s", target) return ConfigFileState.UNCHANGED try: with open(target, "w", encoding="utf-8") as config_fh: config_fh.write(rendered_template) logger.info("Created or changed %s", target) except OSError as exc: logger.error("Could not write config file: %s", exc) return ConfigFileState.ERROR return ConfigFileState.CHANGED def _create_dir(self, dc, vm_path): """ Create directory based on DC name and VM path """ folder_path = os.path.dirname(vm_path).lstrip("/").replace("/", ".") dir_name = f"{self.config_path}/bareos-dir.d/{dc}_{folder_path}" Util.mkdir(dir_name) # This is both the same now, but like this it's easier to change self.fileset_path = dir_name self.job_path = dir_name class BareosConfig: """ Class retrieve items from the Bareos config using bareos.bsock """ def __init__(self): self.directorconsole = None self.vmware_job_module_name = "bareos-fd-vmware" self.vmware_jobs = {} self.configs_by_config_file = {} self.plugin_options = None self._connect() def _connect(self): bconsole_credentials = self._get_bconsole_credentials() self.directorconsole = bareos.bsock.DirectorConsoleJson( address="localhost", port=9101, name=bconsole_credentials["name"], password=bconsole_credentials["password"], ) def _get_bconsole_credentials( self, config_file="/etc/bareos/bareos-dir.d/console/vmwbackup.conf" ): password_rex = re.compile(r"Password\s*=\s*(.+)", re.IGNORECASE) name_rex = re.compile(r"Name\s*=\s*(.+)", re.IGNORECASE) with open(config_file, "r", encoding="utf-8") as f: lines = f.readlines() for line in lines: m = password_rex.search(line) if m: plain_password = m.group(1).strip('"') break for line in lines: m = name_rex.search(line) if m: console_name = m.group(1).strip('"') break return {"password": bareos.bsock.Password(plain_password), "name": console_name} def get_vmware_jobs(self): """ Get information about VMware plugin jobs from Bareos config """ jobs_by_name = self.directorconsole.call("show jobs")["jobs"] filesets_by_name = self.directorconsole.call("show fileset")["filesets"] # pp.pprint(filesets_by_name) # pp.pprint(jobs_by_name) for job_name in jobs_by_name: logger.info("Getting config for job: %s", job_name) fileset = jobs_by_name[job_name].get("fileset") if fileset is None: continue logger.debug(" FileSet: %s", jobs_by_name[job_name]["fileset"]) include = filesets_by_name[fileset].get("include") if include is None: logger.debug(" FileSet %s has no include section", fileset) continue if len(include) > 1: raise RuntimeError( f" ERROR: FileSet {fileset} contains multipe include sections" ) plugin = filesets_by_name[fileset]["include"][0].get("plugin") if plugin is None: logger.debug( " FileSet %s has no plugin definition in include section", fileset ) continue if len(plugin) > 1: raise RuntimeError( f" ERROR: FileSet {fileset} contains multipe plugin definitions" ) self.parse_plugin_definition(plugin[0]) # pp.pprint(self.plugin_options) if self.plugin_options.get("module_name") != self.vmware_job_module_name: continue self.vmware_jobs[job_name] = {} self.vmware_jobs[job_name]["plugin_options"] = self.plugin_options return True def parse_plugin_definition(self, plugindef): """ Parse a plugin definition string """ print(f'DEBUG: plugin def parser called with "{plugindef}"') # Parse plugin options into a dict self.plugin_options = {} plugin_options = plugindef.split(":") while plugin_options: current_option = plugin_options.pop(0) key, sep, val = current_option.partition("=") # See if the last character is a escape of the value string while val[-1:] == "\\": val = val[:-1] + ":" + plugin_options.pop(0) # Replace all '\\' val = val.replace("\\", "") print(f'DEBUG: key:val = "{key}:{val}"') if val == "": continue if key not in self.plugin_options: self.plugin_options[key] = val return True def reload(self): """ Call reload """ success = True try: self.directorconsole.call("reload") except bareos.exceptions.JsonRpcErrorReceivedException: logger.fatal("reload bareos config failed") success = False return success def run_job(self, job=None, client=None): """ Run job """ success = True try: self.directorconsole.call(f"run job={job} client={client}") except bareos.exceptions.JsonRpcErrorReceivedException: logger.error("failed to run job %s on client %s", job, client) success = False return success class VMwareApi: """ VMware vShpere API access class """ def __init__(self): self.si = None self.args = None def connect_vmware(self, args): """ Connect vmware vCenter API """ try: retry_no_ssl_verify = False try: self.si = SmartConnect( host=args.host, user=args.user, pwd=args.password, port=int(args.port), ) except IOError as e: if "CERTIFICATE_VERIFY_FAILED" in e.strerror and not args.sslverify: # Note: pylint complains when using fstring with logging logger.warning( "Connecting host %s with user %s failed: %s", args.host, args.user, str(e), ) logger.info("Retrying without SSL verification") retry_no_ssl_verify = True else: logger.error( "Connecting host %s with user %s failed: %s", args.host, args.user, str(e), ) if retry_no_ssl_verify: try: self.si = SmartConnect( host=args.host, user=args.user, pwd=args.password, port=int(args.port), disableSslCertValidation=True, ) except IOError as e: logger.error( "ERROR: Connecting host %s with user %s failed: %s", args.host, args.user, str(e), ) if not self.si: sys.exit(1) atexit.register(Disconnect, self.si) except vmodl.MethodFault as e: logger.critical("Caught vmodl fault : %s", e.msg) sys.exit(1) except Exception as e: raise RuntimeError("Caught unexpected Exception : " + str(e)) from e self.args = args return True def get_dc_foldertree(self, dcftree): """ Get Folder Tree for DC """ dc_view = self.si.content.viewManager.CreateContainerView( self.si.content.rootFolder, [vim.Datacenter], False ) dc_list = dc_view.view dc_view.Destroy() for dc in dc_list: if dc.name == self.args.datacenter: logger.debug( "Trying to find folder: %s in DC %s", self.args.folder, dc.name ) found_folder = find_vm_folder(self.si, dc, self.args.folder) if found_folder: logger.debug( "Found folder: %s for %s", found_folder.name, self.args.folder ) else: logger.critical( "Could not find folder: %s in DC %s", self.args.folder, dc.name ) sys.exit(1) dcftree[dc.name] = {} self._get_dcftree(dcftree[dc.name], self.args.folder, found_folder) return True def _get_dcftree(self, dcf, folder, vm_folder): """ Recursive function to walk the tree of folders """ for vm_or_folder in vm_folder.childEntity: if isinstance(vm_or_folder, vim.VirtualMachine): dcf[folder + "/" + vm_or_folder.name] = vm_or_folder elif isinstance(vm_or_folder, vim.Folder): self._get_dcftree(dcf, folder + "/" + vm_or_folder.name, vm_or_folder) elif isinstance(vm_or_folder, vim.VirtualApp): # vm_or_folder is a vApp in this case, contains a list a VMs for vapp_vm in vm_or_folder.vm: dcf[folder + "/" + vm_or_folder.name + "/" + vapp_vm.name] = vapp_vm else: logger.warning( "%s is neither Folder nor VirtualMachine nor vApp, ignoring.", vm_or_folder, ) def get_vm(self, dc=None, folder=None, vmname=None): """ Get VM data by given datacenter, folder and VM name """ vm_path = Util.proper_path(f"/{dc}/vm/{folder}/{vmname}") logger.debug("Searching VM path %s", vm_path) found_vm = self.si.content.searchIndex.FindByInventoryPath( inventoryPath=vm_path ) return found_vm class Util: """ Utility class """ @staticmethod def mkdir(directory_name): """ Utility Function for creating directories, works like mkdir -p """ try: os.stat(directory_name) except OSError: os.makedirs(directory_name) @staticmethod def needs_write(content, file_path): """ Check if file needs to be written """ target_file_hash = hashlib.sha256() content_hash = hashlib.sha256() if not os.path.exists(file_path): return True with open(file_path, "rb") as fh: target_file_hash.update(fh.read()) content_hash.update(bytes(content, encoding="utf-8")) return target_file_hash.digest() != content_hash.digest() @staticmethod def proper_path(path): """ Ensure proper path: - removes multiplied slashes - makes sure it starts with slash - makes sure it does not end with slash - if slashes only or the empty string were passed, returns a single slash """ clean_path = slashes_rex.sub("/", "/" + path) clean_path = clean_path.rstrip("/") if clean_path == "": clean_path = "/" return clean_path if __name__ == "__main__": main() # vim: tabstop=4 shiftwidth=4 softtabstop=4 expandtab