detect if backup is already running

capsulflask/db_model.py

···
 
 
   def get_next_vm_to_back_up(self):
+    # TODO:: to support multiple hosts properly, this should probably group by host and return multiple rows for multiple hosts.
     self.cursor.execute(""" 
       SELECT id, email, host, last_backup_started, last_backup_finished, backup_window_start_hour_utc FROM vms 
       WHERE deleted is null AND has_guest_agent = true AND (

capsulflask/hub_model.py

···
     except: 
       pass
 
+    if result_status == "busy":
+      current_app.logger.info(f"failed to start backup on '{id}' on host '{host}' because backup is already running")   
+      return
+
     if not result_status == "success":
-      raise ValueError(f"""failed to start backup on "{id}" on host "{host}" for {email}: {result_json_string}""")      
+      raise ValueError(f"failed to start backup on '{id}' on host '{host}' for {email}: {result_json_string}")      
 
   def destroy(self, email: str, id: str):
     validate_capsul_id(id)

capsulflask/shell_scripts/is-backup-currently-running.sh

···
+#!/bin/sh -e
+                                                                                        
+if ps aux | grep -q '[v]irtnbdbackup'; then
+    echo "yes"
+else
+    echo "no"
+fi
+
+

capsulflask/spoke_api.py

···
       current_app.logger.error(f"/hosts/operation returned 400: {required_property} is required for handle_start_backup")
       return abort(400, f"bad request; {required_property} is required for handle_start_backup")
 
+  return_status = "error"
   try:
-    current_app.config['SPOKE_MODEL'].start_backup(email=request_body['email'], id=request_body['id'])
+    return_status = current_app.config['SPOKE_MODEL'].start_backup(email=request_body['email'], id=request_body['id'])
   except:
     error_message = my_exec_info_message(sys.exc_info())
     params=           f"email='{request_body['email'] if 'email' in request_body else 'KeyError'}', "
···
     current_app.logger.error(f"current_app.config['SPOKE_MODEL'].start_backup({params}) failed: {error_message}")
     return jsonify(dict(assignment_status="assigned", status="error", error_message=error_message))
   
-  return jsonify(dict(assignment_status="assigned", status="success"))
+  return jsonify(dict(assignment_status="assigned", status=return_status))

capsulflask/spoke_model.py

···
   def __init__(self, flask_app):
     self.flask_app = flask_app
 
-  def validate_completed_process(self, completedProcess, email=None):
+  def validate_completed_process(self, completed_process, email=None):
     emailPart = ""
     if email != None:
       emailPart = f"for {email}"
 
-    if completedProcess.returncode != 0:
-      raise RuntimeError(f"""{" ".join(completedProcess.args)} failed {emailPart} with exit code {completedProcess.returncode}
+    if completed_process.returncode != 0:
+      raise RuntimeError(f"""{" ".join(completed_process.args)} failed {emailPart} with exit code {completed_process.returncode}
         stdout:
-        {completedProcess.stdout}
+        {completed_process.stdout}
         stderr:
-        {completedProcess.stderr}
+        {completed_process.stderr}
       """)
 
   def capacity_avaliable(self, additional_ram_bytes):
     my_args=[join(current_app.root_path, 'shell_scripts/capacity-avaliable.sh'), str(additional_ram_bytes)]
-    completedProcess = run(my_args, capture_output=True)
+    completed_process = run(my_args, capture_output=True)
 
-    if completedProcess.returncode != 0:
+    if completed_process.returncode != 0:
       current_app.logger.error(f"""
-      capacity-avaliable.sh exited {completedProcess.returncode} with
+      capacity-avaliable.sh exited {completed_process.returncode} with
         stdout:
-        {completedProcess.stdout}
+        {completed_process.stdout}
         stderr:
-        {completedProcess.stderr}
+        {completed_process.stderr}
       """)
       return False
 
-    lines = completedProcess.stdout.splitlines()
+    lines = completed_process.stdout.splitlines()
     output = lines[len(lines)-1]
     if not output == b"yes":
       current_app.logger.error(f"capacity-avaliable.sh exited 0 and returned {output} but did not return \"yes\" ")
···
 
   def get(self, id, get_ssh_host_keys):
     validate_capsul_id(id)
-    completedProcess = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True)
-    self.validate_completed_process(completedProcess)
-    lines = completedProcess.stdout.splitlines()
+    completed_process = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True)
+    self.validate_completed_process(completed_process)
+    lines = completed_process.stdout.splitlines()
     if len(lines) == 0:
       current_app.logger.warning("shell_scripts/get.sh returned zero lines!")
       return None
···
 
     if get_ssh_host_keys:
       try:
-        completedProcess2 = run([join(current_app.root_path, 'shell_scripts/ssh-keyscan.sh'), ipaddr], capture_output=True)
-        self.validate_completed_process(completedProcess2)
-        ssh_host_keys = json.loads(completedProcess2.stdout.decode("utf-8"))
+        completed_process2 = run([join(current_app.root_path, 'shell_scripts/ssh-keyscan.sh'), ipaddr], capture_output=True)
+        self.validate_completed_process(completed_process2)
+        ssh_host_keys = json.loads(completed_process2.stdout.decode("utf-8"))
         return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr, ssh_host_keys=ssh_host_keys)
       except:
         current_app.logger.warning(f"""
···
 
     ssh_keys_string = "\n".join(ssh_authorized_keys)
 
-    completedProcess = run([
+    completed_process = run([
       join(current_app.root_path, 'shell_scripts/create.sh'),
       id,
       template_image_file_name,
···
       public_ipv4
     ], capture_output=True)
 
-    self.validate_completed_process(completedProcess, email)
-    lines = completedProcess.stdout.splitlines()
+    self.validate_completed_process(completed_process, email)
+    lines = completed_process.stdout.splitlines()
     status = lines[len(lines)-1].decode("utf-8")
 
     vmSettings = f"""
···
       raise ValueError(f"""failed to create vm for {email} with:
         {vmSettings}
         stdout:
-        {completedProcess.stdout}
+        {completed_process.stdout}
         stderr:
-        {completedProcess.stderr}
+        {completed_process.stderr}
       """)
 
   def destroy(self, email: str, id: str):
     validate_capsul_id(id)
-    completedProcess = run([join(current_app.root_path, 'shell_scripts/destroy.sh'), id], capture_output=True)
-    self.validate_completed_process(completedProcess, email)
-    lines = completedProcess.stdout.splitlines()
+    completed_process = run([join(current_app.root_path, 'shell_scripts/destroy.sh'), id], capture_output=True)
+    self.validate_completed_process(completed_process, email)
+    lines = completed_process.stdout.splitlines()
     status = lines[len(lines)-1].decode("utf-8")
     if not status == "success":
       raise ValueError(f"""failed to destroy vm {id} for {email} on {current_app.config["SPOKE_HOST_ID"]}:
         stdout:
-        {completedProcess.stdout}
+        {completed_process.stdout}
         stderr:
-        {completedProcess.stderr}
+        {completed_process.stderr}
       """)
 
   def vm_state_command(self, email: str, id: str, command: str):
···
     if command not in ["stop", "force-stop", "start", "restart"]:
       raise ValueError(f"command ({command}) must be one of stop, force-stop, start, or restart")
 
-    completedProcess = run([join(current_app.root_path, f"shell_scripts/{command}.sh"), id], capture_output=True)
-    self.validate_completed_process(completedProcess, email)
-    returned_string = completedProcess.stdout.decode("utf-8")
+    completed_process = run([join(current_app.root_path, f"shell_scripts/{command}.sh"), id], capture_output=True)
+    self.validate_completed_process(completed_process, email)
+    returned_string = completed_process.stdout.decode("utf-8")
     current_app.logger.info(f"{command} vm {id} for {email} returned: {returned_string}")
   
   def net_set_dhcp(self, email: str, host_id: str, network_name: str, macs: list, remove_ipv4: str, add_ipv4: str):
···
         raise ValueError(f"remove_ipv4 \"{remove_ipv4}\" must match \"^[0-9.]+$\"")
       
       for mac in macs:
-        completedProcess = run([join(current_app.root_path, f"shell_scripts/ip-dhcp-host.sh"), "delete", network_name, mac, remove_ipv4], capture_output=True)
-        self.validate_completed_process(completedProcess, email)
+        completed_process = run([join(current_app.root_path, f"shell_scripts/ip-dhcp-host.sh"), "delete", network_name, mac, remove_ipv4], capture_output=True)
+        self.validate_completed_process(completed_process, email)
 
     if add_ipv4 != None and add_ipv4 != "":
       if not re.match(r"^[0-9.]+$", add_ipv4):
         raise ValueError(f"add_ipv4 \"{add_ipv4}\" must match \"^[0-9.]+$\"")
       
       for mac in macs:
-        completedProcess = run([join(current_app.root_path, f"shell_scripts/ip-dhcp-host.sh"), "add", network_name, mac, add_ipv4], capture_output=True)
-        self.validate_completed_process(completedProcess, email)
+        completed_process = run([join(current_app.root_path, f"shell_scripts/ip-dhcp-host.sh"), "add", network_name, mac, add_ipv4], capture_output=True)
+        self.validate_completed_process(completed_process, email)
 
-  def start_backup(self, email: str, id: str):
+  def start_backup(self, email: str, id: str) -> str:
     validate_capsul_id(id)
     current_app.logger.info(f"start_backup: {id} for {email}")
+
+    completed_process = run([join(current_app.root_path, f"shell_scripts/is-backup-currently-running.sh")], capture_output=True)
+    self.validate_completed_process(completed_process, email)
+
+    if "yes" in completed_process.stdout.decode("utf-8"):
+      current_app.logger.info(f"start_backup: virtnbdbackup process is already running, exiting...")
+      return "busy"
 
     vm_backup_dirs = []
     backup_storage_mounts = list(map(lambda x: x.rstrip("/"), current_app.config['BACKUP_STORAGE_MOUNTS'].split(",")))
···
       current_app.logger.info(f"creating new backup folder w/ new full backup: {new_backup_dir_path}")
       new_thread = threading.Thread(target=self.do_backup, args=(id, "full", new_backup_dir_path))
       new_thread.start()
-      current_app.logger.info(f"new_thread.start() finished")
     else:
       current_app.logger.info(f"performing incremental backup to: {most_recent_backup_dir_string}")
       new_thread = threading.Thread(target=self.do_backup, args=(id, "inc", most_recent_backup_dir_string))
       new_thread.start()
-      current_app.logger.info(f"new_thread.start() finished")
+
+    current_app.logger.info(f"finished spawning background thread for backup")
+    return "success"
 
   def do_backup(self, id, backup_level, backup_dir_path):
     with self.flask_app.app_context():
···
         'Authorization': f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}",
         'Content-Type': "application/json"
       }
-      notify_backup_finished_url = f"{app.config['HUB_URL']}/hub/backup-finished/{current_app.config['SPOKE_HOST_ID']}"
+      notify_backup_finished_url = f"{current_app.config['HUB_URL']}/hub/backup-finished/{current_app.config['SPOKE_HOST_ID']}"
       payload = json.dumps({
         'id': id,
         'command': ' '.join(completed_process.args),