working on support for managing VM state and IP address

capsulflask/admin.py

···
 @bp.route("/")
 @admin_account_required
 def index():
+
+  # first create the hosts list w/ ip allocation visualization 
+  # 
+   
   hosts = get_model().list_hosts_with_networks(None)
   vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
   network_display_width_px = float(270)
···
   {'}'}
   """]
 
+  public_ipv4_by_capsul_id = dict()
+
   for kv in hosts.items():
     host_id = kv[0]
     value = kv[1]
···
       if host_id in vms_by_host_and_network:
         if network['network_name'] in vms_by_host_and_network[host_id]:
           for vm in vms_by_host_and_network[host_id][network['network_name']]:
+            public_ipv4_by_capsul_id[vm['id']] = vm['public_ipv4']
             ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
             if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
               allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
···
 
     display_hosts.append(display_host)
 
+
+  # Now creating the capsuls running status ui
+  # 
+
+  db_vms = get_model().all_vm_ids_with_desired_state()
+  virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state()
+
+  virt_vms_dict = dict()
+  for vm in virt_vms:
+    virt_vms_dict[vm["id"]] = vm["state"]
+  
+  in_db_but_not_in_virt = []
+  needs_to_be_started = []
+  needs_to_be_started_missing_ipv4 = []
+
+  for vm in db_vms:
+    if vm["id"] not in virt_vms_dict:
+      in_db_but_not_in_virt.append(vm["id"])
+    elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
+      if vm["id"] in public_ipv4_by_capsul_id:
+        needs_to_be_started.append({"id": vm["id"], "ipv4": public_ipv4_by_capsul_id[vm["id"]]})
+      else:
+        needs_to_be_started_missing_ipv4.append(vm["id"])
+
   csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
   response_text = render_template(
     "admin.html", 
     display_hosts=display_hosts, 
+    in_db_but_not_in_virt=in_db_but_not_in_virt,
+    needs_to_be_started=needs_to_be_started,
+    needs_to_be_started_missing_ipv4=needs_to_be_started_missing_ipv4,
     network_display_width_px=network_display_width_px,
     csp_inline_style_nonce=csp_inline_style_nonce,
     inline_style='\n'.join(inline_styles)

capsulflask/cli.py

···
 
 
 def ensure_vms_and_db_are_synced():
-  db_ids = get_model().all_non_deleted_vm_ids()
-  virt_ids = current_app.config["HUB_MODEL"].list_ids()
+  db_vms = get_model().all_vm_ids_with_desired_state()
+  virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state()
 
   db_ids_dict = dict()
   virt_ids_dict = dict()
 
-  for id in db_ids:
-    db_ids_dict[id] = True
+  for vm in db_vms:
+    db_ids_dict[vm['id']] = vm['desired_state']
   
-  for id in virt_ids:
-    virt_ids_dict[id] = True
+  for vm in virt_vms:
+    virt_ids_dict[vm['id']] = vm['desired_state']
 
   errors = list()
 
   for id in db_ids_dict:
     if id not in virt_ids_dict:
       errors.append(f"{id} is in the database but not in the virtualization model")
+    elif db_ids_dict[id] != virt_ids_dict[id]:
+      errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
 
   for id in virt_ids_dict:
     if id not in db_ids_dict:

capsulflask/consistency.py

···
+
+from flask import current_app
+from capsulflask.db import get_model
+
+# {
+#   "capsul-123abc45": {
+#     "id": "capsul-123abc45",
+#     "public_ipv4": "123.123.123.123",
+#     "public_ipv6": "::::",
+#     "host": "baikal",
+#     "network_name": "public1",
+#     "virtual_bridge_name": "virbr1",
+#     "state": "running"
+#   },
+#   { ... },
+#   ...
+# }
+def get_all_vms_from_db() -> dict:
+  db_hosts = get_model().list_hosts_with_networks(None)
+  db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
+
+  db_vms_by_id = dict()
+
+  for kv in db_hosts.items():
+    host_id = kv[0]
+    value = kv[1]
+    for network in value['networks']:
+      if host_id in db_vms_by_host_and_network and network['network_name'] in db_vms_by_host_and_network[host_id]:
+        for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
+          vm['network_name'] = network['network_name']
+          vm['virtual_bridge_name'] = network['virtual_bridge_name']
+          vm['host'] = host_id
+          db_vms_by_id[vm['id']] = vm
+ 
+  # for vm in db_vms:
+  #   if vm["id"] not in db_vms_by_id:
+  #     # TODO 
+  #     raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") 
+  #   else:
+  #     db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] 
+
+  return db_vms_by_id
+
+def get_all_vms_from_hosts() -> dict:
+  virt_vms = current_app.config["HUB_MODEL"].virsh_list()
+  virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
+  db_hosts = get_model().list_hosts_with_networks(None)
+  
+  virt_vms_by_id = dict()
+
+  for kv in db_hosts.items():
+    host_id = kv[0]
+    value = kv[1]
+    for network in value['networks']:
+      
+
+  for vm in db_vms:
+    if vm["id"] not in db_vms_by_id:
+      # TODO 
+      raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") 
+    else:
+      db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] 
+
+  virt_vms = current_app.config["HUB_MODEL"].get_vm_()
+
+def ensure_vms_and_db_are_synced():
+  
+  
+
+  # Now creating the capsuls running status ui
+  # 
+
+
+
+  for vm in db_vms:
+    db_ids_dict[vm['id']] = vm['desired_state']
+  
+  for vm in virt_vms:
+    virt_ids_dict[vm['id']] = vm['desired_state']
+
+  errors = list()
+
+  for id in db_ids_dict:
+    if id not in virt_ids_dict:
+      errors.append(f"{id} is in the database but not in the virtualization model")
+    elif db_ids_dict[id] != virt_ids_dict[id]:
+      errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
+
+  for id in virt_ids_dict:
+    if id not in db_ids_dict:
+      errors.append(f"{id} is in the virtualization model but not in the database")
+
+  if len(errors) > 0:
+    email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
+    email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
+
+    current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
+    for error in errors:
+      current_app.logger.info(f"cron_task:      {error}.")
+      
+    current_app.config["FLASK_MAIL_INSTANCE"].send(
+        Message(
+          "Capsul Consistency Check Failed",
+          sender=current_app.config["MAIL_DEFAULT_SENDER"],
+          body="\n".join(errors),
+          recipients=email_addresses
+        )
+    )

capsulflask/console.py

···
   letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
   return f"capsul-{letters_n_nummers}"
 
-def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
+def double_check_capsul_address(id, get_ssh_host_keys):
   try:
     result = current_app.config["HUB_MODEL"].get(id, get_ssh_host_keys)
-    if result != None and result.ipv4 != None and result.ipv4 != ipv4:
-      ipv4 = result.ipv4
-      get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4)
+    # if result != None and result.ipv4 != None and result.ipv4 != ipv4:
+    #   ipv4 = result.ipv4
+    #   get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4)
 
     if result != None and result.ssh_host_keys != None and get_ssh_host_keys:
       get_model().update_vm_ssh_host_keys(email=session["account"], id=id, ssh_host_keys=result.ssh_host_keys)
···
   # for now we are going to check the IP according to the virt model
   # on every request. this could be done by a background job and cached later on...
   for vm in vms:
-    result = double_check_capsul_address(vm["id"], vm["ipv4"], False)
+    result = double_check_capsul_address(vm["id"], False)
     if result is not None:
       vm["ipv4"] = result.ipv4
       vm["state"] = result.state
···
   else:
     needs_ssh_host_keys = "ssh_host_keys" not in vm or len(vm["ssh_host_keys"]) == 0
 
-    vm_from_virt_model = double_check_capsul_address(vm["id"], vm["ipv4"], needs_ssh_host_keys)
+    vm_from_virt_model = double_check_capsul_address(vm["id"], needs_ssh_host_keys)
     
     if vm_from_virt_model is not None:
       vm["ipv4"] = vm_from_virt_model.ipv4

capsulflask/db.py

···
   hasSchemaVersionTable = False
   actionWasTaken = False
   schemaVersion = 0
-  desiredSchemaVersion = 20
+  desiredSchemaVersion = 21
 
   cursor = connection.cursor()
 

capsulflask/db_model.py

···
 
     return hosts
 
-  def all_non_deleted_vm_ids(self):
-    self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
-    return list(map(lambda x: x[0], self.cursor.fetchall()))
+  def all_vm_ids_with_desired_state(self):
+    self.cursor.execute("SELECT id, desired_state FROM vms WHERE deleted IS NULL")
+    return list(map(lambda x: {"id": x[0], "desired_state": x[1]}, self.cursor.fetchall()))
 
   def operating_systems_dict(self):
     self.cursor.execute("SELECT id, template_image_file_name, description FROM os_images WHERE deprecated = FALSE")
···
 
   def list_hosts_with_networks(self, host_id: str):
     query = """
-    SELECT hosts.id, hosts.last_health_check, host_network.network_name, 
+    SELECT hosts.id, hosts.last_health_check, host_network.network_name, host_network.virtual_bridge_name,
     host_network.public_ipv4_cidr_block, host_network.public_ipv4_first_usable_ip, host_network.public_ipv4_last_usable_ip
     FROM hosts
     JOIN host_network ON host_network.host = hosts.id
···
         hosts[row[0]] = dict(last_health_check=row[1], networks=[])
 
       hosts[row[0]]["networks"].append(dict(
-        network_name=row[2], 
-        public_ipv4_cidr_block=row[3], 
-        public_ipv4_first_usable_ip=row[4], 
-        public_ipv4_last_usable_ip=row[5]
+        network_name=row[2],
+        virtual_bridge_name=row[3], 
+        public_ipv4_cidr_block=row[4], 
+        public_ipv4_first_usable_ip=row[5], 
+        public_ipv4_last_usable_ip=row[6]
       ))
 
     return hosts

capsulflask/hub_model.py

···
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4)
 
-  def list_ids(self) -> list:
-    return get_model().all_non_deleted_vm_ids()
+  def get_all_by_host_and_network(self) -> dict:
+    return get_model().non_deleted_vms_by_host_and_network()
 
   def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
···
 
     return None
 
-  def list_ids(self) -> list:
+  def get_all_by_host_and_network(self) -> dict:
     online_hosts = get_model().get_online_hosts()
-    payload = json.dumps(dict(type="list_ids"))
+    payload = json.dumps(dict(type="get_all_by_host_and_network"))
     results = self.synchronous_operation(online_hosts, None, payload)
-    to_return = []
+    to_return = dict()
     for i in range(len(results)):
       host = online_hosts[i]
       result = results[i]
       try:
         result_body = json.loads(result.body)
-        if isinstance(result_body, dict) and 'ids' in result_body and isinstance(result_body['ids'], list):
-          all_valid = True
-          for id in result_body['ids']:
-            try:
-              validate_capsul_id(id)
-              to_return.append(id)
-            except:
-              all_valid = False
-          if not all_valid:
-            current_app.logger.error(f"""error reading ids for list_ids operation, host {host.id}""")
+
+        has_host = isinstance(result_body, dict) and host.id in result_body and isinstance(result_body[host.id], dict)
+        has_networks = has_host and 'networks' in result_body[host.id] and isinstance(result_body[host.id]['networks'], dict)
+        if has_host and has_networks:
+          to_return[host.id] = result_body[host.id]['networks']
         else:
-            result_json_string = json.dumps({"error_message": "invalid response, missing 'ids' list"})
-            current_app.logger.error(f"""missing 'ids' list for list_ids operation, host {host.id}""")
+          # result_json_string = json.dumps({"error_message": "invalid response, missing 'networks' list"})
+          current_app.logger.error(f"""missing 'networks' list for get_all_by_host_and_network operation, host {host.id}""")
       except:
         # no need to do anything here since if it cant be parsed then generic_operation will handle it.
         pass

capsulflask/schema_migrations/21_down_desired_state.sql

···
+
+
+
+ALTER TABLE vms DROP COLUMN desired_state;
+
+UPDATE schemaversion SET version = 20;
+

capsulflask/schema_migrations/21_up_desired_state.sql

···
+
+
+
+ALTER TABLE vms ADD COLUMN desired_state TEXT DEFAULT 'running';
+
+UPDATE schemaversion SET version = 21;
+

capsulflask/shared.py

···
   def get(self, id: str) -> VirtualMachine:
     pass
 
-  def list_ids(self) -> list:
+  def get_all_by_host_and_network(self) -> dict:
     pass
 
   def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list):

capsulflask/shell_scripts/list-ids.sh

···
-#!/bin/sh
-
-virsh list --all | grep running | grep -v ' Id' | grep -v -- '----' | awk '{print $2}' | sort

capsulflask/shell_scripts/ssh-keyscan.sh

···
 fi
 
 printf  '['
-DELIMITER=""
+delimiter=""
 ssh-keyscan "$ip_address" 2>/dev/null | while read -r line; do
   if echo "$line" | grep -qE "^$ip_address"' +(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$'; then
-    KEY_CONTENT="$(echo "$line" | awk '{ print $2 " " $3 }')"
-    FINGERPRINT_OUTPUT="$(echo "$KEY_CONTENT" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')"
-    SHA256_HASH="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $1 }')"
-    KEY_TYPE="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $2 }')"
-    printf '%s\n  {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$DELIMITER" "$KEY_TYPE" "$KEY_CONTENT" "$SHA256_HASH"
-    DELIMITER=","
+    key_content="$(echo "$line" | awk '{ print $2 " " $3 }')"
+    fingerprint_output="$(echo "$key_content" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')"
+    sha256_hash="$(echo "$fingerprint_output" | awk '{ print $1 }')"
+    key_type="$(echo "$fingerprint_output" | awk '{ print $2 }')"
+    printf '%s\n  {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$delimiter" "$key_type" "$key_content" "$sha256_hash"
+    delimiter=","
   fi
 done
 printf '\n]\n'

capsulflask/shell_scripts/virsh-list.sh

···
+#!/bin/sh
+
+printf  '['
+delimiter=""
+virsh list --all | while read -r line; do
+  if echo "$line" | grep -qE '(running)|(shut off)'; then
+    capsul_id="$(echo "$line" | awk '{ print $2 }')"
+    capsul_state="$(echo "$line" | sed -E 's/.*((running)|(shut off))\w*/\1/')"
+    printf '%s\n  {"id":"%s", "state":"%s"}' "$delimiter" "$capsul_id" "$capsul_state"
+    delimiter=","
+  fi
+done
+printf '\n]\n'

capsulflask/shell_scripts/virsh-net-list.sh

···
+#!/bin/sh
+
+virsh net-list --all | tail -n +3 | awk '{ print $1 }'

capsulflask/spoke_api.py

···
     handlers = {
       "capacity_avaliable": handle_capacity_avaliable,
       "get": handle_get,
-      "list_ids": handle_list_ids,
+      "get_all_by_host_and_network": handle_get_all_by_host_and_network,
       "create": handle_create,
       "destroy": handle_destroy,
       "vm_state_command": handle_vm_state_command,
···
   
   return jsonify(dict(assignment_status="assigned", id=vm.id, host=vm.host, state=vm.state, ipv4=vm.ipv4, ipv6=vm.ipv6, ssh_host_keys=vm.ssh_host_keys))
 
-def handle_list_ids(operation_id, request_body):
-  return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].list_ids()))
+def handle_get_all_by_host_and_network(operation_id, request_body):
+  return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].get_all_by_host_and_network()))
 
 def handle_create(operation_id, request_body):
   if not operation_id:

capsulflask/spoke_model.py

···
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
 
-  def list_ids(self) -> list:
-    return get_model().all_non_deleted_vm_ids()
+  def get_all_by_host_and_network(self) -> dict:
+    return get_model().non_deleted_vms_by_host_and_network(None)
 
   def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
     validate_capsul_id(id)
···
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr)
 
-  def list_ids(self) -> list:
-    completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True)
+  def get_all_by_host_and_network(self) -> list:
+    # TODO implement this 
+    completedProcess = run([join(current_app.root_path, 'shell_scripts/virsh-list.sh')], capture_output=True)
     self.validate_completed_process(completedProcess)
-    return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() ))
+    return json.loads(completedProcess.stdout.decode("utf-8"))
 
   def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
     validate_capsul_id(id)