Refactor logging and docs

exploitdb/ctfsetup/qemu/ErrorsAndWorkarounds.md

@@ -24,6 +24,7 @@ Workarounds:
 NOTE:
  - Starting from Linux Kernel 2.4.15 ext3 was available
  - Starting from Linux Kernel 2.6.19 ext4 was available.
+ But a test run on kernel 3.1.2 showed otherwise. The kernel expected a fs of ext3.
  
 ---
 <br>
@@ -69,7 +70,7 @@ Error message:
     make[1]: *** [Makerules:369: libc/inet/if_index.os] Error 1
     
  Cause:
-  - Outdated uclibc
+  - Outdated uclibc/glibc
   
  Workarounds:
   - Nothing yet.

exploitdb/ctfsetup/qemu/debian_image_configuration_chroot.sh

 #!/bin/bash
 
-# Check input parameters
+# This script is meant to be copied into a mounted disk image and run with chroot
+
+# Example run:
+#   $ chroot mnt/ /debian_image_configuration_chroot.sh -u user -p pass
+
 
+# Check input parameters
 while [[ $# -gt 0 ]]; do case $1 in
   -u|--user) USERNAME="$2"; shift;shift;;
   -p|--pass) PASSWORD="$2";shift;shift;;
@@ -59,6 +64,9 @@ EOF
   echo "127.0.0.1       localhost.localdomain ctf" >> "/etc/hosts"
 }
 
+# Some old debian suite have /dev/null bugs. Replace it.
+rm -f /dev/null; mknod -m 666 /dev/null c 1 3
+
 
 verbose_log "Apt install dependencies"
 apt update

exploitdb/ctfsetup/qemu/debian_image_setup.sh

 #!/bin/bash
 
-# Example run:
-# bash debian_image_setup.sh -d <vm_dir>
+# Example runs:
+#   $ bash debian_image_setup.sh -d "./vm01/"
+#   $ bash debian_image_setup.sh -d ./some_parent/diskA" -a x86_64 --user ctf_user -p 123456
 
 set -eu
 

exploitdb/ctfsetup/qemu/debian_setup.md

+# Debian VM disk image setup
+
+The system will automatically setup Debian VM images using debootstrap,
+and configure the image to support basic linux shell commands.
+
+
+Known limitations of debian images:
+ - Latest Debian stable does not support running on kernels with version 
+ earlier than 3.2 as the image itself is built upon glibc (2.26+)
+ https://www.debian.org/releases/stable/i386/release-notes/ch-information.en.html
+ - ext2, 3, 4 file systems are supported by different versions of kernel.
+ Some kernel can only be run on one of these filesystem
+ 
+ 
+ 
+ 
+  
+ 
+
+

exploitdb/ctfsetup/qemu/setup.py

@@ -3,16 +3,16 @@ This creates an emulated system with specific properties
 and runs in Qemu
 """
 
+import logging
 import os
 import subprocess
 import sys
-import logging
-from shutil import copyfile
-from pathlib import Path
 from datetime import datetime
-from utils import get_gcc_version_by_kernel_version, get_available_port
+from pathlib import Path
+from shutil import copyfile
 
 import buildroot_setup
+from utils import get_gcc_version_by_kernel_version, get_available_port, concatenate_msgs, simple_call_shell_command
 
 DEFAULT_ARCH = "i386"
 DEFAULT_KERNEL_VERSION = "3.18.100"
@@ -27,13 +27,21 @@ BUILDROOT_CUSTOM_CONFIG_PATH = "./buildroot_setup_config.txt"
 
 VERBOSE = True
 
+# Set QUIET to True to turn off all log messages
+# NOTE: subprocess quiet is not support yet.
+QUIET = False
+
+
+def log(*msgs):
+    if not QUIET:
+        result = concatenate_msgs(*msgs)
+        print(f"[Setup] - {result}")
+
 
 def verbose_log(*msgs):
-    if VERBOSE:
-        result = str(msgs[0])
-        for msg in msgs[1:]:
-            result += " " + str(msg)
-        print("[{} (verbose)] - {}".format("Setup", result))
+    if VERBOSE and not QUIET:
+        result = concatenate_msgs(*msgs)
+        print(f"[Setup] - (verbose) {result}")
 
 
 def main(*args):
@@ -45,52 +53,54 @@ def main(*args):
     sys.exit()
 
 
+def parse_args(args):
+    args_dict = dict()
+    for arg in args:
+        (k, v) = arg.split("=")
+        args_dict[k] = v
+    return args_dict
+
+
 def setup(**kwargs):
-    verbose_log("Validating and fixing input kwargs")
+    log("Validating and fixing input kwargs")
 
     kwargs = derive_input_kwargs(**kwargs)
 
-    verbose_log("Checking dependencies")
+    log("Checking dependencies")
     check_dependencies()
 
-    verbose_log("Creating VM image directory")
+    log("Creating VM image directory")
     vm_directory = create_vm_directory()
 
-    verbose_log("Setup VM image")
+    log("Setup VM image")
     setup_vm_disk_image(vm_directory, **kwargs)
 
-    verbose_log("Configuring kernel")
+    log("Configuring kernel")
     configure_kernel(**kwargs)
     copy_config_to_buildroot_src(BUILDROOT_CUSTOM_CONFIG_PATH)
 
-    verbose_log("Compiling kernel")
+    log("Compiling kernel")
     compile_kernel()
     copy_kernel_to_vm_directory(vm_directory)
 
-    verbose_log("Booting VM image on QEMU")
+    log("Booting VM image on QEMU")
     # Booting up need an available port
     # Race condition after getting an available port. Need loop.
 
+    pid = -1
+
     for i in range(5):
         try:
             verbose_log("  Get a port for SSH connection")
             port = get_available_port()
 
             verbose_log("  Booting with QEMU", port)
-            boot_with_qemu(vm_directory, port)
+            pid = boot_with_qemu(vm_directory, port)
             break
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             verbose_log("Failed to boot QEMU, trying again with another port...")
 
-    verbose_log("Booted VM image on QEMU")
-
-
-def parse_args(args):
-    args_dict = dict()
-    for arg in args:
-        (k, v) = arg.split("=")
-        args_dict[k] = v
-    return args_dict
+    log("Booted VM image on QEMU", f"(PID: {pid})")
 
 
 def derive_input_kwargs(**kwargs):
@@ -112,13 +122,6 @@ def derive_input_kwargs(**kwargs):
     return kwargs
 
 
-def simple_call_shell_command(command):
-    """
-    Executes a command. There are limitations with this function. Use with caution.
-    """
-    return subprocess.check_call(command.split(' '), stdout=sys.stdout, stderr=subprocess.STDOUT)
-
-
 def check_dependencies():
     try:
         simple_call_shell_command("bash setup_dependencies.sh")
@@ -188,19 +191,17 @@ def boot_with_qemu(vm_directory, port):
     command += f"bash ../../qemu_background_boot.sh -p {port}"
     try:
         subprocess.run(command, shell=True, check=True)
-        print("QEMU VM is booted, and running on port", port)
+        verbose_log("QEMU VM is booted, and running on port", port)
         with open(str(vm_directory) + '/config') as f:
             pid = f.readline()
-        print("PID:", pid)
-        print("Available for ssh for:")
-        print("    user:", GUEST_USERNAME)
-        print("    pass:", GUEST_PASSWORD)
-        return
+        verbose_log("PID:", pid)
+        verbose_log("Available for ssh for:")
+        verbose_log("    user:", GUEST_USERNAME)
+        verbose_log("    pass:", GUEST_PASSWORD)
+        return pid
     except subprocess.CalledProcessError as e:
-        print("Qemu failed to boot")
-        print(e)
-
-    subprocess.run("cd ../../", shell=True)
+        verbose_log("Qemu failed to boot")
+        raise e
 
 
 if __name__ == "__main__":

exploitdb/ctfsetup/qemu/utils.py

@@ -26,6 +26,13 @@ def get_available_linux_dists():
     pass
 
 
+def concatenate_msgs(*msgs):
+    result = str(msgs[0])
+    for msg in msgs[1:]:
+        result += " " + str(msg)
+    return result
+
+
 def get_available_port():
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.bind(("", 0))
@@ -35,9 +42,11 @@ def get_available_port():
     return port
 
 
-def run_bash_command(command, verbose=True):
-    if verbose:
-        subprocess.check_call(command.split(' '), stdout=sys.stdout, stderr=subprocess.STDOUT)
+def simple_call_shell_command(command):
+    """
+    Executes a command. There are limitations with this function. Use with caution.
+    """
+    return subprocess.check_call(command.split(' '), stdout=sys.stdout, stderr=subprocess.STDOUT)
 
 
 def get_gcc_version_by_kernel_version(kernel_version):