Ports: Detect more types of errors in the AvailablePorts.md file

This adds support for detecting incorrect version numbers and links
in the ports list.

Also, unlike before it doesn't parse the package.sh script but executes
it instead which allows us to detect syntax errors.

Meta/lint-ports.py

@@ -3,9 +3,17 @@
 import os
 import re
 import sys
+import subprocess
 
-# Matches e.g. "| [`bash`]..." and captures "bash" in group 1
+# Matches e.g. "| [`bash`](bash/) | GNU Bash | 5.0 | https://www.gnu.org/software/bash/ |"
-PORT_TABLE_REGEX = re.compile(r'^\| \[`([^`]+)`\][^`]+$', re.MULTILINE)
+# and captures "bash" in group 1, "bash/" in group 2, "<spaces>" in group 3, "GNU Bash" in group 4, "5.0" in group 5
+# and "https://www.gnu.org/software/bash/" in group 6.
+PORT_TABLE_REGEX = re.compile(
+    r'^\| \[`([^`]+)`\]\(([^\)]+)\)([^\|]+) \| ([^\|]+) \| ([^\|]+?) \| ([^\|]+) \|+$', re.MULTILINE
+)
+
+# Matches non-abbreviated git hashes
+GIT_HASH_REGEX = re.compile(r'^[0-9a-f]{40}$')
 
 PORT_TABLE_FILE = 'AvailablePorts.md'
 IGNORE_FILES = {
@@ -28,8 +36,19 @@ def read_port_table(filename):
     Returns:
         set: all PORT_TABLE_REGEX matches
     """
+    ports = {}
     with open(filename, 'r') as fp:
-        return set(PORT_TABLE_REGEX.findall(fp.read()))
+        matches = PORT_TABLE_REGEX.findall(fp.read())
+        for match in matches:
+            line_len = sum([len(part) for part in match])
+            ports[match[0]] = {
+                "dir_ref": match[1],
+                "name": match[2].strip(),
+                "version": match[4].strip(),
+                "url": match[5].strip(),
+                "line_len": line_len
+            }
+    return ports
 
 
 def read_port_dirs():
@@ -39,7 +58,7 @@ def read_port_dirs():
         list: all ports (set), no errors encountered (bool)
     """
 
-    ports = set()
+    ports = {}
     all_good = True
     for entry in os.listdir():
         if entry in IGNORE_FILES:
@@ -52,39 +71,107 @@ def read_port_dirs():
             print(f"Ports/{entry}/ is missing its package.sh?!")
             all_good = False
             continue
-        ports.add(entry)
+        ports[entry] = get_port_properties(entry)
 
     return ports, all_good
 
 
+PORT_PROPERTIES = ('port', 'version', 'files', 'auth_type')
+
+
+def get_port_properties(port):
+    """Retrieves common port properties from its package.sh file.
+
+    Returns:
+        dict: keys are values from PORT_PROPERTIES, values are from the package.sh file
+    """
+
+    props = {}
+    for prop in PORT_PROPERTIES:
+        res = subprocess.run(f"cd {port}; exec ./package.sh showproperty {prop}", shell=True, capture_output=True)
+        if res.returncode == 0:
+            props[prop] = res.stdout.decode('utf-8').strip()
+        else:
+            print((
+                f'Executing "./package.sh showproperty {prop}" script for port {port} failed with '
+                f'exit code {res.returncode}, output from stderr:\n{res.stderr.decode("utf-8").strip()}'
+            ))
+            props[prop] = ''
+    return props
+
+
 def check_package_files(ports):
     """Check port package.sh file for required properties.
 
     Args:
-        ports (set): List of all ports to check
+        ports (list): List of all ports to check
 
     Returns:
         bool: no errors encountered
     """
 
-    packages = set()
     all_good = True
     for port in ports:
         package_file = f"{port}/package.sh"
         if not os.path.exists(package_file):
             continue
-        packages.add(package_file)
 
-    properties = ['port', 'version', 'files', 'auth_type']
+        props = get_port_properties(port)
-    for package in packages:
+        for prop in PORT_PROPERTIES:
-        with open(package, 'r') as fp:
+            if prop == 'auth_type' and re.match('^https://github.com/SerenityOS/', props["files"]):
-            data = fp.read()
+                continue
-            for p in properties:
+            if props[prop] == '':
-                if not re.findall(f"^{p}=", data, re.M):
+                print(f"Ports/{port} is missing required property '{prop}'")
-                    if p == 'auth_type' and re.findall('^files="?https://github.com/SerenityOS/', data, re.M):
+                all_good = False
-                        continue
+
-                    print(f"Ports/{package} is missing '{p}'")
+    return all_good
-                    all_good = False
+
+
+def check_available_ports(from_table, ports):
+    """Check AvailablePorts.md for correct properties.
+
+    Args:
+        from_table (dict): Ports table from AvailablePorts.md
+        ports (dict): Dictionary with port properties from package.sh
+
+    Returns:
+        bool: no errors encountered
+    """
+
+    all_good = True
+
+    previous_line_len = None
+
+    for port in from_table.keys():
+        if previous_line_len is None:
+            previous_line_len = from_table[port]["line_len"]
+        if previous_line_len != from_table[port]["line_len"]:
+            print(f"Table row for port {port} is improperly aligned with other rows.")
+            all_good = False
+        else:
+            previous_line_len = from_table[port]["line_len"]
+
+        actual_ref = from_table[port]["dir_ref"]
+        expected_ref = f"{port}/"
+        if actual_ref != expected_ref:
+            print((
+                f'Directory link target in AvailablePorts.md for port {port} is '
+                f'incorrect, expected "{expected_ref}", found "{actual_ref}"'
+            ))
+            all_good = False
+
+        actual_version = from_table[port]["version"]
+        expected_version = ports[port]["version"]
+        if GIT_HASH_REGEX.match(expected_version):
+            expected_version = expected_version[0:7]
+        if expected_version == "git":
+            expected_version = ""
+        if actual_version != expected_version:
+            print((
+                f'Version in AvailablePorts.md for port {port} is incorrect, '
+                f'expected "{expected_version}", found "{actual_version}"'
+            ))
+            all_good = False
 
     return all_good
 
@@ -95,19 +182,25 @@ def run():
     from_table = read_port_table(PORT_TABLE_FILE)
     ports, all_good = read_port_dirs()
 
-    if from_table - ports:
+    from_table_set = set(from_table.keys())
+    ports_set = set(ports.keys())
+
+    if from_table_set - ports_set:
         all_good = False
         print('AvailablePorts.md lists ports that do not appear in the file system:')
         for port in sorted(from_table - ports):
             print(f"    {port}")
 
-    if ports - from_table:
+    if ports_set - from_table_set:
         all_good = False
         print('AvailablePorts.md is missing the following ports:')
-        for port in sorted(ports - from_table):
+        for port in sorted(ports_set - from_table_set):
             print(f"    {port}")
 
-    if not check_package_files(ports):
+    if not check_package_files(ports.keys()):
+        all_good = False
+
+    if not check_available_ports(from_table, ports):
         all_good = False
 
     if not all_good:

Ports/.port_include.sh

@@ -65,7 +65,7 @@ shift
 : "${useconfigure:=false}"
 : "${depends:=}"
 : "${patchlevel:=1}"
-: "${auth_type:=md5}"
+: "${auth_type:=}"
 : "${auth_import_key:=}"
 : "${auth_opts:=}"
 : "${launcher_name:=}"
@@ -375,8 +375,12 @@ do_uninstall() {
     echo "Uninstalling $port!"
     uninstall
 }
-do_showdepends() {
+do_showproperty() {
-    echo -n $depends
+    if [ -z ${!1+x} ]; then
+        echo "Property '$1' is not set." >&2
+        exit 1
+    fi
+    echo ${!1}
 }
 do_all() {
     do_installdepends
@@ -393,8 +397,8 @@ parse_arguments() {
         do_all
     else
         case "$1" in
-            fetch|patch|configure|build|install|installdepends|clean|clean_dist|clean_all|uninstall|showdepends)
+            fetch|patch|configure|build|install|installdepends|clean|clean_dist|clean_all|uninstall|showproperty)
-                do_$1
+                do_$1 $2
                 ;;
             --auto)
                 do_all $1
@@ -405,7 +409,7 @@ parse_arguments() {
                 parse_arguments $@
                 ;;
             *)
-                >&2 echo "I don't understand $1! Supported arguments: fetch, patch, configure, build, install, installdepends, clean, clean_dist, clean_all, uninstall, showdepends."
+                >&2 echo "I don't understand $1! Supported arguments: fetch, patch, configure, build, install, installdepends, clean, clean_dist, clean_all, uninstall, showproperty."
                 exit 1
                 ;;
         esac

Ports/build_all.sh

@@ -49,7 +49,7 @@ for file in *; do
                 popd > /dev/null
                 continue
             fi
-            built_ports="$built_ports $port $(./package.sh showdepends) "
+            built_ports="$built_ports $port $(./package.sh showproperty depends) "
 
             if [ "$clean" == true ]; then
                 if [ "$verbose" == true ]; then