tests/coverage_dungeon.py

   1 #!/usr/bin/env python3
   2 # SPDX-FileCopyrightText: Copyright Eric S. Raymond <esr@thyrsus.com>
   3 # SPDX-License-Identifier: BSD-2-Clause
   4 """
   5 This is the open-adventure dungeon text coverage report generator. It
   6 consumes a YAML description of the dungeon and determines whether the
   7 various strings contained are present within the test check files.
   8
   9 The default HTML output is appropriate for use with Gitlab CI.
  10 You can override it with a command-line argument.
  11
  12 The DANGLING lists are for actions and messages that should be
  13 considered always found even if the checkfile search doesn't find them.
  14 Typically this will because an action emit a templated message that
  15 can't be regression-tested by equality.
  16 """
  17
  18 # pylint: disable=consider-using-f-string,line-too-long,invalid-name,missing-function-docstring,redefined-outer-name
  19
  20 import os
  21 import sys
  22 import re
  23 import yaml
  24
  25 TEST_DIR = "."
  26 YAML_PATH = "../adventure.yaml"
  27 HTML_TEMPLATE_PATH = "../templates/coverage_dungeon.html.tpl"
  28 DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
  29 DANGLING_ACTIONS = ["ACT_VERSION"]
  30 DANGLING_MESSAGES = ["SAVERESUME_DISABLED"]
  31
  32 STDOUT_REPORT_CATEGORY = (
  33     "  {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
  34 )
  35
  36 HTML_SUMMARY_ROW = """
  37     <tr>
  38         <td class="headerItem"><a href="#{name}">{name}:</a></td>
  39         <td class="headerCovTableEntry">{total}</td>
  40         <td class="headerCovTableEntry">{covered}</td>
  41         <td class="headerCovTableEntry">{percent:.1f}%</td>
  42     </tr>
  43 """
  44
  45 HTML_CATEGORY_SECTION = """
  46     <tr id="{id}"></tr>
  47     {rows}
  48     <tr>
  49         <td>&nbsp;</td>
  50     </tr>
  51 """
  52
  53 HTML_CATEGORY_HEADER = """
  54     <tr>
  55         <td class="tableHead" width="60%" colspan="{colspan}">{label}</td>
  56         {cells}
  57     </tr>
  58 """
  59
  60 HTML_CATEGORY_HEADER_CELL = '<td class="tableHead" width="15%">{}</td>\n'
  61
  62 HTML_CATEGORY_COVERAGE_CELL = '<td class="{}">&nbsp;</td>\n'
  63
  64 HTML_CATEGORY_ROW = """
  65     <tr>
  66         <td class="coverFile" colspan="{colspan}">{id}</td>
  67         {cells}
  68     </tr>
  69 """
  70
  71
  72 def search(needle, haystack):
  73     # Search for needle in haystack, first escaping needle for regex, then
  74     # replacing %s, %d, etc. with regex wildcards, so the variable messages
  75     # within the dungeon definition will actually match
  76
  77     if needle is None or needle == "" or needle == "NO_MESSAGE":
  78         # if needle is empty, assume we're going to find an empty string
  79         return True
  80
  81     needle_san = (
  82         re.escape(needle)
  83         .replace("\\n", "\n")
  84         .replace("\\t", "\t")
  85         .replace("%S", ".*")
  86         .replace("%s", ".*")
  87         .replace("%d", ".*")
  88         .replace("%V", ".*")
  89     )
  90
  91     return re.search(needle_san, haystack)
  92
  93
  94 def obj_coverage(objects, text, report):
  95     # objects have multiple descriptions based on state
  96     for _, objouter in enumerate(objects):
  97         (obj_name, obj) = objouter
  98         if obj["descriptions"]:
  99             for j, desc in enumerate(obj["descriptions"]):
 100                 name = "{}[{}]".format(obj_name, j)
 101                 if name not in report["messages"]:
 102                     report["messages"][name] = {"covered": False}
 103                     report["total"] += 1
 104                 if not report["messages"][name]["covered"] and search(desc, text):
 105                     report["messages"][name]["covered"] = True
 106                     report["covered"] += 1
 107
 108
 109 def loc_coverage(locations, text, report):
 110     # locations have a long and a short description, that each have to
 111     # be checked separately
 112     for name, loc in locations:
 113         desc = loc["description"]
 114         if name not in report["messages"]:
 115             report["messages"][name] = {"long": False, "short": False}
 116             report["total"] += 2
 117         if not report["messages"][name]["long"] and search(desc["long"], text):
 118             report["messages"][name]["long"] = True
 119             report["covered"] += 1
 120         if not report["messages"][name]["short"] and search(desc["short"], text):
 121             report["messages"][name]["short"] = True
 122             report["covered"] += 1
 123
 124
 125 def hint_coverage(obituaries, text, report):
 126     # hints have a "question" where the hint is offered, followed
 127     # by the actual hint if the player requests it
 128     for _, hintouter in enumerate(obituaries):
 129         hint = hintouter["hint"]
 130         name = hint["name"]
 131         if name not in report["messages"]:
 132             report["messages"][name] = {"question": False, "hint": False}
 133             report["total"] += 2
 134         if not report["messages"][name]["question"] and search(hint["question"], text):
 135             report["messages"][name]["question"] = True
 136             report["covered"] += 1
 137         if not report["messages"][name]["hint"] and search(hint["hint"], text):
 138             report["messages"][name]["hint"] = True
 139             report["covered"] += 1
 140
 141
 142 def obit_coverage(obituaries, text, report):
 143     # obituaries have a "query" where it asks the player for a resurrection,
 144     # followed by a snarky comment if the player says yes
 145     for name, obit in enumerate(obituaries):
 146         if name not in report["messages"]:
 147             report["messages"][name] = {"query": False, "yes_response": False}
 148             report["total"] += 2
 149         if not report["messages"][name]["query"] and search(obit["query"], text):
 150             report["messages"][name]["query"] = True
 151             report["covered"] += 1
 152         if not report["messages"][name]["yes_response"] and search(
 153             obit["yes_response"], text
 154         ):
 155             report["messages"][name]["yes_response"] = True
 156             report["covered"] += 1
 157
 158
 159 def threshold_coverage(classes, text, report):
 160     # works for class thresholds and turn threshold, which have a "message"
 161     # property
 162     for name, item in enumerate(classes):
 163         if name not in report["messages"]:
 164             report["messages"][name] = {"covered": False}
 165             report["total"] += 1
 166         if not report["messages"][name]["covered"] and search(item["message"], text):
 167             report["messages"][name]["covered"] = True
 168             report["covered"] += 1
 169
 170
 171 def arb_coverage(arb_msgs, text, report):
 172     for name, message in arb_msgs:
 173         if name not in report["messages"]:
 174             report["messages"][name] = {"covered": False}
 175             report["total"] += 1
 176         if not report["messages"][name]["covered"] and (
 177             search(message, text) or name in DANGLING_MESSAGES
 178         ):
 179             report["messages"][name]["covered"] = True
 180             report["covered"] += 1
 181
 182
 183 def actions_coverage(items, text, report):
 184     # works for actions
 185     for name, item in items:
 186         if name not in report["messages"]:
 187             report["messages"][name] = {"covered": False}
 188             report["total"] += 1
 189         if not report["messages"][name]["covered"] and (
 190             search(item["message"], text) or name in DANGLING_ACTIONS
 191         ):
 192             report["messages"][name]["covered"] = True
 193             report["covered"] += 1
 194
 195
 196 def coverage_report(db, check_file_contents):
 197     # Create report for each category, including total items,  number of items
 198     # covered, and a list of the covered messages
 199     report = {}
 200     for name in db.keys():
 201         # initialize each catagory
 202         report[name] = {
 203             "name": name,  # convenience for string formatting
 204             "total": 0,
 205             "covered": 0,
 206             "messages": {},
 207         }
 208
 209     # search for each message in every test check file
 210     for chk in check_file_contents:
 211         arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
 212         hint_coverage(db["hints"], chk, report["hints"])
 213         loc_coverage(db["locations"], chk, report["locations"])
 214         obit_coverage(db["obituaries"], chk, report["obituaries"])
 215         obj_coverage(db["objects"], chk, report["objects"])
 216         actions_coverage(db["actions"], chk, report["actions"])
 217         threshold_coverage(db["classes"], chk, report["classes"])
 218         threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
 219
 220     return report
 221
 222
 223 if __name__ == "__main__":
 224     # load DB
 225     try:
 226         with open(YAML_PATH, "r", encoding="ascii", errors="surrogateescape") as f:
 227             db = yaml.safe_load(f)
 228     except IOError as e:
 229         print("ERROR: could not load %s (%s)" % (YAML_PATH, e.strerror))
 230         sys.exit(-1)
 231
 232     # get contents of all the check files
 233     check_file_contents = []
 234     for filename in os.listdir(TEST_DIR):
 235         if filename.endswith(".chk"):
 236             with open(filename, "r", encoding="ascii", errors="surrogateescape") as f:
 237                 check_file_contents.append(f.read())
 238
 239     # run coverage analysis report on dungeon database
 240     report = coverage_report(db, check_file_contents)
 241
 242     # render report output
 243     categories_html = ""
 244     summary_html = ""
 245     summary_stdout = "adventure.yaml coverage rate:\n"
 246     for name, category in sorted(report.items()):
 247         # ignore categories with zero entries
 248         if category["total"] > 0:
 249             # Calculate percent coverage
 250             category["percent"] = (category["covered"] / float(category["total"])) * 100
 251
 252             # render section header
 253             cat_messages = list(category["messages"].items())
 254             cat_keys = cat_messages[0][1].keys()
 255             headers_html = ""
 256             colspan = 10 - len(cat_keys)
 257             for key in cat_keys:
 258                 headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
 259             category_html = HTML_CATEGORY_HEADER.format(
 260                 colspan=colspan, label=category["name"], cells=headers_html
 261             )
 262
 263             # render message coverage row
 264             for message_id, covered in cat_messages:
 265                 category_html_row = ""
 266                 for key, value in covered.items():
 267                     category_html_row += HTML_CATEGORY_COVERAGE_CELL.format(
 268                         "uncovered" if not value else "covered"
 269                     )
 270                 category_html += HTML_CATEGORY_ROW.format(
 271                     id=message_id, colspan=colspan, cells=category_html_row
 272                 )
 273             categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
 274
 275             # render category summaries
 276             summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
 277             summary_html += HTML_SUMMARY_ROW.format(**category)
 278
 279     # output some quick report stats
 280     print(summary_stdout)
 281
 282     if len(sys.argv) > 1:
 283         html_output_path = sys.argv[1]
 284     else:
 285         html_output_path = DEFAULT_HTML_OUTPUT_PATH
 286
 287     # render HTML report
 288     try:
 289         with open(
 290             HTML_TEMPLATE_PATH, "r", encoding="ascii", errors="surrogateescape"
 291         ) as f:
 292             # read in HTML template
 293             html_template = f.read()
 294     except IOError as e:
 295         print("ERROR: reading HTML report template failed ({})".format(e.strerror))
 296         sys.exit(-1)
 297
 298     # parse template with report and write it out
 299     try:
 300         with open(
 301             html_output_path, "w", encoding="ascii", errors="surrogateescape"
 302         ) as f:
 303             f.write(
 304                 html_template.format(categories=categories_html, summary=summary_html)
 305             )
 306     except IOError as e:
 307         print("ERROR: writing HTML report failed ({})".format(e.strerror))