tests/coverage_dungeon.py

   1 #!/usr/bin/env python3
   2 # SPDX-FileCopyrightText: Eric S. Raymond <esr@thyrsus.com>
   3 # SPDX-License-Identifier: BSD-2-Clause
   4 """
   5 This is the open-adventure dungeon text coverage report generator. It
   6 consumes a YAML description of the dungeon and determines whether the
   7 various strings contained are present within the test check files.
   8
   9 The default HTML output is appropriate for use with Gitlab CI.
  10 You can override it with a command-line argument.
  11
  12 The DANGLING lists are for actions and messages that should be
  13 considered always found even if the checkfile search doesn't find them.
  14 Typically this will because an action emit a templated message that
  15 can't be regression-tested by equality.
  16 """
  17
  18 # pylint: disable=consider-using-f-string,line-too-long,invalid-name,missing-function-docstring,redefined-outer-name
  19
  20 import os
  21 import sys
  22 import re
  23 import yaml
  24
  25 TEST_DIR = "."
  26 YAML_PATH = "../adventure.yaml"
  27 HTML_TEMPLATE_PATH = "../templates/coverage_dungeon.html.tpl"
  28 DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
  29 DANGLING_ACTIONS = ["ACT_VERSION"]
  30 DANGLING_MESSAGES = ["SAVERESUME_DISABLED"]
  31
  32 STDOUT_REPORT_CATEGORY = "  {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
  33
  34 HTML_SUMMARY_ROW = '''
  35     <tr>
  36         <td class="headerItem"><a href="#{name}">{name}:</a></td>
  37         <td class="headerCovTableEntry">{total}</td>
  38         <td class="headerCovTableEntry">{covered}</td>
  39         <td class="headerCovTableEntry">{percent:.1f}%</td>
  40     </tr>
  41 '''
  42
  43 HTML_CATEGORY_SECTION = '''
  44     <tr id="{id}"></tr>
  45     {rows}
  46     <tr>
  47         <td>&nbsp;</td>
  48     </tr>
  49 '''
  50
  51 HTML_CATEGORY_HEADER = '''
  52     <tr>
  53         <td class="tableHead" width="60%" colspan="{colspan}">{label}</td>
  54         {cells}
  55     </tr>
  56 '''
  57
  58 HTML_CATEGORY_HEADER_CELL = '<td class="tableHead" width="15%">{}</td>\n'
  59
  60 HTML_CATEGORY_COVERAGE_CELL = '<td class="{}">&nbsp;</td>\n'
  61
  62 HTML_CATEGORY_ROW = '''
  63     <tr>
  64         <td class="coverFile" colspan="{colspan}">{id}</td>
  65         {cells}
  66     </tr>
  67 '''
  68
  69 def search(needle, haystack):
  70     # Search for needle in haystack, first escaping needle for regex, then
  71     # replacing %s, %d, etc. with regex wildcards, so the variable messages
  72     # within the dungeon definition will actually match
  73
  74     if needle is None or needle == "" or needle == "NO_MESSAGE":
  75         # if needle is empty, assume we're going to find an empty string
  76         return True
  77
  78     needle_san = re.escape(needle) \
  79              .replace("\\n", "\n") \
  80              .replace("\\t", "\t") \
  81              .replace("%S", ".*") \
  82              .replace("%s", ".*") \
  83              .replace("%d", ".*") \
  84              .replace("%V", ".*")
  85
  86     return re.search(needle_san, haystack)
  87
  88 def obj_coverage(objects, text, report):
  89     # objects have multiple descriptions based on state
  90     for _, objouter in enumerate(objects):
  91         (obj_name, obj) = objouter
  92         if obj["descriptions"]:
  93             for j, desc in enumerate(obj["descriptions"]):
  94                 name = "{}[{}]".format(obj_name, j)
  95                 if name not in report["messages"]:
  96                     report["messages"][name] = {"covered" : False}
  97                     report["total"] += 1
  98                 if not report["messages"][name]["covered"] and search(desc, text):
  99                     report["messages"][name]["covered"] = True
 100                     report["covered"] += 1
 101
 102 def loc_coverage(locations, text, report):
 103     # locations have a long and a short description, that each have to
 104     # be checked separately
 105     for name, loc in locations:
 106         desc = loc["description"]
 107         if name not in report["messages"]:
 108             report["messages"][name] = {"long" : False, "short": False}
 109             report["total"] += 2
 110         if not report["messages"][name]["long"] and search(desc["long"], text):
 111             report["messages"][name]["long"] = True
 112             report["covered"] += 1
 113         if not report["messages"][name]["short"] and search(desc["short"], text):
 114             report["messages"][name]["short"] = True
 115             report["covered"] += 1
 116
 117 def hint_coverage(obituaries, text, report):
 118     # hints have a "question" where the hint is offered, followed
 119     # by the actual hint if the player requests it
 120     for _, hintouter in enumerate(obituaries):
 121         hint = hintouter["hint"]
 122         name = hint["name"]
 123         if name not in report["messages"]:
 124             report["messages"][name] = {"question" : False, "hint": False}
 125             report["total"] += 2
 126         if not report["messages"][name]["question"] and search(hint["question"], text):
 127             report["messages"][name]["question"] = True
 128             report["covered"] += 1
 129         if not report["messages"][name]["hint"] and search(hint["hint"], text):
 130             report["messages"][name]["hint"] = True
 131             report["covered"] += 1
 132
 133 def obit_coverage(obituaries, text, report):
 134     # obituaries have a "query" where it asks the player for a resurrection,
 135     # followed by a snarky comment if the player says yes
 136     for name, obit in enumerate(obituaries):
 137         if name not in report["messages"]:
 138             report["messages"][name] = {"query" : False, "yes_response": False}
 139             report["total"] += 2
 140         if not report["messages"][name]["query"] and search(obit["query"], text):
 141             report["messages"][name]["query"] = True
 142             report["covered"] += 1
 143         if not report["messages"][name]["yes_response"] and search(obit["yes_response"], text):
 144             report["messages"][name]["yes_response"] = True
 145             report["covered"] += 1
 146
 147 def threshold_coverage(classes, text, report):
 148     # works for class thresholds and turn threshold, which have a "message"
 149     # property
 150     for name, item in enumerate(classes):
 151         if name not in report["messages"]:
 152             report["messages"][name] = {"covered" : False}
 153             report["total"] += 1
 154         if not report["messages"][name]["covered"] and search(item["message"], text):
 155             report["messages"][name]["covered"] = True
 156             report["covered"] += 1
 157
 158 def arb_coverage(arb_msgs, text, report):
 159     for name, message in arb_msgs:
 160         if name not in report["messages"]:
 161             report["messages"][name] = {"covered" : False}
 162             report["total"] += 1
 163         if not report["messages"][name]["covered"] and (search(message, text) or name in DANGLING_MESSAGES):
 164             report["messages"][name]["covered"] = True
 165             report["covered"] += 1
 166
 167 def actions_coverage(items, text, report):
 168     # works for actions
 169     for name, item in items:
 170         if name not in report["messages"]:
 171             report["messages"][name] = {"covered" : False}
 172             report["total"] += 1
 173         if not report["messages"][name]["covered"] and (search(item["message"], text) or name in DANGLING_ACTIONS):
 174             report["messages"][name]["covered"] = True
 175             report["covered"] += 1
 176
 177 def coverage_report(db, check_file_contents):
 178     # Create report for each category, including total items,  number of items
 179     # covered, and a list of the covered messages
 180     report = {}
 181     for name in db.keys():
 182         # initialize each catagory
 183         report[name] = {
 184             "name" : name, # convenience for string formatting
 185             "total" : 0,
 186             "covered" : 0,
 187             "messages" : {}
 188         }
 189
 190     # search for each message in every test check file
 191     for chk in check_file_contents:
 192         arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
 193         hint_coverage(db["hints"], chk, report["hints"])
 194         loc_coverage(db["locations"], chk, report["locations"])
 195         obit_coverage(db["obituaries"], chk, report["obituaries"])
 196         obj_coverage(db["objects"], chk, report["objects"])
 197         actions_coverage(db["actions"], chk, report["actions"])
 198         threshold_coverage(db["classes"], chk, report["classes"])
 199         threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
 200
 201     return report
 202
 203 if __name__ == "__main__":
 204     # load DB
 205     try:
 206         with open(YAML_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 207             db = yaml.safe_load(f)
 208     except IOError as e:
 209         print('ERROR: could not load %s (%s)' % (YAML_PATH, e.strerror))
 210         sys.exit(-1)
 211
 212     # get contents of all the check files
 213     check_file_contents = []
 214     for filename in os.listdir(TEST_DIR):
 215         if filename.endswith(".chk"):
 216             with open(filename, "r", encoding='ascii', errors='surrogateescape') as f:
 217                 check_file_contents.append(f.read())
 218
 219     # run coverage analysis report on dungeon database
 220     report = coverage_report(db, check_file_contents)
 221
 222     # render report output
 223     categories_html = ""
 224     summary_html = ""
 225     summary_stdout = "adventure.yaml coverage rate:\n"
 226     for name, category in sorted(report.items()):
 227         # ignore categories with zero entries
 228         if category["total"] > 0:
 229             # Calculate percent coverage
 230             category["percent"] = (category["covered"] / float(category["total"])) * 100
 231
 232             # render section header
 233             cat_messages = list(category["messages"].items())
 234             cat_keys = cat_messages[0][1].keys()
 235             headers_html = ""
 236             colspan = 10 - len(cat_keys)
 237             for key in cat_keys:
 238                 headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
 239             category_html = HTML_CATEGORY_HEADER.format(colspan=colspan, label=category["name"], cells=headers_html)
 240
 241             # render message coverage row
 242             for message_id, covered in cat_messages:
 243                 category_html_row = ""
 244                 for key, value in covered.items():
 245                     category_html_row += HTML_CATEGORY_COVERAGE_CELL.format("uncovered" if not value else "covered")
 246                 category_html += HTML_CATEGORY_ROW.format(id=message_id,colspan=colspan, cells=category_html_row)
 247             categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
 248
 249             # render category summaries
 250             summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
 251             summary_html += HTML_SUMMARY_ROW.format(**category)
 252
 253     # output some quick report stats
 254     print(summary_stdout)
 255
 256     if len(sys.argv) > 1:
 257         html_output_path = sys.argv[1]
 258     else:
 259         html_output_path = DEFAULT_HTML_OUTPUT_PATH
 260
 261     # render HTML report
 262     try:
 263         with open(HTML_TEMPLATE_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 264             # read in HTML template
 265             html_template = f.read()
 266     except IOError as e:
 267         print('ERROR: reading HTML report template failed ({})'.format(e.strerror))
 268         sys.exit(-1)
 269
 270     # parse template with report and write it out
 271     try:
 272         with open(html_output_path, "w", encoding='ascii', errors='surrogateescape') as f:
 273             f.write(html_template.format(categories=categories_html, summary=summary_html))
 274     except IOError as e:
 275         print('ERROR: writing HTML report failed ({})'.format(e.strerror))