tests/coverage_dungeon.py

   1 #!/usr/bin/env python3
   2 """
   3 This is the open-adventure dungeon text coverage report generator. It
   4 consumes a YAML description of the dungeon and determines whether the
   5 various strings contained are present within the test check files.
   6
   7 The default HTML output is appropriate for use with Gitlab CI.
   8 You can override it with a command-line argument.
   9
  10 The DANGLING list is for actions that should be considered always found
  11 even if the checkfile search doesn't find them. Typically this will because
  12 they emit a templated message that can't be regression-tested by equality.
  13 """
  14
  15 # pylint: disable=consider-using-f-string
  16
  17 import os
  18 import sys
  19 import re
  20 import yaml
  21
  22 TEST_DIR = "."
  23 YAML_PATH = "../adventure.yaml"
  24 HTML_TEMPLATE_PATH = "../templates/coverage_dungeon.html.tpl"
  25 DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
  26 DANGLING = ["ACT_VERSION"]
  27
  28 STDOUT_REPORT_CATEGORY = "  {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
  29
  30 HTML_SUMMARY_ROW = '''
  31     <tr>
  32         <td class="headerItem"><a href="#{name}">{name}:</a></td>
  33         <td class="headerCovTableEntry">{total}</td>
  34         <td class="headerCovTableEntry">{covered}</td>
  35         <td class="headerCovTableEntry">{percent:.1f}%</td>
  36     </tr>
  37 '''
  38
  39 HTML_CATEGORY_SECTION = '''
  40     <tr id="{id}"></tr>
  41     {rows}
  42     <tr>
  43         <td>&nbsp;</td>
  44     </tr>
  45 '''
  46
  47 HTML_CATEGORY_HEADER = '''
  48     <tr>
  49         <td class="tableHead" width="60%" colspan="{colspan}">{label}</td>
  50         {cells}
  51     </tr>
  52 '''
  53
  54 HTML_CATEGORY_HEADER_CELL = '<td class="tableHead" width="15%">{}</td>\n'
  55
  56 HTML_CATEGORY_COVERAGE_CELL = '<td class="{}">&nbsp;</td>\n'
  57
  58 HTML_CATEGORY_ROW = '''
  59     <tr>
  60         <td class="coverFile" colspan="{colspan}">{id}</td>
  61         {cells}
  62     </tr>
  63 '''
  64
  65 def search(needle, haystack):
  66     # Search for needle in haystack, first escaping needle for regex, then
  67     # replacing %s, %d, etc. with regex wildcards, so the variable messages
  68     # within the dungeon definition will actually match
  69
  70     if needle is None or needle == "" or needle == "NO_MESSAGE":
  71         # if needle is empty, assume we're going to find an empty string
  72         return True
  73
  74     needle_san = re.escape(needle) \
  75              .replace("\\n", "\n") \
  76              .replace("\\t", "\t") \
  77              .replace("%S", ".*") \
  78              .replace("%s", ".*") \
  79              .replace("%d", ".*") \
  80              .replace("%V", ".*")
  81
  82     return re.search(needle_san, haystack)
  83
  84 def obj_coverage(objects, text, report):
  85     # objects have multiple descriptions based on state
  86     for _, objouter in enumerate(objects):
  87         (obj_name, obj) = objouter
  88         if obj["descriptions"]:
  89             for j, desc in enumerate(obj["descriptions"]):
  90                 name = "{}[{}]".format(obj_name, j)
  91                 if name not in report["messages"]:
  92                     report["messages"][name] = {"covered" : False}
  93                     report["total"] += 1
  94                 if not report["messages"][name]["covered"] and search(desc, text):
  95                     report["messages"][name]["covered"] = True
  96                     report["covered"] += 1
  97
  98 def loc_coverage(locations, text, report):
  99     # locations have a long and a short description, that each have to
 100     # be checked seperately
 101     for name, loc in locations:
 102         desc = loc["description"]
 103         if name not in report["messages"]:
 104             report["messages"][name] = {"long" : False, "short": False}
 105             report["total"] += 2
 106         if not report["messages"][name]["long"] and search(desc["long"], text):
 107             report["messages"][name]["long"] = True
 108             report["covered"] += 1
 109         if not report["messages"][name]["short"] and search(desc["short"], text):
 110             report["messages"][name]["short"] = True
 111             report["covered"] += 1
 112
 113 def hint_coverage(obituaries, text, report):
 114     # hints have a "question" where the hint is offered, followed
 115     # by the actual hint if the player requests it
 116     for _, hintouter in enumerate(obituaries):
 117         hint = hintouter["hint"]
 118         name = hint["name"]
 119         if name not in report["messages"]:
 120             report["messages"][name] = {"question" : False, "hint": False}
 121             report["total"] += 2
 122         if not report["messages"][name]["question"] and search(hint["question"], text):
 123             report["messages"][name]["question"] = True
 124             report["covered"] += 1
 125         if not report["messages"][name]["hint"] and search(hint["hint"], text):
 126             report["messages"][name]["hint"] = True
 127             report["covered"] += 1
 128
 129 def obit_coverage(obituaries, text, report):
 130     # obituaries have a "query" where it asks the player for a resurrection,
 131     # followed by a snarky comment if the player says yes
 132     for name, obit in enumerate(obituaries):
 133         if name not in report["messages"]:
 134             report["messages"][name] = {"query" : False, "yes_response": False}
 135             report["total"] += 2
 136         if not report["messages"][name]["query"] and search(obit["query"], text):
 137             report["messages"][name]["query"] = True
 138             report["covered"] += 1
 139         if not report["messages"][name]["yes_response"] and search(obit["yes_response"], text):
 140             report["messages"][name]["yes_response"] = True
 141             report["covered"] += 1
 142
 143 def threshold_coverage(classes, text, report):
 144     # works for class thresholds and turn threshold, which have a "message"
 145     # property
 146     for name, item in enumerate(classes):
 147         if name not in report["messages"]:
 148             report["messages"][name] = {"covered" : "False"}
 149             report["total"] += 1
 150         if not report["messages"][name]["covered"] and search(item["message"], text):
 151             report["messages"][name]["covered"] = True
 152             report["covered"] += 1
 153
 154 def arb_coverage(arb_msgs, text, report):
 155     for name, message in arb_msgs:
 156         if name not in report["messages"]:
 157             report["messages"][name] = {"covered" : False}
 158             report["total"] += 1
 159         if not report["messages"][name]["covered"] and search(message, text):
 160             report["messages"][name]["covered"] = True
 161             report["covered"] += 1
 162
 163 def actions_coverage(items, text, report):
 164     # works for actions
 165     for name, item in items:
 166         if name not in report["messages"]:
 167             report["messages"][name] = {"covered" : False}
 168             report["total"] += 1
 169         if not report["messages"][name]["covered"] and (search(item["message"], text) or name in DANGLING):
 170             report["messages"][name]["covered"] = True
 171             report["covered"] += 1
 172
 173 def coverage_report(db, check_file_contents):
 174     # Create report for each catagory, including total items,  number of items
 175     # covered, and a list of the covered messages
 176     report = {}
 177     for name in db.keys():
 178         # initialize each catagory
 179         report[name] = {
 180             "name" : name, # convenience for string formatting
 181             "total" : 0,
 182             "covered" : 0,
 183             "messages" : {}
 184         }
 185
 186     # search for each message in every test check file
 187     for chk in check_file_contents:
 188         arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
 189         hint_coverage(db["hints"], chk, report["hints"])
 190         loc_coverage(db["locations"], chk, report["locations"])
 191         obit_coverage(db["obituaries"], chk, report["obituaries"])
 192         obj_coverage(db["objects"], chk, report["objects"])
 193         actions_coverage(db["actions"], chk, report["actions"])
 194         threshold_coverage(db["classes"], chk, report["classes"])
 195         threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
 196
 197     return report
 198
 199 if __name__ == "__main__":
 200     # load DB
 201     try:
 202         with open(YAML_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 203             db = yaml.safe_load(f)
 204     except IOError as e:
 205         print('ERROR: could not load %s (%s)' % (YAML_PATH, e.strerror))
 206         sys.exit(-1)
 207
 208     # get contents of all the check files
 209     check_file_contents = []
 210     for filename in os.listdir(TEST_DIR):
 211         if filename.endswith(".chk"):
 212             with open(filename, "r", encoding='ascii', errors='surrogateescape') as f:
 213                 check_file_contents.append(f.read())
 214
 215     # run coverage analysis report on dungeon database
 216     report = coverage_report(db, check_file_contents)
 217
 218     # render report output
 219     categories_html = ""
 220     summary_html = ""
 221     summary_stdout = "adventure.yaml coverage rate:\n"
 222     for name, category in sorted(report.items()):
 223         # ignore categories with zero entries
 224         if category["total"] > 0:
 225             # Calculate percent coverage
 226             category["percent"] = (category["covered"] / float(category["total"])) * 100
 227
 228             # render section header
 229             cat_messages = list(category["messages"].items())
 230             cat_keys = cat_messages[0][1].keys()
 231             headers_html = ""
 232             colspan = 10 - len(cat_keys)
 233             for key in cat_keys:
 234                 headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
 235             category_html = HTML_CATEGORY_HEADER.format(colspan=colspan, label=category["name"], cells=headers_html)
 236
 237             # render message coverage row
 238             for message_id, covered in cat_messages:
 239                 category_html_row = ""
 240                 for key, value in covered.items():
 241                     category_html_row += HTML_CATEGORY_COVERAGE_CELL.format("uncovered" if not value else "covered")
 242                 category_html += HTML_CATEGORY_ROW.format(id=message_id,colspan=colspan, cells=category_html_row)
 243             categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
 244
 245             # render category summaries
 246             summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
 247             summary_html += HTML_SUMMARY_ROW.format(**category)
 248
 249     # output some quick report stats
 250     print(summary_stdout)
 251
 252     if len(sys.argv) > 1:
 253         html_output_path = sys.argv[1]
 254     else:
 255         html_output_path = DEFAULT_HTML_OUTPUT_PATH
 256
 257     # render HTML report
 258     try:
 259         with open(HTML_TEMPLATE_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 260             # read in HTML template
 261             html_template = f.read()
 262     except IOError as e:
 263         print('ERROR: reading HTML report template failed ({})'.format(e.strerror))
 264         sys.exit(-1)
 265
 266     # parse template with report and write it out
 267     try:
 268         with open(html_output_path, "w", encoding='ascii', errors='surrogateescape') as f:
 269             f.write(html_template.format(categories=categories_html, summary=summary_html))
 270     except IOError as e:
 271         print('ERROR: writing HTML report failed ({})'.format(e.strerror))