tests/coverage_dungeon.py

   1 #!/usr/bin/env python3
   2 """
   3 This is the open-adventure dungeon text coverage report generator. It
   4 consumes a YAML description of the dungeon and determines whether the
   5 various strings contained are present within the test check files.
   6
   7 The default HTML output is appropriate for use with Gitlab CI.
   8 You can override it with a command-line argument.
   9
  10 The DANGLING lists are for actions and messages that should be
  11 considered always found even if the checkfile search doesn't find them.
  12 Typically this will because an action emit a templated message that
  13 can't be regression-tested by equality.
  14 """
  15
  16 # pylint: disable=consider-using-f-string,line-too-long,invalid-name,missing-function-docstring,redefined-outer-name
  17
  18 import os
  19 import sys
  20 import re
  21 import yaml
  22
  23 TEST_DIR = "."
  24 YAML_PATH = "../adventure.yaml"
  25 HTML_TEMPLATE_PATH = "../templates/coverage_dungeon.html.tpl"
  26 DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
  27 DANGLING_ACTIONS = ["ACT_VERSION"]
  28 DANGLING_MESSAGES = ["SAVERESUME_DISABLED"]
  29
  30 STDOUT_REPORT_CATEGORY = "  {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
  31
  32 HTML_SUMMARY_ROW = '''
  33     <tr>
  34         <td class="headerItem"><a href="#{name}">{name}:</a></td>
  35         <td class="headerCovTableEntry">{total}</td>
  36         <td class="headerCovTableEntry">{covered}</td>
  37         <td class="headerCovTableEntry">{percent:.1f}%</td>
  38     </tr>
  39 '''
  40
  41 HTML_CATEGORY_SECTION = '''
  42     <tr id="{id}"></tr>
  43     {rows}
  44     <tr>
  45         <td>&nbsp;</td>
  46     </tr>
  47 '''
  48
  49 HTML_CATEGORY_HEADER = '''
  50     <tr>
  51         <td class="tableHead" width="60%" colspan="{colspan}">{label}</td>
  52         {cells}
  53     </tr>
  54 '''
  55
  56 HTML_CATEGORY_HEADER_CELL = '<td class="tableHead" width="15%">{}</td>\n'
  57
  58 HTML_CATEGORY_COVERAGE_CELL = '<td class="{}">&nbsp;</td>\n'
  59
  60 HTML_CATEGORY_ROW = '''
  61     <tr>
  62         <td class="coverFile" colspan="{colspan}">{id}</td>
  63         {cells}
  64     </tr>
  65 '''
  66
  67 def search(needle, haystack):
  68     # Search for needle in haystack, first escaping needle for regex, then
  69     # replacing %s, %d, etc. with regex wildcards, so the variable messages
  70     # within the dungeon definition will actually match
  71
  72     if needle is None or needle == "" or needle == "NO_MESSAGE":
  73         # if needle is empty, assume we're going to find an empty string
  74         return True
  75
  76     needle_san = re.escape(needle) \
  77              .replace("\\n", "\n") \
  78              .replace("\\t", "\t") \
  79              .replace("%S", ".*") \
  80              .replace("%s", ".*") \
  81              .replace("%d", ".*") \
  82              .replace("%V", ".*")
  83
  84     return re.search(needle_san, haystack)
  85
  86 def obj_coverage(objects, text, report):
  87     # objects have multiple descriptions based on state
  88     for _, objouter in enumerate(objects):
  89         (obj_name, obj) = objouter
  90         if obj["descriptions"]:
  91             for j, desc in enumerate(obj["descriptions"]):
  92                 name = "{}[{}]".format(obj_name, j)
  93                 if name not in report["messages"]:
  94                     report["messages"][name] = {"covered" : False}
  95                     report["total"] += 1
  96                 if not report["messages"][name]["covered"] and search(desc, text):
  97                     report["messages"][name]["covered"] = True
  98                     report["covered"] += 1
  99
 100 def loc_coverage(locations, text, report):
 101     # locations have a long and a short description, that each have to
 102     # be checked seperately
 103     for name, loc in locations:
 104         desc = loc["description"]
 105         if name not in report["messages"]:
 106             report["messages"][name] = {"long" : False, "short": False}
 107             report["total"] += 2
 108         if not report["messages"][name]["long"] and search(desc["long"], text):
 109             report["messages"][name]["long"] = True
 110             report["covered"] += 1
 111         if not report["messages"][name]["short"] and search(desc["short"], text):
 112             report["messages"][name]["short"] = True
 113             report["covered"] += 1
 114
 115 def hint_coverage(obituaries, text, report):
 116     # hints have a "question" where the hint is offered, followed
 117     # by the actual hint if the player requests it
 118     for _, hintouter in enumerate(obituaries):
 119         hint = hintouter["hint"]
 120         name = hint["name"]
 121         if name not in report["messages"]:
 122             report["messages"][name] = {"question" : False, "hint": False}
 123             report["total"] += 2
 124         if not report["messages"][name]["question"] and search(hint["question"], text):
 125             report["messages"][name]["question"] = True
 126             report["covered"] += 1
 127         if not report["messages"][name]["hint"] and search(hint["hint"], text):
 128             report["messages"][name]["hint"] = True
 129             report["covered"] += 1
 130
 131 def obit_coverage(obituaries, text, report):
 132     # obituaries have a "query" where it asks the player for a resurrection,
 133     # followed by a snarky comment if the player says yes
 134     for name, obit in enumerate(obituaries):
 135         if name not in report["messages"]:
 136             report["messages"][name] = {"query" : False, "yes_response": False}
 137             report["total"] += 2
 138         if not report["messages"][name]["query"] and search(obit["query"], text):
 139             report["messages"][name]["query"] = True
 140             report["covered"] += 1
 141         if not report["messages"][name]["yes_response"] and search(obit["yes_response"], text):
 142             report["messages"][name]["yes_response"] = True
 143             report["covered"] += 1
 144
 145 def threshold_coverage(classes, text, report):
 146     # works for class thresholds and turn threshold, which have a "message"
 147     # property
 148     for name, item in enumerate(classes):
 149         if name not in report["messages"]:
 150             report["messages"][name] = {"covered" : "False"}
 151             report["total"] += 1
 152         if not report["messages"][name]["covered"] and search(item["message"], text):
 153             report["messages"][name]["covered"] = True
 154             report["covered"] += 1
 155
 156 def arb_coverage(arb_msgs, text, report):
 157     for name, message in arb_msgs:
 158         if name not in report["messages"]:
 159             report["messages"][name] = {"covered" : False}
 160             report["total"] += 1
 161         if not report["messages"][name]["covered"] and search(message, text) or name in DANGLING_MESSAGES:
 162             report["messages"][name]["covered"] = True
 163             report["covered"] += 1
 164
 165 def actions_coverage(items, text, report):
 166     # works for actions
 167     for name, item in items:
 168         if name not in report["messages"]:
 169             report["messages"][name] = {"covered" : False}
 170             report["total"] += 1
 171         if not report["messages"][name]["covered"] and (search(item["message"], text) or name in DANGLING_ACTIONS):
 172             report["messages"][name]["covered"] = True
 173             report["covered"] += 1
 174
 175 def coverage_report(db, check_file_contents):
 176     # Create report for each catagory, including total items,  number of items
 177     # covered, and a list of the covered messages
 178     report = {}
 179     for name in db.keys():
 180         # initialize each catagory
 181         report[name] = {
 182             "name" : name, # convenience for string formatting
 183             "total" : 0,
 184             "covered" : 0,
 185             "messages" : {}
 186         }
 187
 188     # search for each message in every test check file
 189     for chk in check_file_contents:
 190         arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
 191         hint_coverage(db["hints"], chk, report["hints"])
 192         loc_coverage(db["locations"], chk, report["locations"])
 193         obit_coverage(db["obituaries"], chk, report["obituaries"])
 194         obj_coverage(db["objects"], chk, report["objects"])
 195         actions_coverage(db["actions"], chk, report["actions"])
 196         threshold_coverage(db["classes"], chk, report["classes"])
 197         threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
 198
 199     return report
 200
 201 if __name__ == "__main__":
 202     # load DB
 203     try:
 204         with open(YAML_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 205             db = yaml.safe_load(f)
 206     except IOError as e:
 207         print('ERROR: could not load %s (%s)' % (YAML_PATH, e.strerror))
 208         sys.exit(-1)
 209
 210     # get contents of all the check files
 211     check_file_contents = []
 212     for filename in os.listdir(TEST_DIR):
 213         if filename.endswith(".chk"):
 214             with open(filename, "r", encoding='ascii', errors='surrogateescape') as f:
 215                 check_file_contents.append(f.read())
 216
 217     # run coverage analysis report on dungeon database
 218     report = coverage_report(db, check_file_contents)
 219
 220     # render report output
 221     categories_html = ""
 222     summary_html = ""
 223     summary_stdout = "adventure.yaml coverage rate:\n"
 224     for name, category in sorted(report.items()):
 225         # ignore categories with zero entries
 226         if category["total"] > 0:
 227             # Calculate percent coverage
 228             category["percent"] = (category["covered"] / float(category["total"])) * 100
 229
 230             # render section header
 231             cat_messages = list(category["messages"].items())
 232             cat_keys = cat_messages[0][1].keys()
 233             headers_html = ""
 234             colspan = 10 - len(cat_keys)
 235             for key in cat_keys:
 236                 headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
 237             category_html = HTML_CATEGORY_HEADER.format(colspan=colspan, label=category["name"], cells=headers_html)
 238
 239             # render message coverage row
 240             for message_id, covered in cat_messages:
 241                 category_html_row = ""
 242                 for key, value in covered.items():
 243                     category_html_row += HTML_CATEGORY_COVERAGE_CELL.format("uncovered" if not value else "covered")
 244                 category_html += HTML_CATEGORY_ROW.format(id=message_id,colspan=colspan, cells=category_html_row)
 245             categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
 246
 247             # render category summaries
 248             summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
 249             summary_html += HTML_SUMMARY_ROW.format(**category)
 250
 251     # output some quick report stats
 252     print(summary_stdout)
 253
 254     if len(sys.argv) > 1:
 255         html_output_path = sys.argv[1]
 256     else:
 257         html_output_path = DEFAULT_HTML_OUTPUT_PATH
 258
 259     # render HTML report
 260     try:
 261         with open(HTML_TEMPLATE_PATH, "r", encoding='ascii', errors='surrogateescape') as f:
 262             # read in HTML template
 263             html_template = f.read()
 264     except IOError as e:
 265         print('ERROR: reading HTML report template failed ({})'.format(e.strerror))
 266         sys.exit(-1)
 267
 268     # parse template with report and write it out
 269     try:
 270         with open(html_output_path, "w", encoding='ascii', errors='surrogateescape') as f:
 271             f.write(html_template.format(categories=categories_html, summary=summary_html))
 272     except IOError as e:
 273         print('ERROR: writing HTML report failed ({})'.format(e.strerror))