tests/coverage_dungeon.py

   1 #!/usr/bin/env python
   2
   3 # This is the open-adventure dungeon text coverage report generator. It
   4 # consumes a YAML description of the dungeon and determines whether the
   5 # various strings contained are present within the test check files.
   6 #
   7 # The default HTML output is appropriate for use with Gitlab CI.
   8 # You can override it with a command-line argument.
   9
  10 import os
  11 import sys
  12 import yaml
  13 import re
  14
  15 TEST_DIR = "."
  16 YAML_PATH = "../adventure.yaml"
  17 HTML_TEMPLATE_PATH = "coverage_dungeon.html.tpl"
  18 DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
  19
  20 STDOUT_REPORT_CATEGORY = "  {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
  21
  22 HTML_SUMMARY_ROW = """
  23     <tr>
  24         <td class="headerItem"><a href="#{name}">{name}:</a></td>
  25         <td class="headerCovTableEntry">{total}</td>
  26         <td class="headerCovTableEntry">{covered}</td>
  27         <td class="headerCovTableEntry">{percent:.1f}%</td>
  28     </tr>
  29 """
  30
  31 HTML_CATEGORY_SECTION = """
  32     <tr id="{id}"></tr>
  33     {rows}
  34     <tr>
  35         <td>&nbsp;</td>
  36     </tr>
  37 """
  38
  39 HTML_CATEGORY_HEADER = """
  40     <tr>
  41         <td class="tableHead" width="60%" colspan="{colspan}">{label}</td>
  42         {cells}
  43     </tr>
  44 """
  45
  46 HTML_CATEGORY_HEADER_CELL = '<td class="tableHead" width="15%">{}</td>\n'
  47
  48 HTML_CATEGORY_COVERAGE_CELL = '<td class="{}">&nbsp;</td>\n'
  49
  50 HTML_CATEGORY_ROW = """
  51     <tr>
  52         <td class="coverFile" colspan="{colspan}">{id}</td>
  53         {cells}
  54     </tr>
  55 """
  56
  57 def search(needle, haystack):
  58     # Search for needle in haystack, first escaping needle for regex, then
  59     # replacing %s, %d, etc. with regex wildcards, so the variable messages
  60     # within the dungeon definition will actually match
  61     needle = re.escape(needle) \
  62              .replace("\\n", "\n") \
  63              .replace("\\t", "\t") \
  64              .replace("\%S", ".*") \
  65              .replace("\%s", ".*") \
  66              .replace("\%d", ".*") \
  67              .replace("\%V", ".*")
  68
  69     return re.search(needle, haystack)
  70
  71 def obj_coverage(objects, text, report):
  72     # objects have multiple descriptions based on state
  73     for i, objouter in enumerate(objects):
  74         (obj_name, obj) = objouter
  75         if obj["descriptions"]:
  76             for j, desc in enumerate(obj["descriptions"]):
  77                 name = "{}[{}]".format(obj_name, j)
  78                 if name not in report["messages"]:
  79                     report["messages"][name] = {"covered" : False}
  80                     report["total"] += 1
  81                 if report["messages"][name]["covered"] != True:
  82                     if desc == None or desc == '' or search(desc, text):
  83                         report["messages"][name]["covered"] = True
  84                         report["covered"] += 1
  85
  86 def loc_coverage(locations, text, report):
  87     # locations have a long and a short description, that each have to
  88     # be checked seperately
  89     for name, loc in locations:
  90         if name not in report["messages"]:
  91             report["messages"][name] = {"long" : False, "short": False}
  92             report["total"] += 2
  93         if report["messages"][name]["long"] != True:
  94             if loc["description"]["long"] == None or loc["description"]["long"] == '' or search(loc["description"]["long"], text):
  95                 report["messages"][name]["long"] = True
  96                 report["covered"] += 1
  97         if report["messages"][name]["short"] != True:
  98             if loc["description"]["short"] == None or loc["description"]["short"] == '' or search(loc["description"]["short"], text):
  99                 report["messages"][name]["short"] = True
 100                 report["covered"] += 1
 101
 102 def hint_coverage(obituaries, text, report):
 103     # hints have a "question" where the hint is offered, followed
 104     # by the actual hint if the player requests it
 105     for i, hintouter in enumerate(obituaries):
 106         hint = hintouter["hint"]
 107         name = hint["name"]
 108         if name not in report["messages"]:
 109             report["messages"][name] = {"question" : False, "hint": False}
 110             report["total"] += 2
 111         if report["messages"][name]["question"] != True and search(hint["question"], text):
 112             report["messages"][name]["question"] = True
 113             report["covered"] += 1
 114         if report["messages"][name]["hint"] != True and search(hint["hint"], text):
 115             report["messages"][name]["hint"] = True
 116             report["covered"] += 1
 117
 118 def obit_coverage(obituaries, text, report):
 119     # obituaries have a "query" where it asks the player for a resurrection,
 120     # followed by a snarky comment if the player says yes
 121     for name, obit in enumerate(obituaries):
 122         if name not in report["messages"]:
 123             report["messages"][name] = {"query" : False, "yes_response": False}
 124             report["total"] += 2
 125         if report["messages"][name]["query"] != True and search(obit["query"], text):
 126             report["messages"][name]["query"] = True
 127             report["covered"] += 1
 128         if report["messages"][name]["yes_response"] != True and search(obit["yes_response"], text):
 129             report["messages"][name]["yes_response"] = True
 130             report["covered"] += 1
 131
 132 def threshold_coverage(classes, text, report):
 133     # works for class thresholds and turn threshold, which have a "message"
 134     # property
 135     for name, item in enumerate(classes):
 136         if name not in report["messages"]:
 137             report["messages"][name] = {"covered" : "False"}
 138             report["total"] += 1
 139         if report["messages"][name]["covered"] != True:
 140             if item["message"] == None or item["message"] == "NO_MESSAGE" or search(item["message"], text):
 141                 report["messages"][name]["covered"] = True
 142                 report["covered"] += 1
 143
 144 def arb_coverage(arb_msgs, text, report):
 145     for name, message in arb_msgs:
 146         if name not in report["messages"]:
 147             report["messages"][name] = {"covered" : False}
 148             report["total"] += 1
 149         if report["messages"][name]["covered"] != True:
 150             if message == None or search(message, text):
 151                 report["messages"][name]["covered"] = True
 152                 report["covered"] += 1
 153
 154 def specials_actions_coverage(items, text, report):
 155     # works for actions or specials
 156     for name, item in items:
 157         if name not in report["messages"]:
 158             report["messages"][name] = {"covered" : False}
 159             report["total"] += 1
 160         if report["messages"][name]["covered"] != True:
 161             if item["message"] == None or item["message"] == "NO_MESSAGE" or search(item["message"], text):
 162                 report["messages"][name]["covered"] = True
 163                 report["covered"] += 1
 164
 165 def coverage_report(db, check_file_contents):
 166     # Create report for each catagory, including total items,  number of items
 167     # covered, and a list of the covered messages
 168     report = {}
 169     for name in db.keys():
 170         # initialize each catagory
 171         report[name] = {
 172             "name" : name, # convenience for string formatting
 173             "total" : 0,
 174             "covered" : 0,
 175             "messages" : {}
 176         }
 177
 178     # search for each message in ever test check file
 179     for chk in check_file_contents:
 180         arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
 181         hint_coverage(db["hints"], chk, report["hints"])
 182         loc_coverage(db["locations"], chk, report["locations"])
 183         obit_coverage(db["obituaries"], chk, report["obituaries"])
 184         obj_coverage(db["objects"], chk, report["objects"])
 185         specials_actions_coverage(db["actions"], chk, report["actions"])
 186         specials_actions_coverage(db["specials"], chk, report["specials"])
 187         threshold_coverage(db["classes"], chk, report["classes"])
 188         threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
 189
 190     return report
 191
 192 if __name__ == "__main__":
 193     # load DB
 194     try:
 195         with open(YAML_PATH, "r") as f:
 196             db = yaml.load(f)
 197     except IOError as e:
 198         print('ERROR: could not load {} ({}})'.format(YAML_PATH, e.strerror))
 199         exit(-1)
 200
 201     # get contents of all the check files
 202     check_file_contents = []
 203     for filename in os.listdir(TEST_DIR):
 204         if filename.endswith(".chk"):
 205             with open(filename, "r") as f:
 206                 check_file_contents.append(f.read())
 207
 208     # run coverage analysis report on dungeon database
 209     report = coverage_report(db, check_file_contents)
 210
 211     # render report output
 212     categories_html = ""
 213     summary_html = ""
 214     summary_stdout = "adventure.yaml coverage rate:\n"
 215     for name, category in sorted(report.items()):
 216         # ignore categories with zero entries
 217         if category["total"] > 0:
 218             # Calculate percent coverage
 219             category["percent"] = (category["covered"] / float(category["total"])) * 100
 220
 221             # render section header
 222             cat_keys = list(category["messages"].items())[0][1].keys()
 223             headers_html = ""
 224             colspan = 10 - len(cat_keys)
 225             for key in cat_keys:
 226                 headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
 227             category_html = HTML_CATEGORY_HEADER.format(colspan=colspan, label=category["name"], cells=headers_html)
 228
 229             # render message coverage row
 230             for message_id, covered in sorted(category["messages"].items()):
 231                 category_html_row = ""
 232                 for key, value in covered.items():
 233                     category_html_row += HTML_CATEGORY_COVERAGE_CELL.format("uncovered" if value != True else "covered")
 234                 category_html += HTML_CATEGORY_ROW.format(id=message_id,colspan=colspan, cells=category_html_row)
 235             categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
 236
 237             # render category summaries
 238             summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
 239             summary_html += HTML_SUMMARY_ROW.format(**category)
 240
 241     # output some quick report stats
 242     print(summary_stdout)
 243
 244     if len(sys.argv) > 1:
 245         html_output_path = sys.argv[1]
 246     else:
 247         html_output_path = DEFAULT_HTML_OUTPUT_PATH
 248
 249     # render HTML report
 250     try:
 251         with open(HTML_TEMPLATE_PATH, "r") as f:
 252             # read in HTML template
 253             html_template = f.read()
 254     except IOError as e:
 255         print('ERROR: reading HTML report template failed ({})'.format(e.strerror))
 256         exit(-1)
 257
 258     # parse template with report and write it out
 259     try:
 260         with open(html_output_path, "w") as f:
 261             f.write(html_template.format(categories=categories_html, summary=summary_html))
 262     except IOError as e:
 263         print('ERROR: writing HTML report failed ({})'.format(e.strerror))