X-Git-Url: https://jxself.org/git/?p=open-adventure.git;a=blobdiff_plain;f=tests%2Fcoverage_dungeon.py;h=6484fa79b008f0ff15d95bd6e530c0cc5275e3a3;hp=44ce626a11e3dd626c130a7a6b518dd07bb74972;hb=319c5830c835b86cdcaa56e2ba9bdb50549e9186;hpb=0e2c8511318f0df697120296c5e849202c0ac09f
diff --git a/tests/coverage_dungeon.py b/tests/coverage_dungeon.py
index 44ce626..6484fa7 100755
--- a/tests/coverage_dungeon.py
+++ b/tests/coverage_dungeon.py
@@ -4,326 +4,260 @@
# consumes a YAML description of the dungeon and determines whether the
# various strings contained are present within the test check files.
#
-# Currently, only the location descriptions, arbitrary messages, object
-# descriptions, hints, classes and turn thrusholds are supported. This will
-# be expanded in the future.
+# The default HTML output is appropriate for use with Gitlab CI.
+# You can override it with a command-line argument.
import os
+import sys
import yaml
import re
-test_dir = "."
-yaml_name = "../adventure.yaml"
-html_template_path = "coverage_dungeon.html.tpl"
-html_output_path = "../coverage/adventure.yaml.html"
+TEST_DIR = "."
+YAML_PATH = "../adventure.yaml"
+HTML_TEMPLATE_PATH = "../templates/coverage_dungeon.html.tpl"
+DEFAULT_HTML_OUTPUT_PATH = "../coverage/adventure.yaml.html"
-location_row = """
+STDOUT_REPORT_CATEGORY = " {name:.<19}: {percent:5.1f}% covered ({covered} of {total})\n"
+
+HTML_SUMMARY_ROW = '''
- | {} |
- |
- |
+
+
+
+
-"""
+'''
-arb_msg_row = """
+HTML_CATEGORY_SECTION = '''
+ |
+ {rows}
- | {} |
- |
+ |
-"""
+'''
+
+HTML_CATEGORY_HEADER = '''
+
+ | {label} |
+ {cells}
+
+'''
+
+HTML_CATEGORY_HEADER_CELL = '{} | \n'
-object_row = """
+HTML_CATEGORY_COVERAGE_CELL = ' | \n'
+
+HTML_CATEGORY_ROW = '''
- | {} |
- |
+ {id} |
+ {cells}
-"""
+'''
def search(needle, haystack):
# Search for needle in haystack, first escaping needle for regex, then
# replacing %s, %d, etc. with regex wildcards, so the variable messages
# within the dungeon definition will actually match
- needle = re.escape(needle) \
+
+ if needle == None or needle == "" or needle == "NO_MESSAGE":
+ # if needle is empty, assume we're going to find an empty string
+ return True
+
+ needle_san = re.escape(needle) \
+ .replace("\\n", "\n") \
+ .replace("\\t", "\t") \
.replace("\%S", ".*") \
.replace("\%s", ".*") \
.replace("\%d", ".*") \
.replace("\%V", ".*")
- return re.search(needle, haystack)
-
-def loc_coverage(locations, text):
- for locname, loc in locations:
- if loc["description"]["long"] == None or loc["description"]["long"] == '':
- loc["description"]["long"] = True
- if loc["description"]["long"] != True:
- if search(loc["description"]["long"], text):
- loc["description"]["long"] = True
- if loc["description"]["short"] == None or loc["description"]["short"] == '':
- loc["description"]["short"] = True
- if loc["description"]["short"] != True:
- if search(loc["description"]["short"], text):
- loc["description"]["short"] = True
-
-def arb_coverage(arb_msgs, text):
- for i, msg in enumerate(arb_msgs):
- (msg_name, msg_text) = msg
- if msg_text == None or msg_text == '':
- arb_msgs[i] = (msg_name, True)
- elif msg_text != True:
- if search(msg_text, text):
- arb_msgs[i] = (msg_name, True)
-
-def obj_coverage(objects, text):
+ return re.search(needle_san, haystack)
+
+def obj_coverage(objects, text, report):
+ # objects have multiple descriptions based on state
for i, objouter in enumerate(objects):
(obj_name, obj) = objouter
if obj["descriptions"]:
for j, desc in enumerate(obj["descriptions"]):
- if desc == None or desc == '':
- obj["descriptions"][j] = True
- objects[i] = (obj_name, obj)
- elif desc != True:
- if search(desc, text):
- obj["descriptions"][j] = True
- objects[i] = (obj_name, obj)
-
-def hint_coverage(hints, text):
- for name, hint in hints:
- if hint["question"] != True:
- if search(hint["question"], text):
- hint["question"] = True
- if hint["hint"] != True:
- if search(hint["hint"], text):
- hint["hint"] = True
-
-def special_coverage(specials, text):
- for name, special in specials:
- if special["message"] == None:
- special["message"] = True
- if special["message"] != True:
- if search(special["message"], text):
- special["message"] = True
-
-def threshold_coverage(classes, text):
- for i, msg in enumerate(classes):
- if msg["message"] == None:
- msg["message"] = True
- elif msg["message"] != True:
- if search(msg["message"], text):
- msg["message"] = True
-
-def obit_coverage(obituaries, text):
- for i, obit in enumerate(obituaries):
- if obit["query"] != True:
- if search(obit["query"], text):
- obit["query"] = True
- if obit["yes_response"] != True:
- if search(obit["yes_response"], text):
- obit["yes_response"] = True
-
-def actions_coverage(actions, text):
- for name, action in actions:
- if action["message"] == None:
- action["message"] = True
- if action["message"] != True:
- if search(action["message"], text):
- action["message"] = True
+ name = "{}[{}]".format(obj_name, j)
+ if name not in report["messages"]:
+ report["messages"][name] = {"covered" : False}
+ report["total"] += 1
+ if report["messages"][name]["covered"] != True and search(desc, text):
+ report["messages"][name]["covered"] = True
+ report["covered"] += 1
+
+def loc_coverage(locations, text, report):
+ # locations have a long and a short description, that each have to
+ # be checked seperately
+ for name, loc in locations:
+ desc = loc["description"]
+ if name not in report["messages"]:
+ report["messages"][name] = {"long" : False, "short": False}
+ report["total"] += 2
+ if report["messages"][name]["long"] != True and search(desc["long"], text):
+ report["messages"][name]["long"] = True
+ report["covered"] += 1
+ if report["messages"][name]["short"] != True and search(desc["short"], text):
+ report["messages"][name]["short"] = True
+ report["covered"] += 1
+
+def hint_coverage(obituaries, text, report):
+ # hints have a "question" where the hint is offered, followed
+ # by the actual hint if the player requests it
+ for i, hintouter in enumerate(obituaries):
+ hint = hintouter["hint"]
+ name = hint["name"]
+ if name not in report["messages"]:
+ report["messages"][name] = {"question" : False, "hint": False}
+ report["total"] += 2
+ if report["messages"][name]["question"] != True and search(hint["question"], text):
+ report["messages"][name]["question"] = True
+ report["covered"] += 1
+ if report["messages"][name]["hint"] != True and search(hint["hint"], text):
+ report["messages"][name]["hint"] = True
+ report["covered"] += 1
+
+def obit_coverage(obituaries, text, report):
+ # obituaries have a "query" where it asks the player for a resurrection,
+ # followed by a snarky comment if the player says yes
+ for name, obit in enumerate(obituaries):
+ if name not in report["messages"]:
+ report["messages"][name] = {"query" : False, "yes_response": False}
+ report["total"] += 2
+ if report["messages"][name]["query"] != True and search(obit["query"], text):
+ report["messages"][name]["query"] = True
+ report["covered"] += 1
+ if report["messages"][name]["yes_response"] != True and search(obit["yes_response"], text):
+ report["messages"][name]["yes_response"] = True
+ report["covered"] += 1
+
+def threshold_coverage(classes, text, report):
+ # works for class thresholds and turn threshold, which have a "message"
+ # property
+ for name, item in enumerate(classes):
+ if name not in report["messages"]:
+ report["messages"][name] = {"covered" : "False"}
+ report["total"] += 1
+ if report["messages"][name]["covered"] != True and search(item["message"], text):
+ report["messages"][name]["covered"] = True
+ report["covered"] += 1
+
+def arb_coverage(arb_msgs, text, report):
+ for name, message in arb_msgs:
+ if name not in report["messages"]:
+ report["messages"][name] = {"covered" : False}
+ report["total"] += 1
+ if report["messages"][name]["covered"] != True and search(message, text):
+ report["messages"][name]["covered"] = True
+ report["covered"] += 1
+
+def actions_coverage(items, text, report):
+ # works for actions
+ for name, item in items:
+ if name not in report["messages"]:
+ report["messages"][name] = {"covered" : False}
+ report["total"] += 1
+ if report["messages"][name]["covered"] != True and search(item["message"], text):
+ report["messages"][name]["covered"] = True
+ report["covered"] += 1
+
+def coverage_report(db, check_file_contents):
+ # Create report for each catagory, including total items, number of items
+ # covered, and a list of the covered messages
+ report = {}
+ for name in db.keys():
+ # initialize each catagory
+ report[name] = {
+ "name" : name, # convenience for string formatting
+ "total" : 0,
+ "covered" : 0,
+ "messages" : {}
+ }
+
+ # search for each message in ever test check file
+ for chk in check_file_contents:
+ arb_coverage(db["arbitrary_messages"], chk, report["arbitrary_messages"])
+ hint_coverage(db["hints"], chk, report["hints"])
+ loc_coverage(db["locations"], chk, report["locations"])
+ obit_coverage(db["obituaries"], chk, report["obituaries"])
+ obj_coverage(db["objects"], chk, report["objects"])
+ actions_coverage(db["actions"], chk, report["actions"])
+ threshold_coverage(db["classes"], chk, report["classes"])
+ threshold_coverage(db["turn_thresholds"], chk, report["turn_thresholds"])
+
+ return report
if __name__ == "__main__":
- with open(yaml_name, "r") as f:
- db = yaml.load(f)
-
- with open(html_template_path, "r") as f:
- html_template = f.read()
-
- motions = db["motions"]
- locations = db["locations"]
- arb_msgs = db["arbitrary_messages"]
- objects = db["objects"]
- hintsraw = db["hints"]
- classes = db["classes"]
- turn_thresholds = db["turn_thresholds"]
- obituaries = db["obituaries"]
- actions = db["actions"]
- specials = db["specials"]
-
- hints = []
- for hint in hintsraw:
- hints.append((hint["hint"]["name"], {"question" : hint["hint"]["question"],"hint" : hint["hint"]["hint"]}))
-
- text = ""
- for filename in os.listdir(test_dir):
+ # load DB
+ try:
+ with open(YAML_PATH, "r") as f:
+ db = yaml.load(f)
+ except IOError as e:
+ print('ERROR: could not load {} ({}})'.format(YAML_PATH, e.strerror))
+ exit(-1)
+
+ # get contents of all the check files
+ check_file_contents = []
+ for filename in os.listdir(TEST_DIR):
if filename.endswith(".chk"):
- with open(filename, "r") as chk:
- text = chk.read()
- loc_coverage(locations, text)
- arb_coverage(arb_msgs, text)
- obj_coverage(objects, text)
- hint_coverage(hints, text)
- threshold_coverage(classes, text)
- threshold_coverage(turn_thresholds, text)
- obit_coverage(obituaries, text)
- actions_coverage(actions, text)
- special_coverage(specials, text)
-
- location_html = ""
- location_total = len(locations) * 2
- location_covered = 0
- locations.sort()
- for locouter in locations:
- locname = locouter[0]
- loc = locouter[1]
- if loc["description"]["long"] != True:
- long_success = "uncovered"
- else:
- long_success = "covered"
- location_covered += 1
-
- if loc["description"]["short"] != True:
- short_success = "uncovered"
- else:
- short_success = "covered"
- location_covered += 1
-
- location_html += location_row.format(locname, long_success, short_success)
- location_percent = round((location_covered / float(location_total)) * 100, 1)
-
- arb_msgs.sort()
- arb_msg_html = ""
- arb_total = len(arb_msgs)
- arb_covered = 0
- for name, msg in arb_msgs:
- if msg != True:
- success = "uncovered"
- else:
- success = "covered"
- arb_covered += 1
- arb_msg_html += arb_msg_row.format(name, success)
- arb_percent = round((arb_covered / float(arb_total)) * 100, 1)
-
- object_html = ""
- objects_total = 0
- objects_covered = 0
- objects.sort()
- for (obj_name, obj) in objects:
- if obj["descriptions"]:
- for j, desc in enumerate(obj["descriptions"]):
- objects_total += 1
- if desc != True:
- success = "uncovered"
- else:
- success = "covered"
- objects_covered += 1
- object_html += object_row.format("%s[%d]" % (obj_name, j), success)
- objects_percent = round((objects_covered / float(objects_total)) * 100, 1)
-
- hints.sort()
- hints_html = "";
- hints_total = len(hints) * 2
- hints_covered = 0
- for name, hint in hints:
- if hint["question"] != True:
- question_success = "uncovered"
- else:
- question_success = "covered"
- hints_covered += 1
- if hint["hint"] != True:
- hint_success = "uncovered"
- else:
- hint_success = "covered"
- hints_covered += 1
- hints_html += location_row.format(name, question_success, hint_success)
- hints_percent = round((hints_covered / float(hints_total)) * 100, 1)
-
- class_html = ""
- class_total = len(classes)
- class_covered = 0
- for name, msg in enumerate(classes):
- if msg["message"] != True:
- success = "uncovered"
- else:
- success = "covered"
- class_covered += 1
- class_html += arb_msg_row.format(msg["threshold"], success)
- class_percent = round((class_covered / float(class_total)) * 100, 1)
-
- turn_html = ""
- turn_total = len(turn_thresholds)
- turn_covered = 0
- for name, msg in enumerate(turn_thresholds):
- if msg["message"] != True:
- success = "uncovered"
- else:
- success = "covered"
- turn_covered += 1
- turn_html += arb_msg_row.format(msg["threshold"], success)
- turn_percent = round((turn_covered / float(turn_total)) * 100, 1)
-
- obituaries_html = "";
- obituaries_total = len(obituaries) * 2
- obituaries_covered = 0
- for i, obit in enumerate(obituaries):
- if obit["query"] != True:
- query_success = "uncovered"
- else:
- query_success = "covered"
- obituaries_covered += 1
- if obit["yes_response"] != True:
- obit_success = "uncovered"
- else:
- obit_success = "covered"
- obituaries_covered += 1
- obituaries_html += location_row.format(i, query_success, obit_success)
- obituaries_percent = round((obituaries_covered / float(obituaries_total)) * 100, 1)
-
- actions.sort()
- actions_html = "";
- actions_total = len(actions)
- actions_covered = 0
- for name, action in actions:
- if action["message"] != True:
- success = "uncovered"
- else:
- success = "covered"
- actions_covered += 1
- actions_html += arb_msg_row.format(name, success)
- actions_percent = round((actions_covered / float(actions_total)) * 100, 1)
-
- special_html = ""
- special_total = len(specials)
- special_covered = 0
- for name, special in specials:
- if special["message"] != True:
- success = "uncovered"
- else:
- success = "covered"
- special_covered += 1
- special_html += arb_msg_row.format(name, success)
- special_percent = round((special_covered / float(special_total)) * 100, 1)
+ with open(filename, "r") as f:
+ check_file_contents.append(f.read())
+
+ # run coverage analysis report on dungeon database
+ report = coverage_report(db, check_file_contents)
+
+ # render report output
+ categories_html = ""
+ summary_html = ""
+ summary_stdout = "adventure.yaml coverage rate:\n"
+ for name, category in sorted(report.items()):
+ # ignore categories with zero entries
+ if category["total"] > 0:
+ # Calculate percent coverage
+ category["percent"] = (category["covered"] / float(category["total"])) * 100
+
+ # render section header
+ cat_messages = sorted(category["messages"].items())
+ cat_keys = cat_messages[0][1].keys()
+ headers_html = ""
+ colspan = 10 - len(cat_keys)
+ for key in cat_keys:
+ headers_html += HTML_CATEGORY_HEADER_CELL.format(key)
+ category_html = HTML_CATEGORY_HEADER.format(colspan=colspan, label=category["name"], cells=headers_html)
+
+ # render message coverage row
+ for message_id, covered in cat_messages:
+ category_html_row = ""
+ for key, value in covered.items():
+ category_html_row += HTML_CATEGORY_COVERAGE_CELL.format("uncovered" if value != True else "covered")
+ category_html += HTML_CATEGORY_ROW.format(id=message_id,colspan=colspan, cells=category_html_row)
+ categories_html += HTML_CATEGORY_SECTION.format(id=name, rows=category_html)
+
+ # render category summaries
+ summary_stdout += STDOUT_REPORT_CATEGORY.format(**category)
+ summary_html += HTML_SUMMARY_ROW.format(**category)
# output some quick report stats
- print("\nadventure.yaml coverage rate:")
- print(" locations..........: {}% covered ({} of {})".format(location_percent, location_covered, location_total))
- print(" arbitrary_messages.: {}% covered ({} of {})".format(arb_percent, arb_covered, arb_total))
- print(" objects............: {}% covered ({} of {})".format(objects_percent, objects_covered, objects_total))
- print(" hints..............: {}% covered ({} of {})".format(hints_percent, hints_covered, hints_total))
- print(" classes............: {}% covered ({} of {})".format(class_percent, class_covered, class_total))
- print(" turn_thresholds....: {}% covered ({} of {})".format(turn_percent, turn_covered, turn_total))
- print(" obituaries.........: {}% covered ({} of {})".format(obituaries_percent, obituaries_covered, obituaries_total))
- print(" actions............: {}% covered ({} of {})".format(actions_percent, actions_covered, actions_total))
- print(" specials...........: {}% covered ({} of {})".format(special_percent, special_covered, special_total))
+ print(summary_stdout)
+
+ if len(sys.argv) > 1:
+ html_output_path = sys.argv[1]
+ else:
+ html_output_path = DEFAULT_HTML_OUTPUT_PATH
# render HTML report
- with open(html_output_path, "w") as f:
- f.write(html_template.format(
- location_total, location_covered, location_percent,
- arb_total, arb_covered, arb_percent,
- objects_total, objects_covered, objects_percent,
- hints_total, hints_covered, hints_percent,
- class_total, class_covered, class_percent,
- turn_total, turn_covered, turn_percent,
- obituaries_total, obituaries_covered, obituaries_percent,
- actions_total, actions_covered, actions_percent,
- special_total, special_covered, special_percent,
- location_html, arb_msg_html, object_html, hints_html,
- class_html, turn_html, obituaries_html, actions_html, special_html
- ))
+ try:
+ with open(HTML_TEMPLATE_PATH, "r") as f:
+ # read in HTML template
+ html_template = f.read()
+ except IOError as e:
+ print('ERROR: reading HTML report template failed ({})'.format(e.strerror))
+ exit(-1)
+
+ # parse template with report and write it out
+ try:
+ with open(html_output_path, "w") as f:
+ f.write(html_template.format(categories=categories_html, summary=summary_html))
+ except IOError as e:
+ print('ERROR: writing HTML report failed ({})'.format(e.strerror))