Skip to content

Commit f08d800

Browse files
ashwin6-devashwin6-dev
committed
feat: changed marking params to be in answer, not params, for easier frontend config
1 parent 1d4c742 commit f08d800

3 files changed

Lines changed: 84 additions & 68 deletions

File tree

‎docs/dev.md‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,34 @@ Select the sort of evlauation you want to using the check box. Make sure to only
99

1010
```json
1111
{
12-
"response":"<str>",
13-
"answer":"<str>",
14-
"params": {
15-
"truthTable": "<bool>",
16-
"equivalence": "<bool>",
17-
"tautology": "<bool>",
18-
"satisfiability": "<bool>",
19-
}
12+
"response": { "formula": "<str>", "truthTable": null | { "variables": ["<str>"], "cells": [[ "<str>" ]] } },
13+
"answer": {
14+
"satisability": true | false,
15+
"tautology": true | false,
16+
"equivalent": null | "<str>",
17+
"truthTable": null | { }
18+
},
19+
"params": { }
2020
}
2121
```
2222

23+
Exactly one of `satisability`, `tautology`, `equivalent` (non-null), or `truthTable` (non-null) must be set in `answer` to choose the evaluation mode.
24+
2325
### `truthTable`
2426

25-
uses the evaluation for truth tables
27+
When `answer.truthTable` is not null, uses truth table evaluation (response must include `truthTable` with `variables` and `cells`).
2628

27-
### `equivalence`
29+
### `equivalent`
2830

29-
checks if response formula and answer formula are equivalent
31+
When `answer.equivalent` is a string, checks if response formula and that formula are equivalent.
3032

3133
### `tautology`
3234

33-
checks if response formula is a tautology
35+
When `answer.tautology` is true, checks if response formula is a tautology.
3436

35-
### `satisfiability`
37+
### `satisability`
3638

37-
checks if response formula is satisfiabiable
39+
When `answer.satisability` is true, checks if response formula is satisfiable.
3840

3941
## Outputs
4042

‎evaluation_function/evaluation.py‎

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def evaluation_function(
4141

4242

4343
try:
44-
if not isinstance(answer, str):
44+
if not isinstance(answer, dict):
4545
return Result(
4646
is_correct=False,
4747
feedback_items=[("incorrect input", "missing answer object")]
@@ -66,50 +66,64 @@ def evaluation_function(
6666

6767
formula = formula_parser(response_formula)
6868

69-
# check if input is a truth table
70-
truth_table = response.get("truthTable", None)
71-
if truth_table is not None and isinstance(truth_table, dict):
72-
variables = truth_table.get("variables", [])
73-
cells = truth_table.get("cells", [])
69+
# Answer shape: satisability (bool), tautology (bool), equivalent (None|str), truthTable (None|dict)
70+
satisability = answer.get("satisability", False) is True
71+
tautology = answer.get("tautology", False) is True
72+
equivalent = answer.get("equivalent")
73+
if equivalent is not None and not isinstance(equivalent, str):
74+
equivalent = None
75+
elif equivalent is not None and isinstance(equivalent, str) and equivalent.strip() == "":
76+
equivalent = None
77+
answer_truth_table = answer.get("truthTable")
7478

75-
if not isinstance(variables, list) or not isinstance(cells, list):
76-
return Result(
77-
is_correct=False,
78-
feedback_items=[("incorrect input", "truthTable must contain 'variables' and 'cells' arrays")]
79-
)
79+
has_equivalence = equivalent is not None
80+
has_truth_table = answer_truth_table is not None
8081

81-
answer_formula = formula_parser(answer)
82-
num_atoms = len(_extract_atoms(answer_formula))
83-
84-
truth_table_result = evaluate_truth_table(variables, cells, num_atoms)
85-
if not truth_table_result.is_correct:
86-
return truth_table_result
87-
88-
equivalence = params.get("equivalence", False)
89-
tautology = params.get("tautology", False)
90-
satisfiability = params.get("satisfiability", False)
91-
92-
num_selected = sum([equivalence, tautology, satisfiability])
82+
num_selected = sum([satisability, tautology, has_equivalence, has_truth_table])
9383

9484
if num_selected == 0:
9585
return Result(
9686
is_correct=False,
9787
feedback_items=[("invalid param", "please select a param")]
9888
)
99-
elif num_selected > 1:
89+
if num_selected > 1:
10090
return Result(
10191
is_correct=False,
10292
feedback_items=[("invalid param", "please only select 1 param")]
10393
)
10494

95+
# Truth table mode: validate response truth table if present
96+
response_truth_table = response.get("truthTable", None)
97+
if has_truth_table:
98+
if response_truth_table is None or not isinstance(response_truth_table, dict):
99+
return Result(
100+
is_correct=False,
101+
feedback_items=[("incorrect input", "truthTable required when answer expects truth table")]
102+
)
103+
variables = response_truth_table.get("variables", [])
104+
cells = response_truth_table.get("cells", [])
105+
106+
if not isinstance(variables, list) or not isinstance(cells, list):
107+
return Result(
108+
is_correct=False,
109+
feedback_items=[("incorrect input", "truthTable must contain 'variables' and 'cells' arrays")]
110+
)
111+
112+
num_atoms = len(_extract_atoms(formula))
113+
truth_table_result = evaluate_truth_table(variables, cells, num_atoms)
114+
if not truth_table_result.is_correct:
115+
return truth_table_result
116+
105117
is_correct = False
106-
if equivalence:
107-
answer_formula = formula_parser(answer)
118+
if has_equivalence:
119+
answer_formula = formula_parser(equivalent)
108120
is_correct = EquivalenceEvaluator(formula, answer_formula).evaluate()
109121
elif tautology:
110122
is_correct = TautologyEvaluator(formula).evaluate()
111-
elif satisfiability:
123+
elif satisability:
112124
is_correct = SatisfiabilityEvaluator(formula).evaluate()
125+
elif has_truth_table:
126+
is_correct = True # already validated above
113127

114128
return Result(is_correct=is_correct)
115129

‎evaluation_function/evaluation_test.py‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class TestEvaluationFunction(unittest.TestCase):
2323

2424
def test_evaluation_default(self):
2525
response = {"formula": "Hello, World"}
26-
answer = "Hello, World"
26+
answer = "Hello, World" # invalid: answer must be dict
2727
params = Params()
2828

2929
result = evaluation_function(response, answer, params).to_dict()
@@ -32,53 +32,53 @@ def test_evaluation_default(self):
3232

3333
def test_check_tautology(self):
3434
response = {"formula": "p ∨ ¬p"}
35-
answer = ""
36-
params = {"tautology": True}
35+
answer = {"satisability": False, "tautology": True, "equivalent": None, "truthTable": None}
36+
params = Params()
3737

3838
result = evaluation_function(response, answer, params).to_dict()
3939

4040
self.assertTrue(result.get("is_correct"))
4141

4242
def test_check_tautology_fail(self):
4343
response = {"formula": "p ∧ ¬p"}
44-
answer = ""
45-
params = {"tautology": True}
44+
answer = {"satisability": False, "tautology": True, "equivalent": None, "truthTable": None}
45+
params = Params()
4646

4747
result = evaluation_function(response, answer, params).to_dict()
4848

4949
self.assertFalse(result.get("is_correct"))
5050

5151
def test_check_satisfiability(self):
5252
response = {"formula": "p ∧ q"}
53-
answer = ""
54-
params = {"satisfiability": True}
53+
answer = {"satisability": True, "tautology": False, "equivalent": None, "truthTable": None}
54+
params = Params()
5555

5656
result = evaluation_function(response, answer, params).to_dict()
5757

5858
self.assertTrue(result.get("is_correct"))
59-
59+
6060
def test_check_satisfiability_fail(self):
6161
response = {"formula": "p ∧ ¬p"}
62-
answer = ""
63-
params = {"satisfiability": True}
62+
answer = {"satisability": True, "tautology": False, "equivalent": None, "truthTable": None}
63+
params = Params()
6464

6565
result = evaluation_function(response, answer, params).to_dict()
6666

6767
self.assertFalse(result.get("is_correct"))
6868

6969
def test_check_equivalence(self):
7070
response = {"formula": "p ∧ q"}
71-
answer = "p ∧ (q ∨ q)"
72-
params = {"equivalence": True}
71+
answer = {"satisability": False, "tautology": False, "equivalent": "p ∧ (q ∨ q)", "truthTable": None}
72+
params = Params()
7373

7474
result = evaluation_function(response, answer, params).to_dict()
7575

7676
self.assertTrue(result.get("is_correct"))
77-
77+
7878
def test_check_equivalence_fail(self):
7979
response = {"formula": "p ∧ q"}
80-
answer = "p"
81-
params = {"equivalence": True}
80+
answer = {"satisability": False, "tautology": False, "equivalent": "p", "truthTable": None}
81+
params = Params()
8282

8383
result = evaluation_function(response, answer, params).to_dict()
8484

@@ -97,8 +97,8 @@ def test_truth_table_valid(self):
9797
]
9898
}
9999
}
100-
answer = "p ∧ q"
101-
params = {"equivalence": True}
100+
answer = {"satisability": False, "tautology": False, "equivalent": None, "truthTable": {}}
101+
params = Params()
102102

103103
result = evaluation_function(response, answer, params).to_dict()
104104

@@ -117,17 +117,17 @@ def test_truth_table_invalid(self):
117117
]
118118
}
119119
}
120-
answer = "p ∧ q"
121-
params = {"equivalence": True}
120+
answer = {"satisability": False, "tautology": False, "equivalent": None, "truthTable": {}}
121+
params = Params()
122122

123123
result = evaluation_function(response, answer, params).to_dict()
124124

125125
self.assertFalse(result.get("is_correct"))
126126

127127
def test_invalid_response_type(self):
128128
response = "just a string" # Invalid type
129-
answer = "p"
130-
params = {"tautology": True}
129+
answer = {"satisability": False, "tautology": True, "equivalent": None, "truthTable": None}
130+
params = Params()
131131

132132
result = evaluation_function(response, answer, params).to_dict()
133133

@@ -136,26 +136,26 @@ def test_invalid_response_type(self):
136136

137137
def test_missing_formula_field(self):
138138
response = {"wrongField": "p"}
139-
answer = "p"
140-
params = {"tautology": True}
139+
answer = {"satisability": False, "tautology": True, "equivalent": None, "truthTable": None}
140+
params = Params()
141141

142142
result = evaluation_function(response, answer, params).to_dict()
143143

144144
self.assertFalse(result.get("is_correct"))
145145

146146
def test_no_params_selected(self):
147147
response = {"formula": "p"}
148-
answer = "p"
149-
params = {}
148+
answer = {"satisability": False, "tautology": False, "equivalent": None, "truthTable": None}
149+
params = Params()
150150

151151
result = evaluation_function(response, answer, params).to_dict()
152152

153153
self.assertFalse(result.get("is_correct"))
154154

155155
def test_multiple_params_selected(self):
156156
response = {"formula": "p"}
157-
answer = "p"
158-
params = {"tautology": True, "satisfiability": True}
157+
answer = {"satisability": True, "tautology": True, "equivalent": None, "truthTable": None}
158+
params = Params()
159159

160160
result = evaluation_function(response, answer, params).to_dict()
161161

0 commit comments

Comments
 (0)