Skip to content

Commit 8b6345e

Browse files
mame82mame82
authored
gRPC view: Adjusted parsing strategy for deprecated protobuf fields (mitmproxy#4858)
* Adjusted parsing strategy for deprecated protobuf fields * Added test comment * adjusted comment to reflect correct field IDs
1 parent 77cf2ab commit 8b6345e

2 files changed

Lines changed: 78 additions & 3 deletions

File tree

‎mitmproxy/contentviews/grpc.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,8 @@ def parse_message_fields(self, message: bytes) -> List:
172172
tag = pair.field_tag
173173
wt = pair.wire_type
174174
if wt == GoogleProtobuf.Pair.WireTypes.group_start or wt == GoogleProtobuf.Pair.WireTypes.group_end:
175-
# ignore deprecated types without values
176-
continue
175+
# raise error on deprecated types without values
176+
raise ValueError("deprecated field: {}".format(wt))
177177
v: Union[GoogleProtobuf.DelimitedBytes, VlqBase128Le] = pair.value # for WireType bit-32 and bit-64
178178
preferred_decoding = ProtoParser.DecodedTypes.unknown
179179
# see: https://www.oreilly.com/library/view/grpc-up-and/9781492058328/ch04.html

‎test/mitmproxy/contentviews/test_grpc.py‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,81 @@ def test_view_protobuf_custom_parsing_request(tdata):
110110

111111

112112
def test_view_protobuf_custom_parsing_response(tdata):
113+
# expect to parse 1.3.2 and 1.3.3 as string automatically
114+
# even if there is a length delimeted field containing `b"DC"`, which would translate to
115+
# two deprecated fields [8: group_start, 8: group_end] (and thus represent a valid nested message,
116+
# but containing deprecated wire types)
117+
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = None
118+
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = None
119+
120+
v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
121+
p = tdata.path(datadir + "msg3.bin")
122+
123+
with open(p, "rb") as f:
124+
raw = f.read()
125+
view_text, output = v(raw, flow=sim_flow, http_message=sim_flow.response) # simulate response message
126+
assert view_text == "Protobuf (flattened)"
127+
output = list(output) # assure list conversion if generator
128+
assert output == [
129+
[('text', '[message] '), ('text', ' '), ('text', '1 '), ('text', ' ')], # noqa: E501
130+
[('text', '[string] '), ('text', ' '), ('text', '1.1 '), ('text', '\x15 ')], # noqa: E501
131+
[('text', '[string] '), ('text', 'address '), ('text', '1.2 '), ('text', '1650 Pennsylvania Avenue NW, Washington, DC 20502, USA ')], # noqa: E501
132+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
133+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', 'b\'"\' ')], # noqa: E501
134+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '1650 ')], # noqa: E501
135+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '1650 ')], # noqa: E501
136+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
137+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x02' ")], # noqa: E501
138+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Pennsylvania Avenue Northwest ')], # noqa: E501
139+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Pennsylvania Avenue NW ')], # noqa: E501
140+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
141+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x14\\x04' ")], # noqa: E501
142+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Northwest Washington ')], # noqa: E501
143+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Northwest Washington ')], # noqa: E501
144+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
145+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x0c\\x04' ")], # noqa: E501
146+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Washington ')], # noqa: E501
147+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Washington ')], # noqa: E501
148+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
149+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
150+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
151+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
152+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
153+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
154+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501
155+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'US ')], # noqa: E501
156+
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
157+
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x17' ")], # noqa: E501
158+
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '20502 ')], # noqa: E501
159+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '20502 ')], # noqa: E501
160+
[('text', '[message] '), ('text', ' '), ('text', '1.5 '), ('text', ' ')], # noqa: E501
161+
[('text', '[message] '), ('text', 'position '), ('text', '1.5.1 '), ('text', ' ')], # noqa: E501
162+
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.1.1 '), ('text', '38.8970309 ')], # noqa: E501
163+
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.1.2 '), ('text', '-77.03872559999999 ')], # noqa: E501
164+
[('text', '[uint32] '), ('text', ' '), ('text', '1.5.2 '), ('text', '1 ')], # noqa: E501
165+
[('text', '[message] '), ('text', 'position '), ('text', '1.5.3 '), ('text', ' ')], # noqa: E501
166+
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.3.1 '), ('text', '38.8962271697085 ')], # noqa: E501
167+
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.3.2 '), ('text', '-77.0400511802915 ')], # noqa: E501
168+
[('text', '[message] '), ('text', 'position '), ('text', '1.5.4 '), ('text', ' ')], # noqa: E501
169+
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.4.1 '), ('text', '38.8989251302915 ')], # noqa: E501
170+
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.4.2 '), ('text', '-77.03735321970849 ')], # noqa: E501
171+
[('text', '[message] '), ('text', 'position '), ('text', '1.5.5 '), ('text', ' ')], # noqa: E501
172+
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.5.1 '), ('text', '38.896898 ')], # noqa: E501
173+
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.5.2 '), ('text', '-77.03917229999999 ')], # noqa: E501
174+
[('text', '[message] '), ('text', 'position '), ('text', '1.5.6 '), ('text', ' ')], # noqa: E501
175+
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.6.1 '), ('text', '38.8982543 ')], # noqa: E501
176+
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.6.2 '), ('text', '-77.0382321 ')], # noqa: E501
177+
[('text', '[string] '), ('text', ' '), ('text', '1.7 '), ('text', 'ChIJAXiAory3t4kRpkrvas9dYmQ ')], # noqa: E501
178+
[('text', '[message] '), ('text', ' '), ('text', '2 '), ('text', ' ')], # noqa: E501
179+
[('text', '[uint32] '), ('text', ' '), ('text', '2.1 '), ('text', '21 ')], # noqa: E501
180+
]
181+
182+
183+
def test_view_protobuf_custom_parsing_response2(tdata):
184+
# try to parse 1.3.2 and 1.3.3 as string
185+
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.2
186+
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.3
187+
113188
v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
114189
p = tdata.path(datadir + "msg3.bin")
115190

@@ -141,7 +216,7 @@ def test_view_protobuf_custom_parsing_response(tdata):
141216
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
142217
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
143218
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
144-
[('text', '[message] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', ' ')], # noqa: E501
219+
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
145220
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
146221
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
147222
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501

0 commit comments

Comments
 (0)