v2 / vlib / net / http / http_proxy_test.v
394 lines · 356 sloc · 11.31 KB · d942476a842a131eb4cfdf7872028574149b761b
Raw
1// vtest retry: 3
2// vtest vflags: -d use_openssl
3module http
4
5import encoding.base64
6import net
7import net.mbedtls
8import net.urllib
9import os
10import time
11
12const sample_proxy_url = 'https://localhost'
13const sample_auth_proxy_url = 'http://user:pass@localhost:8888'
14
15const sample_host = '127.0.0.1:1337'
16const sample_request = &Request{
17 url: 'http://${sample_host}'
18}
19const sample_path = '/'
20const proxy_https_request_count = 12
21const proxy_https_test_cert_path = @VEXEROOT +
22 '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.crt'
23const proxy_https_test_key_path = @VEXEROOT +
24 '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.key'
25
26fn test_proxy_fields() ? {
27 sample_proxy := new_http_proxy(sample_proxy_url)!
28 sample_auth_proxy := new_http_proxy(sample_auth_proxy_url)!
29
30 assert sample_proxy.scheme == 'https'
31 assert sample_proxy.host == 'localhost:443'
32 assert sample_proxy.hostname == 'localhost'
33 assert sample_proxy.port == 443
34 assert sample_proxy.url == sample_proxy_url
35 assert sample_auth_proxy.scheme == 'http'
36 assert sample_auth_proxy.username == 'user'
37 assert sample_auth_proxy.password == 'pass'
38 assert sample_auth_proxy.host == 'localhost:8888'
39 assert sample_auth_proxy.hostname == 'localhost'
40 assert sample_auth_proxy.port == 8888
41 assert sample_auth_proxy.url == sample_auth_proxy_url
42}
43
44fn test_proxy_headers() ? {
45 sample_proxy := new_http_proxy(sample_proxy_url)!
46 headers := sample_proxy.build_proxy_headers(sample_host)
47
48 assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
49 'Proxy-Connection: Keep-Alive\r\n\r\n'
50}
51
52fn test_proxy_headers_authenticated() ? {
53 sample_proxy := new_http_proxy(sample_auth_proxy_url)!
54 headers := sample_proxy.build_proxy_headers(sample_host)
55
56 auth_token := base64.encode(('${sample_proxy.username}:' + '${sample_proxy.password}').bytes())
57
58 assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
59 'Proxy-Connection: Keep-Alive\r\nProxy-Authorization: Basic ${auth_token}\r\n\r\n'
60}
61
62enum ProxyTunnelCopyResult {
63 data
64 timeout
65 closed
66}
67
68fn count_open_file_descriptors() int {
69 $if windows {
70 return 0
71 } $else {
72 fds := os.ls('/dev/fd') or { return 0 }
73 return fds.len
74 }
75}
76
77fn start_https_proxy_test_target_server() !(int, chan bool) {
78 ready := chan int{cap: 1}
79 done := chan bool{cap: 1}
80 spawn fn [ready, done] () {
81 mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) }
82 port := int((port_listener.addr() or { panic(err) }).port() or { panic(err) })
83 port_listener.close() or {}
84 mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{
85 cert: proxy_https_test_cert_path
86 cert_key: proxy_https_test_key_path
87 validate: false
88 in_memory_verification: false
89 }) or { panic(err) }
90 ready <- port
91 defer {
92 listener.shutdown() or {}
93 done <- true
94 }
95 for _ in 0 .. proxy_https_request_count {
96 mut conn := listener.accept() or { panic(err) }
97 handle_https_proxy_test_target_connection(mut conn)
98 }
99 }()
100 return <-ready, done
101}
102
103fn handle_https_proxy_test_target_connection(mut conn mbedtls.SSLConn) {
104 defer {
105 conn.shutdown() or {}
106 }
107 mut request_buf := []u8{len: 2048}
108 _ = conn.read(mut request_buf) or { return }
109 conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or {
110 return
111 }
112}
113
114fn start_https_proxy_test_server(target_port int) !(int, chan bool) {
115 ready := chan int{cap: 1}
116 done := chan bool{cap: 1}
117 spawn fn [ready, done, target_port] () {
118 mut listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) }
119 port := int((listener.addr() or { panic(err) }).port() or { panic(err) })
120 ready <- port
121 mut workers := []thread{cap: proxy_https_request_count}
122 for _ in 0 .. proxy_https_request_count {
123 mut client := listener.accept() or { panic(err) }
124 workers << spawn handle_https_proxy_test_tunnel(mut client, target_port)
125 }
126 listener.close() or {}
127 workers.wait()
128 done <- true
129 }()
130 return <-ready, done
131}
132
133fn handle_https_proxy_test_tunnel(mut client net.TcpConn, target_port int) {
134 defer {
135 client.close() or {}
136 }
137 request := read_proxy_request(mut client) or { return }
138 if !request.starts_with('CONNECT 127.0.0.1:${target_port} HTTP/1.1\r\n') {
139 client.write_string('HTTP/1.1 400 Bad Request\r\n\r\n') or {}
140 return
141 }
142 mut upstream := net.dial_tcp('127.0.0.1:${target_port}') or {
143 client.write_string('HTTP/1.1 502 Bad Gateway\r\n\r\n') or {}
144 return
145 }
146 defer {
147 upstream.close() or {}
148 }
149 client.write_string('HTTP/1.1 200 Connection Established\r\n\r\n') or { return }
150 client.set_read_timeout(50 * time.millisecond)
151 upstream.set_read_timeout(50 * time.millisecond)
152 deadline := time.now().add(2 * time.second)
153 for time.now() < deadline {
154 client_state := copy_https_proxy_test_tunnel_data(mut client, mut upstream)
155 upstream_state := copy_https_proxy_test_tunnel_data(mut upstream, mut client)
156 if client_state == .closed || upstream_state == .closed {
157 return
158 }
159 }
160}
161
162fn read_proxy_request(mut conn net.TcpConn) !string {
163 mut total_bytes_read := 0
164 mut msg := [4096]u8{}
165 mut buffer := [1]u8{}
166 for total_bytes_read < msg.len {
167 bytes_read := conn.read_ptr(&buffer[0], 1)!
168 if bytes_read == 0 {
169 return error('unexpected EOF while reading proxy request')
170 }
171 msg[total_bytes_read] = buffer[0]
172 total_bytes_read++
173 if total_bytes_read > 3 && msg[total_bytes_read - 1] == `\n`
174 && msg[total_bytes_read - 2] == `\r` && msg[total_bytes_read - 3] == `\n`
175 && msg[total_bytes_read - 4] == `\r` {
176 return msg[..total_bytes_read].bytestr()
177 }
178 }
179 return error('proxy request headers exceeded 4096 bytes')
180}
181
182fn copy_https_proxy_test_tunnel_data(mut src net.TcpConn, mut dst net.TcpConn) ProxyTunnelCopyResult {
183 mut buf := []u8{len: 1024}
184 bytes_read := src.read(mut buf) or {
185 if err.code() == net.err_timed_out_code {
186 return .timeout
187 }
188 return .closed
189 }
190 if bytes_read <= 0 {
191 return .closed
192 }
193 dst.write(buf[..bytes_read]) or { return .closed }
194 return .data
195}
196
197fn test_https_proxy_requests_do_not_leak_sockets() ! {
198 $if windows {
199 return
200 }
201 $if sanitized_job ? {
202 return
203 }
204 $if tinyc {
205 // TinyCC hangs in the bundled mbedtls handshake path on linux CI.
206 return
207 }
208 target_port, target_done := start_https_proxy_test_target_server()!
209 proxy_port, proxy_done := start_https_proxy_test_server(target_port)!
210 baseline_fds := count_open_file_descriptors()
211 proxy := new_http_proxy('http://127.0.0.1:${proxy_port}')!
212 for _ in 0 .. proxy_https_request_count {
213 resp := fetch(
214 method: .get
215 url: 'https://127.0.0.1:${target_port}/'
216 proxy: proxy
217 validate: false
218 )!
219 assert resp.status_code == 200
220 assert resp.body == 'ok'
221 }
222 _ = <-target_done
223 _ = <-proxy_done
224 time.sleep(100 * time.millisecond)
225 final_fds := count_open_file_descriptors()
226 assert final_fds <= baseline_fds + 3
227}
228
229fn test_http_proxy_do() {
230 env := os.environ()
231 mut env_proxy := ''
232
233 for envvar in ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY'] {
234 prox_val := env[envvar] or { continue }
235 if prox_val != '' {
236 env_proxy = env[envvar]
237 }
238 }
239 if env_proxy != '' {
240 println('Has usable proxy env vars')
241 proxy := new_http_proxy(env_proxy)!
242 mut header := new_header(key: .user_agent, value: 'vlib')
243 header.add_custom('X-Vlang-Test', 'proxied')!
244 res := proxy.http_do(urllib.parse('http://httpbin.org/headers')!, Method.get, '/headers', &Request{
245 proxy: proxy
246 header: header
247 }, '', header)!
248 println(res.status_code)
249 println('he4aders ${res.header}')
250 assert res.status_code == 200
251 // assert res.header.data['X-Vlang-Test'] == 'proxied'
252 } else {
253 println('Proxy env vars (HTTP_PROXY or HTTPS_PROXY) not set. Skipping test.')
254 }
255}
256
257const multipart_https_payload_len = 20 * 1024 + 137
258
259fn test_https_multipart_form_preserves_large_binary_body() ! {
260 $if tinyc {
261 // TinyCC hangs in the bundled mbedtls handshake path on linux CI.
262 return
263 }
264 mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0')!
265 port := port_listener.addr()!.port()!
266 port_listener.close()!
267
268 payload := multipart_https_test_payload()
269 form := {
270 'alpha': 'beta'
271 }
272 files := {
273 'file': [
274 FileData{
275 filename: 'payload.bin'
276 content_type: 'application/octet-stream'
277 data: payload
278 },
279 ]
280 }
281 body, boundary := multipart_form_body(form, files)
282
283 mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{
284 cert: proxy_https_test_cert_path
285 cert_key: proxy_https_test_key_path
286 validate: false
287 })!
288 server := spawn multipart_https_serve_once(mut listener, body, boundary, form, files)
289
290 mut header := new_header()
291 header.set(.content_type, 'multipart/form-data; boundary="${boundary}"')
292 resp := fetch(
293 method: .post
294 url: 'https://127.0.0.1:${port}/upload'
295 header: header
296 data: body
297 validate: false
298 )!
299 server.wait()
300
301 assert resp.status_code == 200
302 assert resp.body == 'ok'
303}
304
305fn multipart_https_test_payload() string {
306 mut payload := []u8{len: multipart_https_payload_len, init: u8(((index * 17) % 250) + 1)}
307 payload[127] = 0
308 payload[4096] = 0
309 payload[16 * 1024] = 0
310 payload[payload.len - 1] = `!`
311 return payload.bytestr()
312}
313
314fn multipart_https_serve_once(mut listener mbedtls.SSLListener, expected_body string, boundary string, expected_form map[string]string, expected_files map[string][]FileData) {
315 defer {
316 listener.shutdown() or {}
317 }
318 mut conn := listener.accept() or { panic(err) }
319 conn.set_read_timeout(5 * time.second)
320 defer {
321 conn.shutdown() or {}
322 }
323
324 request_text := read_https_request(mut conn) or { panic(err) }
325 req := parse_request_str(request_text) or { panic(err) }
326
327 assert req.method == .post
328 assert req.url == '/upload'
329 assert req.data == expected_body
330 assert req.data.len == expected_body.len
331 assert req.header.get(.content_length) or { panic(err) } == expected_body.len.str()
332 assert req.header.get(.content_type) or { panic(err) } == 'multipart/form-data; boundary="${boundary}"'
333
334 form, files := parse_multipart_form(req.data, boundary)
335 assert form == expected_form
336 assert files == expected_files
337
338 conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or {
339 panic(err)
340 }
341}
342
343fn read_https_request(mut conn mbedtls.SSLConn) !string {
344 mut request := []u8{}
345 mut buf := []u8{len: 1024}
346 mut content_length := -1
347 mut headers_end := -1
348
349 for {
350 n := conn.read(mut buf) or {
351 if err.code() == net.err_timed_out_code {
352 return error('timed out while reading HTTPS request')
353 }
354 return err
355 }
356 if n <= 0 {
357 break
358 }
359 request << buf[..n]
360
361 request_str := request.bytestr()
362 if headers_end == -1 {
363 headers_end = request_str.index('\r\n\r\n') or { -1 }
364 if headers_end != -1 {
365 headers := request_str[..headers_end]
366 for line in headers.split('\r\n') {
367 if line.to_lower().starts_with('content-length:') {
368 content_length = line.all_after(':').trim_space().int()
369 break
370 }
371 }
372 }
373 }
374
375 if headers_end != -1 && content_length >= 0 {
376 body_start := headers_end + 4
377 if request.len - body_start >= content_length {
378 break
379 }
380 }
381 }
382
383 if headers_end == -1 {
384 return error('HTTPS request did not include a full header block')
385 }
386 if content_length < 0 {
387 return error('HTTPS request did not include Content-Length')
388 }
389 body_start := headers_end + 4
390 if request.len - body_start < content_length {
391 return error('HTTPS request body was truncated: expected ${content_length} bytes, got ${request.len - body_start}')
392 }
393 return request.bytestr()
394}
395