v / vlib / net / http / http_proxy_test.v
395 lines · 357 sloc · 11.33 KB · 8fd2094bf98944a2aff9c00c3ee04b60174d2dd4
Raw
1// vtest retry: 3
2// vtest vflags: -d use_openssl
3// vtest build: !windows
4module http
5
6import encoding.base64
7import net
8import net.mbedtls
9import net.urllib
10import os
11import time
12
13const sample_proxy_url = 'https://localhost'
14const sample_auth_proxy_url = 'http://user:pass@localhost:8888'
15
16const sample_host = '127.0.0.1:1337'
17const sample_request = &Request{
18 url: 'http://${sample_host}'
19}
20const sample_path = '/'
21const proxy_https_request_count = 12
22const proxy_https_test_cert_path = @VEXEROOT +
23 '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.crt'
24const proxy_https_test_key_path = @VEXEROOT +
25 '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.key'
26
27fn test_proxy_fields() ? {
28 sample_proxy := new_http_proxy(sample_proxy_url)!
29 sample_auth_proxy := new_http_proxy(sample_auth_proxy_url)!
30
31 assert sample_proxy.scheme == 'https'
32 assert sample_proxy.host == 'localhost:443'
33 assert sample_proxy.hostname == 'localhost'
34 assert sample_proxy.port == 443
35 assert sample_proxy.url == sample_proxy_url
36 assert sample_auth_proxy.scheme == 'http'
37 assert sample_auth_proxy.username == 'user'
38 assert sample_auth_proxy.password == 'pass'
39 assert sample_auth_proxy.host == 'localhost:8888'
40 assert sample_auth_proxy.hostname == 'localhost'
41 assert sample_auth_proxy.port == 8888
42 assert sample_auth_proxy.url == sample_auth_proxy_url
43}
44
45fn test_proxy_headers() ? {
46 sample_proxy := new_http_proxy(sample_proxy_url)!
47 headers := sample_proxy.build_proxy_headers(sample_host)
48
49 assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
50 'Proxy-Connection: Keep-Alive\r\n\r\n'
51}
52
53fn test_proxy_headers_authenticated() ? {
54 sample_proxy := new_http_proxy(sample_auth_proxy_url)!
55 headers := sample_proxy.build_proxy_headers(sample_host)
56
57 auth_token := base64.encode(('${sample_proxy.username}:' + '${sample_proxy.password}').bytes())
58
59 assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
60 'Proxy-Connection: Keep-Alive\r\nProxy-Authorization: Basic ${auth_token}\r\n\r\n'
61}
62
63enum ProxyTunnelCopyResult {
64 data
65 timeout
66 closed
67}
68
69fn count_open_file_descriptors() int {
70 $if windows {
71 return 0
72 } $else {
73 fds := os.ls('/dev/fd') or { return 0 }
74 return fds.len
75 }
76}
77
78fn start_https_proxy_test_target_server() !(int, chan bool) {
79 ready := chan int{cap: 1}
80 done := chan bool{cap: 1}
81 spawn fn [ready, done] () {
82 mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) }
83 port := int((port_listener.addr() or { panic(err) }).port() or { panic(err) })
84 port_listener.close() or {}
85 mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{
86 cert: proxy_https_test_cert_path
87 cert_key: proxy_https_test_key_path
88 validate: false
89 in_memory_verification: false
90 }) or { panic(err) }
91 ready <- port
92 defer {
93 listener.shutdown() or {}
94 done <- true
95 }
96 for _ in 0 .. proxy_https_request_count {
97 mut conn := listener.accept() or { panic(err) }
98 handle_https_proxy_test_target_connection(mut conn)
99 }
100 }()
101 return <-ready, done
102}
103
104fn handle_https_proxy_test_target_connection(mut conn mbedtls.SSLConn) {
105 defer {
106 conn.shutdown() or {}
107 }
108 mut request_buf := []u8{len: 2048}
109 _ = conn.read(mut request_buf) or { return }
110 conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or {
111 return
112 }
113}
114
115fn start_https_proxy_test_server(target_port int) !(int, chan bool) {
116 ready := chan int{cap: 1}
117 done := chan bool{cap: 1}
118 spawn fn [ready, done, target_port] () {
119 mut listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) }
120 port := int((listener.addr() or { panic(err) }).port() or { panic(err) })
121 ready <- port
122 mut workers := []thread{cap: proxy_https_request_count}
123 for _ in 0 .. proxy_https_request_count {
124 mut client := listener.accept() or { panic(err) }
125 workers << spawn handle_https_proxy_test_tunnel(mut client, target_port)
126 }
127 listener.close() or {}
128 workers.wait()
129 done <- true
130 }()
131 return <-ready, done
132}
133
134fn handle_https_proxy_test_tunnel(mut client net.TcpConn, target_port int) {
135 defer {
136 client.close() or {}
137 }
138 request := read_proxy_request(mut client) or { return }
139 if !request.starts_with('CONNECT 127.0.0.1:${target_port} HTTP/1.1\r\n') {
140 client.write_string('HTTP/1.1 400 Bad Request\r\n\r\n') or {}
141 return
142 }
143 mut upstream := net.dial_tcp('127.0.0.1:${target_port}') or {
144 client.write_string('HTTP/1.1 502 Bad Gateway\r\n\r\n') or {}
145 return
146 }
147 defer {
148 upstream.close() or {}
149 }
150 client.write_string('HTTP/1.1 200 Connection Established\r\n\r\n') or { return }
151 client.set_read_timeout(50 * time.millisecond)
152 upstream.set_read_timeout(50 * time.millisecond)
153 deadline := time.now().add(2 * time.second)
154 for time.now() < deadline {
155 client_state := copy_https_proxy_test_tunnel_data(mut client, mut upstream)
156 upstream_state := copy_https_proxy_test_tunnel_data(mut upstream, mut client)
157 if client_state == .closed || upstream_state == .closed {
158 return
159 }
160 }
161}
162
163fn read_proxy_request(mut conn net.TcpConn) !string {
164 mut total_bytes_read := 0
165 mut msg := [4096]u8{}
166 mut buffer := [1]u8{}
167 for total_bytes_read < msg.len {
168 bytes_read := conn.read_ptr(&buffer[0], 1)!
169 if bytes_read == 0 {
170 return error('unexpected EOF while reading proxy request')
171 }
172 msg[total_bytes_read] = buffer[0]
173 total_bytes_read++
174 if total_bytes_read > 3 && msg[total_bytes_read - 1] == `\n`
175 && msg[total_bytes_read - 2] == `\r` && msg[total_bytes_read - 3] == `\n`
176 && msg[total_bytes_read - 4] == `\r` {
177 return msg[..total_bytes_read].bytestr()
178 }
179 }
180 return error('proxy request headers exceeded 4096 bytes')
181}
182
183fn copy_https_proxy_test_tunnel_data(mut src net.TcpConn, mut dst net.TcpConn) ProxyTunnelCopyResult {
184 mut buf := []u8{len: 1024}
185 bytes_read := src.read(mut buf) or {
186 if err.code() == net.err_timed_out_code {
187 return .timeout
188 }
189 return .closed
190 }
191 if bytes_read <= 0 {
192 return .closed
193 }
194 dst.write(buf[..bytes_read]) or { return .closed }
195 return .data
196}
197
198fn test_https_proxy_requests_do_not_leak_sockets() ! {
199 $if windows {
200 return
201 }
202 $if sanitized_job ? {
203 return
204 }
205 $if tinyc {
206 // TinyCC hangs in the bundled mbedtls handshake path on linux CI.
207 return
208 }
209 target_port, target_done := start_https_proxy_test_target_server()!
210 proxy_port, proxy_done := start_https_proxy_test_server(target_port)!
211 baseline_fds := count_open_file_descriptors()
212 proxy := new_http_proxy('http://127.0.0.1:${proxy_port}')!
213 for _ in 0 .. proxy_https_request_count {
214 resp := fetch(
215 method: .get
216 url: 'https://127.0.0.1:${target_port}/'
217 proxy: proxy
218 validate: false
219 )!
220 assert resp.status_code == 200
221 assert resp.body == 'ok'
222 }
223 _ = <-target_done
224 _ = <-proxy_done
225 time.sleep(100 * time.millisecond)
226 final_fds := count_open_file_descriptors()
227 assert final_fds <= baseline_fds + 3
228}
229
230fn test_http_proxy_do() {
231 env := os.environ()
232 mut env_proxy := ''
233
234 for envvar in ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY'] {
235 prox_val := env[envvar] or { continue }
236 if prox_val != '' {
237 env_proxy = env[envvar]
238 }
239 }
240 if env_proxy != '' {
241 println('Has usable proxy env vars')
242 proxy := new_http_proxy(env_proxy)!
243 mut header := new_header(key: .user_agent, value: 'vlib')
244 header.add_custom('X-Vlang-Test', 'proxied')!
245 res := proxy.http_do(urllib.parse('http://httpbin.org/headers')!, Method.get, '/headers', &Request{
246 proxy: proxy
247 header: header
248 }, '', header)!
249 println(res.status_code)
250 println('he4aders ${res.header}')
251 assert res.status_code == 200
252 // assert res.header.data['X-Vlang-Test'] == 'proxied'
253 } else {
254 println('Proxy env vars (HTTP_PROXY or HTTPS_PROXY) not set. Skipping test.')
255 }
256}
257
258const multipart_https_payload_len = 20 * 1024 + 137
259
260fn test_https_multipart_form_preserves_large_binary_body() ! {
261 $if tinyc {
262 // TinyCC hangs in the bundled mbedtls handshake path on linux CI.
263 return
264 }
265 mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0')!
266 port := port_listener.addr()!.port()!
267 port_listener.close()!
268
269 payload := multipart_https_test_payload()
270 form := {
271 'alpha': 'beta'
272 }
273 files := {
274 'file': [
275 FileData{
276 filename: 'payload.bin'
277 content_type: 'application/octet-stream'
278 data: payload
279 },
280 ]
281 }
282 body, boundary := multipart_form_body(form, files)
283
284 mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{
285 cert: proxy_https_test_cert_path
286 cert_key: proxy_https_test_key_path
287 validate: false
288 })!
289 server := spawn multipart_https_serve_once(mut listener, body, boundary, form, files)
290
291 mut header := new_header()
292 header.set(.content_type, 'multipart/form-data; boundary="${boundary}"')
293 resp := fetch(
294 method: .post
295 url: 'https://127.0.0.1:${port}/upload'
296 header: header
297 data: body
298 validate: false
299 )!
300 server.wait()
301
302 assert resp.status_code == 200
303 assert resp.body == 'ok'
304}
305
306fn multipart_https_test_payload() string {
307 mut payload := []u8{len: multipart_https_payload_len, init: u8(((index * 17) % 250) + 1)}
308 payload[127] = 0
309 payload[4096] = 0
310 payload[16 * 1024] = 0
311 payload[payload.len - 1] = `!`
312 return payload.bytestr()
313}
314
315fn multipart_https_serve_once(mut listener mbedtls.SSLListener, expected_body string, boundary string, expected_form map[string]string, expected_files map[string][]FileData) {
316 defer {
317 listener.shutdown() or {}
318 }
319 mut conn := listener.accept() or { panic(err) }
320 conn.set_read_timeout(5 * time.second)
321 defer {
322 conn.shutdown() or {}
323 }
324
325 request_text := read_https_request(mut conn) or { panic(err) }
326 req := parse_request_str(request_text) or { panic(err) }
327
328 assert req.method == .post
329 assert req.url == '/upload'
330 assert req.data == expected_body
331 assert req.data.len == expected_body.len
332 assert req.header.get(.content_length) or { panic(err) } == expected_body.len.str()
333 assert req.header.get(.content_type) or { panic(err) } == 'multipart/form-data; boundary="${boundary}"'
334
335 form, files := parse_multipart_form(req.data, boundary)
336 assert form == expected_form
337 assert files == expected_files
338
339 conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or {
340 panic(err)
341 }
342}
343
344fn read_https_request(mut conn mbedtls.SSLConn) !string {
345 mut request := []u8{}
346 mut buf := []u8{len: 1024}
347 mut content_length := -1
348 mut headers_end := -1
349
350 for {
351 n := conn.read(mut buf) or {
352 if err.code() == net.err_timed_out_code {
353 return error('timed out while reading HTTPS request')
354 }
355 return err
356 }
357 if n <= 0 {
358 break
359 }
360 request << buf[..n]
361
362 request_str := request.bytestr()
363 if headers_end == -1 {
364 headers_end = request_str.index('\r\n\r\n') or { -1 }
365 if headers_end != -1 {
366 headers := request_str[..headers_end]
367 for line in headers.split('\r\n') {
368 if line.to_lower().starts_with('content-length:') {
369 content_length = line.all_after(':').trim_space().int()
370 break
371 }
372 }
373 }
374 }
375
376 if headers_end != -1 && content_length >= 0 {
377 body_start := headers_end + 4
378 if request.len - body_start >= content_length {
379 break
380 }
381 }
382 }
383
384 if headers_end == -1 {
385 return error('HTTPS request did not include a full header block')
386 }
387 if content_length < 0 {
388 return error('HTTPS request did not include Content-Length')
389 }
390 body_start := headers_end + 4
391 if request.len - body_start < content_length {
392 return error('HTTPS request body was truncated: expected ${content_length} bytes, got ${request.len - body_start}')
393 }
394 return request.bytestr()
395}
396