| 1 | // vtest retry: 3 |
| 2 | // vtest vflags: -d use_openssl |
| 3 | // vtest build: !windows |
| 4 | module http |
| 5 | |
| 6 | import encoding.base64 |
| 7 | import net |
| 8 | import net.mbedtls |
| 9 | import net.urllib |
| 10 | import os |
| 11 | import time |
| 12 | |
| 13 | const sample_proxy_url = 'https://localhost' |
| 14 | const sample_auth_proxy_url = 'http://user:pass@localhost:8888' |
| 15 | |
| 16 | const sample_host = '127.0.0.1:1337' |
| 17 | const sample_request = &Request{ |
| 18 | url: 'http://${sample_host}' |
| 19 | } |
| 20 | const sample_path = '/' |
| 21 | const proxy_https_request_count = 12 |
| 22 | const proxy_https_test_cert_path = @VEXEROOT + |
| 23 | '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.crt' |
| 24 | const proxy_https_test_key_path = @VEXEROOT + |
| 25 | '/vlib/net/websocket/tests/autobahn/fuzzing_server_wss/config/server.key' |
| 26 | |
| 27 | fn test_proxy_fields() ? { |
| 28 | sample_proxy := new_http_proxy(sample_proxy_url)! |
| 29 | sample_auth_proxy := new_http_proxy(sample_auth_proxy_url)! |
| 30 | |
| 31 | assert sample_proxy.scheme == 'https' |
| 32 | assert sample_proxy.host == 'localhost:443' |
| 33 | assert sample_proxy.hostname == 'localhost' |
| 34 | assert sample_proxy.port == 443 |
| 35 | assert sample_proxy.url == sample_proxy_url |
| 36 | assert sample_auth_proxy.scheme == 'http' |
| 37 | assert sample_auth_proxy.username == 'user' |
| 38 | assert sample_auth_proxy.password == 'pass' |
| 39 | assert sample_auth_proxy.host == 'localhost:8888' |
| 40 | assert sample_auth_proxy.hostname == 'localhost' |
| 41 | assert sample_auth_proxy.port == 8888 |
| 42 | assert sample_auth_proxy.url == sample_auth_proxy_url |
| 43 | } |
| 44 | |
| 45 | fn test_proxy_headers() ? { |
| 46 | sample_proxy := new_http_proxy(sample_proxy_url)! |
| 47 | headers := sample_proxy.build_proxy_headers(sample_host) |
| 48 | |
| 49 | assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' + |
| 50 | 'Proxy-Connection: Keep-Alive\r\n\r\n' |
| 51 | } |
| 52 | |
| 53 | fn test_proxy_headers_authenticated() ? { |
| 54 | sample_proxy := new_http_proxy(sample_auth_proxy_url)! |
| 55 | headers := sample_proxy.build_proxy_headers(sample_host) |
| 56 | |
| 57 | auth_token := base64.encode(('${sample_proxy.username}:' + '${sample_proxy.password}').bytes()) |
| 58 | |
| 59 | assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' + |
| 60 | 'Proxy-Connection: Keep-Alive\r\nProxy-Authorization: Basic ${auth_token}\r\n\r\n' |
| 61 | } |
| 62 | |
| 63 | enum ProxyTunnelCopyResult { |
| 64 | data |
| 65 | timeout |
| 66 | closed |
| 67 | } |
| 68 | |
| 69 | fn count_open_file_descriptors() int { |
| 70 | $if windows { |
| 71 | return 0 |
| 72 | } $else { |
| 73 | fds := os.ls('/dev/fd') or { return 0 } |
| 74 | return fds.len |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | fn start_https_proxy_test_target_server() !(int, chan bool) { |
| 79 | ready := chan int{cap: 1} |
| 80 | done := chan bool{cap: 1} |
| 81 | spawn fn [ready, done] () { |
| 82 | mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) } |
| 83 | port := int((port_listener.addr() or { panic(err) }).port() or { panic(err) }) |
| 84 | port_listener.close() or {} |
| 85 | mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{ |
| 86 | cert: proxy_https_test_cert_path |
| 87 | cert_key: proxy_https_test_key_path |
| 88 | validate: false |
| 89 | in_memory_verification: false |
| 90 | }) or { panic(err) } |
| 91 | ready <- port |
| 92 | defer { |
| 93 | listener.shutdown() or {} |
| 94 | done <- true |
| 95 | } |
| 96 | for _ in 0 .. proxy_https_request_count { |
| 97 | mut conn := listener.accept() or { panic(err) } |
| 98 | handle_https_proxy_test_target_connection(mut conn) |
| 99 | } |
| 100 | }() |
| 101 | return <-ready, done |
| 102 | } |
| 103 | |
| 104 | fn handle_https_proxy_test_target_connection(mut conn mbedtls.SSLConn) { |
| 105 | defer { |
| 106 | conn.shutdown() or {} |
| 107 | } |
| 108 | mut request_buf := []u8{len: 2048} |
| 109 | _ = conn.read(mut request_buf) or { return } |
| 110 | conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or { |
| 111 | return |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | fn start_https_proxy_test_server(target_port int) !(int, chan bool) { |
| 116 | ready := chan int{cap: 1} |
| 117 | done := chan bool{cap: 1} |
| 118 | spawn fn [ready, done, target_port] () { |
| 119 | mut listener := net.listen_tcp(.ip, '127.0.0.1:0') or { panic(err) } |
| 120 | port := int((listener.addr() or { panic(err) }).port() or { panic(err) }) |
| 121 | ready <- port |
| 122 | mut workers := []thread{cap: proxy_https_request_count} |
| 123 | for _ in 0 .. proxy_https_request_count { |
| 124 | mut client := listener.accept() or { panic(err) } |
| 125 | workers << spawn handle_https_proxy_test_tunnel(mut client, target_port) |
| 126 | } |
| 127 | listener.close() or {} |
| 128 | workers.wait() |
| 129 | done <- true |
| 130 | }() |
| 131 | return <-ready, done |
| 132 | } |
| 133 | |
| 134 | fn handle_https_proxy_test_tunnel(mut client net.TcpConn, target_port int) { |
| 135 | defer { |
| 136 | client.close() or {} |
| 137 | } |
| 138 | request := read_proxy_request(mut client) or { return } |
| 139 | if !request.starts_with('CONNECT 127.0.0.1:${target_port} HTTP/1.1\r\n') { |
| 140 | client.write_string('HTTP/1.1 400 Bad Request\r\n\r\n') or {} |
| 141 | return |
| 142 | } |
| 143 | mut upstream := net.dial_tcp('127.0.0.1:${target_port}') or { |
| 144 | client.write_string('HTTP/1.1 502 Bad Gateway\r\n\r\n') or {} |
| 145 | return |
| 146 | } |
| 147 | defer { |
| 148 | upstream.close() or {} |
| 149 | } |
| 150 | client.write_string('HTTP/1.1 200 Connection Established\r\n\r\n') or { return } |
| 151 | client.set_read_timeout(50 * time.millisecond) |
| 152 | upstream.set_read_timeout(50 * time.millisecond) |
| 153 | deadline := time.now().add(2 * time.second) |
| 154 | for time.now() < deadline { |
| 155 | client_state := copy_https_proxy_test_tunnel_data(mut client, mut upstream) |
| 156 | upstream_state := copy_https_proxy_test_tunnel_data(mut upstream, mut client) |
| 157 | if client_state == .closed || upstream_state == .closed { |
| 158 | return |
| 159 | } |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | fn read_proxy_request(mut conn net.TcpConn) !string { |
| 164 | mut total_bytes_read := 0 |
| 165 | mut msg := [4096]u8{} |
| 166 | mut buffer := [1]u8{} |
| 167 | for total_bytes_read < msg.len { |
| 168 | bytes_read := conn.read_ptr(&buffer[0], 1)! |
| 169 | if bytes_read == 0 { |
| 170 | return error('unexpected EOF while reading proxy request') |
| 171 | } |
| 172 | msg[total_bytes_read] = buffer[0] |
| 173 | total_bytes_read++ |
| 174 | if total_bytes_read > 3 && msg[total_bytes_read - 1] == `\n` |
| 175 | && msg[total_bytes_read - 2] == `\r` && msg[total_bytes_read - 3] == `\n` |
| 176 | && msg[total_bytes_read - 4] == `\r` { |
| 177 | return msg[..total_bytes_read].bytestr() |
| 178 | } |
| 179 | } |
| 180 | return error('proxy request headers exceeded 4096 bytes') |
| 181 | } |
| 182 | |
| 183 | fn copy_https_proxy_test_tunnel_data(mut src net.TcpConn, mut dst net.TcpConn) ProxyTunnelCopyResult { |
| 184 | mut buf := []u8{len: 1024} |
| 185 | bytes_read := src.read(mut buf) or { |
| 186 | if err.code() == net.err_timed_out_code { |
| 187 | return .timeout |
| 188 | } |
| 189 | return .closed |
| 190 | } |
| 191 | if bytes_read <= 0 { |
| 192 | return .closed |
| 193 | } |
| 194 | dst.write(buf[..bytes_read]) or { return .closed } |
| 195 | return .data |
| 196 | } |
| 197 | |
| 198 | fn test_https_proxy_requests_do_not_leak_sockets() ! { |
| 199 | $if windows { |
| 200 | return |
| 201 | } |
| 202 | $if sanitized_job ? { |
| 203 | return |
| 204 | } |
| 205 | $if tinyc { |
| 206 | // TinyCC hangs in the bundled mbedtls handshake path on linux CI. |
| 207 | return |
| 208 | } |
| 209 | target_port, target_done := start_https_proxy_test_target_server()! |
| 210 | proxy_port, proxy_done := start_https_proxy_test_server(target_port)! |
| 211 | baseline_fds := count_open_file_descriptors() |
| 212 | proxy := new_http_proxy('http://127.0.0.1:${proxy_port}')! |
| 213 | for _ in 0 .. proxy_https_request_count { |
| 214 | resp := fetch( |
| 215 | method: .get |
| 216 | url: 'https://127.0.0.1:${target_port}/' |
| 217 | proxy: proxy |
| 218 | validate: false |
| 219 | )! |
| 220 | assert resp.status_code == 200 |
| 221 | assert resp.body == 'ok' |
| 222 | } |
| 223 | _ = <-target_done |
| 224 | _ = <-proxy_done |
| 225 | time.sleep(100 * time.millisecond) |
| 226 | final_fds := count_open_file_descriptors() |
| 227 | assert final_fds <= baseline_fds + 3 |
| 228 | } |
| 229 | |
| 230 | fn test_http_proxy_do() { |
| 231 | env := os.environ() |
| 232 | mut env_proxy := '' |
| 233 | |
| 234 | for envvar in ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY'] { |
| 235 | prox_val := env[envvar] or { continue } |
| 236 | if prox_val != '' { |
| 237 | env_proxy = env[envvar] |
| 238 | } |
| 239 | } |
| 240 | if env_proxy != '' { |
| 241 | println('Has usable proxy env vars') |
| 242 | proxy := new_http_proxy(env_proxy)! |
| 243 | mut header := new_header(key: .user_agent, value: 'vlib') |
| 244 | header.add_custom('X-Vlang-Test', 'proxied')! |
| 245 | res := proxy.http_do(urllib.parse('http://httpbin.org/headers')!, Method.get, '/headers', &Request{ |
| 246 | proxy: proxy |
| 247 | header: header |
| 248 | }, '', header)! |
| 249 | println(res.status_code) |
| 250 | println('he4aders ${res.header}') |
| 251 | assert res.status_code == 200 |
| 252 | // assert res.header.data['X-Vlang-Test'] == 'proxied' |
| 253 | } else { |
| 254 | println('Proxy env vars (HTTP_PROXY or HTTPS_PROXY) not set. Skipping test.') |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | const multipart_https_payload_len = 20 * 1024 + 137 |
| 259 | |
| 260 | fn test_https_multipart_form_preserves_large_binary_body() ! { |
| 261 | $if tinyc { |
| 262 | // TinyCC hangs in the bundled mbedtls handshake path on linux CI. |
| 263 | return |
| 264 | } |
| 265 | mut port_listener := net.listen_tcp(.ip, '127.0.0.1:0')! |
| 266 | port := port_listener.addr()!.port()! |
| 267 | port_listener.close()! |
| 268 | |
| 269 | payload := multipart_https_test_payload() |
| 270 | form := { |
| 271 | 'alpha': 'beta' |
| 272 | } |
| 273 | files := { |
| 274 | 'file': [ |
| 275 | FileData{ |
| 276 | filename: 'payload.bin' |
| 277 | content_type: 'application/octet-stream' |
| 278 | data: payload |
| 279 | }, |
| 280 | ] |
| 281 | } |
| 282 | body, boundary := multipart_form_body(form, files) |
| 283 | |
| 284 | mut listener := mbedtls.new_ssl_listener('127.0.0.1:${port}', mbedtls.SSLConnectConfig{ |
| 285 | cert: proxy_https_test_cert_path |
| 286 | cert_key: proxy_https_test_key_path |
| 287 | validate: false |
| 288 | })! |
| 289 | server := spawn multipart_https_serve_once(mut listener, body, boundary, form, files) |
| 290 | |
| 291 | mut header := new_header() |
| 292 | header.set(.content_type, 'multipart/form-data; boundary="${boundary}"') |
| 293 | resp := fetch( |
| 294 | method: .post |
| 295 | url: 'https://127.0.0.1:${port}/upload' |
| 296 | header: header |
| 297 | data: body |
| 298 | validate: false |
| 299 | )! |
| 300 | server.wait() |
| 301 | |
| 302 | assert resp.status_code == 200 |
| 303 | assert resp.body == 'ok' |
| 304 | } |
| 305 | |
| 306 | fn multipart_https_test_payload() string { |
| 307 | mut payload := []u8{len: multipart_https_payload_len, init: u8(((index * 17) % 250) + 1)} |
| 308 | payload[127] = 0 |
| 309 | payload[4096] = 0 |
| 310 | payload[16 * 1024] = 0 |
| 311 | payload[payload.len - 1] = `!` |
| 312 | return payload.bytestr() |
| 313 | } |
| 314 | |
| 315 | fn multipart_https_serve_once(mut listener mbedtls.SSLListener, expected_body string, boundary string, expected_form map[string]string, expected_files map[string][]FileData) { |
| 316 | defer { |
| 317 | listener.shutdown() or {} |
| 318 | } |
| 319 | mut conn := listener.accept() or { panic(err) } |
| 320 | conn.set_read_timeout(5 * time.second) |
| 321 | defer { |
| 322 | conn.shutdown() or {} |
| 323 | } |
| 324 | |
| 325 | request_text := read_https_request(mut conn) or { panic(err) } |
| 326 | req := parse_request_str(request_text) or { panic(err) } |
| 327 | |
| 328 | assert req.method == .post |
| 329 | assert req.url == '/upload' |
| 330 | assert req.data == expected_body |
| 331 | assert req.data.len == expected_body.len |
| 332 | assert req.header.get(.content_length) or { panic(err) } == expected_body.len.str() |
| 333 | assert req.header.get(.content_type) or { panic(err) } == 'multipart/form-data; boundary="${boundary}"' |
| 334 | |
| 335 | form, files := parse_multipart_form(req.data, boundary) |
| 336 | assert form == expected_form |
| 337 | assert files == expected_files |
| 338 | |
| 339 | conn.write_string('HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') or { |
| 340 | panic(err) |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | fn read_https_request(mut conn mbedtls.SSLConn) !string { |
| 345 | mut request := []u8{} |
| 346 | mut buf := []u8{len: 1024} |
| 347 | mut content_length := -1 |
| 348 | mut headers_end := -1 |
| 349 | |
| 350 | for { |
| 351 | n := conn.read(mut buf) or { |
| 352 | if err.code() == net.err_timed_out_code { |
| 353 | return error('timed out while reading HTTPS request') |
| 354 | } |
| 355 | return err |
| 356 | } |
| 357 | if n <= 0 { |
| 358 | break |
| 359 | } |
| 360 | request << buf[..n] |
| 361 | |
| 362 | request_str := request.bytestr() |
| 363 | if headers_end == -1 { |
| 364 | headers_end = request_str.index('\r\n\r\n') or { -1 } |
| 365 | if headers_end != -1 { |
| 366 | headers := request_str[..headers_end] |
| 367 | for line in headers.split('\r\n') { |
| 368 | if line.to_lower().starts_with('content-length:') { |
| 369 | content_length = line.all_after(':').trim_space().int() |
| 370 | break |
| 371 | } |
| 372 | } |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | if headers_end != -1 && content_length >= 0 { |
| 377 | body_start := headers_end + 4 |
| 378 | if request.len - body_start >= content_length { |
| 379 | break |
| 380 | } |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | if headers_end == -1 { |
| 385 | return error('HTTPS request did not include a full header block') |
| 386 | } |
| 387 | if content_length < 0 { |
| 388 | return error('HTTPS request did not include Content-Length') |
| 389 | } |
| 390 | body_start := headers_end + 4 |
| 391 | if request.len - body_start < content_length { |
| 392 | return error('HTTPS request body was truncated: expected ${content_length} bytes, got ${request.len - body_start}') |
| 393 | } |
| 394 | return request.bytestr() |
| 395 | } |
| 396 | |