v / vlib / net / s3 / integration_test.v
287 lines · 267 sloc · 8.57 KB · 4142432483c4e8de44ab7b0d6ac944f3251e03c8
Raw
1// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
2// Use of this source code is governed by an MIT license
3// that can be found in the LICENSE file.
4module s3
5
6// Integration tests against a live S3-compatible endpoint. They are skipped
7// unless the `S3_INTEGRATION` env var is set, and the `S3_HOST`, `S3_KEY_ID`
8// and `S3_KEY_SECRET` env vars are populated. `v test .` stays fast and
9// offline by default.
10//
11// Run with:
12// S3_INTEGRATION=1 \
13// S3_HOST=s3.example.com \
14// S3_KEY_ID=... S3_KEY_SECRET=... \
15// S3_BUCKET=vs3-tests \
16// v test .
17import os
18import rand
19import time
20import net.http
21
22// http_dispatch_creds_present returns true when the env exposes the AWS-style
23// names that http.fetch's bridge will pick up via `Credentials.from_env()`.
24// The integration test below relies on this — `S3_HOST` / `S3_KEY_ID` /
25// `S3_KEY_SECRET` (used by the rest of this file) aren't recognised by
26// `from_env()`, so we map them on the fly when needed.
27fn ensure_aws_env_for_bridge() {
28 if os.getenv('S3_ACCESS_KEY_ID') == '' {
29 os.setenv('S3_ACCESS_KEY_ID', os.getenv('S3_KEY_ID'), true)
30 }
31 if os.getenv('S3_SECRET_ACCESS_KEY') == '' {
32 os.setenv('S3_SECRET_ACCESS_KEY', os.getenv('S3_KEY_SECRET'), true)
33 }
34 if os.getenv('S3_ENDPOINT') == '' {
35 host := os.getenv('S3_HOST')
36 endpoint := if host.contains('://') { host } else { 'https://${host}' }
37 os.setenv('S3_ENDPOINT', endpoint, true)
38 }
39}
40
41fn integration_enabled() bool {
42 return os.getenv('S3_INTEGRATION') != ''
43}
44
45fn live_client() ?Client {
46 if !integration_enabled() {
47 return none
48 }
49 host := os.getenv('S3_HOST')
50 key_id := os.getenv('S3_KEY_ID')
51 secret := os.getenv('S3_KEY_SECRET')
52 if host == '' || key_id == '' || secret == '' {
53 return none
54 }
55 endpoint := if host.contains('://') { host } else { 'https://${host}' }
56 bucket := os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
57 return Client{
58 credentials: Credentials{
59 endpoint: endpoint
60 access_key_id: key_id
61 secret_access_key: secret
62 bucket: bucket
63 region: 'us-east-1'
64 }
65 }
66}
67
68fn rand_key(prefix string) string {
69 return '${prefix}/${time.now().unix()}-${rand.u64()}.txt'
70}
71
72fn test_integration_roundtrip() {
73 c := live_client() or { return }
74 key := rand_key('it/roundtrip')
75 defer {
76 c.delete(key) or {}
77 }
78 c.put(key, 'hi'.bytes(), content_type: 'text/plain') or { panic(err) }
79 got := c.get_string(key) or { panic(err) }
80 assert got == 'hi'
81 stat := c.stat(key) or { panic(err) }
82 assert stat.size == 2
83 assert stat.content_type.starts_with('text/plain')
84}
85
86fn test_integration_range_read() {
87 c := live_client() or { return }
88 key := rand_key('it/range')
89 defer {
90 c.delete(key) or {}
91 }
92 c.put(key, 'abcdefghijklmnop'.bytes()) or { panic(err) }
93 first := c.get(key, range: 'bytes=0-4') or { panic(err) }
94 assert first.bytestr() == 'abcde', 'got: ${first.bytestr()}'
95 tail := c.get(key, range: 'bytes=10-') or { panic(err) }
96 assert tail.bytestr() == 'klmnop', 'got: ${tail.bytestr()}'
97}
98
99fn test_integration_presign_get() {
100 c := live_client() or { return }
101 key := rand_key('it/presign')
102 defer {
103 c.delete(key) or {}
104 }
105 c.put(key, 'PRESIGN_OK'.bytes()) or { panic(err) }
106 url := c.presign(key, expires_in: 60) or { panic(err) }
107 body := fetch_url(url) or { panic(err) }
108 assert body.contains('PRESIGN_OK'), 'got: ${body}'
109}
110
111fn test_integration_list_with_prefix() {
112 c := live_client() or { return }
113 prefix := 'it/list-${rand.u64()}/'
114 defer {
115 // Best-effort cleanup
116 if res := c.list(prefix: prefix, max_keys: 100) {
117 for o in res.objects {
118 c.delete(o.key) or {}
119 }
120 }
121 }
122 for i in 0 .. 3 {
123 c.put('${prefix}item-${i}.txt', 'x'.bytes()) or { panic(err) }
124 }
125 res := c.list(prefix: prefix, fetch_owner: false) or { panic(err) }
126 assert res.objects.len == 3
127 for o in res.objects {
128 assert o.key.starts_with(prefix)
129 assert o.size == 1
130 }
131}
132
133fn test_integration_exists_and_delete() {
134 c := live_client() or { return }
135 key := rand_key('it/exists')
136 c.put(key, 'present'.bytes()) or { panic(err) }
137 assert c.exists(key) or { panic(err) }
138 c.delete(key) or { panic(err) }
139 assert !(c.exists(key) or { panic(err) })
140}
141
142fn test_integration_fetch_helper() {
143 c := live_client() or { return }
144 key := rand_key('it/fetch')
145 url := 's3://${c.credentials.bucket}/${key}'
146 defer {
147 c.delete(key) or {}
148 }
149 put := fetch(url,
150 method: .put
151 body: 'fetch_works'.bytes()
152 credentials: c.credentials
153 ) or { panic(err) }
154 assert put.status_code == 200
155 get := fetch(url, credentials: c.credentials) or { panic(err) }
156 assert get.body.bytestr() == 'fetch_works'
157}
158
159fn test_integration_multipart_upload_bytes() {
160 c := live_client() or { return }
161 // 6 MiB triggers multipart (one full part + a tail).
162 mut data := []u8{len: 6 * 1024 * 1024}
163 for i in 0 .. data.len {
164 data[i] = u8(i & 0xFF)
165 }
166 key := rand_key('it/multipart')
167 defer {
168 c.delete(key) or {}
169 }
170 c.upload_bytes_multipart(key, data) or { panic(err) }
171 stat := c.stat(key) or { panic(err) }
172 assert stat.size == data.len
173 got := c.get(key) or { panic(err) }
174 assert got.len == data.len
175 for i in 0 .. data.len {
176 if got[i] != data[i] {
177 assert false, 'byte mismatch at ${i}'
178 break
179 }
180 }
181}
182
183fn test_integration_multipart_parallel_many_parts() {
184 c := live_client() or { return }
185 // Force 8 parts at the 5 MiB minimum + tail. With queue_size=5 (default)
186 // we get genuine parallel dispatch and out-of-order completion that the
187 // dispatcher must re-order before CompleteMultipartUpload.
188 mut data := []u8{len: 8 * 5 * 1024 * 1024 + 1024}
189 for i in 0 .. data.len {
190 data[i] = u8((i * 31 + 7) & 0xFF)
191 }
192 key := rand_key('it/parallel')
193 defer {
194 c.delete(key) or {}
195 }
196 c.upload_bytes_multipart(key, data) or { panic(err) }
197 stat := c.stat(key) or { panic(err) }
198 assert stat.size == data.len, 'size mismatch: got ${stat.size}'
199 // Verify a tail range, where corruption from out-of-order parts would land.
200 tail := c.get(key, range: 'bytes=${data.len - 1024}-') or { panic(err) }
201 for i in 0 .. tail.len {
202 if tail[i] != data[data.len - 1024 + i] {
203 assert false, 'tail byte mismatch at offset ${i}'
204 break
205 }
206 }
207}
208
209fn test_integration_bucket_lifecycle() {
210 c := live_client() or { return }
211 // Random suffix keeps reruns idempotent.
212 name := 's3v-it-${rand.u64():08x}'.to_lower()
213 c.create_bucket(bucket: name) or {
214 // Some providers require region constraint or have different defaults
215 // — surface but don't fail the suite for that one provider quirk.
216 eprintln('skip: create_bucket(${name}) failed: ${err}')
217 return
218 }
219 defer {
220 c.delete_bucket(bucket: name) or {}
221 }
222 exists := c.bucket_exists(bucket: name) or { panic(err) }
223 assert exists, 'bucket ${name} should exist after create'
224 // Delete once explicitly so we can assert it disappeared.
225 c.delete_bucket(bucket: name) or { panic(err) }
226 gone := c.bucket_exists(bucket: name) or { panic(err) }
227 assert !gone, 'bucket ${name} should be gone after delete'
228}
229
230fn test_integration_invalid_credentials_yields_clean_error() {
231 if !integration_enabled() {
232 return
233 }
234 host := os.getenv('S3_HOST')
235 if host == '' {
236 return
237 }
238 c := Client{
239 credentials: Credentials{
240 endpoint: 'https://${host}'
241 access_key_id: 'AKIADOESNOTEXIST'
242 secret_access_key: 'badbadbadbadbadbadbadbadbadbadbadbadbadbad'
243 bucket: os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
244 }
245 }
246 if _ := c.stat('any-key') {
247 assert false, 'expected error'
248 } else {
249 assert err is S3Error, 'got non-S3 error: ${typeof(err).name}'
250 }
251}
252
253// fetch_url is a tiny `http.get`-style helper used only in the presign test.
254fn fetch_url(url string) !string {
255 resp := http.get(url)!
256 return resp.body
257}
258
259// Exercises the `http.fetch(url: 's3://...')` bridge end-to-end. The s3
260// module's `init()` registers itself with net.http; this test goes through
261// that path (no direct `s3.fetch` call) to prove the dispatch works.
262fn test_integration_http_fetch_dispatch_to_s3() {
263 c := live_client() or { return }
264 ensure_aws_env_for_bridge()
265 bucket := c.credentials.bucket
266 key := rand_key('it/http-bridge')
267 url := 's3://${bucket}/${key}'
268
269 put := http.fetch(url: url, method: .put, data: 'via-http.fetch') or {
270 assert false, 'PUT through bridge failed: ${err}'
271 return
272 }
273 assert put.status_code == 200, 'unexpected PUT status: ${put.status_code}'
274
275 get := http.fetch(url: url) or {
276 assert false, 'GET through bridge failed: ${err}'
277 return
278 }
279 assert get.status_code == 200, 'unexpected GET status: ${get.status_code}'
280 assert get.body == 'via-http.fetch', 'body mismatch: ${get.body}'
281
282 del := http.fetch(url: url, method: .delete) or {
283 assert false, 'DELETE through bridge failed: ${err}'
284 return
285 }
286 assert del.status_code in [200, 204], 'unexpected DELETE status: ${del.status_code}'
287}
288