| 1 | // Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. |
| 2 | // Use of this source code is governed by an MIT license |
| 3 | // that can be found in the LICENSE file. |
| 4 | module s3 |
| 5 | |
| 6 | // Integration tests against a live S3-compatible endpoint. They are skipped |
| 7 | // unless the `S3_INTEGRATION` env var is set, and the `S3_HOST`, `S3_KEY_ID` |
| 8 | // and `S3_KEY_SECRET` env vars are populated. `v test .` stays fast and |
| 9 | // offline by default. |
| 10 | // |
| 11 | // Run with: |
| 12 | // S3_INTEGRATION=1 \ |
| 13 | // S3_HOST=s3.example.com \ |
| 14 | // S3_KEY_ID=... S3_KEY_SECRET=... \ |
| 15 | // S3_BUCKET=vs3-tests \ |
| 16 | // v test . |
| 17 | import os |
| 18 | import rand |
| 19 | import time |
| 20 | import net.http |
| 21 | |
| 22 | // http_dispatch_creds_present returns true when the env exposes the AWS-style |
| 23 | // names that http.fetch's bridge will pick up via `Credentials.from_env()`. |
| 24 | // The integration test below relies on this — `S3_HOST` / `S3_KEY_ID` / |
| 25 | // `S3_KEY_SECRET` (used by the rest of this file) aren't recognised by |
| 26 | // `from_env()`, so we map them on the fly when needed. |
| 27 | fn ensure_aws_env_for_bridge() { |
| 28 | if os.getenv('S3_ACCESS_KEY_ID') == '' { |
| 29 | os.setenv('S3_ACCESS_KEY_ID', os.getenv('S3_KEY_ID'), true) |
| 30 | } |
| 31 | if os.getenv('S3_SECRET_ACCESS_KEY') == '' { |
| 32 | os.setenv('S3_SECRET_ACCESS_KEY', os.getenv('S3_KEY_SECRET'), true) |
| 33 | } |
| 34 | if os.getenv('S3_ENDPOINT') == '' { |
| 35 | host := os.getenv('S3_HOST') |
| 36 | endpoint := if host.contains('://') { host } else { 'https://${host}' } |
| 37 | os.setenv('S3_ENDPOINT', endpoint, true) |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | fn integration_enabled() bool { |
| 42 | return os.getenv('S3_INTEGRATION') != '' |
| 43 | } |
| 44 | |
| 45 | fn live_client() ?Client { |
| 46 | if !integration_enabled() { |
| 47 | return none |
| 48 | } |
| 49 | host := os.getenv('S3_HOST') |
| 50 | key_id := os.getenv('S3_KEY_ID') |
| 51 | secret := os.getenv('S3_KEY_SECRET') |
| 52 | if host == '' || key_id == '' || secret == '' { |
| 53 | return none |
| 54 | } |
| 55 | endpoint := if host.contains('://') { host } else { 'https://${host}' } |
| 56 | bucket := os.getenv_opt('S3_BUCKET') or { 'vs3-tests' } |
| 57 | return Client{ |
| 58 | credentials: Credentials{ |
| 59 | endpoint: endpoint |
| 60 | access_key_id: key_id |
| 61 | secret_access_key: secret |
| 62 | bucket: bucket |
| 63 | region: 'us-east-1' |
| 64 | } |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | fn rand_key(prefix string) string { |
| 69 | return '${prefix}/${time.now().unix()}-${rand.u64()}.txt' |
| 70 | } |
| 71 | |
| 72 | fn test_integration_roundtrip() { |
| 73 | c := live_client() or { return } |
| 74 | key := rand_key('it/roundtrip') |
| 75 | defer { |
| 76 | c.delete(key) or {} |
| 77 | } |
| 78 | c.put(key, 'hi'.bytes(), content_type: 'text/plain') or { panic(err) } |
| 79 | got := c.get_string(key) or { panic(err) } |
| 80 | assert got == 'hi' |
| 81 | stat := c.stat(key) or { panic(err) } |
| 82 | assert stat.size == 2 |
| 83 | assert stat.content_type.starts_with('text/plain') |
| 84 | } |
| 85 | |
| 86 | fn test_integration_range_read() { |
| 87 | c := live_client() or { return } |
| 88 | key := rand_key('it/range') |
| 89 | defer { |
| 90 | c.delete(key) or {} |
| 91 | } |
| 92 | c.put(key, 'abcdefghijklmnop'.bytes()) or { panic(err) } |
| 93 | first := c.get(key, range: 'bytes=0-4') or { panic(err) } |
| 94 | assert first.bytestr() == 'abcde', 'got: ${first.bytestr()}' |
| 95 | tail := c.get(key, range: 'bytes=10-') or { panic(err) } |
| 96 | assert tail.bytestr() == 'klmnop', 'got: ${tail.bytestr()}' |
| 97 | } |
| 98 | |
| 99 | fn test_integration_presign_get() { |
| 100 | c := live_client() or { return } |
| 101 | key := rand_key('it/presign') |
| 102 | defer { |
| 103 | c.delete(key) or {} |
| 104 | } |
| 105 | c.put(key, 'PRESIGN_OK'.bytes()) or { panic(err) } |
| 106 | url := c.presign(key, expires_in: 60) or { panic(err) } |
| 107 | body := fetch_url(url) or { panic(err) } |
| 108 | assert body.contains('PRESIGN_OK'), 'got: ${body}' |
| 109 | } |
| 110 | |
| 111 | fn test_integration_list_with_prefix() { |
| 112 | c := live_client() or { return } |
| 113 | prefix := 'it/list-${rand.u64()}/' |
| 114 | defer { |
| 115 | // Best-effort cleanup |
| 116 | if res := c.list(prefix: prefix, max_keys: 100) { |
| 117 | for o in res.objects { |
| 118 | c.delete(o.key) or {} |
| 119 | } |
| 120 | } |
| 121 | } |
| 122 | for i in 0 .. 3 { |
| 123 | c.put('${prefix}item-${i}.txt', 'x'.bytes()) or { panic(err) } |
| 124 | } |
| 125 | res := c.list(prefix: prefix, fetch_owner: false) or { panic(err) } |
| 126 | assert res.objects.len == 3 |
| 127 | for o in res.objects { |
| 128 | assert o.key.starts_with(prefix) |
| 129 | assert o.size == 1 |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | fn test_integration_exists_and_delete() { |
| 134 | c := live_client() or { return } |
| 135 | key := rand_key('it/exists') |
| 136 | c.put(key, 'present'.bytes()) or { panic(err) } |
| 137 | assert c.exists(key) or { panic(err) } |
| 138 | c.delete(key) or { panic(err) } |
| 139 | assert !(c.exists(key) or { panic(err) }) |
| 140 | } |
| 141 | |
| 142 | fn test_integration_fetch_helper() { |
| 143 | c := live_client() or { return } |
| 144 | key := rand_key('it/fetch') |
| 145 | url := 's3://${c.credentials.bucket}/${key}' |
| 146 | defer { |
| 147 | c.delete(key) or {} |
| 148 | } |
| 149 | put := fetch(url, |
| 150 | method: .put |
| 151 | body: 'fetch_works'.bytes() |
| 152 | credentials: c.credentials |
| 153 | ) or { panic(err) } |
| 154 | assert put.status_code == 200 |
| 155 | get := fetch(url, credentials: c.credentials) or { panic(err) } |
| 156 | assert get.body.bytestr() == 'fetch_works' |
| 157 | } |
| 158 | |
| 159 | fn test_integration_multipart_upload_bytes() { |
| 160 | c := live_client() or { return } |
| 161 | // 6 MiB triggers multipart (one full part + a tail). |
| 162 | mut data := []u8{len: 6 * 1024 * 1024} |
| 163 | for i in 0 .. data.len { |
| 164 | data[i] = u8(i & 0xFF) |
| 165 | } |
| 166 | key := rand_key('it/multipart') |
| 167 | defer { |
| 168 | c.delete(key) or {} |
| 169 | } |
| 170 | c.upload_bytes_multipart(key, data) or { panic(err) } |
| 171 | stat := c.stat(key) or { panic(err) } |
| 172 | assert stat.size == data.len |
| 173 | got := c.get(key) or { panic(err) } |
| 174 | assert got.len == data.len |
| 175 | for i in 0 .. data.len { |
| 176 | if got[i] != data[i] { |
| 177 | assert false, 'byte mismatch at ${i}' |
| 178 | break |
| 179 | } |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | fn test_integration_multipart_parallel_many_parts() { |
| 184 | c := live_client() or { return } |
| 185 | // Force 8 parts at the 5 MiB minimum + tail. With queue_size=5 (default) |
| 186 | // we get genuine parallel dispatch and out-of-order completion that the |
| 187 | // dispatcher must re-order before CompleteMultipartUpload. |
| 188 | mut data := []u8{len: 8 * 5 * 1024 * 1024 + 1024} |
| 189 | for i in 0 .. data.len { |
| 190 | data[i] = u8((i * 31 + 7) & 0xFF) |
| 191 | } |
| 192 | key := rand_key('it/parallel') |
| 193 | defer { |
| 194 | c.delete(key) or {} |
| 195 | } |
| 196 | c.upload_bytes_multipart(key, data) or { panic(err) } |
| 197 | stat := c.stat(key) or { panic(err) } |
| 198 | assert stat.size == data.len, 'size mismatch: got ${stat.size}' |
| 199 | // Verify a tail range, where corruption from out-of-order parts would land. |
| 200 | tail := c.get(key, range: 'bytes=${data.len - 1024}-') or { panic(err) } |
| 201 | for i in 0 .. tail.len { |
| 202 | if tail[i] != data[data.len - 1024 + i] { |
| 203 | assert false, 'tail byte mismatch at offset ${i}' |
| 204 | break |
| 205 | } |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | fn test_integration_bucket_lifecycle() { |
| 210 | c := live_client() or { return } |
| 211 | // Random suffix keeps reruns idempotent. |
| 212 | name := 's3v-it-${rand.u64():08x}'.to_lower() |
| 213 | c.create_bucket(bucket: name) or { |
| 214 | // Some providers require region constraint or have different defaults |
| 215 | // — surface but don't fail the suite for that one provider quirk. |
| 216 | eprintln('skip: create_bucket(${name}) failed: ${err}') |
| 217 | return |
| 218 | } |
| 219 | defer { |
| 220 | c.delete_bucket(bucket: name) or {} |
| 221 | } |
| 222 | exists := c.bucket_exists(bucket: name) or { panic(err) } |
| 223 | assert exists, 'bucket ${name} should exist after create' |
| 224 | // Delete once explicitly so we can assert it disappeared. |
| 225 | c.delete_bucket(bucket: name) or { panic(err) } |
| 226 | gone := c.bucket_exists(bucket: name) or { panic(err) } |
| 227 | assert !gone, 'bucket ${name} should be gone after delete' |
| 228 | } |
| 229 | |
| 230 | fn test_integration_invalid_credentials_yields_clean_error() { |
| 231 | if !integration_enabled() { |
| 232 | return |
| 233 | } |
| 234 | host := os.getenv('S3_HOST') |
| 235 | if host == '' { |
| 236 | return |
| 237 | } |
| 238 | c := Client{ |
| 239 | credentials: Credentials{ |
| 240 | endpoint: 'https://${host}' |
| 241 | access_key_id: 'AKIADOESNOTEXIST' |
| 242 | secret_access_key: 'badbadbadbadbadbadbadbadbadbadbadbadbadbad' |
| 243 | bucket: os.getenv_opt('S3_BUCKET') or { 'vs3-tests' } |
| 244 | } |
| 245 | } |
| 246 | if _ := c.stat('any-key') { |
| 247 | assert false, 'expected error' |
| 248 | } else { |
| 249 | assert err is S3Error, 'got non-S3 error: ${typeof(err).name}' |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | // fetch_url is a tiny `http.get`-style helper used only in the presign test. |
| 254 | fn fetch_url(url string) !string { |
| 255 | resp := http.get(url)! |
| 256 | return resp.body |
| 257 | } |
| 258 | |
| 259 | // Exercises the `http.fetch(url: 's3://...')` bridge end-to-end. The s3 |
| 260 | // module's `init()` registers itself with net.http; this test goes through |
| 261 | // that path (no direct `s3.fetch` call) to prove the dispatch works. |
| 262 | fn test_integration_http_fetch_dispatch_to_s3() { |
| 263 | c := live_client() or { return } |
| 264 | ensure_aws_env_for_bridge() |
| 265 | bucket := c.credentials.bucket |
| 266 | key := rand_key('it/http-bridge') |
| 267 | url := 's3://${bucket}/${key}' |
| 268 | |
| 269 | put := http.fetch(url: url, method: .put, data: 'via-http.fetch') or { |
| 270 | assert false, 'PUT through bridge failed: ${err}' |
| 271 | return |
| 272 | } |
| 273 | assert put.status_code == 200, 'unexpected PUT status: ${put.status_code}' |
| 274 | |
| 275 | get := http.fetch(url: url) or { |
| 276 | assert false, 'GET through bridge failed: ${err}' |
| 277 | return |
| 278 | } |
| 279 | assert get.status_code == 200, 'unexpected GET status: ${get.status_code}' |
| 280 | assert get.body == 'via-http.fetch', 'body mismatch: ${get.body}' |
| 281 | |
| 282 | del := http.fetch(url: url, method: .delete) or { |
| 283 | assert false, 'DELETE through bridge failed: ${err}' |
| 284 | return |
| 285 | } |
| 286 | assert del.status_code in [200, 204], 'unexpected DELETE status: ${del.status_code}' |
| 287 | } |
| 288 | |