From ec1f95ac5024aa944e7887de927492271457bf95 Mon Sep 17 00:00:00 2001
From: Delyan Angelov <delian66@gmail.com>
Date: Mon, 11 Nov 2024 19:18:58 +0200
Subject: [PATCH] v.util: implement tokenize_to_args/1, use it in
 util.join_env_vflags_and_os_args/0, add tests (#22831)

---
 vlib/v/util/util.v               | 24 ++----------
 vlib/v/util/vflags/vflags.v      | 66 ++++++++++++++++++++++++++++++++
 vlib/v/util/vflags/vflags_test.v | 47 +++++++++++++++++++++++
 3 files changed, 116 insertions(+), 21 deletions(-)
 create mode 100644 vlib/v/util/vflags/vflags.v
 create mode 100644 vlib/v/util/vflags/vflags_test.v

diff --git a/vlib/v/util/util.v b/vlib/v/util/util.v
index d5771e2d0..8441c3916 100644
--- a/vlib/v/util/util.v
+++ b/vlib/v/util/util.v
@@ -11,6 +11,7 @@ import time
 import v.pref
 import v.vmod
 import v.util.recompilation
+import v.util.vflags
 import runtime
 
 // math.bits is needed by strconv.ftoa
@@ -441,11 +442,7 @@ pub fn quote_path(s string) string {
 }
 
 pub fn args_quote_paths(args []string) string {
-	mut res := []string{}
-	for a in args {
-		res << quote_path(a)
-	}
-	return res.join(' ')
+	return args.map(quote_path(it)).join(' ')
 }
 
 pub fn path_of_executable(path string) string {
@@ -512,22 +509,7 @@ pub fn replace_op(s string) string {
 
 // join_env_vflags_and_os_args returns all the arguments (the ones from the env variable VFLAGS too), passed on the command line.
 pub fn join_env_vflags_and_os_args() []string {
-	// TODO: use a proper parser, instead of splitting on ' '
-	vosargs := os.getenv('VOSARGS')
-	if vosargs != '' {
-		return vosargs.split(' ')
-	}
-	mut args := []string{}
-	vflags := os.getenv('VFLAGS')
-	if vflags != '' {
-		args << os.args[0]
-		args << vflags.split(' ')
-		if os.args.len > 1 {
-			args << os.args[1..]
-		}
-		return args
-	}
-	return os.args
+	return vflags.join_env_vflags_and_os_args()
 }
 
 pub fn check_module_is_installed(modulename string, is_verbose bool, need_update bool) !bool {
diff --git a/vlib/v/util/vflags/vflags.v b/vlib/v/util/vflags/vflags.v
new file mode 100644
index 000000000..e5736a828
--- /dev/null
+++ b/vlib/v/util/vflags/vflags.v
@@ -0,0 +1,66 @@
+// Copyright (c) 2019-2024 Delyan Angelov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module vflags
+
+import os
+import strings
+
+// join_env_vflags_and_os_args returns all the arguments (the ones from the env variable VFLAGS too), passed on the command line.
+pub fn join_env_vflags_and_os_args() []string {
+	vosargs := os.getenv('VOSARGS')
+	if vosargs != '' {
+		return tokenize_to_args(vosargs)
+	}
+	vflags := os.getenv('VFLAGS')
+	if vflags != '' {
+		mut args := []string{}
+		args << os.args[0]
+		args << tokenize_to_args(vflags)
+		args << os.args#[1..]
+		return args
+	}
+	return os.args
+}
+
+// tokenize_to_args converts the input `s`, into an array of arguments.
+// The arguments are separated by one or more spaces in the input `s`.
+// The separating spaces are ignored.
+// It supports quoted arguments, where "several little words" for example,
+// will become a *single argument* in the output.
+// The quotes can be single or double ones.
+pub fn tokenize_to_args(s string) []string {
+	mut tokens := []string{}
+	mut ctoken := strings.new_builder(20)
+	mut in_quotes := false
+	mut quote_char := ` `
+	for i in 0 .. s.len {
+		c := s[i]
+		if !in_quotes && c in [`"`, `'`] {
+			in_quotes = true
+			quote_char = c
+		} else if in_quotes && c == quote_char {
+			if i > 0 && s[i - 1] == `\\` {
+				// support escaping a quote with a \
+				ctoken.go_back(1)
+				ctoken.write_rune(c)
+			} else {
+				in_quotes = false
+				tokens << ctoken.str()
+			}
+		} else if c.is_space() && !in_quotes {
+			// space outside quotes means end of a token
+			if ctoken.len > 0 {
+				tokens << ctoken.str()
+			}
+		} else {
+			// part of a token
+			ctoken.write_rune(c)
+		}
+	}
+	// add the potential remaining token too
+	if ctoken.len > 0 {
+		tokens << ctoken.str()
+	}
+	return tokens
+}
diff --git a/vlib/v/util/vflags/vflags_test.v b/vlib/v/util/vflags/vflags_test.v
new file mode 100644
index 000000000..0c0567f48
--- /dev/null
+++ b/vlib/v/util/vflags/vflags_test.v
@@ -0,0 +1,47 @@
+import os
+import v.util.vflags
+
+fn test_tokenize_to_args() {
+	assert vflags.tokenize_to_args('-cc gcc -ldflags ""') == ['-cc', 'gcc', '-ldflags', '']
+	assert vflags.tokenize_to_args('abc def xyz') == ['abc', 'def', 'xyz']
+	assert vflags.tokenize_to_args('abc -def --xyz') == ['abc', '-def', '--xyz']
+	assert vflags.tokenize_to_args('abc  -def   --xyz') == ['abc', '-def', '--xyz']
+	assert vflags.tokenize_to_args('abc  "a b"   --xyz') == ['abc', 'a b', '--xyz'], 'Using double quote should work'
+	assert vflags.tokenize_to_args("abc  'a b'   --xyz") == ['abc', 'a b', '--xyz'], 'Using single quote instead of a double " should work too'
+	assert vflags.tokenize_to_args('abc  "a quote: \\" . The end."   --xyz') == [
+		'abc',
+		'a quote: " . The end.',
+		'--xyz',
+	], 'escaping " should work'
+}
+
+fn test_join_env_vflags_and_os_args() {
+	os.unsetenv('VFLAGS')
+	os.unsetenv('VOSARGS')
+
+	start := vflags.join_env_vflags_and_os_args()
+	// dump(start)
+	assert start.len > 0
+
+	os.setenv('VFLAGS', r'-cc gcc -ldflags ""', true)
+	x := vflags.join_env_vflags_and_os_args()
+	// dump(x)
+	assert x.len > 0
+	assert 'gcc' in x
+	assert '-cc' in x
+	assert '-ldflags' in x
+	assert x.last() == ''
+
+	os.setenv('VFLAGS', r'-cc   gcc -ldflags   "-lpthread -lm"', true)
+	y := vflags.join_env_vflags_and_os_args()
+	// dump(y)
+	assert '-ldflags' in y
+	assert y.last() == '-lpthread -lm'
+
+	os.setenv('VFLAGS', r'-cc gcc -ldflags "-lpthread -lm"    -showcc', true)
+	z := vflags.join_env_vflags_and_os_args()
+	// dump(z)
+	assert '-ldflags' in z
+	assert z#[-2..-1][0] == '-lpthread -lm'
+	assert z.last() == '-showcc'
+}
-- 
2.39.5