feat: impl format tool.

kclvm/tools/Cargo.toml

@@ -15,6 +15,7 @@ kclvm-ast = {path = "../ast", version = "0.1.0"}
 kclvm-error = {path = "../error", version = "0.1.0"}
 kclvm-parser = {path = "../parser", version = "0.1.0"}
 kclvm-sema = {path = "../sema", version = "0.1.0"}
+kclvm-config = {path = "../config", version = "0.1.0"}
 
 [dev-dependencies]
 pretty_assertions = "1.2.1"

kclvm/tools/benches/benchmark.rs

@@ -1,7 +1,8 @@
 use criterion::{criterion_group, criterion_main, Criterion};
+use kclvm_tools::format::{format, FormatOptions};
 use kclvm_tools::query::override_file;
 
-pub fn criterion_benchmark(c: &mut Criterion) {
+pub fn criterion_benchmark_override(c: &mut Criterion) {
     c.bench_function("override", |b| {
         b.iter(|| {
             override_file(
@@ -14,5 +15,17 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, criterion_benchmark);
+pub fn criterion_benchmark_format(c: &mut Criterion) {
+    c.bench_function("format", |b| {
+        b.iter(|| {
+            format("./benches/test_data/format.k", &FormatOptions::default()).unwrap();
+        })
+    });
+}
+
+criterion_group!(
+    benches,
+    criterion_benchmark_override,
+    criterion_benchmark_format
+);
 criterion_main!(benches);

kclvm/tools/benches/test_data/format.k

@@ -0,0 +1,10 @@
+schema Data:
+    id?: int = 0
+    value?: str = "value"
+
+schema Config:
+    image: str
+    data?: Data
+
+config = Config {image = "image/image:v1"}
+

kclvm/tools/src/format/mod.rs

@@ -0,0 +1,90 @@
+//! [kclvm_tools::format] module mainly contains some functions of language formatting,
+//! the main API function is `format`, which accepts a path to be formatted and
+//! formatted options.
+//!
+//! The basic principle is to call the [kclvm_parser::parse_file] function to parse the
+//! AST Module, and then use the AST printer [kclvm_tools::printer::print_ast_module]
+//! to print it as source code string.
+use anyhow::{anyhow, Result};
+use std::path::Path;
+
+use crate::{printer::print_ast_module, util::get_kcl_files};
+use kclvm_parser::parse_file;
+
+#[cfg(test)]
+mod tests;
+
+/// FormatOptions contains two options:
+/// - is_stdout: whether to output the formatted result to stdout.
+/// - recursively: whether to recursively traverse a folder and format all KCL files in it.
+#[derive(Debug, Default)]
+pub struct FormatOptions {
+    pub is_stdout: bool,
+    pub recursively: bool,
+}
+
+/// Formats kcl file or directory path contains kcl files and
+/// returns the changed file paths.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kclvm_tools::format::{format, FormatOptions};
+///
+/// // Format a single file.
+/// format("path_to_a_single_file.k", &FormatOptions::default()).unwrap();
+/// // Format a folder contains kcl files
+/// format("path_to_a_folder", &FormatOptions::default()).unwrap();
+/// ```
+pub fn format<P: AsRef<Path>>(path: P, opts: &FormatOptions) -> Result<Vec<String>> {
+    let mut changed_paths: Vec<String> = vec![];
+    let path_ref = path.as_ref();
+    if path_ref.is_dir() {
+        for file in &get_kcl_files(path, opts.recursively)? {
+            if format_file(file, opts)? {
+                changed_paths.push(file.clone())
+            }
+        }
+    } else if path_ref.is_file() {
+        let file = path_ref.to_str().unwrap().to_string();
+        if format_file(&file, opts)? {
+            changed_paths.push(file)
+        }
+    }
+    if !opts.is_stdout {
+        let n = changed_paths.len();
+        println!(
+            "KCL format done and {} {} formatted:",
+            n,
+            if n <= 1 { "file was" } else { "files were" }
+        );
+        for p in &changed_paths {
+            println!("{}", p);
+        }
+    }
+    Ok(changed_paths)
+}
+
+/// Formats a file and returns whether the file has been formatted and modified.
+fn format_file(file: &str, opts: &FormatOptions) -> Result<bool> {
+    let src = std::fs::read_to_string(file)?;
+    let (source, is_formatted) = format_source(&src)?;
+    if opts.is_stdout {
+        println!("{}", source);
+    } else {
+        std::fs::write(file, &source)?
+    }
+    Ok(is_formatted)
+}
+
+/// Formats a code source and returns the formatted source and
+/// whether the source is changed.
+fn format_source(src: &str) -> Result<(String, bool)> {
+    let module = match parse_file("", Some(src.to_string())) {
+        Ok(module) => module,
+        Err(err) => return Err(anyhow!("{}", err)),
+    };
+    let formatted_src = print_ast_module(&module);
+    let is_formatted = src != formatted_src;
+    Ok((formatted_src, is_formatted))
+}

kclvm/tools/src/format/test_data/fail_format_data/syntax_err.k

@@ -0,0 +1 @@
+a = 

kclvm/tools/src/format/test_data/format_data/assert.golden

@@ -0,0 +1,4 @@
+assert True if True, "message"
+# Comment
+assert False if data, "message"
+assert 1

kclvm/tools/src/format/test_data/format_data/assert.input

@@ -0,0 +1,3 @@
+assert True     if True, "message"
+assert  False   if   data   ,  "message"  # Comment
+assert   1

kclvm/tools/src/format/test_data/format_data/blankline.golden

@@ -0,0 +1,4 @@
+a = 1
+b = 2
+c = 3
+d = 4

kclvm/tools/src/format/test_data/format_data/blankline.input

@@ -0,0 +1,12 @@
+
+a=1
+
+
+b= 2
+
+
+c =3
+
+
+
+d  =  4

kclvm/tools/src/format/test_data/format_data/breakline.golden

@@ -0,0 +1,9 @@
+import math
+schema Base:
+    name: str
+
+schema Person(Base):
+    age: int
+
+person = Person {}
+

kclvm/tools/src/format/test_data/format_data/breakline.input

@@ -0,0 +1,6 @@
+import math
+schema Base:
+    name: str
+schema Person(Base):
+    age: int
+person = Person{}

kclvm/tools/src/format/test_data/format_data/check.golden

@@ -0,0 +1,13 @@
+schema Person:
+    firstName: str = "John"
+    lastName: str
+    times: int
+
+    check:
+        len(lastName) > 0 if times > 5
+
+JohnDoe = Person {
+    "lastName": "Doe"
+    "times": 10
+}
+

kclvm/tools/src/format/test_data/format_data/check.input

@@ -0,0 +1,12 @@
+
+schema Person:
+    firstName: str = "John"
+    lastName: str
+    times: int
+    check:
+        len(lastName)    > 0 if          times >     5
+
+JohnDoe = Person {
+    "lastName": "Doe"
+    "times":10
+}

kclvm/tools/src/format/test_data/format_data/codelayout.golden

@@ -0,0 +1,80 @@
+import math as alias_math
+schema Person(Base):
+    # inline comment
+    name: str
+    age: int
+
+    check:
+        age > 0 if age, "age must > 0"
+
+person = Person {
+    name: "Alice"
+    age: 18
+}
+
+if True:
+    a = 1
+elif True:
+    b = 2
+else:
+    c = 3
+d = 1 + 2
+e = (1 + 2)
+f = [1, 2, 3]
+g = {"key": "value"}
+# block comment
+print(1)
+dct = {"key": "value"}
+lst = [1, 2, 3]
+h = dct['key']
+i = lst[1]
+x = 1
+y = 2
+long_variable = 3
+i = i + 1
+submitted += 1
+x = x * 2 - 1
+hypot2 = x * x + y * y
+_c = (a + b) * (a - b)
+_b = 2
+_c = 3
+_d = 4
+_value = (1 + 2 * 3)
+_value = (1 + 2 * 3)
+_value = 1 + -2 * ~3
+_list = [1, 2, 3]
+_list = [*_list, [4, 5, 6]]
+_list = [*_list, [4, 5, 6]]
+_dict = {**{"k": "v"}, **{"k": "v"}}
+a = [1, 2, 3]
+b = [
+    1
+    2
+    3
+    4
+    5
+    6
+]
+_dict = {
+    "k1": "v1"
+    "k2": "v2"
+    "k3": "v3"
+    "k4": "v4"
+    "k5": "v5"
+}
+foo = 1
+if foo is not None:
+    _a = 1
+    _dict |= {}
+    hello = "world{}".format(1)[2:4:].lower()
+    range_int = [i for i in range(10) if i > 1]
+
+op = 1 + 2 - -3 + (3 - 1) // 3
+op += 1
+op -= 12 + 23
+print("  ", end='')
+log = math.log(12)
+aa = 1
+assert aa == 1 if aa, "message"
+aaaa = (1 + 2 / 2) if _a == 2 + +134.3 else ("a" * 3)
+bbbb = "{}".format(a)

kclvm/tools/src/format/test_data/format_data/codelayout.input

@@ -0,0 +1,76 @@
+
+
+
+import           math   as  alias_math
+schema Person ( Base):
+  name:str# inline comment
+  age:int
+  check  :
+   age   >0 if   age,     "age must > 0"
+person = Person{
+        name:"Alice"
+        age:18
+}
+if True:
+    a = 1
+elif True:
+  b = 2
+else:
+      c = 3
+d    =    1    +   2
+e = ( 1 + 2 )
+f=[ 1, 2, 3 ]
+g = { "key" : "value" }
+# block comment
+print (1)
+dct={"key": "value"}
+lst=[1,2,3]
+h = dct [   'key'   ]
+i = lst [  1 ]
+x             = 1
+y             = 2
+long_variable = 3
+i =   i+1
+submitted+=1
+x = x*2 - 1
+hypot2 = x*x + y*y
+_c = (a+b) * (a-b)
+_b=2
+_c= 3
+_d =4     
+
+_value = (1 + 2 * 3)
+_value = (1+2*3)
+_value =1+ - 2 * ~ 3
+_list = [1, 2, 3]
+_list = [*_list, [4, 5 ,6]]
+_list = [* _list, [4, 5 ,6]]
+
+_dict = {** {"k": "v"}, ** {"k": "v"}}
+a = [1,2,3]
+b = [
+    1,2,3,
+    4,5,6,
+]
+_dict={
+  "k1":"v1"
+  "k2" :"v2"
+  "k3": "v3"
+  "k4" : "v4"
+  "k5"  :  "v5"
+}
+foo=1
+if foo   is   not   None:
+    _a = 1
+    _dict|={}
+    hello = "world{}" . format( 1 )[2 :    4] . lower(   )
+    range_int       = [     i for   i   in   range( 10 )   if i > 1 ]
+op = 1+2 -   -  3   +  (3 - 1) //   3
+op += 1
+op -= 12 +   23
+print(  "  "  ,   end=   '')
+log = math.  log(12)
+aa = 1
+assert aa     == 1  if  aa,"message"
+aaaa = (1 + 2 / 2) if    _a == 2   +  +   134.3    else ("a"*3)  
+bbbb = "{}".    format(a)

kclvm/tools/src/format/test_data/format_data/collection_if.golden

@@ -0,0 +1,32 @@
+dataDict0 = {
+    if True:
+        age = 101
+    elif True:
+        age = 123
+    else:
+        age = 111
+}
+dataDict1 = {
+    if True:
+        age: 101
+    elif True:
+        age: 123
+    else:
+        age: 111
+}
+dataList0 = [
+    if True:
+        1
+    elif False:
+        2
+    else:
+        3
+]
+dataList1 = [
+    if True:
+        *[1]
+    elif False:
+        2
+    else:
+        3
+]

kclvm/tools/src/format/test_data/format_data/collection_if.input

@@ -0,0 +1,32 @@
+dataDict0    =    {
+    if    True:
+        age = 101
+    elif    True:
+          age    =    123
+    else:
+        age = 111
+}
+dataDict1    =    {
+    if    True:
+        age : 101
+    elif    True:
+          age    :    123
+    else:
+        age : 111
+}
+dataList0 = [
+    if   True:
+        1
+    elif   False:
+          2
+    else   :
+            3
+]
+dataList1 = [
+    if   True:
+        * [1]
+    elif   False:
+          2
+    else   :
+            3
+]

kclvm/tools/src/format/test_data/format_data/comment.golden

@@ -0,0 +1,12 @@
+# Block comment
+# Inline comment
+a = 1
+schema Person:
+    """
+    Schema doc string
+    """
+    # Inline comment in schema
+    name: str = "Alice"
+    # Block comment in schema
+    age: int = 18
+

kclvm/tools/src/format/test_data/format_data/comment.input

@@ -0,0 +1,9 @@
+# Block comment
+a = 1# Inline comment
+schema Person:
+    """
+    Schema doc string
+    """
+    name:str="Alice"# Inline comment in schema
+    # Block comment in schema
+    age:int=18

kclvm/tools/src/format/test_data/format_data/comp_for.golden

@@ -0,0 +1,2 @@
+data0 = [i + 1 for i in range(10) if i > 1]
+data1 = [i + 1 for i in range(10) if i > 1]

kclvm/tools/src/format/test_data/format_data/comp_for.input

@@ -0,0 +1,7 @@
+data0 = [
+    i + 1 for i in range(10)    if i > 1
+]
+data1 = [
+    i + 1 for i      in range(10)
+    if   i > 1
+]

kclvm/tools/src/format/test_data/format_data/import.golden

@@ -0,0 +1,6 @@
+import a
+import b
+import c
+import d
+import e as e
+a = 1

kclvm/tools/src/format/test_data/format_data/import.input

@@ -0,0 +1,10 @@
+import a
+
+import b
+
+import c
+import d
+
+import e as e
+
+a = 1

kclvm/tools/src/format/test_data/format_data/indent.golden

@@ -0,0 +1,14 @@
+if True:
+    a = 1
+elif True:
+    b = 2
+    if True:
+        c = 3
+    else:
+        d = 4
+else:
+    e = 8
+schema Person:
+    name: str
+    age: int
+

kclvm/tools/src/format/test_data/format_data/indent.input

@@ -0,0 +1,14 @@
+if True:
+ a = 1
+elif True:
+ b = 2
+ if True:
+        c = 3
+ else:
+   d = 4
+else:
+  e =  8
+
+schema Person:
+ name: str
+ age: int

kclvm/tools/src/format/test_data/format_data/inline_comment.golden

@@ -0,0 +1,17 @@
+# Inline comment 1
+# Inline comment 2
+# Inline comment 3
+a = 1
+# Inline comment 4
+# Inline comment 5
+# 
+# 
+# Inline comment 6
+# Inline comment 7
+# 
+# Inline comment 8
+b = 2
+# Same inline comment
+# Same inline comment
+# Same inline comment
+c = b + 1

kclvm/tools/src/format/test_data/format_data/inline_comment.input

@@ -0,0 +1,17 @@
+# Inline comment 1
+# Inline comment 2
+# Inline comment 3
+a = 1
+# Inline comment 4
+# Inline comment 5
+# 
+# 
+# Inline comment 6
+# Inline comment 7
+# 
+# Inline comment 8
+b=2
+# Same inline comment
+# Same inline comment
+# Same inline comment
+c=b+1

kclvm/tools/src/format/test_data/format_data/lambda.golden

@@ -0,0 +1,8 @@
+f0 = lambda {
+    1 + 1
+
+}
+f1 = lambda x: int, y: int -> int {
+    x + y
+
+}

kclvm/tools/src/format/test_data/format_data/lambda.input

@@ -0,0 +1,6 @@
+f0 =   lambda    {
+    1 +    1
+}
+f1 = lambda   x  : int , y  : int ->int{
+    x + y
+}

kclvm/tools/src/format/test_data/format_data/quant.golden

@@ -0,0 +1,12 @@
+a = all x in [-1, 0, 1, 2, 3] {
+    x >= 1 if x > 0
+}
+b = any x in {k1 = "v1", k2 = "v2"} {
+    x in ["k1", "v2"]
+}
+c = map x in {k1 = "v1", k2 = "v2"} {
+    x
+}
+d = filter x in [1, 2, 3] {
+    x > 1
+}

kclvm/tools/src/format/test_data/format_data/quant.input

@@ -0,0 +1,8 @@
+a = all x in [-1, 0, 1, 2, 3] {
+    x >= 1 if x > 0
+}
+b = any x in {k1 = "v1", k2 = "v2"} {x in ["k1", "v2"]}
+c = map x in {k1 = "v1", k2 = "v2"} {x}
+d = filter x in [1, 2, 3] {
+    x > 1
+}

kclvm/tools/src/format/test_data/format_data/schema.golden

@@ -0,0 +1,25 @@
+import math
+mixin XXMixin:
+    nameVar: str
+
+schema Base:
+    """
+    Base schema doc string        
+    """
+    mixin [
+        XXMixin,
+        XXMixin
+    ]
+    name: str = "Alice"
+    labels: {str:str} = None
+
+schema Person[para1: str = "value", para2 = "value"](Base):
+    age: int = 18
+    name = para
+
+    check:
+        True
+        bool(math.log(10))
+
+person = Person(para1="12") {}
+

kclvm/tools/src/format/test_data/format_data/schema.input

@@ -0,0 +1,17 @@
+import     math
+schema XXMixin:
+    nameVar: str
+schema Base:
+    """
+    Base schema doc string        
+    """
+    mixin[XXMixin,XXMixin]
+    name:str="Alice"
+    labels:        {str : str}=None
+schema Person[para1:str="value",para2="value"] ( Base )  : 
+    age:int=18
+    name=para
+    check    :
+        True
+        bool (math.   log(10))
+person = Person(para1 = "12"){}

kclvm/tools/src/format/test_data/format_data/string.golden

@@ -0,0 +1,4 @@
+strA = '123'
+strB = '''
+long string
+'''

kclvm/tools/src/format/test_data/format_data/string.input

@@ -0,0 +1,4 @@
+strA='123'
+strB= '''
+long string
+'''

kclvm/tools/src/format/test_data/format_data/type_alias.golden

@@ -0,0 +1,7 @@
+schema Person:
+    name: str = "kcl"
+    age: int = 1
+
+type PersonOther = Person
+type Int = int
+type UnionType = int|float|str

kclvm/tools/src/format/test_data/format_data/type_alias.input

@@ -0,0 +1,7 @@
+schema    Person:
+    name:   str = "kcl"
+    age:   int = 1
+
+type    PersonOther =    Person
+type    Int = int
+type UnionType    =  int  |    float | str

kclvm/tools/src/format/test_data/format_data/unary.golden

@@ -0,0 +1,3 @@
+x = not True or not False
+y = +1 + -1
+z = ~17

kclvm/tools/src/format/test_data/format_data/unary.input

@@ -0,0 +1,3 @@
+x = not   True   or   not   False
+y = +1   +   -1
+z = ~  0x11

kclvm/tools/src/format/test_data/format_path_data/folder/first.k

@@ -0,0 +1 @@
+a = 1

kclvm/tools/src/format/test_data/format_path_data/folder/second.k

@@ -0,0 +1 @@
+b = 2

kclvm/tools/src/format/test_data/format_path_data/if.k

@@ -0,0 +1,14 @@
+if True:
+ a = 1
+elif True:
+ b = 2
+ if True:
+        c = 3
+ else:
+   d = 4
+else:
+  e =  8
+
+schema Person:
+ name: str
+ age: int

kclvm/tools/src/format/test_data/format_path_data/internal_pkg/test.k

@@ -0,0 +1,14 @@
+if True:
+ a = 1
+elif True:
+ b = 2
+ if True:
+        c = 3
+ else:
+   d = 4
+else:
+  e =  8
+
+schema Person:
+ name: str
+ age: int

kclvm/tools/src/format/test_data/format_path_data/single_file.k

@@ -0,0 +1,2 @@
+a = 1
+b = "2"

kclvm/tools/src/format/tests.rs

@@ -0,0 +1,86 @@
+use super::*;
+use pretty_assertions::assert_eq;
+
+const FILE_INPUT_SUFFIX: &str = ".input";
+const FILE_OUTPUT_SUFFIX: &str = ".golden";
+const TEST_CASES: &[&'static str; 18] = &[
+    "assert",
+    "check",
+    "blankline",
+    "breakline",
+    "codelayout",
+    "collection_if",
+    "comment",
+    "comp_for",
+    "empty",
+    "import",
+    "indent",
+    "inline_comment",
+    "lambda",
+    "quant",
+    "schema",
+    "string",
+    "type_alias",
+    "unary",
+];
+
+fn read_data(data_name: &str) -> (String, String) {
+    let src = std::fs::read_to_string(&format!(
+        "./src/format/test_data/format_data/{}{}",
+        data_name, FILE_INPUT_SUFFIX
+    ))
+    .unwrap();
+
+    (
+        format_source(&src).unwrap().0,
+        std::fs::read_to_string(&format!(
+            "./src/format/test_data/format_data/{}{}",
+            data_name, FILE_OUTPUT_SUFFIX
+        ))
+        .unwrap(),
+    )
+}
+
+#[test]
+fn test_format_source() {
+    for case in TEST_CASES {
+        let (data_input, data_output) = read_data(case);
+        assert_eq!(data_input, data_output, "Test failed on {}", case);
+    }
+}
+
+#[test]
+fn test_format_single_file() {
+    assert!(format(
+        "./src/format/test_data/format_path_data/single_file.k",
+        &FormatOptions::default()
+    )
+    .is_ok());
+}
+
+#[test]
+fn test_format_folder() {
+    assert!(format(
+        "./src/format/test_data/format_path_data/folder",
+        &FormatOptions::default()
+    )
+    .is_ok());
+}
+
+#[test]
+fn test_format_with_stdout_option() {
+    let opts = FormatOptions {
+        is_stdout: true,
+        recursively: false,
+    };
+    let changed_files = format("./src/format/test_data/format_path_data/if.k", &opts).unwrap();
+    assert_eq!(changed_files.len(), 1);
+    let changed_files = format("./src/format/test_data/format_path_data/", &opts).unwrap();
+    assert_eq!(changed_files.len(), 1);
+    let opts = FormatOptions {
+        is_stdout: true,
+        recursively: true,
+    };
+    let changed_files = format("./src/format/test_data/format_path_data/", &opts).unwrap();
+    assert_eq!(changed_files.len(), 2);
+}

kclvm/tools/src/lib.rs

@@ -1,5 +1,7 @@
+pub mod format;
 pub mod printer;
 pub mod query;
+mod util;
 
 #[macro_use]
 extern crate kclvm_error;

kclvm/tools/src/util/mod.rs

@@ -0,0 +1,19 @@
+use anyhow::Result;
+use kclvm_config::modfile::KCL_FILE_SUFFIX;
+use std::path::Path;
+use walkdir::WalkDir;
+
+/// Get kcl files from path.
+pub(crate) fn get_kcl_files<P: AsRef<Path>>(path: P, recursively: bool) -> Result<Vec<String>> {
+    let mut files = vec![];
+    for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) {
+        let path = entry.path();
+        if path.is_file() {
+            let file = path.to_str().unwrap();
+            if file.ends_with(KCL_FILE_SUFFIX) && (recursively || entry.depth() == 1) {
+                files.push(file.to_string())
+            }
+        }
+    }
+    Ok(files)
+}