improve rule for distinguishing tuples (arg lists) from parenthesized blocks

a parenthesized expression is a tuple when at least one of these is true:

- it is empty
- there is a comma
- it begins with `x...` and isn't just `(x...)`
- it begins with `(;` (corresponding to the positional part being empty)

add line numbers to blocks written as `(a; b; c)`

Author: JeffBezanson

src/ast.scm

@@ -406,6 +406,7 @@
                            (eq? (cadr (caddr x)) 'Vararg)))))
 (define (trans?  x) (and (pair? x) (eq? (car x) '|.'|)))
 (define (ctrans? x) (and (pair? x) (eq? (car x) '|'|)))
+(define (linenum? x) (and (pair? x) (eq? (car x) 'line)))
 
 (define (make-assignment l r) `(= ,l ,r))
 (define (assignment? e) (and (pair? e) (eq? (car e) '=)))

src/julia-parser.scm

@@ -675,8 +675,7 @@
                              (memv (peek-token s) ops))
                          (loop ex #f (peek-token s))
                          (if (and add-linenums
-                                  (not (and (pair? (car ex))
-                                            (eq? (caar ex) 'line))))
+                                  (not (linenum? (car ex))))
                              (let ((loc (line-number-node s)))
                                (loop (list* (down s) loc ex) #f (peek-token s)))
                              (loop (cons (down s) ex) #f (peek-token s))))))))))
@@ -1350,7 +1349,7 @@
               (blk  (parse-block s (lambda (s) (parse-docstring s parse-eq)))))
           (expect-end s word)
           (let ((blk  (if (and (length> blk 1)
-                               (pair? (cadr blk)) (eq? (caadr blk) 'line))
+                               (linenum? (cadr blk)))
                           (list* 'block loc (cddr blk))
                           blk)))
             (if (eq? word 'quote)
@@ -1374,7 +1373,7 @@
               (error "let variables should end in \";\" or newline"))
           (let* ((ex (begin0 (parse-block s)
                              (expect-end s word)))
-                 (ex (if (and (length= ex 2) (pair? (cadr ex)) (eq? (caadr ex) 'line))
+                 (ex (if (and (length= ex 2) (linenum? (cadr ex)))
                          `(block)  ;; don't need line info in an empty let block
                          ex)))
             `(let ,(if (and (length= binds 1) (or (assignment? (car binds)) (decl? (car binds))
@@ -1527,8 +1526,7 @@
                                 catch-block
                                 `(block ,loc ,var
                                         ,@(if (and (length= catch-block 2)
-                                                   (pair? (cadr catch-block))
-                                                   (eq? (caadr catch-block) 'line))
+                                                   (linenum? (cadr catch-block)))
                                               '()
                                               (cdr catch-block))))
                             (if var? var 'false)
@@ -1725,7 +1723,7 @@
 ;; . an extra comma at the end is allowed
 ;; . expressions after a ; are enclosed in (parameters ...)
 ;; . an expression followed by ... becomes (... x)
-(define (parse-arglist s closer)
+(define (parse-arglist s closer (add-linenums #f))
 (with-bindings ((range-colon-enabled #t)
                 (space-sensitive #f)
                 (where-enabled #t)
@@ -1739,12 +1737,17 @@
                      (to-kws (reverse! lst))
                      (reverse! lst)))
           (if (eqv? t #\;)
-              (begin (take-token s)
-                     (let ((params (loop '()))
+              (begin (take-token s) (require-token s)
+                     (let ((loc (line-number-node s))
+                           (params (loop '()))
                            (lst    (if (eqv? closer #\) )
                                        (to-kws (reverse lst))
                                        (reverse lst))))
-                       (cons (cons 'parameters params)
+                       (cons `(parameters
+                               ,@(if add-linenums
+                                     (list loc)
+                                     '())
+                               ,@params)
                              lst)))
               (let* ((nxt (parse-eq* s))
                      (c (require-token s)))
@@ -1910,20 +1913,36 @@
 ;; this allows us to first parse tuples using parse-arglist
 (define (parameters-to-block e)
   (if (and (pair? e) (eq? (car e) 'parameters))
-      (cond ((length= e 1) '())
-            ((length= e 2) (parameters-to-block (cadr e)))
-            ((length= e 3)
-             (let ((fst (cadr e))
-                   (snd (caddr e)))
-               (if (and (pair? fst) (eq? (car fst) 'parameters))
-                   (let ((rec (parameters-to-block fst))
-                         (snd (parameters-to-block snd)))
-                     (and rec snd
-                          (cons (car snd) rec)))
-                   #f)))
-            (else #f))
+      (let ((e2   (filter (lambda (x) (not (linenum? x))) e))
+            (lnum (if (and (pair? (cdr e))
+                           (linenum? (cadr e)))
+                      (cadr e)
+                      #f)))
+        (cond ((length= e2 1) '())
+              ((length= e2 2)
+               (let ((rec (parameters-to-block (cadr e2))))
+                 (if (null? rec)
+                     rec
+                     (cons lnum rec))))
+              ((length= e2 3)
+               (let ((fst (cadr e2))
+                     (snd (caddr e2)))
+                 (if (and (pair? fst) (eq? (car fst) 'parameters))
+                     (let ((rec (parameters-to-block fst))
+                           (snd (parameters-to-block snd)))
+                       (and rec snd
+                            (append (if lnum (list lnum) '()) (cons (car snd) rec))))
+                     #f)))
+              (else #f)))
       (list (kw-to-= e))))
 
+(define (rm-linenums e)
+  (if (atom? e) e
+      (map rm-linenums
+           (if (eq? (car e) 'parameters)
+               (filter (lambda (x) (not (linenum? x))) e)
+               e))))
+
 ;; convert an arglist to a tuple or block expr
 ;; leading-semi? means we saw (; ...)
 ;; comma? means there was a comma after the first expression
@@ -1938,9 +1957,10 @@
                                    `(block))))))  ;; all semicolons inside ()
           (and (null? first) (null? args) (not comma?)
                `(block))  ;; this case is (;)
-          (if (and (pair? args) (pair? (car args)) (eq? (caar args) 'parameters))
-              `(tuple ,(car args) ,@first ,@(map kw-to-= (cdr args)))
-              `(tuple ,@first ,@(map kw-to-= args))))))
+          (rm-linenums
+           (if (and (pair? args) (pair? (car args)) (eq? (caar args) 'parameters))
+               `(tuple ,(car args) ,@first ,@(map kw-to-= (cdr args)))
+               `(tuple ,@first ,@(map kw-to-= args)))))))
 
 (define (tuple-to-arglist e)
   (cond ((eq? (car e) 'tuple)  (map =-to-kw (cdr e)))
@@ -1986,8 +2006,8 @@
        (take-token s)  ;; take #\)
        '(|::| . #f))
       ((eqv? nxt #\;)
-       (cons (arglist-to-tuple #t #f (parse-arglist s #\) ))
-             #t))
+       (let ((ex (arglist-to-tuple #t #f (parse-arglist s #\) ))))
+         (cons ex (eq? (car ex) 'tuple))))
       (else
        ;; here we parse the first subexpression separately, so
        ;; we can look for a comma to see if it's a tuple.
@@ -1997,7 +2017,7 @@
          (cond ((eqv? t #\) )
                 (take-token s)
                 ;; value in parentheses (x)
-                (if (and (pair? ex) (eq? (car ex) '...))
+                (if (vararg? ex)
                     (let ((lineno (input-port-line (ts:port s))))
                       (if (or accept-dots-without-comma (eq? (with-bindings ((whitespace-newline #f))
                                                                             (peek-token s))
@@ -2008,6 +2028,19 @@
                                                  (string "(" (deparse (cadr ex)) "...,)"))
                                  (cons `(tuple ,ex) #t))))
                     (cons ex #f)))
+               ((eqv? t #\,)
+                ;; tuple (x,) (x,y) etc.
+                (take-token s)
+                (cons (arglist-to-tuple #f #t (parse-arglist s #\) ) ex)
+                      #t))
+               ((eqv? t #\;)
+                (cons (arglist-to-tuple
+                       #f
+                       ;; consider `(x...; ` the start of an arglist, since it's not useful as a block
+                       (vararg? ex)
+                       (parse-arglist s #\) #t)
+                       ex)
+                      (vararg? ex)))
                ((eq? t 'for)
                 (expect-space-before s 'for)
                 (take-token s)
@@ -2017,13 +2050,7 @@
                       (error "expected \")\""))
                   (cons gen #f)))
                (else
-                ;; tuple (x,) (x,y) (x...) etc.
-                (if (eqv? t #\, )
-                    (take-token s)
-                    (if (not (eqv? t #\;))
-                        (error "missing comma or ) in argument list")))
-                (cons (arglist-to-tuple #f (eqv? t #\,) (parse-arglist s #\) ) ex)
-                      #t)))))))))
+                (error "missing comma or ) in argument list")))))))))
 
 (define (not-eof-for delim c)
   (if (eof-object? c)

src/julia-syntax.scm

@@ -766,7 +766,7 @@
   (if (and (pair? body) (eq? (car body) 'block))
       (cond ((atom? (cdr body))
              `(block ,stmt (null)))
-            ((and (pair? (cadr body)) (eq? (caadr body) 'line))
+            ((linenum? (cadr body))
              `(block ,(cadr body) ,stmt ,@(cddr body)))
             (else
              `(block ,stmt ,@(cdr body))))
@@ -805,8 +805,7 @@
                       ,(ctor-body body curlyargs))))))
 
 (define (function-body-lineno body)
-  (let ((lnos (filter (lambda (e) (and (pair? e) (eq? (car e) 'line)))
-                      body)))
+  (let ((lnos (filter linenum? body)))
     (if (null? lnos) '() (car lnos))))
 
 ;; rewrite calls to `new( ... )` to `new` expressions on the appropriate
@@ -863,7 +862,7 @@
    (fields defs) (separate (lambda (x) (or (symbol? x) (decl? x)))
                            fields0)
    (let* ((defs        (filter (lambda (x) (not (effect-free? x))) defs))
-          (locs        (if (and (pair? fields0) (pair? (car fields0)) (eq? (caar fields0) 'line))
+          (locs        (if (and (pair? fields0) (linenum? (car fields0)))
                            (list (car fields0))
                            '()))
           (field-names (map decl-var fields))
@@ -1103,7 +1102,7 @@
                               (set! a (cadr w))))
                     #f))
          (argl (if (pair? a)
-                   (tuple-to-arglist a)
+                   (tuple-to-arglist (filter (lambda (x) (not (linenum? x))) a))
                    (list a)))
          ;; TODO: always use a specific special name like #anon# or _, then ignore
          ;; this as a local variable name.
@@ -1238,7 +1237,7 @@
       (if (null? f)
           '()
           (let ((x (car f)))
-            (cond ((or (symbol? x) (decl? x) (and (pair? x) (eq? (car x) 'line)))
+            (cond ((or (symbol? x) (decl? x) (linenum? x))
                    (loop (cdr f)))
                   ((and (assignment? x) (or (symbol? (cadr x)) (decl? (cadr x))))
                    (error (string "\"" (deparse x) "\" inside type definition is reserved")))
@@ -1893,8 +1892,7 @@
    (lambda (e)
      (cond ((null? (cdr e)) '(null))
            ((and (null? (cddr e))
-                 (not (and (pair? (cadr e))
-                           (eq? (car (cadr e)) 'line))))
+                 (not (linenum? (cadr e))))
             (expand-forms (cadr e)))
            (else
             (cons 'block
@@ -3608,7 +3606,7 @@ f(x) = yt(x)
             ((block body)
              (let* ((last-fname filename)
                     (fnm        (first-non-meta e))
-                    (fname      (if (and (length> e 1) (pair? fnm) (eq? (car fnm) 'line)
+                    (fname      (if (and (length> e 1) (linenum? fnm)
                                          (length> fnm 2))
                                     (caddr fnm)
                                     filename))

test/syntax.jl

@@ -176,13 +176,21 @@ macro test999_str(args...); args; end
 @test_throws ParseError Meta.parse("(,)")
 @test_throws ParseError Meta.parse("(;,)")
 @test_throws ParseError Meta.parse("(,;)")
+# TODO: would be nice to make these errors, but needed to parse e.g. `(x;y,)->x`
+#@test_throws ParseError Meta.parse("(1;2,)")
+#@test_throws ParseError Meta.parse("(1;2,;)")
+#@test_throws ParseError Meta.parse("(1;2,;3)")
 @test Meta.parse("(x;)") == Expr(:block, :x)
 @test Meta.parse("(;x)") == Expr(:tuple, Expr(:parameters, :x))
 @test Meta.parse("(;x,)") == Expr(:tuple, Expr(:parameters, :x))
 @test Meta.parse("(x,)") == Expr(:tuple, :x)
 @test Meta.parse("(x,;)") == Expr(:tuple, Expr(:parameters), :x)
-@test Meta.parse("(x;y)") == Expr(:block, :x, :y)
-@test Meta.parse("(x=1;y=2)") == Expr(:block, Expr(:(=), :x, 1), Expr(:(=), :y, 2))
+@test Meta.parse("(x;y)") == Expr(:block, :x, LineNumberNode(1,:none), :y)
+@test Meta.parse("(x...;)") == Expr(:tuple, Expr(:parameters), Expr(:(...), :x))
+@test Meta.parse("(;x...)") == Expr(:tuple, Expr(:parameters, Expr(:(...), :x)))
+@test Meta.parse("(x...;y)") == Expr(:tuple, Expr(:parameters, :y), Expr(:(...), :x))
+@test Meta.parse("(x;y...)") == Expr(:block, :x, LineNumberNode(1,:none), Expr(:(...), :y))
+@test Meta.parse("(x=1;y=2)") == Expr(:block, Expr(:(=), :x, 1), LineNumberNode(1,:none), Expr(:(=), :y, 2))
 @test Meta.parse("(x,;y)") == Expr(:tuple, Expr(:parameters, :y), :x)
 @test Meta.parse("(x,;y=1)") == Expr(:tuple, Expr(:parameters, Expr(:kw, :y, 1)), :x)
 @test Meta.parse("(x,a;y=1)") == Expr(:tuple, Expr(:parameters, Expr(:kw, :y, 1)), :x, :a)
@@ -1310,13 +1318,24 @@ end
 #                  f(x) = x", 1)[1] === "x"
 
 # issue #26137
-@test Meta.parse("-()^2")     == Expr(:call, :^, Expr(:call, :-), 2)
-@test Meta.parse("-(x)^2")    == Expr(:call, :-, Expr(:call, :^, :x, 2))
-@test Meta.parse("-(x,)^2")   == Expr(:call, :^, Expr(:call, :-, :x), 2)
-@test Meta.parse("-(x,y)^2")  == Expr(:call, :^, Expr(:call, :-, :x, :y), 2)
-@test Meta.parse("+((1,2))")  == Expr(:call, :+, Expr(:tuple, 1, 2))
-@test Meta.parse("-(x;y)^2")  == Expr(:call, :^, Expr(:call, :-, Expr(:parameters, :y), :x), 2)
-@test Meta.parse("-(x...)^2") == Expr(:call, :^, Expr(:call, :-, Expr(:(...), :x)), 2)
+# cases where parens enclose argument lists
+@test Meta.parse("-()^2")      == Expr(:call, :^, Expr(:call, :-), 2)
+@test Meta.parse("-(x,)^2")    == Expr(:call, :^, Expr(:call, :-, :x), 2)
+@test Meta.parse("-(x,;)^2")   == Expr(:call, :^, Expr(:call, :-, Expr(:parameters), :x), 2)
+@test Meta.parse("-(;x)^2")    == Expr(:call, :^, Expr(:call, :-, Expr(:parameters, :x)), 2)
+@test Meta.parse("-(x,y)^2")   == Expr(:call, :^, Expr(:call, :-, :x, :y), 2)
+@test Meta.parse("-(x...)^2")  == Expr(:call, :^, Expr(:call, :-, Expr(:(...), :x)), 2)
+@test Meta.parse("-(x...;)^2") == Expr(:call, :^, Expr(:call, :-, Expr(:parameters), Expr(:(...), :x)), 2)
+@test Meta.parse("-(x...;)")   == Expr(:call, :-, Expr(:parameters), Expr(:(...), :x))
+
+# cases where parens are just grouping
+@test Meta.parse("-(x)^2")     == Expr(:call, :-, Expr(:call, :^, :x, 2))
+@test Meta.parse("-(a=1)^2")   == Expr(:call, :-, Expr(:call, :^, Expr(:(=), :a, 1), 2))
+@test Meta.parse("-(x;y)^2")   == Expr(:call, :-, Expr(:call, :^, Expr(:block, :x, LineNumberNode(1,:none), :y), 2))
+@test Meta.parse("-(;)^2")     == Expr(:call, :-, Expr(:call, :^, Expr(:block), 2))
+@test Meta.parse("-(;;;;)^2")  == Expr(:call, :-, Expr(:call, :^, Expr(:block), 2))
+@test Meta.parse("-(x;;;)^2")  == Expr(:call, :-, Expr(:call, :^, Expr(:block, :x), 2))
+@test Meta.parse("+((1,2))")   == Expr(:call, :+, Expr(:tuple, 1, 2))
 
 @test_throws ParseError("space before \"(\" not allowed in \"+ (\"") Meta.parse("1 -+ (a=1, b=2)")