Skip to content

Commit 5fe449f

Browse files
flavorjonesflavorjones
committed
regenerate lexical scanner using rexical 1.0.7
related to #1915
1 parent 6777008 commit 5fe449f

File tree

1 file changed

+104
-103
lines changed

1 file changed

+104
-103
lines changed

‎lib/nokogiri/css/tokenizer.rb

+104-103
Original file line numberDiff line numberDiff line change
@@ -1,151 +1,152 @@
11
#--
22
# DO NOT MODIFY!!!!
3-
# This file is automatically generated by rex 1.0.5
3+
# This file is automatically generated by rex 1.0.7
44
# from lexical definition file "lib/nokogiri/css/tokenizer.rex".
55
#++
66

77
module Nokogiri
88
module CSS
99
class Tokenizer # :nodoc:
10-
require 'strscan'
10+
require 'strscan'
1111

12-
class ScanError < StandardError ; end
12+
class ScanError < StandardError ; end
1313

14-
attr_reader :lineno
15-
attr_reader :filename
16-
attr_accessor :state
14+
attr_reader :lineno
15+
attr_reader :filename
16+
attr_accessor :state
1717

18-
def scan_setup(str)
19-
@ss = StringScanner.new(str)
20-
@lineno = 1
21-
@state = nil
22-
end
18+
def scan_setup(str)
19+
@ss = StringScanner.new(str)
20+
@lineno = 1
21+
@state = nil
22+
end
2323

24-
def action
25-
yield
26-
end
24+
def action
25+
yield
26+
end
2727

28-
def scan_str(str)
29-
scan_setup(str)
30-
do_parse
31-
end
32-
alias :scan :scan_str
28+
def scan_str(str)
29+
scan_setup(str)
30+
do_parse
31+
end
32+
alias :scan :scan_str
3333

34-
def load_file( filename )
35-
@filename = filename
36-
open(filename, "r") do |f|
37-
scan_setup(f.read)
38-
end
39-
end
34+
def load_file( filename )
35+
@filename = filename
36+
File.open(filename, "r") do |f|
37+
scan_setup(f.read)
38+
end
39+
end
4040

41-
def scan_file( filename )
42-
load_file(filename)
43-
do_parse
44-
end
41+
def scan_file( filename )
42+
load_file(filename)
43+
do_parse
44+
end
4545

4646

47-
def next_token
48-
return if @ss.eos?
49-
50-
# skips empty actions
51-
until token = _next_token or @ss.eos?; end
52-
token
53-
end
47+
def next_token
48+
return if @ss.eos?
5449

55-
def _next_token
56-
text = @ss.peek(1)
57-
@lineno += 1 if text == "\n"
58-
token = case @state
59-
when nil
60-
case
61-
when (text = @ss.scan(/has\([\s]*/))
62-
action { [:HAS, text] }
50+
# skips empty actions
51+
until token = _next_token or @ss.eos?; end
52+
token
53+
end
6354

64-
when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
65-
action { [:FUNCTION, text] }
55+
def _next_token
56+
text = @ss.peek(1)
57+
@lineno += 1 if text == "\n"
58+
token = case @state
59+
when nil
60+
case
61+
when (text = @ss.scan(/has\([\s]*/))
62+
action { [:HAS, text] }
6663

67-
when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
68-
action { [:IDENT, text] }
64+
when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
65+
action { [:FUNCTION, text] }
6966

70-
when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
71-
action { [:HASH, text] }
67+
when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
68+
action { [:IDENT, text] }
7269

73-
when (text = @ss.scan(/[\s]*~=[\s]*/))
74-
action { [:INCLUDES, text] }
70+
when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
71+
action { [:HASH, text] }
7572

76-
when (text = @ss.scan(/[\s]*\|=[\s]*/))
77-
action { [:DASHMATCH, text] }
73+
when (text = @ss.scan(/[\s]*~=[\s]*/))
74+
action { [:INCLUDES, text] }
7875

79-
when (text = @ss.scan(/[\s]*\^=[\s]*/))
80-
action { [:PREFIXMATCH, text] }
76+
when (text = @ss.scan(/[\s]*\|=[\s]*/))
77+
action { [:DASHMATCH, text] }
8178

82-
when (text = @ss.scan(/[\s]*\$=[\s]*/))
83-
action { [:SUFFIXMATCH, text] }
79+
when (text = @ss.scan(/[\s]*\^=[\s]*/))
80+
action { [:PREFIXMATCH, text] }
8481

85-
when (text = @ss.scan(/[\s]*\*=[\s]*/))
86-
action { [:SUBSTRINGMATCH, text] }
82+
when (text = @ss.scan(/[\s]*\$=[\s]*/))
83+
action { [:SUFFIXMATCH, text] }
8784

88-
when (text = @ss.scan(/[\s]*!=[\s]*/))
89-
action { [:NOT_EQUAL, text] }
85+
when (text = @ss.scan(/[\s]*\*=[\s]*/))
86+
action { [:SUBSTRINGMATCH, text] }
9087

91-
when (text = @ss.scan(/[\s]*=[\s]*/))
92-
action { [:EQUAL, text] }
88+
when (text = @ss.scan(/[\s]*!=[\s]*/))
89+
action { [:NOT_EQUAL, text] }
9390

94-
when (text = @ss.scan(/[\s]*\)/))
95-
action { [:RPAREN, text] }
91+
when (text = @ss.scan(/[\s]*=[\s]*/))
92+
action { [:EQUAL, text] }
9693

97-
when (text = @ss.scan(/\[[\s]*/))
98-
action { [:LSQUARE, text] }
94+
when (text = @ss.scan(/[\s]*\)/))
95+
action { [:RPAREN, text] }
9996

100-
when (text = @ss.scan(/[\s]*\]/))
101-
action { [:RSQUARE, text] }
97+
when (text = @ss.scan(/\[[\s]*/))
98+
action { [:LSQUARE, text] }
10299

103-
when (text = @ss.scan(/[\s]*\+[\s]*/))
104-
action { [:PLUS, text] }
100+
when (text = @ss.scan(/[\s]*\]/))
101+
action { [:RSQUARE, text] }
105102

106-
when (text = @ss.scan(/[\s]*>[\s]*/))
107-
action { [:GREATER, text] }
103+
when (text = @ss.scan(/[\s]*\+[\s]*/))
104+
action { [:PLUS, text] }
108105

109-
when (text = @ss.scan(/[\s]*,[\s]*/))
110-
action { [:COMMA, text] }
106+
when (text = @ss.scan(/[\s]*>[\s]*/))
107+
action { [:GREATER, text] }
111108

112-
when (text = @ss.scan(/[\s]*~[\s]*/))
113-
action { [:TILDE, text] }
109+
when (text = @ss.scan(/[\s]*,[\s]*/))
110+
action { [:COMMA, text] }
114111

115-
when (text = @ss.scan(/\:not\([\s]*/))
116-
action { [:NOT, text] }
112+
when (text = @ss.scan(/[\s]*~[\s]*/))
113+
action { [:TILDE, text] }
117114

118-
when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
119-
action { [:NUMBER, text] }
115+
when (text = @ss.scan(/\:not\([\s]*/))
116+
action { [:NOT, text] }
120117

121-
when (text = @ss.scan(/[\s]*\/\/[\s]*/))
122-
action { [:DOUBLESLASH, text] }
118+
when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
119+
action { [:NUMBER, text] }
123120

124-
when (text = @ss.scan(/[\s]*\/[\s]*/))
125-
action { [:SLASH, text] }
121+
when (text = @ss.scan(/[\s]*\/\/[\s]*/))
122+
action { [:DOUBLESLASH, text] }
126123

127-
when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
128-
action {[:UNICODE_RANGE, text] }
124+
when (text = @ss.scan(/[\s]*\/[\s]*/))
125+
action { [:SLASH, text] }
129126

130-
when (text = @ss.scan(/[\s]+/))
131-
action { [:S, text] }
127+
when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
128+
action {[:UNICODE_RANGE, text] }
132129

133-
when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*(?<!\\)(?:\\{2})*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*(?<!\\)(?:\\{2})*'/))
134-
action { [:STRING, text] }
130+
when (text = @ss.scan(/[\s]+/))
131+
action { [:S, text] }
135132

136-
when (text = @ss.scan(/./))
137-
action { [text, text] }
133+
when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*(?<!\\)(?:\\{2})*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*(?<!\\)(?:\\{2})*'/))
134+
action { [:STRING, text] }
138135

139-
else
140-
text = @ss.string[@ss.pos .. -1]
141-
raise ScanError, "can not match: '" + text + "'"
142-
end # if
136+
when (text = @ss.scan(/./))
137+
action { [text, text] }
143138

144-
else
145-
raise ScanError, "undefined state: '" + state.to_s + "'"
146-
end # case state
147-
token
148-
end # def _next_token
139+
140+
else
141+
text = @ss.string[@ss.pos .. -1]
142+
raise ScanError, "can not match: '" + text + "'"
143+
end # if
144+
145+
else
146+
raise ScanError, "undefined state: '" + state.to_s + "'"
147+
end # case state
148+
token
149+
end # def _next_token
149150

150151
end # class
151152
end

0 commit comments

Comments
 (0)