Module | Wirble::Colorize::Tokenizer |
In: |
lib/wirble.rb
|
Tokenize an inspection string.
# File lib/wirble.rb, line 131 131: def self.tokenize(str) 132: raise 'missing block' unless block_given? 133: chars = str.split(//) 134: 135: # $stderr.puts "DEBUG: chars = #{chars.join(',')}" 136: 137: state, val, i, lc = [], '', 0, nil 138: while i <= chars.size 139: repeat = false 140: c = chars[i] 141: 142: # $stderr.puts "DEBUG: state = #{state}" 143: 144: case state[-1] 145: when nil 146: case c 147: when ':' 148: state << :symbol 149: when '"' 150: state << :string 151: when '#' 152: state << :object 153: when /[a-z]/i 154: state << :keyword 155: repeat = true 156: when /[0-9-]/ 157: state << :number 158: repeat = true 159: when '{' 160: yield :open_hash, '{' 161: when '[' 162: yield :open_array, '[' 163: when ']' 164: yield :close_array, ']' 165: when '}' 166: yield :close_hash, '}' 167: when /\s/ 168: yield :whitespace, c 169: when ',' 170: yield :comma, ',' 171: when '>' 172: yield :refers, '=>' if lc == '=' 173: when '.' 174: yield :range, '..' if lc == '.' 175: when '=' 176: # ignore these, they're used elsewhere 177: nil 178: else 179: # $stderr.puts "DEBUG: ignoring char #{c}" 180: end 181: when :symbol 182: case c 183: # XXX: should have =, but that messes up foo=>bar 184: when /[a-z0-9_!?]/ 185: val << c 186: else 187: yield :symbol_prefix, ':' 188: yield state[-1], val 189: state.pop; val = '' 190: repeat = true 191: end 192: when :string 193: case c 194: when '"' 195: if lc == "\\" 196: val[-1] = ?" 197: else 198: yield :open_string, '"' 199: yield state[-1], val 200: state.pop; val = '' 201: yield :close_string, '"' 202: end 203: else 204: val << c 205: end 206: when :keyword 207: case c 208: when /[a-z0-9_]/i 209: val << c 210: else 211: # is this a class? 212: st = val =~ /^[A-Z]/ ? :class : state[-1] 213: 214: yield st, val 215: state.pop; val = '' 216: repeat = true 217: end 218: when :number 219: case c 220: when /[0-9e-]/ 221: val << c 222: when '.' 223: if lc == '.' 224: val[/\.$/] = '' 225: yield state[-1], val 226: state.pop; val = '' 227: yield :range, '..' 228: else 229: val << c 230: end 231: else 232: yield state[-1], val 233: state.pop; val = '' 234: repeat = true 235: end 236: when :object 237: case c 238: when '<' 239: yield :open_object, '#<' 240: state << :object_class 241: when ':' 242: state << :object_addr 243: when '@' 244: state << :object_line 245: when '>' 246: yield :close_object, '>' 247: state.pop; val = '' 248: end 249: when :object_class 250: case c 251: when ':' 252: yield state[-1], val 253: state.pop; val = '' 254: repeat = true 255: else 256: val << c 257: end 258: when :object_addr 259: case c 260: when '>' 261: when '@' 262: yield :object_addr_prefix, ':' 263: yield state[-1], val 264: state.pop; val = '' 265: repeat = true 266: else 267: val << c 268: end 269: when :object_line 270: case c 271: when '>' 272: yield :object_line_prefix, '@' 273: yield state[-1], val 274: state.pop; val = '' 275: repeat = true 276: else 277: val << c 278: end 279: else 280: raise "unknown state #{state}" 281: end 282: 283: unless repeat 284: i += 1 285: lc = c 286: end 287: end 288: end