1:- module('tokenizer', [tokenize/2]). 2:- set_prolog_flag(double_quotes,chars). 3:- initialization(main). 4:- include(common_grammar). 5 6tokenize(Text,Output) :- phrase(tokenize(Output),Text). 7tokenize([A]) --> string_literal(A). 8tokenize([A]) --> an_int(A). 9tokenize([A]) --> symbol(A). 10tokenize([";"]) --> ";". 11tokenize(["+="]) --> "+=". 12tokenize(["*="]) --> "*=". 13tokenize(["+"]) --> "+". 14tokenize(["-"]) --> "-". 15tokenize([">"]) --> ">". 16tokenize(["<"]) --> "<". 17tokenize([")"]) --> ")". 18tokenize(["("]) --> "(". 19tokenize(["!"]) --> "!". 20tokenize(["@"]) --> "@". 21tokenize(["="]) --> "=". 22tokenize(["*"]) --> "*". 23tokenize(["/"]) --> "/". 24tokenize(["|"]) --> "|". 25tokenize(["&"]) --> "&". 26tokenize(["."]) --> ".". 27tokenize(["."]) --> "?". 28tokenize(["^"]) --> "^". 29tokenize(["?"]) --> "?". 30tokenize(["$"]) --> "$". 31tokenize(["#"]) --> "#". 32tokenize(["%"]) --> "%". 33tokenize([A|B]) --> tokenize([A]),skip_space,tokenize(B). 34skip_space --> 35 [C], {code_type(C, space)}, skip_space. 36skip_space --> []