From 25314fd2aed81aa5281bc3d93dbcc7919368ebe1 Mon Sep 17 00:00:00 2001
From: Bicker <bickerkards@tuta.io>
Date: Wed, 22 May 2019 17:45:37 +0200
Subject: Initial commit

---
 ql/parser/lexer.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 153 insertions(+)
 create mode 100644 ql/parser/lexer.py

(limited to 'ql/parser/lexer.py')

diff --git a/ql/parser/lexer.py b/ql/parser/lexer.py
new file mode 100644
index 0000000..6e53f12
--- /dev/null
+++ b/ql/parser/lexer.py
@@ -0,0 +1,153 @@
+from re import findall
+
+from ply.lex import lex
+
+
+class QLLexer:
+    def __init__(self):
+        self.__errors = []
+        self.__lexer = None
+
+    @property
+    def lexer(self):
+        return self.__lexer
+
+    @property
+    def errors(self):
+        return self.__errors
+
+    def build(self):
+        self.__lexer = lex(module=self)
+
+    def tokenize(self, data):
+        self.__errors = []
+        self.lexer.input(data)
+        token = self.lexer.token()
+        tokens = [token]
+
+        while token:
+            token = self.lexer.token()
+            tokens.append(token)
+
+        return tokens
+
+    tokens = [
+        'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'COLON',
+        'ASSIGN',
+        'EQUALS', 'NOT_EQUALS',
+        'LESS_EQUALS', 'LESS_THAN', 'GREATER_EQUALS', 'GREATER_THAN',
+        'AND', 'OR',
+        'DOLLAR', 'RUBLE',
+        'LEFT_BRACE', 'RIGHT_BRACE',
+        'LEFT_BRACKET', 'RIGHT_BRACKET',
+        'INTEGER_LITERAL', 'DECIMAL_LITERAL',
+        'TRUE', 'FALSE',
+        'DATE_LITERAL',
+        'STRING_LITERAL',
+        'IDENTIFIER',
+    ]
+
+    reserved_keywords = {
+        'form':     'FORM',
+        'if':       'IF',
+        'not':      'NOT',
+
+        # Types
+        'boolean':  'BOOLEAN',
+        'date':     'DATE',
+        'decimal':  'DECIMAL',
+        'integer':  'INTEGER',
+        'money':    'MONEY',
+        'string':   'STRING',
+    }
+
+    tokens += list(reserved_keywords.values())
+
+    # Regular expression rules for simple tokens
+    t_ignore = ' \t'
+
+    t_PLUS = r'\+'
+    t_MINUS = r'-'
+    t_TIMES = r'\*'
+    t_DIVIDE = r'/'
+    t_COLON = r':'
+    t_ASSIGN = r'='
+
+    t_LESS_EQUALS = r'<='
+    t_LESS_THAN = r'<'
+    t_GREATER_EQUALS = r'>='
+    t_GREATER_THAN = r'>'
+    t_EQUALS = r'=='
+    t_NOT_EQUALS = r'!='
+    t_AND = r'&&'
+    t_OR = r'\|\|'
+
+    t_DOLLAR = r'\$'
+    t_RUBLE = r'\₽'
+
+    t_LEFT_BRACE = r'\{'
+    t_RIGHT_BRACE = r'\}'
+
+    t_LEFT_BRACKET = r'\('
+    t_RIGHT_BRACKET = r'\)'
+
+    # Literals
+    @staticmethod
+    def t_FALSE(token):
+        r'False'
+        token.value = False
+        return token
+
+    @staticmethod
+    def t_TRUE(token):
+        r'True'
+        token.value = True
+        return token
+
+    def t_DATE_LITERAL(self, token):
+        r'date\(\s*\d{1,2}\s*,\s*\d{1,2}\s*,\s*\d{1,4}\s*\)'
+
+        try:
+            token.value = findall(r'\d\d*', token.value)
+            return token
+        except SyntaxError:
+            self.errors.append('Invalid date.')
+
+    @staticmethod
+    def t_DECIMAL_LITERAL(token):
+        r'\d+[.]\d+'
+        token.value = token.value
+        return token
+
+    @staticmethod
+    def t_INTEGER_LITERAL(token):
+        r'\d+'
+        token.value = token.value
+        return token
+
+    @staticmethod
+    def t_STRING_LITERAL(token):
+        r'\"(.+?)\"'
+        token.value = token.value[1:-1]
+        return token
+
+    # Other
+    def t_IDENTIFIER(self, token):
+        r'[a-z][a-zA-Z_0-9]*'
+        token.type = self.reserved_keywords.get(token.value, 'IDENTIFIER')
+        return token
+
+    @staticmethod
+    def t_comment(token):
+        r'//.*'
+        pass
+
+    @staticmethod
+    def t_newline(token):
+        r'\n+'
+        token.lexer.lineno += len(token.value)
+
+    # Error handling
+    def t_error(self, token):
+        self.errors.append("Illegal character '%s'" % token.value[0])
+        token.lexer.skip(1)
-- 
cgit v1.2.3