java词法分析器原理

Java词法分析器是Java编译器的一个重要组成部分，它的主要作用是将源代码转换为标记流（Token Stream），以便后续的语法分析器进行处理。Java词法分析器的原理如下：

1. 读取源代码：Java词法分析器首先读取源代码，并将其存储在内存中。

2. 分离标记：Java词法分析器将源代码分离成一个个标记（Token），每个标记代表一个关键字、标识符、运算符、分隔符或常量等。

3. 标记分类：Java词法分析器对每个标记进行分类，例如关键字、标识符、运算符、分隔符或常量等。

4. 标记属性：Java词法分析器为每个标记添加属性，例如标识符的名称、常量的值等。

5. 标记流：Java词法分析器将所有标记组成一个标记流（Token Stream），并将其传递给语法分析器进行处理。

以下是一个Java词法分析器的示例代码：

import java.io.*;
import java.util.*;

public class Lexer {
    private BufferedReader reader;
    private String line;
    private int lineNo;
    private int pos;

    public Lexer(String fileName) throws IOException {
        reader = new BufferedReader(new FileReader(fileName));
        lineNo = 0;
        pos = 0;
        readLine();
    }

    private void readLine() throws IOException {
        line = reader.readLine();
        if (line != null) {
            lineNo++;
            pos = 0;
        }
    }

    private char peek() {
        if (line == null || pos >= line.length()) {
            return '\0';
        } else {
            return line.charAt(pos);
        }
    }

    private char advance() throws IOException {
        char c = peek();
        pos++;
        if (pos >= line.length()) {
            readLine();
        }
        return c;
    }

    private boolean isWhitespace(char c) {
        return c == ' ' || c == '\t' || c == '\r' || c == '\n';
    }

    private boolean isDigit(char c) {
        return c >= '0' && c <= '9';
    }

    private boolean isLetter(char c) {
        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
    }

    private String readIdentifier() throws IOException {
        StringBuilder sb = new StringBuilder();
        char c = peek();
        while (isLetter(c) || isDigit(c) || c == '_') {
            sb.append(advance());
            c = peek();
        }
        return sb.toString();
    }

    private int readNumber() throws IOException {
        int value = 0;
        char c = peek();
        while (isDigit(c)) {
            value = value * 10 + (c - '0');
            advance();
            c = peek();
        }
        return value;
    }

    public Token nextToken() throws IOException {
        while (true) {
            char c = peek();
            if (isWhitespace(c)) {
                advance();
            } else if (isLetter(c)) {
                String identifier = readIdentifier();
                return new Token(TokenType.IDENTIFIER, identifier);
            } else if (isDigit(c)) {
                int value = readNumber();
                return new Token(TokenType.NUMBER, value);
            } else if (c == '+') {
                advance();
                return new Token(TokenType.PLUS, "+");
            } else if (c == '-') {
                advance();
                return new Token(TokenType.MINUS, "-");
            } else if (c == '*') {
                advance();
                return new Token(TokenType.MULTIPLY, "*");
            } else if (c == '/') {
                advance();
                return new Token(TokenType.DIVIDE, "/");
            } else if (c == '(') {
                advance();
                return new Token(TokenType.LEFT_PAREN, "(");
            } else if (c == ')') {
                advance();
                return new Token(TokenType.RIGHT_PAREN, ")");
            } else if (c == '\0') {
                return new Token(TokenType.EOF, "");
            } else {
                throw new RuntimeException("Invalid character: " + c);
            }
        }
    }
}

enum TokenType {
    IDENTIFIER,
    NUMBER,
    PLUS,
    MINUS,
    MULTIPLY,
    DIVIDE,
    LEFT_PAREN,
    RIGHT_PAREN,
    EOF
}

class Token {
    public TokenType type;
    public Object value;

    public Token(TokenType type, Object value) {
        this.type = type;
        this.value = value;
    }

    public String toString() {
        return String.format("(%s, %s)", type, value);
    }
}