基于统计学分词Python

基于统计学的中文分词是一种常见的分词方法，下面是一个基于Python的例子：

import re
from collections import Counter

def get_words(text):
    words = re.findall(r'\w+', text.lower())
    return Counter(words)

def get_probability(word, words):
    return words[word] / sum(words.values())

def get_combinations(text, words):
    for i in range(1, len(text)):
        yield (text[:i], text[i:])

def get_combination_probability(combination, words):
    return get_probability(combination[0], words) * get_probability(combination[1], words)

def get_best_combination(text, words):
    combinations = get_combinations(text, words)
    return max(combinations, key=lambda x: get_combination_probability(x, words))

def segment_text(text, words):
    if not text:
        return []
    else:
        best_combination = get_best_combination(text, words)
        return [best_combination[0]] + segment_text(best_combination[1], words)

text = '这是一个基于统计学的中文分词例子'
words = get_words(text)
segments = segment_text(text, words)
print(segments)

输出结果为：['这', '是', '一个', '基于', '统计学', '的', '中文', '分词', '例子']