Python实现 代码 双向最大匹配法 规则分词 正向最大匹配法 逆向最大匹配法 中文分词技术

"""正向最大匹配算法MM"""
class MaxMatch(object):
    def __init__(self):
        self.window_size = 3

    def cut(self, text):
        result = []
        index = 0
        text_length = len(text)
        dic = ['研究', '研究生', '生命', '命', '的', '起源']
        while text_length > index:
            for size in range(self.window_size + index, index, -1):
                piece = text[index:size]
                if piece in dic:
                    index = size - 1
                    break
            index = index + 1
            result.append(piece)
        # print(result)
        return result

"""逆向最大匹配算法RMM"""
class ReverseMaxMatch(object):
    def __init__(self):
        self.window_size = 3

    def cut(self, text):
        result = []
        index = len(text)
        dic = ['研究', '研究生', '生命', '命', '的', '起源']
        while index > 0:
            for size in range(index - self.window_size, index):
                piece = text[size:index]
                if piece in dic:
                    index = size + 1
                    break
            index = index - 1
            result.append(piece)
        result.reverse()#由于append为添加至末尾，故需反向打印
        # print(result)
        return result

"""双向最大匹配法"""
class BiDirectionMatch(object):
    def cut(self, text):
        MM = MaxMatch()
        RMM = ReverseMaxMatch()

        MMMatch = MM.cut(text)
        RMMMatch = RMM.cut(text)

        #返回分词数较少者
        if(len(MMMatch) != len(RMMMatch)):
            if(len(MMMatch) < len(RMMMatch)):
                return MMMatch
            else:
                return RMMMatch
        #若分词数量相同，则进一步判断
        else:
            MMsingle = 0
            RMMsingle = 0
            isEqual = True #标记结果是否相同

            for i in range(len(MMMatch)):
                if(MMMatch[i] != RMMMatch):
                    isEqual = False
                #统计单字数
                if(len(MMMatch[i]) == 1):
                    MMsingle += 1
                if(len(RMMMatch[i]) == 1):
                    RMMsingle += 1
                #如果两个结果一样，随便返回一个
                if isEqual:
                    return MMMatch
                #如果两个结果不一样，则返回单字数最少的那个
                elif MMsingle < RMMsingle:
                    return MMMatch
                else:
                    return RMMMatch


if __name__ == '__main__':
    text = '研究生命的起源'
    tokenizer = BiDirectionMatch()
    print(tokenizer.cut(text))