python - 转换矩阵中的多个 ngram，概率不加到 1

Question

我正在尝试找到一种方法，使用 python 和 numpy 为给定文本使用 unigrams、bigrams 和 trigrams 制作转换矩阵。每行的概率应该等于 1。我先用二元组做了这个，效果很好：

distinct_words = list(word_dict.keys())
dwc = len(distinct_words)

matrix = np.zeros((dwc, dwc), dtype=np.float)
for i in range(len(distinct_words)):
    word = distinct_words[i]
    first_word_idx = i
    total = 0
    for bigram, count in ngrams.items():
        word_1, word_2 = bigram.split(" ")
        if word_1 == word:
            total += count
    for bigram, count in ngrams.items():
        word_1, word_2 = bigram.split(" ")
        if word_1 == word:
            second_word_idx = index_dict[word_2]
            matrix[first_word_idx,second_word_idx] = count / total

但现在我想添加 unigrams 和 trigrams 并加权它们的概率（trigrams * .6、bigrams * .2、unigrams *.2）。我不认为我的 python 非常简洁，这是一个问题，但我也不知道如何使用多个 n-gram（和权重，虽然老实说权重是次要的），所以我仍然可以获得所有概率从任何给定的行加起来为一个。

distinct_words = list(word_dict.keys())
dwc = len(distinct_words)

matrix = np.zeros((dwc, dwc), dtype=np.float)
for i in range(len(distinct_words)):
  word = distinct_words[i]
  first_word_index = i 
  bi_total = 0
  tri_total=0
  tri_prob = 0
  bi_prob = 0
  uni_prob = word_dict[word] / len(distinct_words)
  if i < len(distinct_words)-1:
    for trigram, count in trigrams.items():
      word_1, word_2, word_3 = trigram.split()
      if word_1 + word_2 == word + distinct_words[i+1]:
        tri_total += count
    for trigram, count in trigrams.items():
      word_1, word_2, word_3 = trigram.split()
      if word_1 + word_2 == word + distinct_words[i+1]:
        second_word_index = index_dict[word_2]
        tri_prob = count/bigrams[word_1 + " " + word_2]
  for bigram, count in bigrams.items():
    word_1, word_2 = bigram.split(" ")
    if word_1 == word:
      bi_total += count
  for bigram, count in bigrams.items():
    word_1, word_2 = bigram.split(" ")
    if word_1 == word:
      second_word_index = index_dict[word_2]
      bi_prob = count / bi_total
      matrix[first_word_index,second_word_index] = (tri_prob * .4) + (bi_prob * .2) + (word_dict[word]/len(word_dict) *.2)

我正在阅读本讲座以了解如何设置我的概率矩阵，这似乎很有意义，但我不确定我哪里出错了。

如果它有帮助，我的 n_grams 就是来自这个——它只是产生一个 n_gram 的字典作为一个字符串和它的计数。

def get_ngram(words, n):
    word_dict = {}
    for i, word in enumerate(words):
        if i > (n-2):
            n_gram = []
            for num in range(n):
                index = i - num
                n_gram.append(words[index])
            if len(n_gram) > 1:
                formatted_gram = ""
                for word in reversed(n_gram):
                    formatted_gram += word + " "
            else:
                formatted_gram = n_gram[0]
            stripped = formatted_gram.strip() if formatted_gram else n_gram[0]
            if stripped in word_dict:
                word_dict[stripped] += 1
            else:
                word_dict[stripped] = 1
    return word_dict

标签： pythonnumpynlpprobabilityprobability-theory

Answer 1

我已经实现了一个用于计算 unigrams、bigrams 和 trigrams 的示例。您可以zip轻松地用于连接项目。此外，Counter用于计数项目并defaultdict用于项目的概率。defaultdict当键未在集合中映射时很重要，返回零。否则，您必须添加 if 子句以避免None.

from collections import Counter, defaultdict

def calculate_grams(items_list):
  # count items in list
  counts = Counter()
  for item in items_list:
    counts[item] += 1

  # calculate probabilities, defaultdict returns 0 if not found
  prob = defaultdict(float)
  for item, count in counts.most_common():
    prob[item] = count / len(items_list)

  return prob

def calculate_bigrams(words):
  # tuple first and second items
  return calculate_grams(list(zip(words, words[1:])))

def calculate_trigrams(words):
  # tuple first, second and third items
  return calculate_grams(list(zip(words, words[1:], words[2:])))


dataset = ['a', 'b', 'b', 'c', 'a', 'a', 'a', 'b', 'e', 'e', 'c']

# create dictionary
dictionary = set(dataset)
print("Dictionary", dictionary)

unigrams = calculate_grams(dataset)
print("Unigrams", unigrams)

bigrams = calculate_bigrams(dataset)
print("Bigrams", bigrams)

trigrams = calculate_trigrams(dataset)
print("Trigrams", trigrams)

# Testing
test_words = ['a', 'b']
print("Testing", test_words)

for c in dictionary:
  # calculate each probabilities
  unigram_prob = unigrams[c]
  bigram_prob = bigrams[(test_words[-1], c)]
  trigram_prob = bigrams[(test_words[-2], test_words[-1], c)]
  # calculate total probability
  prob = .2 * unigram_prob + .2 * bigram_prob + .4 * trigram_prob
  print(c, prob)

输出：

Unigrams defaultdict(<class 'float'>, {'a': 0.36363636363636365, 'b': 0.2727272727272727, 'c': 0.18181818181818182, 'e': 0.18181818181818182})
Bigrams defaultdict(<class 'float'>, {('a', 'b'): 0.2, ('a', 'a'): 0.2, ('b', 'b'): 0.1, ('b', 'c'): 0.1, ('c', 'a'): 0.1, ('b', 'e'): 0.1, ('e', 'e'): 0.1, ('e', 'c'): 0.1})
Trigrams defaultdict(<class 'float'>, {('a', 'b', 'b'): 0.1111111111111111, ('b', 'b', 'c'): 0.1111111111111111, ('b', 'c', 'a'): 0.1111111111111111, ('c', 'a', 'a'): 0.1111111111111111, ('a', 'a', 'a'): 0.1111111111111111, ('a', 'a', 'b'): 0.1111111111111111, ('a', 'b', 'e'): 0.1111111111111111, ('b', 'e', 'e'): 0.1111111111111111, ('e', 'e', 'c'): 0.1111111111111111})

Testing ['a', 'b']
e 0.05636363636363637
b 0.07454545454545455
c 0.05636363636363637
a 0.07272727272727274