时间戳

首页 > 解决方案 > 大型矩阵上的高效双循环

python - 大型矩阵上的高效双循环

问题描述

我有以下代码，我需要多次运行它。目前，它需要的时间太长。有没有一种有效的方法来编写这两个 for 循环。

ErrorEst=[]
for i in range(len(embedingFea)):#17000
    temp=[]
    for j in range(len(emedingEnt)):#15000
        if cooccurrenceCount[i][j]>0:
            #print(coaccuranceCount[i][j]/ count_max)
            weighting_factor = np.min(
                        [1.0,
                        math.pow(np.float32(cooccurrenceCount[i][j]/ count_max), scaling_factor)])

            embedding_product = (np.multiply(emedingEnt[j], embedingFea[i]), 1)
            #tf.log(tf.to_float(self.__cooccurrence_count))
            log_cooccurrences =np.log (np.float32(cooccurrenceCount[i][j]))

            distance_expr = np.square(([
                embedding_product+
                focal_bias[i],
                context_bias[j],
                -(log_cooccurrences)]))

            single_losses =(weighting_factor* distance_expr)
            temp.append(single_losses)
    ErrorEst.append(np.sum(temp))

标签： pythonalgorithmnumpy

解决方案

您可以使用 Numba 或 Cython

首先，确保尽可能避免使用列表，并使用显式循环编写简单易读的代码，就像在 C 中所做的那样。所有输入和输出都只是 numpy 数组或标量。

你的代码

import numpy as np
import numba as nb
import math

def your_func(embedingFea,emedingEnt,cooccurrenceCount,count_max,scaling_factor,focal_bias,context_bias):
    ErrorEst=[]
    for i in range(len(embedingFea)):#17000
        temp=[]
        for j in range(len(emedingEnt)):#15000
            if cooccurrenceCount[i][j]>0:
                weighting_factor = np.min([1.0,math.pow(np.float32(cooccurrenceCount[i][j]/ count_max), scaling_factor)])
                embedding_product = (np.multiply(emedingEnt[j], embedingFea[i]), 1)
                log_cooccurrences =np.log (np.float32(cooccurrenceCount[i][j]))

                distance_expr = np.square(([embedding_product+focal_bias[i],context_bias[j],-(log_cooccurrences)]))

                single_losses =(weighting_factor* distance_expr)
                temp.append(single_losses)
        ErrorEst.append(np.sum(temp))
    return ErrorEst

麻省密码

@nb.njit(fastmath=True,error_model="numpy",parallel=True)
def your_func_2(embedingFea,emedingEnt,cooccurrenceCount,count_max,scaling_factor,focal_bias,context_bias):
    ErrorEst=np.empty((embedingFea.shape[0],2))
    for i in nb.prange(embedingFea.shape[0]):
        temp_1=0.
        temp_2=0.
        for j in range(emedingEnt.shape[0]):
            if cooccurrenceCount[i,j]>0:
                weighting_factor=(cooccurrenceCount[i,j]/ count_max)**scaling_factor
                if weighting_factor>1.:
                    weighting_factor=1.

                embedding_product = emedingEnt[j]*embedingFea[i]
                log_cooccurrences =np.log(cooccurrenceCount[i,j])

                temp_1+=weighting_factor*(embedding_product+focal_bias[i])**2
                temp_1+=weighting_factor*(context_bias[j])**2
                temp_1+=weighting_factor*(log_cooccurrences)**2

                temp_2+=weighting_factor*(1.+focal_bias[i])**2
                temp_2+=weighting_factor*(context_bias[j])**2
                temp_2+=weighting_factor*(log_cooccurrences)**2


        ErrorEst[i,0]=temp_1
        ErrorEst[i,1]=temp_2
    return ErrorEst

计时

embedingFea=np.random.rand(1700)+1
emedingEnt=np.random.rand(1500)+1
cooccurrenceCount=np.random.rand(1700,1500)+1
focal_bias=np.random.rand(1700)
context_bias=np.random.rand(1500)
count_max=100
scaling_factor=2.5

%timeit res_1=your_func(embedingFea,emedingEnt,cooccurrenceCount,count_max,scaling_factor,focal_bias,context_bias)
1min 1s ± 346 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
%timeit res_2=your_func_2(embedingFea,emedingEnt,cooccurrenceCount,count_max,scaling_factor,focal_bias,context_bias)
17.6 ms ± 2.81 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

推荐阅读