Kaggle案例分析1--Bestbuy

#!/usr/bin/env python
import csv  # 用于读取csv格式的数据
import re
import operator
import Levenshtein  # 用于计算字符串之间的Levenshtein距离(又称编辑距离)

# 将下载下来的训练/测试数据集放在data文件夹中(data文件夹与本程序同路径)
__train__ = './data/train.csv'
__test__ = './data/test.csv'

# 此函数对csv数据进行读取, path: csv文件路径; cols: 要读取的列索引;
# ignore_header=True : 头部首行不读取
def read_data(path, cols, ignore_header=True):
    csv_file_object = csv.reader(open(path, 'rb'))
    if ignore_header:  # 数据的首行为属性值,跳过首行
        header = csv_file_object.next()
    x = []
    for row in csv_file_object:
        r = []
        for col in cols:
            r.append(row[col])
        x.append(r)
    return x

# 对字符串进行规则化,大写都变成小写,去除空格,去除除数字/字母之外的字符
def string_normalize(s):
    res = s.lower()  # 大写都弄成小写
    res = res.replace(' ', '')  # 去除空格
    res = ''.join(c for c in res if c.isalnum())
    return res

# 对数据进行预处理(主要是对搜索关键词进行预处理)
def preprocess_data(raw_data, col):
    for row in raw_data:
        row[col] = string_normalize(row[col])
# 计算keys里面与query之间最相似的一个key
def best_match_key(keys, query):
    similarity = 0
    best_key = None
    for key in keys:
        sim = Levenshtein.ratio(key, query)  # 计算两个查询字符串之间的Levenshtein相似度
        if sim > similarity:
            similarity = sim
            best_key = key
    return (best_key, similarity)

def create_match(x, thresh=.85):
    match = {}
    for row in x:
        matched_key = None
        sku, query = row
        # Fuzzy matching.
        best_key, similarity = best_match_key(match.keys(), query)
        if similarity > thresh:
            matched_key = best_key
        else:
            match[query] = {sku : 1}
        if matched_key is None:
            continue
        if not match[matched_key].has_key(sku):
            match[matched_key][sku] = 1
        else:
            match[matched_key][sku] += 1

    # Sorts the dictionary.
    for key in match.keys():
        tmp_dict = match[key]
        tmp_dict = sorted(tmp_dict.iteritems(), key=operator.itemgetter(1))
        tmp_dict.reverse()
        match[key] = tmp_dict
    return match

# 统计训练集当中每个sku出现的次数, 
def get_top(x):
    sku_count_dict = {}
    for row in x:
        if not sku_count_dict.has_key(row[0]):
            sku_count_dict[row[0]] = 1
        else:
            sku_count_dict[row[0]] += 1
    sorted_dict = sorted(sku_count_dict.iteritems(), key=operator.itemgetter(1))
    sorted_dict.reverse()

    res = []
    for i in range(len(sorted_dict)):
        res.append(sorted_dict[i][0])
    return res;

def predict(match, top, query, k, thresh=.7):
    res = []
    matched_key, similarity = best_match_key(match.keys(), query)
    # if similarity < 0.8:
    #     print 'matched_key = %s, query = %s, sim = %s' \
    #         % (matched_key, query, similarity)
    if similarity > thresh:
        for i in range(min(k, len(match[matched_key]))):
            res.append(match[matched_key][i][0])
    if len(res) < k:
        for i in range(len(top)):
            if top[i] not in res:
                res.append(top[i])
                if len(res) == k:
                    break
    return res

# 程序从这里开始执行
if __name__ == '__main__':
    # 读取训练数据
    print 'Reading and preprocessing data...'
    x = read_data(__train__, [1, 3])  # [1,3]的含义是只读取sku和query这两列属性值
    preprocess_data(x, 1)

    # 将数据划分成训练集+验证集
    l = int(len(x) * 1)
    x_cv = x[l - 10 : :]
    x = x[0 : l]

    top = get_top(x) # 出现的sku次数从大到小排序

    # Predicts on cv.
    print 'Predicting...'
    k = 5
    # thresh_match = [0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
    # thresh_predict = [0, 0.2, 0.4, 0.6, 0.7, 0.8]
    thresh_match = [0.75]
    thresh_predict = [0]
    best_match = None
    thresh1 = 0
    thresh2 = 0
    accuracy = -1
    for t1 in thresh_match:
        for t2 in thresh_predict:
            match = create_match(x, t1)
            p_cv = []
            for row in x_cv:
                q = row[1]
                p_cv.append(predict(match, top, q, k, t2))
            correct = 0.
            for i in range(len(x_cv)):
                if x_cv[i][0] in p_cv[i]:
                    correct += 1.
            ac = correct / len(x_cv)
            print 't1 = %f, t2 = %f, accuracy = %f' % (t1, t2, ac)
            if ac > accuracy:
                accuracy = ac
                thresh1 = t1
                thresh2 = t2
                best_match = match
    print 'thresh1 = %f, thresh2 = %f, accuracy = %f' \
        % (thresh1, thresh2, accuracy)

    # 读取测试数据
    x_test = read_data(__test__, [2])
    preprocess_data(x_test, 0)  # 数据的预处理

    # 对测试数据进行预测
    res = []
    k = 5
    for row in x_test:
        q = row[0]
        res.append(predict(best_match, top, q, k, thresh2))

    # 将结果写入result.csv当中
    open_file_object = csv.writer(open("./data/result.csv", "wb"))
    open_file_object.writerow(['sku'])
    for p in res:
        open_file_object.writerow([' '.join(p)])

import sys

sys.path
Out[4]: 
['',
 'D:\\Anaconda2\\python27.zip',
 'D:\\Anaconda2\\DLLs',
 'D:\\Anaconda2\\lib',
 'D:\\Anaconda2\\lib\\plat-win',
 'D:\\Anaconda2\\lib\\lib-tk',
 'D:\\Anaconda2',
 'd:\\anaconda2\\lib\\site-packages\\sphinx-1.4.1-py2.7.egg',
 'd:\\anaconda2\\lib\\site-packages\\setuptools-23.0.0-py2.7.egg',
 'D:\\Anaconda2\\lib\\site-packages',
 'D:\\Anaconda2\\lib\\site-packages\\win32',
 'D:\\Anaconda2\\lib\\site-packages\\win32\\lib',
 'D:\\Anaconda2\\lib\\site-packages\\Pythonwin',
 'D:\\Anaconda2\\lib\\site-packages\\IPython\\extensions',
 'C:\\Users\\WXC\\.ipython']

sys.path.append('your path')

 1 def add(num):
 2     num[0] = 10
 3 
 4 def add2(num):
 5     num = 10
 6 
 7 d = [2]
 8 add(d)
 9 print d     # [12]
10 
11 d2 = 2
12 add2(d2)
13 print d2    # 2

1 def changeValue(a):
2     for row in a:
3         row = 10    # row的类型为:<type 'int'>
4 
5 b = [1,8,9,2,0]
6 changeValue(b)    # b的值并没有发生改变
7 print b
8 [1, 8, 9, 2, 0]

1 def changeValue(a):
2     for row in a:
3         row[0] = 10
4 
5 c = [[1,2],[5,8]]
6 changeValue(c)
7 print c
8 [[10, 2], [10, 8]]

1 a = [5,1,0,2,5]
2 b = a.sort()
3 print a
4 [0,1,2,5,5]
5 print b
6 None

a = [5,1,0,2,5]

print sorted(a)
[0, 1, 2, 5, 5]

print a
[5, 1, 0, 2, 5]

from operator import itemgetter
# 后面的操作对象也要用()圈起来噻
print itemgetter(1)([1,2,3])   # 2
print itemgetter(1)('ABCDEFR')  # B
print itemgetter(1,3,4)('ABCDEFR')  # ('B', 'D', 'E')

#! /usr/bin/env python
#coding=utf-8

from operator import itemgetter

inventory = [('apple',3),('banana',2),('pear',5),('orange',1)]
getcount = itemgetter(1)

print map(getcount, inventory)
输出: [3, 2, 5, 1]
print sorted(inventory, key = getcount)  # 用getcount指定key
输出: [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)]

In[64]: students = [('Jace','A',10),('Mara','A',52),('Mich','B',5)]
students
Out[66]: [('Jace', 'A', 10), ('Mara', 'A', 52), ('Mich', 'B', 5)]
sorted(students, key=lambda stud:stud[2])
Out[67]: [('Mich', 'B', 5), ('Jace', 'A', 10), ('Mara', 'A', 52)]

1 f = lambda x, y : x + y    # lambda表达式, : 左边为输入参数, : 右边为运算式
2 
3 f(1,5)
4 Out[60]: 6

1 p = map(lambda x: x*x, [y for y in range(10)])
2 
3 p
4 Out[62]: [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]

# 方式1: 利用循环
items = [1,2,3,4,5]
squared = []
for i in items:
    squared.append(i**2)
# 方式2: 利用匿名函数和map
items = [1,2,3,4,5]
squared = map(lambda x:x**2, items)  # [1,4,9,16,25]

 1 def multiply(x):
 2     return x * x
 3 
 4 def add(x):
 5     return x + x
 6 
 7 funcs = [multiply, add]
 8 for i in range(5):
 9     value = map(lambda x:x(i), funcs) # funcs里面的每个函数带入x,求x(i)就是求multiply(i)和add(i)
10     print value
11 
12 [0, 0]
13 [1, 2]
14 [4, 4]
15 [9, 6]
16 [16, 8]

#! /usr/bin/env python
#coding=utf-8

ls = range(-3, 3)
print ls

less_than_zero1 = map(lambda x:x<0, ls)
print less_than_zero1

less_than_zero2 = filter(lambda x:x<0, ls)
print less_than_zero2

[-3, -2, -1, 0, 1, 2]
[True, True, True, False, False, False]
[-3, -2, -1]

product = reduce(lambda x, y: x*y, [1,2,5,1])

product
Out[15]: 10