python - 为 reddit 抓取代码放入 break 循环

问题描述

我希望通过 python 获取特定日期之间的所有 reddit 评论。下面的代码是我正在尝试的，它从 start_time (= int(dt.datetime(2020,5,1,0,0).timestamp())) 开始，但是当它到达 end_time (= end_time = int(dt.datetime(2020,4,20,0,0).timestamp())) 的四月一日。

有谁知道如何更新它以导致循环在到达这个 end_time 时中断？

submission_df = pd.DataFrame(None, columns=["id", "title", "score",
           "author", "author_flair_text", "body", "total_awards_received", "awarders", 
           "created_utc", "full_link", "num_comments"
                                           ])
values = []
def downloadFromUrl(df, object_type):
    count = 0
    previous_epoch = int(start_time) #change this to just start_time?
    if len(values) > 0:
        previous_epoch = values[-1]['created_utc'] - 1
    print(f"Saving {object_type}s")


    while True:
        new_url = url.format(object_type, subreddit) + str(previous_epoch)
        json = requests.get(new_url)
        try:
            json_data = json.json()
        except:
            print(json)
        else:
            if 'data' not in json_data:
                break
            objects = json_data['data']
            if len(objects) == 0:
                break

            for submission in objects:
                previous_epoch = submission['created_utc'] - 1
                count += 1
                if object_type == 'comment':
                    try:
                        handle.write(str(submission['score']))
                        handle.write(" : ")
                        handle.write(datetime.fromtimestamp(submission['created_utc']).strftime("%Y-%m-%d"))
                        handle.write("\n")
                        text = submission['body']
                        textASCII = text.encode(encoding='ascii', errors='ignore').decode()
                        handle.write(textASCII)
                        handle.write("\n-------------------------------\n")
                    except Exception as err:
                        print(f"Couldn't print comment: https://www.reddit.com{submission['permalink']}")
                        print(traceback.format_exc())

                elif object_type == 'submission':
                    if 'awarders' in submission:
                        awarders = submission['awarders']
                    else:
                        awarders = None
                    if 'total_awards_received' in submission:
                        total_awards = submission['total_awards_received']
                    else:
                        total_awards = None
                    if 'body' in submission:
                        body = submission['body']
                    else:
                        body = None

                    try:
                        values.append(dict(zip(submission_df.columns, [submission['id'], submission['title'], submission['score'], submission['author'],
                                        submission['author_flair_text'], body,
                                        total_awards, awarders, submission['created_utc'],
                                        submission['full_link'], submission['num_comments'], submission['over_18']])))
                    except Exception as err:
                        print(f"Couldn't print post: {submission['url']}")
                        print(traceback.format_exc())
                    
        print(f"Saved {len(values)} of {count} {object_type}s through {datetime.fromtimestamp(previous_epoch).strftime('%Y-%m-%d')}")

    print(f"Saved {count} {object_type}s")
    return values
    # handle.close()")
    
new_values = downloadFromUrl(submission_df, "submission")

标签： pythonreddit

python - 为 reddit 抓取代码放入 break 循环

问题描述

解决方案

推荐阅读