python - 如何在scrapy中使用meta获取所有数据并解析?
问题描述
我想将所有数据保存在 json 文件中。如何使用元解析我的数据?我不知道我的元格式是否可以。最后产生一个json文件中的所有数据(我通过meta和我parse_v)帮助我解决这个问题,拜托。
现在我添加完整的代码。希望你能发现我的问题
import json
import scrapy
import time
import chompjs
from scrapy import Request
from scrapy.crawler import CrawlerProcess
class TrendyolSpider(scrapy.Spider):
name = 'data'
start_urls = ['https://www.trendyol.com/join-us/straplez-firfirli-simli-astarli-triko-elbise-krem-p-41896200']
def final_parse(self, response):
abc = response.xpath("//p/script[contains(@type,'application/ld+json')]/text()").extract_first()
json_text = json.loads(abc)
img = json_text.get('image')
products = response.css('div.pd-app-container')
for product in products:
category = product.css('div.breadcrumb>a:nth-child(3)+ a.breadcrumb-item span::text').get(),
product_name = product.css("h1.pr-new-br ::text").getall(),
price = product.css('div.pr-bx-nm span.prc-org::text').get().replace("TL", ""),
discount_price = product.css('div.pr-bx-nm span.prc-slg::text').get().replace("TL", ""),
brand = response.css("div.sl-nm a::text").get(),
image = img,
size = product.css("div.pr-in-at-sp ::text").getall(),
product_information = product.css("div.pr-in-dt-cn ::text").getall(),
product_features = product.css("div.pr-prop-content ::text").getall(),
all_info = response.xpath("//script[contains(@type,'application/javascript')]/text()").extract_first()
product_json = chompjs.parse_js_object(all_info)
ides = product_json['product']['productGroupId']
varient_url = "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + str(ides)
yield Request(url=varient_url, callback=self.parse_v, meta={
'category': category,
'product_name': product_name,
'price': price,
'discount_price': discount_price,
'brand': brand,
'image': image,
'size': size,
'product_information': product_information,
'product_features': product_features,
})
def parse_v(self, response):
json_tex5 = json.loads(response.body)
dataa = json_tex5.get('result').get("slicingAttributes")[0].get("attributes")
yield {
'category': response.meta['category'],
'product_name': response.meta['product_name'],
'price': response.meta['price'],
'discount_price': response.meta['discount_price'],
'brand': response.meta['brand'],
'image': response.meta['image'],
'size': response.meta['size'],
'product_information': response.meta['product_information'],
'product_features': response.meta['product_features'],
'renk': dataa
}
解决方案
这是根据您的问题的答案:
如果你想使用 meta 将数据从一个解析方法传输到另一个解析方法,你需要为每个值创建 key 并使用 meta 在 Request 中注入每个 key-value 对,毕竟在 parse_v 方法中,你必须重新创建 key 并使用 response.meta 获取上一个键,它是新的键值对来产生像“类别”这样的数据: response.meta['cat']
class TrendyolSpider(scrapy.Spider):
name = 'data'
start_urls = [
'https://www.trendyol.com/olalook/kadin-siyah-cepli-minik-beyaz-cicekli-klos-elbise-elb-19000480-p-6635101']
def parse(self, response):
text = response.xpath(
"//p/script[contains(@type,'application/ld+json')]/text()").extract_first()
json_text = json.loads(text)
items = TrendyolItem()
products = response.css('div.pd-app-container')
for product in products:
category = product.css(
'div.breadcrumb>a:nth-child(3)+ a.breadcrumb-item span::text').get(),
product_name = product.css('div.pr-in-cn h1.pr-new-br::text').get() + " " + product.css(
'div.pr-in-cn h1.pr-new-br span::text').get(),
price = product.css(
'div.pr-bx-nm span.prc-org::text').get().replace("TL", ""),
discount_price = product.css(
'div.pr-bx-nm span.prc-slg::text').get().replace("TL", ""),
brand = response.css("div.sl-nm a::text").get(),
image = json_text.get('image'),
size = product.css("div.pr-in-at-sp ::text").getall(),
product_information = product.css(
"div.pr-in-dt-cn ::text").getall(),
product_features = product.css(
"div.pr-prop-content ::text").getall(),
items['category'] = category
items['product_name'] = product_name
items['price'] = price
items['discount_price'] = discount_price
items['brand'] = brand
items['image'] = image
items['size'] = size
items['product_information'] = product_information
items['product_features'] = product_features
all_info = response.xpath(
"//script[contains(@type,'application/javascript')]/text()").extract_first()
product_json = chompjs.parse_js_object(all_info)
ides = product_json['product']['productGroupId']
varient_url = "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + \
str(ides)
yield Request(
url=varient_url,
callback=self.parse_v,
meta={'cat': category, 'pro_name': product_name,'p': price, 'dis_price': discount_price,
'bra': brand,'ima':image,'si':size, 'porduct_info':product_information,'features':product_features
}
)
def parse_v(self, response):
#items = response.meta['items']
json_tex5 = json.loads(response.body)
dataa = json_tex5.get('result').get(
"slicingAttributes")[0].get("attributes")
for i in dataa:
all_info = self.start_urls + i['contents'][0]['url'] + "https://cdn.dsmcdn.com"+i['contents'][0]['imageUrl']\
+ i['contents'][0]['price']['discountedPrice']['text'] + \
i['contents'][0]['price']['originalPrice']['text']
yield {
'Category': response.meta['cat'],
'Product_name': response.meta['pro_name'],
'Price': response.meta['p'],
'Discount_price': response.meta['dis_price'],
'Brand': response.meta['bra'],
'Image': response.meta['ima'],
'Size': response.meta['si'],
'Product_information': response.meta['porduct_info'],
'Product_features': response.meta['features'],
'rank': all_info
}
推荐阅读
- javascript - 从 javascript 对象中查找最小的数字
- c++ - 将文本从文件读入数组时跳过整数
- scikit-learn - 查看 VowpalWabbit sklearn 模型的特征名称和权重
- reactjs - MUI 数据表删除行
- php - 为什么我有意外的语法 IF?
- apex - 尝试保存按钮时尝试取消引用 null 单击此错误触发触发
- oracle - Oracle Apex:交互式报表的帮助文本选项
- javascript - 将变量传递给私有 npm 包
- google-apps-script - 在 Google Apps 脚本中使用 Metabase API 会话令牌时出现错误 401
- java - 构建成功,但我不断收到“线程“主”java.lang.StringIndexOutOfBoundsException 中的异常:字符串索引超出范围:1”