FilesExpand file tree

master

/

mongo_spider.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

39 lines (32 loc) · 1011 Bytes

master

/

mongo_spider.py

Copy path

Top

File metadata and controls

• Code
• Blame

39 lines (32 loc) · 1011 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

# -*- coding: utf-8 -*-

"""

Created on 2021-02-08 16:06:12

---------

@summary:

---------

@author: Boris

"""

import feapder

from feapder import Item, UpdateItem

class TestMongo(feapder.AirSpider):

__custom_setting__ = dict(

ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"],

MONGO_IP="localhost",

MONGO_PORT=27017,

MONGO_DB="feapder",

MONGO_USER_NAME="",

MONGO_USER_PASS="",

)

def start_requests(self):

yield feapder.Request("https://www.baidu.com")

def parse(self, request, response):

title = response.xpath("//title/text()").extract_first() # 取标题

for i in range(10):

item = Item() # 声明一个item

item.table_name = "test_mongo"

item.title = title + str(666) # 给item属性赋值

item.i = i + 5

item.c = "777"

yield item # 返回item， item会自动批量入库

if __name__ == "__main__":

TestMongo().start()