Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit dbb1c63

Browse filesBrowse files
author
Boris
committed
修复去重问题
1 parent 27e3961 commit dbb1c63
Copy full SHA for dbb1c63

2 files changed

+11-7Lines changed: 11 additions & 7 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎feapder/buffer/item_buffer.py‎

Copy file name to clipboardExpand all lines: feapder/buffer/item_buffer.py
+9-1Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,15 @@ def __export_to_db(self, tab_item, datas, is_update=False, update_keys=()):
269269

270270
# 若是任务表, 且上面的pipeline里没mysql,则需调用mysql更新任务
271271
if not self._have_mysql_pipeline and is_update and to_table == self._task_table:
272-
self.mysql_pipeline.update_items(to_table, datas, update_keys=update_keys)
272+
if not self.mysql_pipeline.update_items(
273+
to_table, datas, update_keys=update_keys
274+
):
275+
log.error(
276+
f"{pipeline.__class__.__name__} 更新数据失败. table: {to_table} items: {datas}"
277+
)
278+
return False
279+
280+
return True
273281

274282
def __add_item_to_db(
275283
self, items, update_items, requests, callbacks, items_fingerprints
Collapse file

‎feapder/pipelines/mysql_pipeline.py‎

Copy file name to clipboardExpand all lines: feapder/pipelines/mysql_pipeline.py
+2-6Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,7 @@ def save_items(self, table, items: List[Dict]) -> bool:
4141
sql, datas = tools.make_batch_sql(table, items)
4242
add_count = self.to_db.add_batch(sql, datas)
4343
datas_size = len(datas)
44-
if add_count is None:
45-
log.error("导出数据到表 %s 失败" % (table))
46-
else:
44+
if add_count:
4745
log.info(
4846
"共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, table, datas_size - add_count)
4947
)
@@ -67,9 +65,7 @@ def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool:
6765
table, items, update_columns=update_keys or list(items[0].keys())
6866
)
6967
update_count = self.to_db.add_batch(sql, datas)
70-
if update_count is None:
71-
log.error("更新表 %s 数据失败" % (table))
72-
else:
68+
if update_count:
7369
msg = "共更新 %s 条数据 到 %s" % (update_count // 2, table)
7470
if update_keys:
7571
msg += " 更新字段为 {}".format(update_keys)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.