lcdevelop · hailiang-wang · Jan 29, 2017 · Jan 29, 2017 · Jan 29, 2017
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,19 @@
+# maven
+chatbotv1/target/
+
+# py
+__pycache__/
+*.pyc
+
+# vim
+*.swp
+*.swo
+
+# idea
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# tmp files
+subtitle/subtitle_crawler/result/
diff --git a/subtitle/README.md b/subtitle/README.md
@@ -0,0 +1,16 @@
+# 爬取字幕语料
+
+* 依赖curl
+* 不支持 Windows
+* Tested with py2.7
+
+## Install
+```
+pip install scrapy
+```
+
+## Download Data
+```
+cd ChatBotCourse/subtitle/subtitle_crawler
+scrapy crawl subtitle
+```
diff --git a/subtitle/subtitle_crawler/__init__.pyc b/subtitle/subtitle_crawler/__init__.pyc
diff --git a/subtitle/subtitle_crawler/items.pyc b/subtitle/subtitle_crawler/items.pyc
diff --git a/subtitle/subtitle_crawler/pipelines.py b/subtitle/subtitle_crawler/pipelines.py
@@ -4,14 +4,17 @@
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+import os
+from subprocess import call

+DOWNLOAD_CMD = os.path.join(os.path.dirname(
+    os.path.realpath(__file__)), 'save.sh')

 class SubtitleCrawlerPipeline(object):
+    '''
+    Download Subtitle with curl
+    '''
    def process_item(self, item, spider):
        url = item['url']
-        file_name = url.replace('/','_').replace(':','_')
-        fp = open('result/'+file_name, 'w')
-        fp.write(item['body'])
-        fp.close()
+        call([DOWNLOAD_CMD, url])
        return item
-
diff --git a/subtitle/subtitle_crawler/pipelines.pyc b/subtitle/subtitle_crawler/pipelines.pyc
diff --git a/subtitle/subtitle_crawler/save.sh b/subtitle/subtitle_crawler/save.sh
@@ -0,0 +1,19 @@
+#! /bin/bash 
+###########################################
+# Save file with CURL
+###########################################
+
+# constants
+baseDir=$(cd `dirname "$0"`;pwd)
+SAVE_PATH=$baseDir/result
+# functions
+
+# main 
+[ -z "${BASH_SOURCE[0]}" -o "${BASH_SOURCE[0]}" = "$0" ] || return
+if [ ! -d "$SAVE_PATH" ]; then
+  # Control will enter here if $DIRECTORY doesn't exist.
+  mkdir -p $SAVE_PATH
+fi
+
+cd $SAVE_PATH
+curl -JLO $1
diff --git a/subtitle/subtitle_crawler/settings.pyc b/subtitle/subtitle_crawler/settings.pyc
diff --git a/subtitle/subtitle_crawler/spiders/__init__.pyc b/subtitle/subtitle_crawler/spiders/__init__.pyc
diff --git a/subtitle/subtitle_crawler/spiders/subtitle_spider.pyc b/subtitle/subtitle_crawler/spiders/subtitle_spider.pyc