componentjs
diff --git a/‎lib/crawl.js
Copy file name to clipboardExpand all lines: lib/crawl.js
+41-29Lines changed: 41 additions & 29 deletions b/‎lib/crawl.js
Copy file name to clipboardExpand all lines: lib/crawl.js
+41-29Lines changed: 41 additions & 29 deletions
diff --git a/‎package.json
Copy file name to clipboardExpand all lines: package.json
+2-1Lines changed: 2 additions & 1 deletion b/‎package.json
Copy file name to clipboardExpand all lines: package.json
+2-1Lines changed: 2 additions & 1 deletion
@@ -9,6 +9,7 @@ var channel = require('chanel');
 var request = require('cogent').extend({
   auth: GITHUB_USERNAME + ':' + GITHUB_PASSWORD,
 });
+var parseLink = require('parse-link-header');
 
 var store = require('./store');
 var log = require('./log');
@@ -31,42 +32,53 @@ module.exports = function* (user) {
     }
   }
 
-  // get all the user's repos
-  var res = yield* request('https://api.github.com/search/repositories?q=user:' + user, true);
-  if (res.statusCode === 404) {
-    res.resume();
-    log.write({
-      context: 'user',
-      user: user,
-      type: 'error',
-      message: 'User "' + user + '" not found.',
-    })
-    return;
-  }
-  if (res.statusCode !== 200) {
-    res.resume();
-    log.write({
-      context: 'user',
-      user: user,
-      type: 'error',
-      message: 'Error searching "' + user + '"\'s repositories.',
-    })
-    return;
-  }
-
-  // store the user info
-  var json = store.json.users[user] = res.body;
-
+  var page = 1;
+  var items = [];
   // iterate through each repo with concurrency
   var ch = channel({
     concurrency: CONCURRENCY,
     discard: true,
   });
-  var items = json.items;
-  for (var i = 0; i < items.length; i++) {
-    ch.push(co(crawlRepository(user, items[i])));
+
+  while (true) {
+    // get all the user's repos
+    var res = yield* request('https://api.github.com/search/repositories?q=user:' + user + '&page=' + page, true);
+    if (res.statusCode === 404) {
+      res.resume();
+      log.write({
+        context: 'user',
+        user: user,
+        type: 'error',
+        message: 'User "' + user + '" not found.',
+      })
+      return;
+    }
+    if (res.statusCode !== 200) {
+      res.resume();
+      log.write({
+        context: 'user',
+        user: user,
+        type: 'error',
+        message: 'Error searching "' + user + '"\'s repositories.',
+      })
+      return;
+    }
+    var _items = res.body.items;
+    items = items.concat(res.body.items);
+
+    for (var i = 0; i < _items.length; i++) {
+      ch.push(co(crawlRepository(user, _items[i])));
+    }
+
+    if (!res.headers.link) break;
+    var links = parseLink(res.headers.link);
+    if (!links.next) break;
+
+    page++;
   }
+
   yield ch(true);
+  store.json.users[user] = items;
   yield* store.put();
 
   log.write({
 
@@ -29,7 +29,8 @@
     "koa-static": "^1.4.3",
     "koa-logger": "^1.2.0",
     "koa-compress": "^1.0.3",
-    "finished": "^1.1.1"
+    "finished": "^1.1.1",
+    "parse-link-header": "~0.1.0"
   },
   "devDependencies": {
     "supertest": "~0.9.2",