From 504ba139c192f81058fe5797fc5f487149817368 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Tue, 26 Jul 2016 14:17:50 -0700 Subject: [PATCH 1/4] Generate script: Don't write .cc or .h files if unchanged The generation script now checks whether the file has changed before writing it to `/src` or `/include`. This is to improve compilation time when testing changes to generated code. It works by creating a `/temp` directory, writing the generated code to `/temp/src` and `/temp/include`, then syncing those folders with `/src` and `/include` by deleting files that no longer exist and copying files that have changed or been added since the last code generation. Finally the `/temp` directory is deleted If `/src` and `/include` don't exist (i.e. it's the first time the generation script has been run), `/temp/src` will be copied to `/src` and `/temp/include` will be copied to `/include`. --- .gitignore | 1 + generate/scripts/generateMissingTests.js | 2 +- generate/scripts/generateNativeCode.js | 71 +++++++++++--------- generate/scripts/utils.js | 84 +++++++++++++++++++++++- package.json | 3 +- 5 files changed, 127 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 5529ab114..82837852e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /lib/nodegit.js /node_modules/ /src/ +/temp/ /test/coverage/ /test/home/ /test/repos/ diff --git a/generate/scripts/generateMissingTests.js b/generate/scripts/generateMissingTests.js index 4aef70f4d..bfd93aec0 100644 --- a/generate/scripts/generateMissingTests.js +++ b/generate/scripts/generateMissingTests.js @@ -12,7 +12,7 @@ module.exports = function generateMissingTests() { var testFilePath = path.join(testFilesPath, idef.filename + ".js"); var result = {}; - var file = utils.readFile(testFilePath); + var file = utils.readLocalFile(testFilePath); if (file) { var fieldsResult = []; var functionsResult = []; diff --git a/generate/scripts/generateNativeCode.js b/generate/scripts/generateNativeCode.js index 56d52672c..9088fb72e 100644 --- a/generate/scripts/generateNativeCode.js +++ b/generate/scripts/generateNativeCode.js @@ -24,27 +24,27 @@ module.exports = function generateNativeCode() { }; var partials = { - asyncFunction: utils.readFile("templates/partials/async_function.cc"), - callbackHelpers: utils.readFile("templates/partials/callback_helpers.cc"), - convertFromV8: utils.readFile("templates/partials/convert_from_v8.cc"), - convertToV8: utils.readFile("templates/partials/convert_to_v8.cc"), - doc: utils.readFile("templates/partials/doc.cc"), - fields: utils.readFile("templates/partials/fields.cc"), - guardArguments: utils.readFile("templates/partials/guard_arguments.cc"), - syncFunction: utils.readFile("templates/partials/sync_function.cc"), - fieldAccessors: utils.readFile("templates/partials/field_accessors.cc"), - traits: utils.readFile("templates/partials/traits.h") + asyncFunction: utils.readLocalFile("templates/partials/async_function.cc"), + callbackHelpers: utils.readLocalFile("templates/partials/callback_helpers.cc"), + convertFromV8: utils.readLocalFile("templates/partials/convert_from_v8.cc"), + convertToV8: utils.readLocalFile("templates/partials/convert_to_v8.cc"), + doc: utils.readLocalFile("templates/partials/doc.cc"), + fields: utils.readLocalFile("templates/partials/fields.cc"), + guardArguments: utils.readLocalFile("templates/partials/guard_arguments.cc"), + syncFunction: utils.readLocalFile("templates/partials/sync_function.cc"), + fieldAccessors: utils.readLocalFile("templates/partials/field_accessors.cc"), + traits: utils.readLocalFile("templates/partials/traits.h") }; var templates = { - class_content: utils.readFile("templates/templates/class_content.cc"), - struct_content: utils.readFile("templates/templates/struct_content.cc"), - class_header: utils.readFile("templates/templates/class_header.h"), - struct_header: utils.readFile("templates/templates/struct_header.h"), - binding: utils.readFile("templates/templates/binding.gyp"), - nodegitCC: utils.readFile("templates/templates/nodegit.cc"), - nodegitJS: utils.readFile("templates/templates/nodegit.js"), - enums: utils.readFile("templates/templates/enums.js") + class_content: utils.readLocalFile("templates/templates/class_content.cc"), + struct_content: utils.readLocalFile("templates/templates/struct_content.cc"), + class_header: utils.readLocalFile("templates/templates/class_header.h"), + struct_header: utils.readLocalFile("templates/templates/struct_header.h"), + binding: utils.readLocalFile("templates/templates/binding.gyp"), + nodegitCC: utils.readLocalFile("templates/templates/nodegit.cc"), + nodegitJS: utils.readLocalFile("templates/templates/nodegit.js"), + enums: utils.readLocalFile("templates/templates/enums.js") }; var filters = { @@ -99,28 +99,32 @@ module.exports = function generateNativeCode() { return !idef.ignore; }); + const tempDirPath = path.resolve(__dirname, "../../temp"); + const tempSrcDirPath = path.join(tempDirPath, "src"); + const tempIncludeDirPath = path.join(tempDirPath, "include"); - fse.remove(path.resolve(__dirname, "../../src")).then(function() { - return fse.remove(path.resolve(__dirname, "../../include")); - }).then(function() { - return fse.copy(path.resolve(__dirname, "../templates/manual/include"), path.resolve(__dirname, "../../include")); + const finalSrcDirPath = path.join(__dirname, '../../src'); + const finalIncludeDirPath = path.join(__dirname, '../../include'); + + fse.remove(tempDirPath).then(function() { + return fse.copy(path.resolve(__dirname, "../templates/manual/include"), tempIncludeDirPath); }).then(function() { - return fse.copy(path.resolve(__dirname, "../templates/manual/src"), path.resolve(__dirname, "../../src")); + return fse.copy(path.resolve(__dirname, "../templates/manual/src"), tempSrcDirPath); }).then(function() { // Write out single purpose templates. utils.writeFile("../binding.gyp", beautify(templates.binding.render(enabled)), "binding.gyp"); - utils.writeFile("../src/nodegit.cc", templates.nodegitCC.render(enabled), "nodegit.cc"); + utils.writeFile("../temp/src/nodegit.cc", templates.nodegitCC.render(enabled), "nodegit.cc"); utils.writeFile("../lib/nodegit.js", beautify(templates.nodegitJS.render(enabled)), "nodegit.js"); // Write out all the classes. enabled.forEach(function(idef) { if (idef.type && idef.type != "enum") { utils.writeFile( - "../src/" + idef.filename + ".cc", + "../temp/src/" + idef.filename + ".cc", templates[idef.type + "_content"].render(idef), idef.type + "_content.cc" ); utils.writeFile( - "../include/" + idef.filename + ".h", + "../temp/include/" + idef.filename + ".h", templates[idef.type + "_header"].render(idef), idef.type + "_header.h" ); @@ -133,17 +137,24 @@ module.exports = function generateNativeCode() { if (astyle) { return exec( "astyle --options=\".astylerc\" " - + path.resolve(__dirname, "../../src") + "/*.cc " - + path.resolve(__dirname, "../../include") + "/*.h" + + tempSrcDirPath + "/*.cc " + + tempIncludeDirPath + "/*.h" ).then(function() { return exec( "rm " - + path.resolve(__dirname, "../../src") + "/*.cc.orig " - + path.resolve(__dirname, "../../include") + "/*.h.orig " + + tempSrcDirPath + "/*.cc.orig " + + tempIncludeDirPath + "/*.h.orig " ); }); } }, function() {}) + }).then(function() { + return Promise.all([ + utils.syncDirs(tempSrcDirPath, finalSrcDirPath), + utils.syncDirs(tempIncludeDirPath, finalIncludeDirPath), + ]); + }).then(function() { + return fse.remove(tempDirPath); }).catch(console.log); }; diff --git a/generate/scripts/utils.js b/generate/scripts/utils.js index 6868bf653..ef92f1815 100644 --- a/generate/scripts/utils.js +++ b/generate/scripts/utils.js @@ -1,4 +1,5 @@ const fse = require("fs-extra"); +const walk = require("walk"); const fs = require("fs"); const path = require("path"); @@ -10,7 +11,7 @@ var util = { pointerRegex: /\s*\*\s*/, doublePointerRegex: /\s*\*\*\s*/, - readFile: function(file) { + readLocalFile: function(file) { try { return fs.readFileSync(local(file)).toString(); } @@ -19,6 +20,15 @@ var util = { } }, + readFile: function(filePath) { + try { + return fs.readFileSync(filePath).toString(); + } + catch (unhandledException) { + return ""; + } + }, + writeFile: function(file, content, header) { try { var file = local(file); @@ -62,14 +72,84 @@ var util = { }).join(""); }, + getFilePathsRelativeToDir: function(dir) { + const files = []; + const walker = walk.walk(dir, { followLinks: false }); + if (!util.isDirectory(dir)) { + return Promise.resolve([]); + } + + return new Promise(function(resolve, reject) { + walker.on('file', function(root, stat, next) { + files.push(path.relative(dir, path.join(root, stat.name))); + next(); + }); + + walker.on('end', function() { + resolve(files); + }); + + walker.on('errors', function() { + reject(); + }); + }); + }, + + isFile: function(path) { + var isFile; + try { + isFile = fse.statSync(path).isFile(); + } catch(e) { + isFile = false; + } + + return isFile; + }, + + isDirectory: function(path) { + var isDirectory; + try { + isDirectory = fse.statSync(path).isDirectory(); + } catch(e) { + isDirectory = false; + } + + return isDirectory; + }, + isPointer: function(type) { return util.pointerRegex.test(type) || util.doublePointerRegex.test(type); }, isDoublePointer: function(type) { return util.doublePointerRegex.test(type); - } + }, + syncDirs: function(fromDir, toDir) { + return Promise.all([ + util.getFilePathsRelativeToDir(toDir), + util.getFilePathsRelativeToDir(fromDir) + ]).then(function(filePaths) { + const toFilePaths = filePaths[0]; + const fromFilePaths = filePaths[1]; + + // Delete files that aren't in fromDir + toFilePaths.forEach(function(filePath) { + if (!util.isFile(path.join(fromDir, filePath))) { + fse.remove(path.join(toDir, filePath)); + } + }); + + // Copy files that don't exist in toDir or have different contents + fromFilePaths.forEach(function(filePath) { + const toFilePath = path.join(toDir, filePath); + const fromFilePath = path.join(fromDir, filePath); + if (!util.isFile(toFilePath) || util.readFile(toFilePath) !== util.readFile(fromFilePath)) { + fse.copy(fromFilePath, toFilePath); + } + }); + }); + } }; module.exports = util; diff --git a/package.json b/package.json index 99a566a81..c8e06853a 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "js-beautify": "~1.5.10", "jshint": "~2.8.0", "lcov-result-merger": "~1.0.2", - "mocha": "~2.3.4" + "mocha": "~2.3.4", + "walk": "^2.3.9" }, "vendorDependencies": { "libssh2": "1.7.0", From 880d2d1765644cb73be9b455c8c1c52a2ca5c67a Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Sat, 12 Nov 2016 15:50:04 -0700 Subject: [PATCH 2/4] Generate script: write temp generated files to OS specific temp dir --- .gitignore | 1 - generate/scripts/generateJson.js | 3 +-- generate/scripts/generateMissingTests.js | 2 +- generate/scripts/generateNativeCode.js | 10 ++++++---- generate/scripts/utils.js | 7 +++---- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 82837852e..5529ab114 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ /lib/nodegit.js /node_modules/ /src/ -/temp/ /test/coverage/ /test/home/ /test/repos/ diff --git a/generate/scripts/generateJson.js b/generate/scripts/generateJson.js index f1afeb956..09bc901a1 100644 --- a/generate/scripts/generateJson.js +++ b/generate/scripts/generateJson.js @@ -231,8 +231,7 @@ module.exports = function generateJson() { } - utils.writeFile("output/idefs.json", output); - + utils.writeFile(path.join(__dirname, "..", "output", "idefs.json"), output); }; if (require.main === module) { diff --git a/generate/scripts/generateMissingTests.js b/generate/scripts/generateMissingTests.js index bfd93aec0..669f6929c 100644 --- a/generate/scripts/generateMissingTests.js +++ b/generate/scripts/generateMissingTests.js @@ -58,7 +58,7 @@ module.exports = function generateMissingTests() { Promise.all(promises).then( function() { - utils.writeFile("output/missing-tests.json", output); + utils.writeFile(path.join(__dirname, "..", "/output/missing-tests.json"), output); }, function(fail) { console.error(fail); diff --git a/generate/scripts/generateNativeCode.js b/generate/scripts/generateNativeCode.js index 9088fb72e..3cd19c93b 100644 --- a/generate/scripts/generateNativeCode.js +++ b/generate/scripts/generateNativeCode.js @@ -1,6 +1,7 @@ const path = require("path"); const promisify = require("promisify-node"); const fse = promisify(require("fs-extra")); +const os = require('os'); const exec = require('../../utils/execPromise'); const utils = require("./utils"); @@ -99,7 +100,7 @@ module.exports = function generateNativeCode() { return !idef.ignore; }); - const tempDirPath = path.resolve(__dirname, "../../temp"); + const tempDirPath = path.join(os.tmpdir(), 'nodegit_build'); const tempSrcDirPath = path.join(tempDirPath, "src"); const tempIncludeDirPath = path.join(tempDirPath, "include"); @@ -113,18 +114,19 @@ module.exports = function generateNativeCode() { }).then(function() { // Write out single purpose templates. utils.writeFile("../binding.gyp", beautify(templates.binding.render(enabled)), "binding.gyp"); - utils.writeFile("../temp/src/nodegit.cc", templates.nodegitCC.render(enabled), "nodegit.cc"); + utils.writeFile(path.join(tempSrcDirPath, "nodegit.cc"), templates.nodegitCC.render(enabled), "nodegit.cc"); utils.writeFile("../lib/nodegit.js", beautify(templates.nodegitJS.render(enabled)), "nodegit.js"); // Write out all the classes. enabled.forEach(function(idef) { if (idef.type && idef.type != "enum") { utils.writeFile( - "../temp/src/" + idef.filename + ".cc", + path.format({dir: tempSrcDirPath, name: idef.filename, ext: ".cc"}), templates[idef.type + "_content"].render(idef), idef.type + "_content.cc" ); + utils.writeFile( - "../temp/include/" + idef.filename + ".h", + path.format({dir: tempIncludeDirPath, name: idef.filename, ext: ".h"}), templates[idef.type + "_header"].render(idef), idef.type + "_header.h" ); diff --git a/generate/scripts/utils.js b/generate/scripts/utils.js index ef92f1815..df4130bf9 100644 --- a/generate/scripts/utils.js +++ b/generate/scripts/utils.js @@ -29,9 +29,8 @@ var util = { } }, - writeFile: function(file, content, header) { + writeFile: function(filePath, content, header) { try { - var file = local(file); if (typeof content == "object") { content = JSON.stringify(content, null, 2) } @@ -45,8 +44,8 @@ var util = { content; } - fse.ensureFileSync(file); - fse.writeFileSync(file, content); + fse.ensureFileSync(filePath); + fse.writeFileSync(filePath, content); return true; } catch (exception) { From 272c37b800c78dda01889e5495cbe644b3b907f8 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Sat, 12 Nov 2016 16:32:10 -0700 Subject: [PATCH 3/4] Generate utils: separated writeFile into writeFile and writeLocalFile --- generate/scripts/generateJson.js | 2 +- generate/scripts/generateMissingTests.js | 2 +- generate/scripts/generateNativeCode.js | 6 +++--- generate/scripts/utils.js | 13 ++++++------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/generate/scripts/generateJson.js b/generate/scripts/generateJson.js index 09bc901a1..323247e13 100644 --- a/generate/scripts/generateJson.js +++ b/generate/scripts/generateJson.js @@ -231,7 +231,7 @@ module.exports = function generateJson() { } - utils.writeFile(path.join(__dirname, "..", "output", "idefs.json"), output); + utils.writeLocalFile("output/idefs.json", output); }; if (require.main === module) { diff --git a/generate/scripts/generateMissingTests.js b/generate/scripts/generateMissingTests.js index 669f6929c..29b6474b3 100644 --- a/generate/scripts/generateMissingTests.js +++ b/generate/scripts/generateMissingTests.js @@ -58,7 +58,7 @@ module.exports = function generateMissingTests() { Promise.all(promises).then( function() { - utils.writeFile(path.join(__dirname, "..", "/output/missing-tests.json"), output); + utils.writeLocalFile("/output/missing-tests.json", output); }, function(fail) { console.error(fail); diff --git a/generate/scripts/generateNativeCode.js b/generate/scripts/generateNativeCode.js index 3cd19c93b..3775a43e9 100644 --- a/generate/scripts/generateNativeCode.js +++ b/generate/scripts/generateNativeCode.js @@ -113,9 +113,9 @@ module.exports = function generateNativeCode() { return fse.copy(path.resolve(__dirname, "../templates/manual/src"), tempSrcDirPath); }).then(function() { // Write out single purpose templates. - utils.writeFile("../binding.gyp", beautify(templates.binding.render(enabled)), "binding.gyp"); + utils.writeLocalFile("../binding.gyp", beautify(templates.binding.render(enabled)), "binding.gyp"); utils.writeFile(path.join(tempSrcDirPath, "nodegit.cc"), templates.nodegitCC.render(enabled), "nodegit.cc"); - utils.writeFile("../lib/nodegit.js", beautify(templates.nodegitJS.render(enabled)), "nodegit.js"); + utils.writeLocalFile("../lib/nodegit.js", beautify(templates.nodegitJS.render(enabled)), "nodegit.js"); // Write out all the classes. enabled.forEach(function(idef) { if (idef.type && idef.type != "enum") { @@ -133,7 +133,7 @@ module.exports = function generateNativeCode() { } }); - utils.writeFile("../lib/enums.js", beautify(templates.enums.render(enabled)), "enums.js"); + utils.writeLocalFile("../lib/enums.js", beautify(templates.enums.render(enabled)), "enums.js"); }).then(function() { return exec("command -v astyle").then(function(astyle) { if (astyle) { diff --git a/generate/scripts/utils.js b/generate/scripts/utils.js index df4130bf9..385274400 100644 --- a/generate/scripts/utils.js +++ b/generate/scripts/utils.js @@ -11,13 +11,8 @@ var util = { pointerRegex: /\s*\*\s*/, doublePointerRegex: /\s*\*\*\s*/, - readLocalFile: function(file) { - try { - return fs.readFileSync(local(file)).toString(); - } - catch (unhandledException) { - return ""; - } + readLocalFile: function(filePath) { + return util.readFile(local(filePath)); }, readFile: function(filePath) { @@ -29,6 +24,10 @@ var util = { } }, + writeLocalFile: function(filePath, content, header) { + return util.writeFile(local(filePath), content, header); + }, + writeFile: function(filePath, content, header) { try { if (typeof content == "object") { From 02060bfbe029875486c2c62853866acb386c7d3f Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Sat, 12 Nov 2016 17:20:08 -0700 Subject: [PATCH 4/4] Remove path.format since it's not available in all node versions --- generate/scripts/generateNativeCode.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/scripts/generateNativeCode.js b/generate/scripts/generateNativeCode.js index 3775a43e9..a82bef4e5 100644 --- a/generate/scripts/generateNativeCode.js +++ b/generate/scripts/generateNativeCode.js @@ -120,13 +120,13 @@ module.exports = function generateNativeCode() { enabled.forEach(function(idef) { if (idef.type && idef.type != "enum") { utils.writeFile( - path.format({dir: tempSrcDirPath, name: idef.filename, ext: ".cc"}), + path.join(tempSrcDirPath, idef.filename + ".cc"), templates[idef.type + "_content"].render(idef), idef.type + "_content.cc" ); utils.writeFile( - path.format({dir: tempIncludeDirPath, name: idef.filename, ext: ".h"}), + path.join(tempIncludeDirPath, idef.filename + ".h"), templates[idef.type + "_header"].render(idef), idef.type + "_header.h" );