wrench.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532
  1. /* wrench.js
  2. *
  3. * A collection of various utility functions I've found myself in need of
  4. * for use with Node.js (http://nodejs.org/). This includes things like:
  5. *
  6. * - Recursively deleting directories in Node.js (Sync, not Async)
  7. * - Recursively copying directories in Node.js (Sync, not Async)
  8. * - Recursively chmoding a directory structure from Node.js (Sync, not Async)
  9. * - Other things that I'll add here as time goes on. Shhhh...
  10. *
  11. * ~ Ryan McGrath (ryan [at] venodesigns.net)
  12. */
  13. var fs = require("fs"),
  14. _path = require("path"),
  15. isWindows = !!process.platform.match(/^win/);
  16. /* wrench.readdirSyncRecursive("directory_path");
  17. *
  18. * Recursively dives through directories and read the contents of all the
  19. * children directories.
  20. */
  21. exports.readdirSyncRecursive = function(baseDir) {
  22. baseDir = baseDir.replace(/\/$/, '');
  23. var readdirSyncRecursive = function(baseDir) {
  24. var files = [],
  25. curFiles,
  26. nextDirs,
  27. isDir = function(fname){
  28. return fs.existsSync(_path.join(baseDir, fname)) ? fs.statSync( _path.join(baseDir, fname) ).isDirectory() : false;
  29. },
  30. prependBaseDir = function(fname){
  31. return _path.join(baseDir, fname);
  32. };
  33. curFiles = fs.readdirSync(baseDir);
  34. nextDirs = curFiles.filter(isDir);
  35. curFiles = curFiles.map(prependBaseDir);
  36. files = files.concat( curFiles );
  37. while (nextDirs.length) {
  38. files = files.concat( readdirSyncRecursive( _path.join(baseDir, nextDirs.shift()) ) );
  39. }
  40. return files;
  41. };
  42. // convert absolute paths to relative
  43. var fileList = readdirSyncRecursive(baseDir).map(function(val){
  44. return _path.relative(baseDir, val);
  45. });
  46. return fileList;
  47. };
  48. /* wrench.readdirRecursive("directory_path", function(error, files) {});
  49. *
  50. * Recursively dives through directories and read the contents of all the
  51. * children directories.
  52. *
  53. * Asynchronous, so returns results/error in callback.
  54. * Callback receives the of files in currently recursed directory.
  55. * When no more directories are left, callback is called with null for all arguments.
  56. *
  57. */
  58. exports.readdirRecursive = function(baseDir, fn) {
  59. baseDir = baseDir.replace(/\/$/, '');
  60. var waitCount = 0;
  61. function readdirRecursive(curDir) {
  62. var prependcurDir = function(fname){
  63. return _path.join(curDir, fname);
  64. };
  65. waitCount++;
  66. fs.readdir(curDir, function(e, curFiles) {
  67. if (e) {
  68. fn(e);
  69. return;
  70. }
  71. waitCount--;
  72. curFiles = curFiles.map(prependcurDir);
  73. curFiles.forEach(function(it) {
  74. waitCount++;
  75. fs.stat(it, function(e, stat) {
  76. waitCount--;
  77. if (e) {
  78. fn(e);
  79. } else {
  80. if (stat.isDirectory()) {
  81. readdirRecursive(it);
  82. }
  83. }
  84. if (waitCount == 0) {
  85. fn(null, null);
  86. }
  87. });
  88. });
  89. fn(null, curFiles.map(function(val) {
  90. // convert absolute paths to relative
  91. return _path.relative(baseDir, val);
  92. }));
  93. if (waitCount == 0) {
  94. fn(null, null);
  95. }
  96. });
  97. };
  98. readdirRecursive(baseDir);
  99. };
  100. /* wrench.rmdirSyncRecursive("directory_path", failSilent);
  101. *
  102. * Recursively dives through directories and obliterates everything about it. This is a
  103. * Sync-function, which blocks things until it's done. No idea why anybody would want an
  104. * Asynchronous version. :\
  105. */
  106. exports.rmdirSyncRecursive = function(path, failSilent) {
  107. var files;
  108. try {
  109. files = fs.readdirSync(path);
  110. } catch (err) {
  111. if(failSilent) return;
  112. throw new Error(err.message);
  113. }
  114. /* Loop through and delete everything in the sub-tree after checking it */
  115. for(var i = 0; i < files.length; i++) {
  116. var file = _path.join(path, files[i]);
  117. var currFile = fs.lstatSync(file);
  118. if(currFile.isDirectory()) {
  119. // Recursive function back to the beginning
  120. exports.rmdirSyncRecursive(file);
  121. } else if(currFile.isSymbolicLink()) {
  122. // Unlink symlinks
  123. if (isWindows) {
  124. fs.chmodSync(file, 666) // Windows needs this unless joyent/node#3006 is resolved..
  125. }
  126. fs.unlinkSync(file);
  127. } else {
  128. // Assume it's a file - perhaps a try/catch belongs here?
  129. if (isWindows) {
  130. fs.chmodSync(file, 666) // Windows needs this unless joyent/node#3006 is resolved..
  131. }
  132. fs.unlinkSync(file);
  133. }
  134. }
  135. /* Now that we know everything in the sub-tree has been deleted, we can delete the main
  136. directory. Huzzah for the shopkeep. */
  137. return fs.rmdirSync(path);
  138. };
  139. function isFileIncluded(opts, dir, filename) {
  140. function isMatch(filter) {
  141. if (typeof filter === 'function') {
  142. return filter(filename, dir) === true;
  143. }
  144. else {
  145. // Maintain backwards compatibility and use just the filename
  146. return filename.match(filter);
  147. }
  148. }
  149. if (opts.include || opts.exclude) {
  150. if (opts.exclude) {
  151. if (isMatch(opts.exclude)) {
  152. return false;
  153. }
  154. }
  155. if (opts.include) {
  156. if (isMatch(opts.include)) {
  157. return true;
  158. }
  159. else {
  160. return false;
  161. }
  162. }
  163. return true;
  164. }
  165. else if (opts.filter) {
  166. var filter = opts.filter;
  167. if (!opts.whitelist) {
  168. // if !opts.whitelist is false every file or directory
  169. // which does match opts.filter will be ignored
  170. return isMatch(filter) ? false : true;
  171. } else {
  172. // if opts.whitelist is true every file or directory
  173. // which doesn't match opts.filter will be ignored
  174. return !isMatch(filter) ? false : true;
  175. }
  176. }
  177. return true;
  178. }
  179. /* wrench.copyDirSyncRecursive("directory_to_copy", "new_directory_location", opts);
  180. *
  181. * Recursively dives through a directory and moves all its files to a new location. This is a
  182. * Synchronous function, which blocks things until it's done. If you need/want to do this in
  183. * an Asynchronous manner, look at wrench.copyDirRecursively() below. Specify forceDelete to force directory overwrite.
  184. *
  185. * Note: Directories should be passed to this function without a trailing slash.
  186. */
  187. exports.copyDirSyncRecursive = function(sourceDir, newDirLocation, opts) {
  188. opts = opts || {};
  189. try {
  190. if(fs.statSync(newDirLocation).isDirectory()) {
  191. if(opts.forceDelete) {
  192. exports.rmdirSyncRecursive(newDirLocation);
  193. } else if(!opts.preserveFiles) {
  194. return new Error('You are trying to copy a directory onto a directory that already exists. Specify forceDelete or preserveFiles in the opts argument to specify desired behavior');
  195. }
  196. }
  197. } catch(e) { }
  198. /* Create the directory where all our junk is moving to; read the mode of the source directory and mirror it */
  199. var checkDir = fs.statSync(sourceDir);
  200. try {
  201. fs.mkdirSync(newDirLocation, checkDir.mode);
  202. } catch (e) {
  203. //if the directory already exists, that's okay
  204. if (e.code !== 'EEXIST') throw e;
  205. }
  206. var files = fs.readdirSync(sourceDir);
  207. var hasFilter = opts.filter || opts.include || opts.exclude;
  208. var preserveFiles = opts.preserveFiles === true;
  209. var preserveTimestamps = opts.preserveTimestamps === true;
  210. for(var i = 0; i < files.length; i++) {
  211. // ignores all files or directories which match the RegExp in opts.filter
  212. if(typeof opts !== 'undefined') {
  213. if (hasFilter) {
  214. if (!isFileIncluded(opts, sourceDir, files[i])) {
  215. continue;
  216. }
  217. }
  218. if (opts.excludeHiddenUnix && /^\./.test(files[i])) continue;
  219. }
  220. var currFile = fs.lstatSync(_path.join(sourceDir, files[i]));
  221. var fCopyFile = function(srcFile, destFile) {
  222. if(typeof opts !== 'undefined' && opts.preserveFiles && fs.existsSync(destFile)) return;
  223. var contents = fs.readFileSync(srcFile);
  224. fs.writeFileSync(destFile, contents);
  225. var stat = fs.lstatSync(srcFile);
  226. fs.chmodSync(destFile, stat.mode);
  227. if (preserveTimestamps) {
  228. fs.utimesSync(destFile, stat.atime, stat.mtime)
  229. }
  230. };
  231. if(currFile.isDirectory()) {
  232. /* recursion this thing right on back. */
  233. exports.copyDirSyncRecursive(_path.join(sourceDir, files[i]), _path.join(newDirLocation, files[i]), opts);
  234. } else if(currFile.isSymbolicLink()) {
  235. var symlinkFull = fs.readlinkSync(_path.join(sourceDir, files[i]));
  236. symlinkFull = _path.resolve(fs.realpathSync(sourceDir), symlinkFull);
  237. if (typeof opts !== 'undefined' && !opts.inflateSymlinks) {
  238. fs.symlinkSync(symlinkFull, _path.join(newDirLocation, files[i]));
  239. continue;
  240. }
  241. var tmpCurrFile = fs.lstatSync(symlinkFull);
  242. if (tmpCurrFile.isDirectory()) {
  243. exports.copyDirSyncRecursive(symlinkFull, _path.join(newDirLocation, files[i]), opts);
  244. } else {
  245. /* At this point, we've hit a file actually worth copying... so copy it on over. */
  246. fCopyFile(symlinkFull, _path.join(newDirLocation, files[i]));
  247. }
  248. } else {
  249. /* At this point, we've hit a file actually worth copying... so copy it on over. */
  250. fCopyFile(_path.join(sourceDir, files[i]), _path.join(newDirLocation, files[i]));
  251. }
  252. }
  253. };
  254. /* wrench.chmodSyncRecursive("directory", filemode);
  255. *
  256. * Recursively dives through a directory and chmods everything to the desired mode. This is a
  257. * Synchronous function, which blocks things until it's done.
  258. *
  259. * Note: Directories should be passed to this function without a trailing slash.
  260. */
  261. exports.chmodSyncRecursive = function(sourceDir, filemode) {
  262. var files = fs.readdirSync(sourceDir);
  263. for(var i = 0; i < files.length; i++) {
  264. var currFile = fs.lstatSync(_path.join(sourceDir, files[i]));
  265. if(currFile.isDirectory()) {
  266. /* ...and recursion this thing right on back. */
  267. exports.chmodSyncRecursive(_path.join(sourceDir, files[i]), filemode);
  268. } else {
  269. /* At this point, we've hit a file actually worth copying... so copy it on over. */
  270. fs.chmod(_path.join(sourceDir, files[i]), filemode);
  271. }
  272. }
  273. /* Finally, chmod the parent directory */
  274. fs.chmod(sourceDir, filemode);
  275. };
  276. /* wrench.chownSyncRecursive("directory", uid, gid);
  277. *
  278. * Recursively dives through a directory and chowns everything to the desired user and group. This is a
  279. * Synchronous function, which blocks things until it's done.
  280. *
  281. * Note: Directories should be passed to this function without a trailing slash.
  282. */
  283. exports.chownSyncRecursive = function(sourceDir, uid, gid) {
  284. var files = fs.readdirSync(sourceDir);
  285. for(var i = 0; i < files.length; i++) {
  286. var currFile = fs.lstatSync(_path.join(sourceDir, files[i]));
  287. if(currFile.isDirectory()) {
  288. /* ...and recursion this thing right on back. */
  289. exports.chownSyncRecursive(_path.join(sourceDir, files[i]), uid, gid);
  290. } else {
  291. /* At this point, we've hit a file actually worth chowning... so own it. */
  292. fs.chownSync(_path.join(sourceDir, files[i]), uid, gid);
  293. }
  294. }
  295. /* Finally, chown the parent directory */
  296. fs.chownSync(sourceDir, uid, gid);
  297. };
  298. /* wrench.rmdirRecursive("directory_path", callback);
  299. *
  300. * Recursively dives through directories and obliterates everything about it.
  301. */
  302. exports.rmdirRecursive = function rmdirRecursive(dir, failSilent, clbk){
  303. if(clbk === null || typeof clbk == 'undefined')
  304. clbk = function(err) {};
  305. fs.readdir(dir, function(err, files) {
  306. if(err && typeof failSilent === 'boolean' && !failSilent)
  307. return clbk(err);
  308. if(typeof failSilent === 'function')
  309. clbk = failSilent;
  310. (function rmFile(err){
  311. if (err) return clbk(err);
  312. var filename = files.shift();
  313. if (filename === null || typeof filename == 'undefined')
  314. return fs.rmdir(dir, clbk);
  315. var file = dir+'/'+filename;
  316. fs.lstat(file, function(err, stat){
  317. if (err) return clbk(err);
  318. if (stat.isDirectory())
  319. rmdirRecursive(file, rmFile);
  320. else
  321. fs.unlink(file, rmFile);
  322. });
  323. })();
  324. });
  325. };
  326. /* wrench.copyDirRecursive("directory_to_copy", "new_location", {forceDelete: bool}, callback);
  327. *
  328. * Recursively dives through a directory and moves all its files to a new
  329. * location. Specify forceDelete to force directory overwrite.
  330. *
  331. * Note: Directories should be passed to this function without a trailing slash.
  332. */
  333. exports.copyDirRecursive = function copyDirRecursive(srcDir, newDir, opts, clbk) {
  334. var originalArguments = Array.prototype.slice.apply(arguments);
  335. srcDir = _path.normalize(srcDir);
  336. newDir = _path.normalize(newDir);
  337. fs.stat(newDir, function(err, newDirStat) {
  338. if(!err) {
  339. if(typeof opts !== 'undefined' && typeof opts !== 'function' && opts.forceDelete)
  340. return exports.rmdirRecursive(newDir, function(err) {
  341. copyDirRecursive.apply(this, originalArguments);
  342. });
  343. else if(typeof opts !== 'undefined' && typeof opts !== 'function' && !opts.preserveFiles) {
  344. return clbk(new Error('You are trying to copy a directory onto a directory that already exists. Specify forceDelete or preserveFiles in the opts argument to specify desired behavior'));
  345. }
  346. }
  347. if(typeof opts === 'function')
  348. clbk = opts;
  349. fs.stat(srcDir, function(err, srcDirStat){
  350. if (err) return clbk(err);
  351. fs.mkdir(newDir, srcDirStat.mode, function(err){
  352. if (err) return clbk(err);
  353. fs.readdir(srcDir, function(err, files){
  354. if (err) return clbk(err);
  355. (function copyFiles(err){
  356. if (err) return clbk(err);
  357. var filename = files.shift();
  358. if (filename === null || typeof filename == 'undefined')
  359. return clbk(null);
  360. var file = srcDir+'/'+filename,
  361. newFile = newDir+'/'+filename;
  362. fs.stat(file, function(err, fileStat){
  363. if (err) return clbk(err);
  364. if (fileStat.isDirectory())
  365. copyDirRecursive(file, newFile, copyFiles, clbk);
  366. else if (fileStat.isSymbolicLink())
  367. fs.readlink(file, function(err, link){
  368. if (err) return clbk(err);
  369. fs.symlink(link, newFile, copyFiles);
  370. });
  371. else
  372. fs.readFile(file, function(err, data){
  373. if (err) return clbk(err);
  374. fs.writeFile(newFile, data, copyFiles);
  375. });
  376. });
  377. })();
  378. });
  379. });
  380. });
  381. });
  382. };
  383. var mkdirSyncRecursive = function(path, mode) {
  384. var self = this;
  385. path = _path.normalize(path)
  386. try {
  387. fs.mkdirSync(path, mode);
  388. } catch(err) {
  389. if(err.code == "ENOENT") {
  390. var slashIdx = path.lastIndexOf(_path.sep);
  391. if(slashIdx > 0) {
  392. var parentPath = path.substring(0, slashIdx);
  393. mkdirSyncRecursive(parentPath, mode);
  394. mkdirSyncRecursive(path, mode);
  395. } else {
  396. throw err;
  397. }
  398. } else if(err.code == "EEXIST") {
  399. return;
  400. } else {
  401. throw err;
  402. }
  403. }
  404. };
  405. exports.mkdirSyncRecursive = mkdirSyncRecursive;
  406. exports.LineReader = function(filename, bufferSize) {
  407. this.bufferSize = bufferSize || 8192;
  408. this.buffer = "";
  409. this.fd = fs.openSync(filename, "r");
  410. this.currentPosition = 0;
  411. };
  412. exports.LineReader.prototype = {
  413. close: function() {
  414. return fs.closeSync(this.fd);
  415. },
  416. getBufferAndSetCurrentPosition: function(position) {
  417. var res = fs.readSync(this.fd, this.bufferSize, position, "ascii");
  418. this.buffer += res[0];
  419. if(res[1] === 0) {
  420. this.currentPosition = -1;
  421. } else {
  422. this.currentPosition = position + res[1];
  423. }
  424. return this.currentPosition;
  425. },
  426. hasNextLine: function() {
  427. while(this.buffer.indexOf('\n') === -1) {
  428. this.getBufferAndSetCurrentPosition(this.currentPosition);
  429. if(this.currentPosition === -1) return false;
  430. }
  431. if(this.buffer.indexOf("\n") > -1 || this.buffer.length !== 0) return true;
  432. return false;
  433. },
  434. getNextLine: function() {
  435. var lineEnd = this.buffer.indexOf("\n"),
  436. result = this.buffer.substring(0, lineEnd != -1 ? lineEnd : this.buffer.length);
  437. this.buffer = this.buffer.substring(result.length + 1, this.buffer.length);
  438. return result;
  439. }
  440. };
  441. // vim: et ts=4 sw=4