diff --git a/.travis.yml b/.travis.yml index e20bedc3..694a62f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: node_js node_js: - - 0.8 - - "0.10" - - 0.11 + - 4 + - 6 + - 7 diff --git a/Readme.md b/Readme.md index e103c3dd..bbe73553 100644 --- a/Readme.md +++ b/Readme.md @@ -1,13 +1,15 @@ # Formidable -[![Build Status](https://secure.travis-ci.org/felixge/node-formidable.png?branch=master)](http://travis-ci.org/felixge/node-formidable) +[![Build Status](https://travis-ci.org/felixge/node-formidable.svg?branch=master)](https://travis-ci.org/felixge/node-formidable) ## Purpose -A node.js module for parsing form data, especially file uploads. +A Node.js module for parsing form data, especially file uploads. ## Current status +**Maintainers Wanted:** Please see https://github.com/felixge/node-formidable/issues/412 + This module was developed for [Transloadit](http://transloadit.com/), a service focused on uploading and encoding images and videos. It has been battle-tested against hundreds of GB of file uploads from a large variety of clients and is considered production-ready. @@ -22,19 +24,12 @@ a large variety of clients and is considered production-ready. ## Installation -This is a low level package, and if you're using a high level framework such as Express, chances are it's already included in it. You can [read this discussion](http://stackoverflow.com/questions/11295554/how-to-disable-express-bodyparser-for-file-uploads-node-js) about how Formidable is integrated with Express. - -Via [npm](http://github.com/isaacs/npm): -``` -npm install formidable@latest -``` -Manually: -``` -git clone git://github.com/felixge/node-formidable.git formidable -vim my.js -# var formidable = require('./formidable'); +```sh +npm i -S formidable ``` +This is a low-level package, and if you're using a high-level framework it may already be included. However, [Express v4](http://expressjs.com) does not include any multipart handling, nor does [body-parser](https://github.com/expressjs/body-parser). + Note: Formidable requires [gently](http://github.com/felixge/node-gently) to run the unit tests, but you won't need it for just using the library. ## Example @@ -87,7 +82,7 @@ Sets encoding for incoming form fields. form.uploadDir = "/my/dir"; ``` Sets the directory for placing file uploads in. You can move them later on using -`fs.rename()`. The default is `os.tmpDir()`. +`fs.rename()`. The default is `os.tmpdir()`. ```javascript form.keepExtensions = false; @@ -204,15 +199,20 @@ If hash calculation was set, you can read the hex digest out of this var. #### 'progress' + +Emitted after each incoming chunk of data that has been parsed. Can be used to roll your own progress bar. + ```javascript form.on('progress', function(bytesReceived, bytesExpected) { }); ``` -Emitted after each incoming chunk of data that has been parsed. Can be used to roll your own progress bar. #### 'field' + +Emitted whenever a field / value pair has been received. + ```javascript form.on('field', function(name, value) { }); @@ -220,7 +220,10 @@ form.on('field', function(name, value) { #### 'fileBegin' -Emitted whenever a field / value pair has been received. +Emitted whenever a new file is detected in the upload stream. Use this event if +you want to stream the file to somewhere else while buffering the upload on +the file system. + ```javascript form.on('fileBegin', function(name, file) { }); @@ -228,11 +231,8 @@ form.on('fileBegin', function(name, file) { #### 'file' -Emitted whenever a new file is detected in the upload stream. Use this even if -you want to stream the file to somewhere else while buffering the upload on -the file system. - Emitted whenever a field / file pair has been received. `file` is an instance of `File`. + ```javascript form.on('file', function(name, file) { }); @@ -241,6 +241,7 @@ form.on('file', function(name, file) { #### 'error' Emitted when there is an error processing the incoming form. A request that experiences an error is automatically paused, you will have to manually call `request.resume()` if you want the request to continue firing `'data'` events. + ```javascript form.on('error', function(err) { }); @@ -266,7 +267,18 @@ Emitted when the entire request has been received, and all contained files have ## Changelog -### v1.0.14 +### v1.1.1 (2017-01-15) + + * Fix DeprecationWarning about os.tmpDir() (Christian) + * Update `buffer.write` order of arguments for Node 7 (Kornel Lesiński) + * JSON Parser emits error events to the IncomingForm (alessio.montagnani) + * Improved Content-Disposition parsing (Sebastien) + * Access WriteStream of fs during runtime instead of include time (Jonas Amundsen) + * Use built-in toString to convert buffer to hex (Charmander) + * Add hash to json if present (Nick Stamas) + * Add license to package.json (Simen Bekkhus) + +### v1.0.14 (2013-05-03) * Add failing hash tests. (Ben Trask) * Enable hash calculation again (Eugene Girshov) @@ -298,120 +310,12 @@ Emitted when the entire request has been received, and all contained files have * Remove support for Node.js 0.4 & 0.6 (Andrew Kelley) * Documentation improvements (Sven Lito, Andre Azevedo) * Add support for application/octet-stream (Ion Lupascu, Chris Scribner) -* Use os.tmpDir() to get tmp directory (Andrew Kelley) +* Use os.tmpdir() to get tmp directory (Andrew Kelley) * Improve package.json (Andrew Kelley, Sven Lito) * Fix benchmark script (Andrew Kelley) * Fix scope issue in incoming_forms (Sven Lito) * Fix file handle leak on error (OrangeDog) -### v1.0.11 - -* Calculate checksums for incoming files (sreuter) -* Add definition parameters to "IncomingForm" as an argument (Math-) - -### v1.0.10 - -* Make parts to be proper Streams (Matt Robenolt) - -### v1.0.9 - -* Emit progress when content length header parsed (Tim Koschützki) -* Fix Readme syntax due to GitHub changes (goob) -* Replace references to old 'sys' module in Readme with 'util' (Peter Sugihara) - -### v1.0.8 - -* Strip potentially unsafe characters when using `keepExtensions: true`. -* Switch to utest / urun for testing -* Add travis build - -### v1.0.7 - -* Remove file from package that was causing problems when installing on windows. (#102) -* Fix typos in Readme (Jason Davies). - -### v1.0.6 - -* Do not default to the default to the field name for file uploads where - filename="". - -### v1.0.5 - -* Support filename="" in multipart parts -* Explain unexpected end() errors in parser better - -**Note:** Starting with this version, formidable emits 'file' events for empty -file input fields. Previously those were incorrectly emitted as regular file -input fields with value = "". - -### v1.0.4 - -* Detect a good default tmp directory regardless of platform. (#88) - -### v1.0.3 - -* Fix problems with utf8 characters (#84) / semicolons in filenames (#58) -* Small performance improvements -* New test suite and fixture system - -### v1.0.2 - -* Exclude node\_modules folder from git -* Implement new `'aborted'` event -* Fix files in example folder to work with recent node versions -* Make gently a devDependency - -[See Commits](https://github.com/felixge/node-formidable/compare/v1.0.1...v1.0.2) - -### v1.0.1 - -* Fix package.json to refer to proper main directory. (#68, Dean Landolt) - -[See Commits](https://github.com/felixge/node-formidable/compare/v1.0.0...v1.0.1) - -### v1.0.0 - -* Add support for multipart boundaries that are quoted strings. (Jeff Craig) - -This marks the beginning of development on version 2.0 which will include -several architectural improvements. - -[See Commits](https://github.com/felixge/node-formidable/compare/v0.9.11...v1.0.0) - -### v0.9.11 - -* Emit `'progress'` event when receiving data, regardless of parsing it. (Tim Koschützki) -* Use [W3C FileAPI Draft](http://dev.w3.org/2006/webapi/FileAPI/) properties for File class - -**Important:** The old property names of the File class will be removed in a -future release. - -[See Commits](https://github.com/felixge/node-formidable/compare/v0.9.10...v0.9.11) - -### Older releases - -These releases were done before starting to maintain the above Changelog: - -* [v0.9.10](https://github.com/felixge/node-formidable/compare/v0.9.9...v0.9.10) -* [v0.9.9](https://github.com/felixge/node-formidable/compare/v0.9.8...v0.9.9) -* [v0.9.8](https://github.com/felixge/node-formidable/compare/v0.9.7...v0.9.8) -* [v0.9.7](https://github.com/felixge/node-formidable/compare/v0.9.6...v0.9.7) -* [v0.9.6](https://github.com/felixge/node-formidable/compare/v0.9.5...v0.9.6) -* [v0.9.5](https://github.com/felixge/node-formidable/compare/v0.9.4...v0.9.5) -* [v0.9.4](https://github.com/felixge/node-formidable/compare/v0.9.3...v0.9.4) -* [v0.9.3](https://github.com/felixge/node-formidable/compare/v0.9.2...v0.9.3) -* [v0.9.2](https://github.com/felixge/node-formidable/compare/v0.9.1...v0.9.2) -* [v0.9.1](https://github.com/felixge/node-formidable/compare/v0.9.0...v0.9.1) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.9.0](https://github.com/felixge/node-formidable/compare/v0.8.0...v0.9.0) -* [v0.1.0](https://github.com/felixge/node-formidable/commits/v0.1.0) - ## License Formidable is licensed under the MIT license. diff --git a/benchmark/bench-multipart-parser.js b/benchmark/bench-multipart-parser.js index eab4c3c7..1978e588 100644 --- a/benchmark/bench-multipart-parser.js +++ b/benchmark/bench-multipart-parser.js @@ -59,8 +59,8 @@ function createMultipartBuffer(boundary, size) { , tail = '\r\n--'+boundary+'--\r\n' , buffer = new Buffer(size); - buffer.write(head, 'ascii', 0); - buffer.write(tail, 'ascii', buffer.length - tail.length); + buffer.write(head, 0, 'ascii'); + buffer.write(tail, buffer.length - tail.length, 'ascii'); return buffer; } diff --git a/example/post.js b/example/post.js index f6c15a64..8458fbdc 100644 --- a/example/post.js +++ b/example/post.js @@ -1,7 +1,8 @@ -require('../test/common'); +var common = require('../test/common'); var http = require('http'), util = require('util'), - formidable = require('formidable'), + formidable = common.formidable, + port = common.port, server; server = http.createServer(function(req, res) { @@ -38,6 +39,6 @@ server = http.createServer(function(req, res) { res.end('404'); } }); -server.listen(TEST_PORT); +server.listen(port); -console.log('listening on http://localhost:'+TEST_PORT+'/'); +console.log('listening on http://localhost:'+port+'/'); diff --git a/example/upload.js b/example/upload.js index 050cdd9d..0b498d66 100644 --- a/example/upload.js +++ b/example/upload.js @@ -1,7 +1,9 @@ -require('../test/common'); +var common = require('../test/common'); var http = require('http'), util = require('util'), - formidable = require('formidable'), + os = require('os'), + formidable = common.formidable, + port = common.port, server; server = http.createServer(function(req, res) { @@ -19,7 +21,7 @@ server = http.createServer(function(req, res) { files = [], fields = []; - form.uploadDir = TEST_TMP; + form.uploadDir = os.tmpdir(); form .on('field', function(field, value) { @@ -43,6 +45,6 @@ server = http.createServer(function(req, res) { res.end('404'); } }); -server.listen(TEST_PORT); +server.listen(port); -console.log('listening on http://localhost:'+TEST_PORT+'/'); +console.log('listening on http://localhost:'+port+'/'); diff --git a/lib/file.js b/lib/file.js index e34c10e4..50d34c09 100644 --- a/lib/file.js +++ b/lib/file.js @@ -1,7 +1,7 @@ if (global.GENTLY) require = GENTLY.hijack(require); var util = require('util'), - WriteStream = require('fs').WriteStream, + fs = require('fs'), EventEmitter = require('events').EventEmitter, crypto = require('crypto'); @@ -31,11 +31,11 @@ module.exports = File; util.inherits(File, EventEmitter); File.prototype.open = function() { - this._writeStream = new WriteStream(this.path); + this._writeStream = new fs.WriteStream(this.path); }; File.prototype.toJSON = function() { - return { + var json = { size: this.size, path: this.path, name: this.name, @@ -45,6 +45,10 @@ File.prototype.toJSON = function() { filename: this.filename, mime: this.mime }; + if (this.hash && this.hash != "") { + json.hash = this.hash; + } + return json; }; File.prototype.write = function(buffer, cb) { @@ -52,6 +56,11 @@ File.prototype.write = function(buffer, cb) { if (self.hash) { self.hash.update(buffer); } + + if (this._writeStream.closed) { + return cb(); + } + this._writeStream.write(buffer, function() { self.lastModifiedDate = new Date(); self.size += buffer.length; diff --git a/lib/incoming_form.js b/lib/incoming_form.js index b4234456..8d96bfe1 100644 --- a/lib/incoming_form.js +++ b/lib/incoming_form.js @@ -25,8 +25,9 @@ function IncomingForm(opts) { this.maxFields = opts.maxFields || 1000; this.maxFieldsSize = opts.maxFieldsSize || 2 * 1024 * 1024; + this.maxFileSize = opts.maxFileSize || 2 * 1024 * 1024; this.keepExtensions = opts.keepExtensions || false; - this.uploadDir = opts.uploadDir || os.tmpDir(); + this.uploadDir = opts.uploadDir || (os.tmpdir && os.tmpdir()) || os.tmpDir(); this.encoding = opts.encoding || 'utf-8'; this.headers = null; this.type = null; @@ -39,6 +40,7 @@ function IncomingForm(opts) { this._parser = null; this._flushing = 0; this._fieldsSize = 0; + this._fileSize = 0; this.openedFiles = []; return this; @@ -180,6 +182,7 @@ IncomingForm.prototype.onPart = function(part) { IncomingForm.prototype.handlePart = function(part) { var self = this; + // This MUST check exactly for undefined. You can not change it to !part.filename. if (part.filename === undefined) { var value = '' , decoder = new StringDecoder(this.encoding); @@ -214,6 +217,11 @@ IncomingForm.prototype.handlePart = function(part) { this.openedFiles.push(file); part.on('data', function(buffer) { + self._fileSize += buffer.length; + if (self._fileSize > self.maxFileSize) { + self._error(new Error('maxFileSize exceeded, received '+self._fileSize+' bytes of file data')); + return; + } if (buffer.length == 0) { return; } @@ -352,10 +360,11 @@ IncomingForm.prototype._initMultipart = function(boundary) { headerField = headerField.toLowerCase(); part.headers[headerField] = headerValue; - var m = headerValue.match(/\bname="([^"]+)"/i); + // matches either a quoted-string or a token (RFC 2616 section 19.5.1) + var m = headerValue.match(/\bname=("([^"]*)"|([^\(\)<>@,;:\\"\/\[\]\?=\{\}\s\t/]+))/i); if (headerField == 'content-disposition') { if (m) { - part.name = m[1]; + part.name = m[2] || m[3] || ''; } part.filename = self._fileName(headerValue); @@ -421,10 +430,12 @@ IncomingForm.prototype._initMultipart = function(boundary) { }; IncomingForm.prototype._fileName = function(headerValue) { - var m = headerValue.match(/\bfilename="(.*?)"($|; )/i); + // matches either a quoted-string or a token (RFC 2616 section 19.5.1) + var m = headerValue.match(/\bfilename=("(.*?)"|([^\(\)<>@,;:\\"\/\[\]\?=\{\}\s\t/]+))($|;\s)/i); if (!m) return; - var filename = m[1].substr(m[1].lastIndexOf('\\') + 1); + var match = m[2] || m[3] || ''; + var filename = match.substr(match.lastIndexOf('\\') + 1); filename = filename.replace(/%22/g, '"'); filename = filename.replace(/&#([\d]{4});/g, function(m, code) { return String.fromCharCode(code); @@ -509,7 +520,7 @@ IncomingForm.prototype._initOctetStream = function() { IncomingForm.prototype._initJSONencoded = function() { this.type = 'json'; - var parser = new JSONParser() + var parser = new JSONParser(this) , self = this; if (this.bytesExpected) { @@ -529,11 +540,8 @@ IncomingForm.prototype._initJSONencoded = function() { }; IncomingForm.prototype._uploadPath = function(filename) { - var name = 'upload_'; var buf = crypto.randomBytes(16); - for (var i = 0; i < buf.length; ++i) { - name += ('0' + buf[i].toString(16)).slice(-2); - } + var name = 'upload_' + buf.toString('hex'); if (this.keepExtensions) { var ext = path.extname(filename); @@ -552,4 +560,3 @@ IncomingForm.prototype._maybeEnd = function() { this.emit('end'); }; - diff --git a/lib/json_parser.js b/lib/json_parser.js index db39c310..24ef63b1 100644 --- a/lib/json_parser.js +++ b/lib/json_parser.js @@ -2,7 +2,8 @@ if (global.GENTLY) require = GENTLY.hijack(require); var Buffer = require('buffer').Buffer; -function JSONParser() { +function JSONParser(parent) { + this.parent = parent; this.data = new Buffer(''); this.bytesWritten = 0; } @@ -28,7 +29,9 @@ JSONParser.prototype.end = function() { for (var field in fields) { this.onField(field, fields[field]); } - } catch (e) {} + } catch (e) { + this.parent.emit('error', e); + } this.data = null; this.onEnd(); diff --git a/package.json b/package.json index 4ec587e2..7f0ee0ff 100644 --- a/package.json +++ b/package.json @@ -2,14 +2,15 @@ "name": "formidable", "description": "A node.js module for parsing form data, especially file uploads.", "homepage": "https://github.com/felixge/node-formidable", - "version": "1.0.17", + "license": "MIT", + "version": "1.1.1", "devDependencies": { - "gently": "0.8.0", - "findit": "0.1.1", - "hashish": "0.0.4", - "urun": "~0.0.6", - "utest": "0.0.3", - "request": "~2.11.4" + "gently": "^0.8.0", + "findit": "^0.1.2", + "hashish": "^0.0.4", + "urun": "^0.0.6", + "utest": "^0.0.8", + "request": "^2.11.4" }, "directories": { "lib": "./lib" diff --git a/test/legacy/integration/test-multipart-parser.js b/test/legacy/integration/test-multipart-parser.js index 2ddea476..45047fef 100644 --- a/test/legacy/integration/test-multipart-parser.js +++ b/test/legacy/integration/test-multipart-parser.js @@ -50,7 +50,7 @@ Object.keys(fixtures).forEach(function(name) { endCalled = true; }; - buffer.write(fixture.raw, 'binary', 0); + buffer.write(fixture.raw, 0, undefined, 'binary'); while (offset < buffer.length) { if (offset + CHUNK_LENGTH < buffer.length) { diff --git a/test/legacy/simple/test-multipart-parser.js b/test/legacy/simple/test-multipart-parser.js index bf2cd5e1..c732d32f 100644 --- a/test/legacy/simple/test-multipart-parser.js +++ b/test/legacy/simple/test-multipart-parser.js @@ -34,7 +34,7 @@ test(function parserError() { buffer = new Buffer(5); parser.initWithBoundary(boundary); - buffer.write('--ad', 'ascii', 0); + buffer.write('--ad', 0); assert.equal(parser.write(buffer), 5); });