77 lines
2.4 KiB
JavaScript
77 lines
2.4 KiB
JavaScript
|
var util = require('util');
|
||
|
|
var es = require('event-stream');
|
||
|
|
var request = require('request');
|
||
|
|
var File = require('vinyl');
|
||
|
|
var through2 = require('through2');
|
||
|
|
var extend = require('node.extend');
|
||
|
|
|
||
|
|
module.exports = function(urls, options) {
|
||
|
|
if (options === undefined) {
|
||
|
|
options = {};
|
||
|
|
}
|
||
|
|
|
||
|
|
if (typeof options.base !== 'string' ) {
|
||
|
|
options.base = '/';
|
||
|
|
}
|
||
|
|
|
||
|
|
if (typeof options.buffer !== 'boolean') {
|
||
|
|
options.buffer = true;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!util.isArray(urls)) {
|
||
|
|
urls = [urls];
|
||
|
|
}
|
||
|
|
|
||
|
|
var allowedRequestOptions = ['qs', 'headers', 'auth', 'followRedirect', 'followAllRedirects', 'maxRedirects', 'timeout', 'proxy',
|
||
|
|
'strictSSL', 'aws', 'gzip']
|
||
|
|
|
||
|
|
var requestBaseOptions = {};
|
||
|
|
for (var i = allowedRequestOptions.length - 1; i >= 0; i--) {
|
||
|
|
var k = allowedRequestOptions[i];
|
||
|
|
if (k in options) {
|
||
|
|
requestBaseOptions[k] = options[k];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (options.requestOptions) {
|
||
|
|
for (var k in options.requestOptions) {
|
||
|
|
requestBaseOptions[k] = options.requestOptions[k];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return es.readArray(urls).pipe(es.map(function(data, cb) {
|
||
|
|
var url = options.base + data, requestOptions = extend({url: url}, requestBaseOptions);
|
||
|
|
|
||
|
|
if (!options.buffer) {
|
||
|
|
var file = new File({
|
||
|
|
cwd: '/',
|
||
|
|
base: options.base,
|
||
|
|
path: url,
|
||
|
|
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
|
||
|
|
contents: request(requestOptions).pipe(through2())
|
||
|
|
});
|
||
|
|
|
||
|
|
cb(null, file);
|
||
|
|
} else {
|
||
|
|
// set encoding to `null` to return the body as buffer
|
||
|
|
requestOptions.encoding = null;
|
||
|
|
|
||
|
|
request(requestOptions, function(error, response, body) {
|
||
|
|
if (!error && response.statusCode == 200) {
|
||
|
|
var file = new File({
|
||
|
|
cwd: '/',
|
||
|
|
base: options.base,
|
||
|
|
path: url,
|
||
|
|
contents: body
|
||
|
|
});
|
||
|
|
cb(null, file);
|
||
|
|
} else {
|
||
|
|
if (!error) {
|
||
|
|
error = new Error("GET " + url + " failed with status code:" + response.statusCode);
|
||
|
|
}
|
||
|
|
cb(error);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
};
|