needle.js 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. //////////////////////////////////////////
  2. // Needle -- HTTP Client for Node.js
  3. // Written by Tomás Pollak <tomas@forkhq.com>
  4. // (c) 2012-2023 - Fork Ltd.
  5. // MIT Licensed
  6. //////////////////////////////////////////
  7. var fs = require('fs'),
  8. http = require('http'),
  9. https = require('https'),
  10. url = require('url'),
  11. stream = require('stream'),
  12. debug = require('util').debuglog('needle'),
  13. stringify = require('./querystring').build,
  14. multipart = require('./multipart'),
  15. auth = require('./auth'),
  16. cookies = require('./cookies'),
  17. parsers = require('./parsers'),
  18. decoder = require('./decoder'),
  19. utils = require('./utils');
  20. //////////////////////////////////////////
  21. // variabilia
  22. var version = require('../package.json').version;
  23. var user_agent = 'Needle/' + version;
  24. user_agent += ' (Node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')';
  25. var tls_options = 'pfx key passphrase cert ca ciphers rejectUnauthorized secureProtocol checkServerIdentity family';
  26. // older versions of node (< 0.11.4) prevent the runtime from exiting
  27. // because of connections in keep-alive state. so if this is the case
  28. // we'll default new requests to set a Connection: close header.
  29. var close_by_default = !http.Agent || http.Agent.defaultMaxSockets != Infinity;
  30. // see if we have Object.assign. otherwise fall back to util._extend
  31. var extend = Object.assign ? Object.assign : require('util')._extend;
  32. // these are the status codes that Needle interprets as redirects.
  33. var redirect_codes = [301, 302, 303, 307, 308];
  34. //////////////////////////////////////////
  35. // decompressors for gzip/deflate/br bodies
  36. function bind_opts(fn, options) {
  37. return fn.bind(null, options);
  38. }
  39. var decompressors = {};
  40. try {
  41. var zlib = require('zlib');
  42. // Enable Z_SYNC_FLUSH to avoid Z_BUF_ERROR errors (Node PR #2595)
  43. var zlib_options = {
  44. flush: zlib.Z_SYNC_FLUSH,
  45. finishFlush: zlib.Z_SYNC_FLUSH
  46. };
  47. var br_options = {
  48. flush: zlib.BROTLI_OPERATION_FLUSH,
  49. finishFlush: zlib.BROTLI_OPERATION_FLUSH
  50. };
  51. decompressors['x-deflate'] = bind_opts(zlib.Inflate, zlib_options);
  52. decompressors['deflate'] = bind_opts(zlib.Inflate, zlib_options);
  53. decompressors['x-gzip'] = bind_opts(zlib.Gunzip, zlib_options);
  54. decompressors['gzip'] = bind_opts(zlib.Gunzip, zlib_options);
  55. if (typeof zlib.BrotliDecompress === 'function') {
  56. decompressors['br'] = bind_opts(zlib.BrotliDecompress, br_options);
  57. }
  58. } catch(e) { /* zlib not available */ }
  59. //////////////////////////////////////////
  60. // options and aliases
  61. var defaults = {
  62. // data
  63. boundary : '--------------------NODENEEDLEHTTPCLIENT',
  64. encoding : 'utf8',
  65. parse_response : 'all', // same as true. valid options: 'json', 'xml' or false/null
  66. proxy : null,
  67. // agent & headers
  68. agent : null,
  69. headers : {},
  70. accept : '*/*',
  71. user_agent : user_agent,
  72. // numbers
  73. open_timeout : 10000,
  74. response_timeout : 0,
  75. read_timeout : 0,
  76. follow_max : 0,
  77. stream_length : -1,
  78. // abort signal
  79. signal : null,
  80. // booleans
  81. compressed : false,
  82. decode_response : true,
  83. parse_cookies : true,
  84. follow_set_cookies : false,
  85. follow_set_referer : false,
  86. follow_keep_method : false,
  87. follow_if_same_host : false,
  88. follow_if_same_protocol : false,
  89. follow_if_same_location : false,
  90. use_proxy_from_env_var : true
  91. }
  92. var aliased = {
  93. options: {
  94. decode : 'decode_response',
  95. parse : 'parse_response',
  96. timeout : 'open_timeout',
  97. follow : 'follow_max'
  98. },
  99. inverted: {}
  100. }
  101. // only once, invert aliased keys so we can get passed options.
  102. Object.keys(aliased.options).map(function(k) {
  103. var value = aliased.options[k];
  104. aliased.inverted[value] = k;
  105. });
  106. //////////////////////////////////////////
  107. // helpers
  108. function keys_by_type(type) {
  109. return Object.keys(defaults).map(function(el) {
  110. if (defaults[el] !== null && defaults[el].constructor == type)
  111. return el;
  112. }).filter(function(el) { return el })
  113. }
  114. //////////////////////////////////////////
  115. // the main act
  116. function Needle(method, uri, data, options, callback) {
  117. // if (!(this instanceof Needle)) {
  118. // return new Needle(method, uri, data, options, callback);
  119. // }
  120. if (typeof uri !== 'string')
  121. throw new TypeError('URL must be a string, not ' + uri);
  122. this.method = method.toLowerCase();
  123. this.uri = uri;
  124. this.data = data;
  125. if (typeof options == 'function') {
  126. this.callback = options;
  127. this.options = {};
  128. } else {
  129. this.callback = callback;
  130. this.options = options;
  131. }
  132. }
  133. Needle.prototype.setup = function(uri, options) {
  134. function get_option(key, fallback) {
  135. // if original is in options, return that value
  136. if (typeof options[key] != 'undefined') return options[key];
  137. // otherwise, return value from alias or fallback/undefined
  138. return typeof options[aliased.inverted[key]] != 'undefined'
  139. ? options[aliased.inverted[key]] : fallback;
  140. }
  141. function check_value(expected, key) {
  142. var value = get_option(key),
  143. type = typeof value;
  144. if (type != 'undefined' && type != expected)
  145. throw new TypeError(type + ' received for ' + key + ', but expected a ' + expected);
  146. return (type == expected) ? value : defaults[key];
  147. }
  148. //////////////////////////////////////////////////
  149. // the basics
  150. var config = {
  151. http_opts : {
  152. agent: get_option('agent', defaults.agent),
  153. localAddress: get_option('localAddress', undefined),
  154. lookup: get_option('lookup', undefined),
  155. signal: get_option('signal', defaults.signal)
  156. }, // passed later to http.request() directly
  157. headers : {},
  158. output : options.output,
  159. proxy : get_option('proxy', defaults.proxy),
  160. parser : get_option('parse_response', defaults.parse_response),
  161. encoding : options.encoding || (options.multipart ? 'binary' : defaults.encoding)
  162. }
  163. keys_by_type(Boolean).forEach(function(key) {
  164. config[key] = check_value('boolean', key);
  165. })
  166. keys_by_type(Number).forEach(function(key) {
  167. config[key] = check_value('number', key);
  168. })
  169. if (config.http_opts.signal && !(config.http_opts.signal instanceof AbortSignal))
  170. throw new TypeError(typeof config.http_opts.signal + ' received for signal, but expected an AbortSignal');
  171. // populate http_opts with given TLS options
  172. tls_options.split(' ').forEach(function(key) {
  173. if (typeof options[key] != 'undefined') {
  174. if (config.http_opts.agent) { // pass option to existing agent
  175. config.http_opts.agent.options[key] = options[key];
  176. } else {
  177. config.http_opts[key] = options[key];
  178. }
  179. }
  180. });
  181. //////////////////////////////////////////////////
  182. // headers, cookies
  183. for (var key in defaults.headers)
  184. config.headers[key] = defaults.headers[key];
  185. config.headers['accept'] = options.accept || defaults.accept;
  186. config.headers['user-agent'] = options.user_agent || defaults.user_agent;
  187. if (options.content_type)
  188. config.headers['content-type'] = options.content_type;
  189. // set connection header if opts.connection was passed, or if node < 0.11.4 (close)
  190. if (options.connection || close_by_default)
  191. config.headers['connection'] = options.connection || 'close';
  192. if ((options.compressed || defaults.compressed) && typeof zlib != 'undefined')
  193. config.headers['accept-encoding'] = decompressors['br'] ? 'gzip, deflate, br' : 'gzip, deflate';
  194. if (options.cookies)
  195. config.headers['cookie'] = cookies.write(options.cookies);
  196. //////////////////////////////////////////////////
  197. // basic/digest auth
  198. if (uri.match(/[^\/]@/)) { // url contains user:pass@host, so parse it.
  199. var parts = (url.parse(uri).auth || '').split(':');
  200. options.username = parts[0];
  201. options.password = parts[1];
  202. }
  203. if (options.username) {
  204. if (options.auth && (options.auth == 'auto' || options.auth == 'digest')) {
  205. config.credentials = [options.username, options.password];
  206. } else {
  207. config.headers['authorization'] = auth.basic(options.username, options.password);
  208. }
  209. }
  210. if (config.use_proxy_from_env_var) {
  211. var env_proxy = utils.get_env_var(['HTTP_PROXY', 'HTTPS_PROXY'], true);
  212. if (!config.proxy && env_proxy) config.proxy = env_proxy;
  213. }
  214. // if proxy is present, set auth header from either url or proxy_user option.
  215. if (config.proxy) {
  216. if (!config.use_proxy_from_env_var || utils.should_proxy_to(uri)) {
  217. if (config.proxy.indexOf('http') === -1)
  218. config.proxy = 'http://' + config.proxy;
  219. if (config.proxy.indexOf('@') !== -1) {
  220. var proxy = (url.parse(config.proxy).auth || '').split(':');
  221. options.proxy_user = proxy[0];
  222. options.proxy_pass = proxy[1];
  223. }
  224. if (options.proxy_user)
  225. config.headers['proxy-authorization'] = auth.basic(options.proxy_user, options.proxy_pass);
  226. } else {
  227. delete config.proxy;
  228. }
  229. }
  230. // now that all our headers are set, overwrite them if instructed.
  231. for (var h in options.headers)
  232. config.headers[h.toLowerCase()] = options.headers[h];
  233. config.uri_modifier = get_option('uri_modifier', null);
  234. return config;
  235. }
  236. Needle.prototype.start = function() {
  237. var out = new stream.PassThrough({ objectMode: false }),
  238. uri = this.uri,
  239. data = this.data,
  240. method = this.method,
  241. callback = (typeof this.options == 'function') ? this.options : this.callback,
  242. options = this.options || {};
  243. // if no 'http' is found on URL, prepend it.
  244. if (uri.indexOf('http') === -1)
  245. uri = uri.replace(/^(\/\/)?/, 'http://');
  246. var self = this, body, waiting = false, config = this.setup(uri, options);
  247. // unless options.json was set to false, assume boss also wants JSON if content-type matches.
  248. var json = options.json || (options.json !== false && config.headers['content-type'] == 'application/json');
  249. if (data) {
  250. if (options.multipart) { // boss says we do multipart. so we do it.
  251. var boundary = options.boundary || defaults.boundary;
  252. waiting = true;
  253. multipart.build(data, boundary, function(err, parts) {
  254. if (err) throw(err);
  255. config.headers['content-type'] = 'multipart/form-data; boundary=' + boundary;
  256. next(parts);
  257. });
  258. } else if (utils.is_stream(data)) {
  259. if (method == 'get')
  260. throw new Error('Refusing to pipe() a stream via GET. Did you mean .post?');
  261. if (config.stream_length > 0 || (config.stream_length === 0 && data.path)) {
  262. // ok, let's get the stream's length and set it as the content-length header.
  263. // this prevents some servers from cutting us off before all the data is sent.
  264. waiting = true;
  265. utils.get_stream_length(data, config.stream_length, function(length) {
  266. data.length = length;
  267. next(data);
  268. })
  269. } else {
  270. // if the boss doesn't want us to get the stream's length, or if it doesn't
  271. // have a file descriptor for that purpose, then just head on.
  272. body = data;
  273. }
  274. } else if (Buffer.isBuffer(data)) {
  275. body = data; // use the raw buffer as request body.
  276. } else if (method == 'get' && !json) {
  277. // append the data to the URI as a querystring.
  278. uri = uri.replace(/\?.*|$/, '?' + stringify(data));
  279. } else { // string or object data, no multipart.
  280. // if string, leave it as it is, otherwise, stringify.
  281. body = (typeof(data) === 'string') ? data
  282. : json ? JSON.stringify(data) : stringify(data);
  283. // ensure we have a buffer so bytecount is correct.
  284. body = Buffer.from(body, config.encoding);
  285. }
  286. }
  287. function next(body) {
  288. if (body) {
  289. if (body.length) config.headers['content-length'] = body.length;
  290. // if no content-type was passed, determine if json or not.
  291. if (!config.headers['content-type']) {
  292. config.headers['content-type'] = json
  293. ? 'application/json; charset=utf-8'
  294. : 'application/x-www-form-urlencoded'; // no charset says W3 spec.
  295. }
  296. }
  297. // unless a specific accept header was set, assume json: true wants JSON back.
  298. if (options.json && (!options.accept && !(options.headers || {}).accept))
  299. config.headers['accept'] = 'application/json';
  300. self.send_request(1, method, uri, config, body, out, callback);
  301. }
  302. if (!waiting) next(body);
  303. return out;
  304. }
  305. Needle.prototype.get_request_opts = function(method, uri, config) {
  306. var opts = config.http_opts,
  307. proxy = config.proxy,
  308. remote = proxy ? url.parse(proxy) : url.parse(uri);
  309. opts.protocol = remote.protocol;
  310. opts.host = remote.hostname;
  311. opts.port = remote.port || (remote.protocol == 'https:' ? 443 : 80);
  312. opts.path = proxy ? uri : remote.pathname + (remote.search || '');
  313. opts.method = method;
  314. opts.headers = config.headers;
  315. if (!opts.headers['host']) {
  316. // if using proxy, make sure the host header shows the final destination
  317. var target = proxy ? url.parse(uri) : remote;
  318. opts.headers['host'] = target.hostname;
  319. // and if a non standard port was passed, append it to the port header
  320. if (target.port && [80, 443].indexOf(target.port) === -1) {
  321. opts.headers['host'] += ':' + target.port;
  322. }
  323. }
  324. return opts;
  325. }
  326. Needle.prototype.should_follow = function(location, config, original) {
  327. if (!location) return false;
  328. // returns true if location contains matching property (host or protocol)
  329. function matches(property) {
  330. var property = original[property];
  331. return location.indexOf(property) !== -1;
  332. }
  333. // first, check whether the requested location is actually different from the original
  334. if (!config.follow_if_same_location && location === original)
  335. return false;
  336. if (config.follow_if_same_host && !matches('host'))
  337. return false; // host does not match, so not following
  338. if (config.follow_if_same_protocol && !matches('protocol'))
  339. return false; // procotol does not match, so not following
  340. return true;
  341. }
  342. Needle.prototype.send_request = function(count, method, uri, config, post_data, out, callback) {
  343. if (typeof config.uri_modifier === 'function') {
  344. var modified_uri = config.uri_modifier(uri);
  345. debug('Modifying request URI', uri + ' => ' + modified_uri);
  346. uri = modified_uri;
  347. }
  348. var request,
  349. timer,
  350. returned = 0,
  351. self = this,
  352. request_opts = this.get_request_opts(method, uri, config),
  353. protocol = request_opts.protocol == 'https:' ? https : http,
  354. signal = request_opts.signal;
  355. function unlisten_errors() {
  356. request.removeListener('error', had_error);
  357. // An error can still be fired after closing. In particular, on macOS.
  358. // See also:
  359. // - https://github.com/tomas/needle/issues/391
  360. // - https://github.com/less/less.js/issues/3693
  361. // - https://github.com/nodejs/node/issues/27916
  362. request.once('error', function() {});
  363. }
  364. function done(err, resp) {
  365. if (returned++ > 0)
  366. return debug('Already finished, stopping here.');
  367. if (timer) clearTimeout(timer);
  368. out.done = true;
  369. unlisten_errors();
  370. if (callback)
  371. return callback(err, resp, resp ? resp.body : undefined);
  372. // NOTE: this event used to be called 'end', but the behaviour was confusing
  373. // when errors ocurred, because the stream would still emit an 'end' event.
  374. out.emit('done', err);
  375. // trigger the 'done' event on streams we're being piped to, if any
  376. var pipes = out._readableState.pipes || [];
  377. if (!pipes.forEach) pipes = [pipes];
  378. pipes.forEach(function(st) { st.emit('done', err); })
  379. }
  380. function had_error(err) {
  381. debug('Request error', err);
  382. out.emit('err', err);
  383. done(err || new Error('Unknown error when making request.'));
  384. }
  385. function abort_handler() {
  386. out.emit('err', new Error('Aborted by signal.'));
  387. request.destroy();
  388. }
  389. function set_timeout(type, milisecs) {
  390. if (timer) clearTimeout(timer);
  391. if (milisecs <= 0) return;
  392. timer = setTimeout(function() {
  393. out.emit('timeout', type);
  394. request.destroy();
  395. // also invoke done() to terminate job on read_timeout
  396. if (type == 'read') done(new Error(type + ' timeout'));
  397. signal && signal.removeEventListener('abort', abort_handler);
  398. }, milisecs);
  399. }
  400. debug('Making request #' + count, request_opts);
  401. request = protocol.request(request_opts, function(resp) {
  402. var headers = resp.headers;
  403. debug('Got response', resp.statusCode, headers);
  404. out.emit('response', resp);
  405. set_timeout('read', config.read_timeout);
  406. // if we got cookies, parse them unless we were instructed not to. make sure to include any
  407. // cookies that might have been set on previous redirects.
  408. if (config.parse_cookies && (headers['set-cookie'] || config.previous_resp_cookies)) {
  409. resp.cookies = extend(config.previous_resp_cookies || {}, cookies.read(headers['set-cookie']));
  410. debug('Got cookies', resp.cookies);
  411. }
  412. // if redirect code is found, determine if we should follow it according to the given options.
  413. if (redirect_codes.indexOf(resp.statusCode) !== -1 && self.should_follow(headers.location, config, uri)) {
  414. // clear timer before following redirects to prevent unexpected setTimeout consequence
  415. clearTimeout(timer);
  416. if (count <= config.follow_max) {
  417. out.emit('redirect', headers.location);
  418. // unless 'follow_keep_method' is true, rewrite the request to GET before continuing.
  419. if (!config.follow_keep_method) {
  420. method = 'GET';
  421. post_data = null;
  422. delete config.headers['content-length']; // in case the original was a multipart POST request.
  423. }
  424. if (utils.host_and_ports_match(headers.location, uri)) {
  425. // if follow_set_cookies is true, insert cookies in the next request's headers.
  426. // we set both the original request cookies plus any response cookies we might have received.
  427. if (config.follow_set_cookies) {
  428. var request_cookies = cookies.read(config.headers['cookie']);
  429. config.previous_resp_cookies = resp.cookies;
  430. if (Object.keys(request_cookies).length || Object.keys(resp.cookies || {}).length) {
  431. config.headers['cookie'] = cookies.write(extend(request_cookies, resp.cookies));
  432. }
  433. } else {
  434. // set response cookies if present, otherwise remove header
  435. // if (resp.cookies && Object.keys(resp.cookies).length)
  436. // config.headers['cookie'] = cookies.write(resp.cookies);
  437. // else
  438. delete config.headers['cookie'];
  439. }
  440. } else {
  441. delete config.headers['cookie'];
  442. delete config.headers['authorization'];
  443. delete config.headers['proxy-authorization'];
  444. }
  445. if (config.follow_set_referer)
  446. config.headers['referer'] = encodeURI(uri); // the original, not the destination URL.
  447. config.headers['host'] = null; // clear previous Host header to avoid conflicts.
  448. var redirect_url = utils.resolve_url(headers.location, uri);
  449. debug('Redirecting to ' + redirect_url.toString());
  450. unlisten_errors();
  451. return self.send_request(++count, method, redirect_url.toString(), config, post_data, out, callback);
  452. } else if (config.follow_max > 0) {
  453. return done(new Error('Max redirects reached. Possible loop in: ' + headers.location));
  454. }
  455. }
  456. // if auth is requested and credentials were not passed, resend request, provided we have user/pass.
  457. if (resp.statusCode == 401 && headers['www-authenticate'] && config.credentials) {
  458. if (!config.headers['authorization']) { // only if authentication hasn't been sent
  459. var auth_header = auth.header(headers['www-authenticate'], config.credentials, request_opts);
  460. if (auth_header) {
  461. config.headers['authorization'] = auth_header;
  462. return self.send_request(count, method, uri, config, post_data, out, callback);
  463. }
  464. }
  465. }
  466. // ok, so we got a valid (non-redirect & authorized) response. let's notify the stream guys.
  467. out.emit('header', resp.statusCode, headers);
  468. out.emit('headers', headers);
  469. var pipeline = [],
  470. mime = utils.parse_content_type(headers['content-type']),
  471. text_response = mime.type && (mime.type.indexOf('text/') != -1 || !!mime.type.match(/(\/|\+)(xml|json)$/));
  472. // To start, if our body is compressed and we're able to inflate it, do it.
  473. if (headers['content-encoding'] && decompressors[headers['content-encoding']]) {
  474. var decompressor = decompressors[headers['content-encoding']]();
  475. // make sure we catch errors triggered by the decompressor.
  476. decompressor.on('error', had_error);
  477. pipeline.push(decompressor);
  478. }
  479. // If parse is enabled and we have a parser for it, then go for it.
  480. if (config.parser && parsers[mime.type]) {
  481. // If a specific parser was requested, make sure we don't parse other types.
  482. var parser_name = config.parser.toString().toLowerCase();
  483. if (['xml', 'json'].indexOf(parser_name) == -1 || parsers[mime.type].name == parser_name) {
  484. // OK, so either we're parsing all content types or the one requested matches.
  485. out.parser = parsers[mime.type].name;
  486. pipeline.push(parsers[mime.type].fn());
  487. // Set objectMode on out stream to improve performance.
  488. out._writableState.objectMode = true;
  489. out._readableState.objectMode = true;
  490. }
  491. // If we're not parsing, and unless decoding was disabled, we'll try
  492. // decoding non UTF-8 bodies to UTF-8, using the iconv-lite library.
  493. } else if (text_response && config.decode_response && mime.charset) {
  494. pipeline.push(decoder(mime.charset));
  495. }
  496. // And `out` is the stream we finally push the decoded/parsed output to.
  497. pipeline.push(out);
  498. // Now, release the kraken!
  499. utils.pump_streams([resp].concat(pipeline), function(err) {
  500. if (err) debug(err)
  501. // on node v8.x, if an error ocurrs on the receiving end,
  502. // then we want to abort the request to avoid having dangling sockets
  503. if (err && err.message == 'write after end') request.destroy();
  504. });
  505. // If the user has requested and output file, pipe the output stream to it.
  506. // In stream mode, we will still get the response stream to play with.
  507. if (config.output && resp.statusCode == 200) {
  508. // for some reason, simply piping resp to the writable stream doesn't
  509. // work all the time (stream gets cut in the middle with no warning).
  510. // so we'll manually need to do the readable/write(chunk) trick.
  511. var file = fs.createWriteStream(config.output);
  512. file.on('error', had_error);
  513. out.on('end', function() {
  514. if (file.writable) file.end();
  515. });
  516. file.on('close', function() {
  517. delete out.file;
  518. })
  519. out.on('readable', function() {
  520. var chunk;
  521. while ((chunk = this.read()) !== null) {
  522. if (file.writable) file.write(chunk);
  523. // if callback was requested, also push it to resp.body
  524. if (resp.body) resp.body.push(chunk);
  525. }
  526. })
  527. out.file = file;
  528. }
  529. // Only aggregate the full body if a callback was requested.
  530. if (callback) {
  531. resp.raw = [];
  532. resp.body = [];
  533. resp.bytes = 0;
  534. // Gather and count the amount of (raw) bytes using a PassThrough stream.
  535. var clean_pipe = new stream.PassThrough();
  536. clean_pipe.on('readable', function() {
  537. var chunk;
  538. while ((chunk = this.read()) != null) {
  539. resp.bytes += chunk.length;
  540. resp.raw.push(chunk);
  541. }
  542. })
  543. utils.pump_streams([resp, clean_pipe], function(err) {
  544. if (err) debug(err);
  545. });
  546. // Listen on the 'readable' event to aggregate the chunks, but only if
  547. // file output wasn't requested. Otherwise we'd have two stream readers.
  548. if (!config.output || resp.statusCode != 200) {
  549. out.on('readable', function() {
  550. var chunk;
  551. while ((chunk = this.read()) !== null) {
  552. // We're either pushing buffers or objects, never strings.
  553. if (typeof chunk == 'string') chunk = Buffer.from(chunk);
  554. // Push all chunks to resp.body. We'll bind them in resp.end().
  555. resp.body.push(chunk);
  556. }
  557. })
  558. }
  559. }
  560. // And set the .body property once all data is in.
  561. out.on('end', function() {
  562. if (resp.body) { // callback mode
  563. // we want to be able to access to the raw data later, so keep a reference.
  564. resp.raw = Buffer.concat(resp.raw);
  565. // if parse was successful, we should have an array with one object
  566. if (resp.body[0] !== undefined && !Buffer.isBuffer(resp.body[0])) {
  567. // that's our body right there.
  568. resp.body = resp.body[0];
  569. // set the parser property on our response. we may want to check.
  570. if (out.parser) resp.parser = out.parser;
  571. } else { // we got one or several buffers. string or binary.
  572. resp.body = Buffer.concat(resp.body);
  573. // if we're here and parsed is true, it means we tried to but it didn't work.
  574. // so given that we got a text response, let's stringify it.
  575. if (text_response || out.parser) {
  576. resp.body = resp.body.toString();
  577. }
  578. }
  579. }
  580. // if an output file is being written to, make sure the callback
  581. // is triggered after all data has been written to it.
  582. if (out.file) {
  583. out.file.on('close', function() {
  584. done(null, resp);
  585. })
  586. } else { // elvis has left the building.
  587. done(null, resp);
  588. }
  589. });
  590. // out.on('error', function(err) {
  591. // had_error(err);
  592. // if (err.code == 'ERR_STREAM_DESTROYED' || err.code == 'ERR_STREAM_PREMATURE_CLOSE') {
  593. // request.abort();
  594. // }
  595. // })
  596. }); // end request call
  597. // unless open_timeout was disabled, set a timeout to abort the request.
  598. set_timeout('open', config.open_timeout);
  599. // handle errors on the request object. things might get bumpy.
  600. request.on('error', had_error);
  601. // make sure timer is cleared if request is aborted (issue #257)
  602. request.once('abort', function() {
  603. if (timer) clearTimeout(timer);
  604. })
  605. // set response timeout once we get a valid socket
  606. request.once('socket', function(socket) {
  607. if (socket.connecting) {
  608. socket.once('connect', function() {
  609. set_timeout('response', config.response_timeout);
  610. })
  611. } else {
  612. set_timeout('response', config.response_timeout);
  613. }
  614. })
  615. if (post_data) {
  616. if (utils.is_stream(post_data)) {
  617. utils.pump_streams([post_data, request], function(err) {
  618. if (err) debug(err);
  619. });
  620. } else {
  621. request.write(post_data, config.encoding);
  622. request.end();
  623. }
  624. } else {
  625. request.end();
  626. }
  627. if (signal) { // abort signal given, so handle it
  628. if (signal.aborted === true) {
  629. abort_handler();
  630. } else {
  631. signal.addEventListener('abort', abort_handler, { once: true });
  632. }
  633. }
  634. out.abort = function() { request.destroy() }; // easier access
  635. out.request = request;
  636. return out;
  637. }
  638. //////////////////////////////////////////
  639. // exports
  640. if (typeof Promise !== 'undefined') {
  641. module.exports = function() {
  642. var verb, args = [].slice.call(arguments);
  643. if (args[0].match(/\.|\//)) // first argument looks like a URL
  644. verb = (args.length > 2) ? 'post' : 'get';
  645. else
  646. verb = args.shift();
  647. if (verb.match(/get|head/i) && args.length == 2)
  648. args.splice(1, 0, null); // assume no data if head/get with two args (url, options)
  649. return new Promise(function(resolve, reject) {
  650. module.exports.request(verb, args[0], args[1], args[2], function(err, resp) {
  651. return err ? reject(err) : resolve(resp);
  652. });
  653. })
  654. }
  655. }
  656. module.exports.version = version;
  657. module.exports.defaults = function(obj) {
  658. for (var key in obj) {
  659. var target_key = aliased.options[key] || key;
  660. if (defaults.hasOwnProperty(target_key) && typeof obj[key] != 'undefined') {
  661. if (target_key != 'parse_response' && target_key != 'proxy' && target_key != 'agent' && target_key != 'signal') {
  662. // ensure type matches the original, except for proxy/parse_response that can be null/bool or string, and signal that can be null/AbortSignal
  663. var valid_type = defaults[target_key].constructor.name;
  664. if (obj[key].constructor.name != valid_type)
  665. throw new TypeError('Invalid type for ' + key + ', should be ' + valid_type);
  666. } else if (target_key === 'signal' && obj[key] !== null && !(obj[key] instanceof AbortSignal)) {
  667. throw new TypeError('Invalid type for ' + key + ', should be AbortSignal');
  668. }
  669. defaults[target_key] = obj[key];
  670. } else {
  671. throw new Error('Invalid property for defaults:' + target_key);
  672. }
  673. }
  674. return defaults;
  675. }
  676. 'head get'.split(' ').forEach(function(method) {
  677. module.exports[method] = function(uri, options, callback) {
  678. return new Needle(method, uri, options.query, options, callback).start();
  679. }
  680. })
  681. 'post put patch delete'.split(' ').forEach(function(method) {
  682. module.exports[method] = function(uri, data, options, callback) {
  683. return new Needle(method, uri, data, options, callback).start();
  684. }
  685. })
  686. module.exports.request = function(method, uri, data, opts, callback) {
  687. return new Needle(method, uri, data, opts, callback).start();
  688. };