Skip to content

Commit c566dc2

Browse files
tomastomas
committed
Add utils module and refactor should_proxy_to() based on PR #411. Fixes #383 
1 parent e5dee92 commit c566dc2

File tree

4 files changed

+231
-100
lines changed

4 files changed

+231
-100
lines changed

‎lib/needle.js

+13-98
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ var fs = require('fs'),
1616
auth = require('./auth'),
1717
cookies = require('./cookies'),
1818
parsers = require('./parsers'),
19-
decoder = require('./decoder');
19+
decoder = require('./decoder'),
20+
utils = require('./utils');
2021

2122
//////////////////////////////////////////
2223
// variabilia
@@ -127,99 +128,13 @@ Object.keys(aliased.options).map(function(k) {
127128
//////////////////////////////////////////
128129
// helpers
129130

130-
function get_env_var(keys, try_lower) {
131-
var val, i = -1, env = process.env;
132-
while (!val && i < keys.length-1) {
133-
val = env[keys[++i]];
134-
if (!val && try_lower) {
135-
val = env[keys[i].toLowerCase()];
136-
}
137-
}
138-
return val;
139-
}
140-
141131
function keys_by_type(type) {
142132
return Object.keys(defaults).map(function(el) {
143133
if (defaults[el] !== null && defaults[el].constructor == type)
144134
return el;
145135
}).filter(function(el) { return el })
146136
}
147137

148-
function parse_content_type(header) {
149-
if (!header || header === '') return {};
150-
151-
var found, charset = 'utf8', arr = header.split(';');
152-
153-
if (arr.length > 1 && (found = arr[1].match(/charset=(.+)/)))
154-
charset = found[1];
155-
156-
return { type: arr[0], charset: charset };
157-
}
158-
159-
function is_stream(obj) {
160-
return typeof obj.pipe === 'function';
161-
}
162-
163-
function get_stream_length(stream, given_length, cb) {
164-
if (given_length > 0)
165-
return cb(given_length);
166-
167-
if (stream.end !== void 0 && stream.end !== Infinity && stream.start !== void 0)
168-
return cb((stream.end + 1) - (stream.start || 0));
169-
170-
fs.stat(stream.path, function(err, stat) {
171-
cb(stat ? stat.size - (stream.start || 0) : null);
172-
});
173-
}
174-
175-
function resolve_url(href, base) {
176-
if (url.URL)
177-
return new url.URL(href, base);
178-
179-
// older Node version (< v6.13)
180-
return base ? url.resolve(base, href) : href;
181-
}
182-
183-
function host_and_ports_match(url1, url2) {
184-
if (url1.indexOf('http') < 0) url1 = 'http://' + url1;
185-
if (url2.indexOf('http') < 0) url2 = 'http://' + url2;
186-
var a = url.parse(url1), b = url.parse(url2);
187-
188-
return a.host == b.host
189-
&& String(a.port || (a.protocol == 'https:' ? 443 : 80))
190-
== String(b.port || (b.protocol == 'https:' ? 443 : 80));
191-
}
192-
193-
// returns false if a no_proxy host matches given url
194-
function should_proxy_to(url) {
195-
var no_proxy = get_env_var(['NO_PROXY'], true);
196-
if (!no_proxy) return true;
197-
198-
var host, hosts = no_proxy.split(',');
199-
for (var i in hosts) {
200-
host = hosts[i];
201-
if (host_and_ports_match(host, url)) {
202-
return false;
203-
}
204-
}
205-
206-
return true;
207-
}
208-
209-
function pump_streams(streams, cb) {
210-
if (stream.pipeline)
211-
return stream.pipeline.apply(null, streams.concat(cb));
212-
213-
var tmp = streams.shift();
214-
while (streams.length) {
215-
tmp = tmp.pipe(streams.shift());
216-
tmp.once('error', function(e) {
217-
cb && cb(e);
218-
cb = null;
219-
})
220-
}
221-
}
222-
223138
//////////////////////////////////////////
224139
// the main act
225140

@@ -340,12 +255,12 @@ Needle.prototype.setup = function(uri, options) {
340255
}
341256
}
342257

343-
var env_proxy = get_env_var(['HTTP_PROXY', 'HTTPS_PROXY'], true);
258+
var env_proxy = utils.get_env_var(['HTTP_PROXY', 'HTTPS_PROXY'], true);
344259
if (!config.proxy && env_proxy) config.proxy = env_proxy;
345260

346261
// if proxy is present, set auth header from either url or proxy_user option.
347262
if (config.proxy) {
348-
if (should_proxy_to(uri)) {
263+
if (utils.should_proxy_to(uri)) {
349264
if (config.proxy.indexOf('http') === -1)
350265
config.proxy = 'http://' + config.proxy;
351266

@@ -402,7 +317,7 @@ Needle.prototype.start = function() {
402317
next(parts);
403318
});
404319

405-
} else if (is_stream(data)) {
320+
} else if (utils.is_stream(data)) {
406321

407322
if (method == 'get')
408323
throw new Error('Refusing to pipe() a stream via GET. Did you mean .post?');
@@ -411,7 +326,7 @@ Needle.prototype.start = function() {
411326
// ok, let's get the stream's length and set it as the content-length header.
412327
// this prevents some servers from cutting us off before all the data is sent.
413328
waiting = true;
414-
get_stream_length(data, config.stream_length, function(length) {
329+
utils.get_stream_length(data, config.stream_length, function(length) {
415330
data.length = length;
416331
next(data);
417332
})
@@ -609,7 +524,7 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
609524

610525
// if follow_set_cookies is true, insert cookies in the next request's headers.
611526
// we set both the original request cookies plus any response cookies we might have received.
612-
if (config.follow_set_cookies && host_and_ports_match(headers.location, uri)) {
527+
if (config.follow_set_cookies && utils.host_and_ports_match(headers.location, uri)) {
613528
var request_cookies = cookies.read(config.headers['cookie']);
614529
config.previous_resp_cookies = resp.cookies;
615530
if (Object.keys(request_cookies).length || Object.keys(resp.cookies || {}).length) {
@@ -625,7 +540,7 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
625540

626541
config.headers['host'] = null; // clear previous Host header to avoid conflicts.
627542

628-
var redirect_url = resolve_url(headers.location, uri);
543+
var redirect_url = utils.resolve_url(headers.location, uri);
629544
debug('Redirecting to ' + redirect_url.toString());
630545
return self.send_request(++count, method, redirect_url.toString(), config, post_data, out, callback);
631546
} else if (config.follow_max > 0) {
@@ -650,7 +565,7 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
650565
out.emit('headers', headers);
651566

652567
var pipeline = [],
653-
mime = parse_content_type(headers['content-type']),
568+
mime = utils.parse_content_type(headers['content-type']),
654569
text_response = mime.type && (mime.type.indexOf('text/') != -1 || !!mime.type.match(/(\/|\+)(xml|json)$/));
655570

656571
// To start, if our body is compressed and we're able to inflate it, do it.
@@ -689,7 +604,7 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
689604
pipeline.push(out);
690605

691606
// Now, release the kraken!
692-
pump_streams([resp].concat(pipeline), function(err) {
607+
utils.pump_streams([resp].concat(pipeline), function(err) {
693608
if (err) debug(err)
694609

695610
// on node v8.x, if an error ocurrs on the receiving end,
@@ -745,7 +660,7 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
745660
}
746661
})
747662

748-
pump_streams([resp, clean_pipe], function(err) {
663+
utils.pump_streams([resp, clean_pipe], function(err) {
749664
if (err) debug(err);
750665
});
751666

@@ -836,8 +751,8 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
836751
})
837752

838753
if (post_data) {
839-
if (is_stream(post_data)) {
840-
pump_streams([post_data, request], function(err) {
754+
if (utils.is_stream(post_data)) {
755+
utils.pump_streams([post_data, request], function(err) {
841756
if (err) debug(err);
842757
});
843758
} else {

‎lib/utils.js

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
var fs = require('fs'),
2+
url = require('url'),
3+
stream = require('stream');
4+
5+
function resolve_url(href, base) {
6+
if (url.URL)
7+
return new url.URL(href, base);
8+
9+
// older Node version (< v6.13)
10+
return base ? url.resolve(base, href) : href;
11+
}
12+
13+
function host_and_ports_match(url1, url2) {
14+
if (url1.indexOf('http') < 0) url1 = 'http://' + url1;
15+
if (url2.indexOf('http') < 0) url2 = 'http://' + url2;
16+
var a = url.parse(url1), b = url.parse(url2);
17+
18+
return a.host == b.host
19+
&& String(a.port || (a.protocol == 'https:' ? 443 : 80))
20+
== String(b.port || (b.protocol == 'https:' ? 443 : 80));
21+
}
22+
23+
// returns false if a no_proxy host or pattern matches given url
24+
function should_proxy_to(uri) {
25+
var no_proxy = get_env_var(['NO_PROXY'], true);
26+
if (!no_proxy) return true;
27+
28+
// previous (naive, simple) strategy
29+
// var host, hosts = no_proxy.split(',');
30+
// for (var i in hosts) {
31+
// host = hosts[i];
32+
// if (host_and_ports_match(host, uri)) {
33+
// return false;
34+
// }
35+
// }
36+
37+
var pattern, pattern_list = no_proxy.split(/[\s,]+/);
38+
for (var i in pattern_list) {
39+
pattern = pattern_list[i];
40+
if (pattern.trim().length == 0) continue;
41+
42+
// replace leading dot by asterisk, escape dots and finally replace asterisk by .*
43+
var regex = new RegExp(pattern.replace(/^\./, "*").replace(/[.]/g, '\\$&').replace(/\*/g, '.*'))
44+
if (uri.match(regex)) return false;
45+
}
46+
47+
return true;
48+
}
49+
50+
function get_env_var(keys, try_lower) {
51+
var val, i = -1, env = process.env;
52+
while (!val && i < keys.length-1) {
53+
val = env[keys[++i]];
54+
if (!val && try_lower) {
55+
val = env[keys[i].toLowerCase()];
56+
}
57+
}
58+
return val;
59+
}
60+
61+
function parse_content_type(header) {
62+
if (!header || header === '') return {};
63+
64+
var found, charset = 'utf8', arr = header.split(';');
65+
66+
if (arr.length > 1 && (found = arr[1].match(/charset=(.+)/)))
67+
charset = found[1];
68+
69+
return { type: arr[0], charset: charset };
70+
}
71+
72+
function is_stream(obj) {
73+
return typeof obj.pipe === 'function';
74+
}
75+
76+
function get_stream_length(stream, given_length, cb) {
77+
if (given_length > 0)
78+
return cb(given_length);
79+
80+
if (stream.end !== void 0 && stream.end !== Infinity && stream.start !== void 0)
81+
return cb((stream.end + 1) - (stream.start || 0));
82+
83+
fs.stat(stream.path, function(err, stat) {
84+
cb(stat ? stat.size - (stream.start || 0) : null);
85+
});
86+
}
87+
88+
function pump_streams(streams, cb) {
89+
if (stream.pipeline)
90+
return stream.pipeline.apply(null, streams.concat(cb));
91+
92+
var tmp = streams.shift();
93+
while (streams.length) {
94+
tmp = tmp.pipe(streams.shift());
95+
tmp.once('error', function(e) {
96+
cb && cb(e);
97+
cb = null;
98+
})
99+
}
100+
}
101+
102+
module.exports = {
103+
resolve_url: resolve_url,
104+
get_env_var: get_env_var,
105+
host_and_ports_match: host_and_ports_match,
106+
should_proxy_to: should_proxy_to,
107+
parse_content_type: parse_content_type,
108+
is_stream: is_stream,
109+
get_stream_length: get_stream_length,
110+
pump_streams: pump_streams
111+
}

‎test/proxy_spec.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,9 @@ describe('proxy option', function() {
137137
}))
138138
})
139139

140-
it('proxies request if matching host in list but different port', function(done) {
140+
it('does not proxy request if matching host in list and just has a different port', function(done) {
141141
process.env.NO_PROXY = 'localhost';
142-
send_request({ proxy: nonexisting_host + ':123/done' }, proxied(nonexisting_host, '123', function() {
142+
send_request({ proxy: nonexisting_host + ':123/done' }, not_proxied(function() {
143143
delete process.env.NO_PROXY;
144144
done();
145145
}))

0 commit comments

Comments
 (0)