DWQA QuestionsCategory: ServerCurl error when node executes HTTP request
Bai Yi Zi asked 1 month ago

In the latest node0.12.4, HTTP requests are executed. When there are multiple requests, there is an error at the bottom layer. When there is a problem, the error reported is too funny:
Curl error when node executes HTTP request
It seems that the underlying layer of the new node is HTTP requests made with curl.

function get(url,params,callback,pool) {
    var options = {
        qs:params
    };
    if (pool) {
        options.pool = pool;
    }
    request.get(url,options,function(error,response,body) {
        parseResponse(url,'',error,response,body,callback);
    });
}
exports.get = get;

Code 1 httptest.js

var testhttp = require('../testhttp');
var count = 0;
var keepAliveAgent = require('../Agent.js'); 
var agent = new keepAliveAgent({ maxSockets: 100 }); 
for(var i= 0;i<1000;i++) {
    testhttp.get('http://www.baidu.com',{},function(err,body) {
        console.log(count++, err);
    },agent);
}

Code 2 test get baidu.js

/**
 * Created by Administrator on 14-4-30.
 */
var http = require('http');
var EventEmitter = require('events').EventEmitter;
var net = require('net');
var util = require('util');

var Agent = function(options)
{
    var self = this;
    //Options configuration
    self.options  = options || {};
    //Save all requested hostname
    self.requests = {};
    //Number of socket connections created
    self.sockets = {};
    //Unused socket
    self.unusedSockets = {};
    //Maximum number of socket connections
    self.maxSockets = self.options.maxSockets || Agent.defaultMaxSockets;
    self.on('free',function(socket,host,port){
        var hostname = host + ':' + port;
        //If there is a host being requested
        if(self.requests[hostname] && self.requests[hostname].length)
        {
            self.requests[hostname].shift().onSocket(socket);
        }
        else
        {
            //If there are no requests, destroy the socket connection and remove it from the connection pool
            if(!self.unusedSockets[hostname])
            {
                self.unusedSockets[hostname] = [];
            }
            self.unusedSockets[hostname].push(socket);
        }
    });
    self.createConnection = net.createConnection;
};
util.inherits(Agent,EventEmitter);

Agent.defaultMaxSockets = 10;

Agent.prototype.defaultPort = 80;

Agent.prototype.addRequest = function(req,host,port)
{
    var hostname = host + ':' + port;
    if(this.unusedSockets[hostname] && this.unusedSockets[hostname].length)
    {
        req.onSocket(this.unusedSockets[hostname].shift());
        return;
    }
    if(!this.sockets[hostname])
    {
        this.sockets[hostname] = [];
    }
    if(this.sockets[hostname].length < this.maxSockets)
    {
        req.onSocket(this.createSocket(hostname,host,port));
    }
    else
    {
        if(!this.requests[hostname])
        {
            this.requests[hostname] = [];
        }
        this.requests[hostname].push(req);
    }
};

Agent.prototype.createSocket = function(name, host, port) {
    var self = this;
    var s = self.createConnection(port, host, self.options);
    if (!self.sockets[name]) {
        self.sockets[name] = [];
    }
    this.sockets[name].push(s);
    var onFree = function() {
        self.emit('free', s, host, port);
    }
    s.on('free', onFree);
    var onClose = function(err) {
        //This is the only place to remove the socket proxy. If you want to remove the socket from the connection pool, close the connection. All socket errors will cause the connection to close
        self.removeSocket(s, name, host, port);
    }
    s.on('close', onClose);
    var onRemove = function() {
        // We need this function for cases like HTTP "upgrade"
        // (defined by WebSockets) where we need to remove a socket from the pool
        //  because it'll be locked up indefinitely
        self.removeSocket(s, name, host, port);
        s.removeListener('close', onClose);
        s.removeListener('free', onFree);
        s.removeListener('agentRemove', onRemove);
    }
    s.on('agentRemove', onRemove);
    return s;
};
Agent.prototype.removeSocket = function(s, name, host, port) {
    if (this.sockets[name]) {
        var index = this.sockets[name].indexOf(s);
        if (index !== -1) {
            this.sockets[name].splice(index, 1);
        }
    } else if (this.sockets[name] && this.sockets[name].length === 0) {
        delete this.sockets[name];
        delete this.requests[name];
    }
    if (this.requests[name] && this.requests[name].length) {
        // If we have pending requests and a socket gets closed a new one
        // needs to be created to take over in the pool for the one that closed.
        this.createSocket(name, host, port).emit('free');
    }
};

module.exports = Agent;

Code 3 agent.js
The entry file istest_get_baidu.jsFirst of all, I am surprised that the underlying node uses curl to complete HTTP requests. Second, I want to know whether this error can be avoided.

1 Answers
strucoder answered 1 month ago

The curl request is synchronous, while the HTTP request of nodejs is asynchronous.
If you implement curl in nodejs, there are also relatedpackage