Node.js never exits after pasting into couchbase, opposite most node questions

My problem seems to be the opposite of any node.js query :-) I have a simple forEach loop to read a list of files and insert them into the Couchbase database. This works fine, but never exits after reading all the lines. So I added a counter to disconnect the couchbase connection after all the inserts have completed. It works.

This process is designed to download hundreds of thousands of files, so I brought the asynchronization module into the mix for batch insertion into groups of 100. Async.eachLimit is used to iterate over the array and insert documents into packages. Now, the Orin problem has returned. No matter what magic is used by async.eachLimit to recognize the process, it does not.

I have passed javascript validation, callbacks, async, etc. Google searches affect keywords, but not this problem. I reduced the code to the next test test. To test, create three files and add their names to testlist.txt.

Async.eachLimit in place runs until it reaches the limit and then freezes. Comment on this and uncomment the array.forEach array and it works. Thanks in advance!

var fs = require('fs');
var couchbase = require('couchbase');
var async = require('async');

var filelist = 'testlist.txt';
var key_count = 0;
var cb_config = { host: 'localhost:8091', bucket: 'default'};
var db = new couchbase.Connection(cb_config, function(err) {
  if (err) {
    console.log('ERRR connect to couchbase at config['+cb_config+']');
    throw err;
  }
});

var insertFile=function(line) {
    console.log('LOAD ['+line+']');

    fs.readFile(line, function(file_err, f_doc) {
        if(file_err) throw file_err;

        db.set(line, f_doc, function(db_err, db_res){
            if (db_err) {
                console.log('FAIL ['+line+'] err['+db_err+']');
            } else {
                console.log('PASS ['+line+']');
            }

            key_count--;
            if (key_count == 0) {
                console.log('DONE Shutting down client, no more keys');
                db.shutdown();
            }
        });
    });
}

// read list of files into data array from file filelist
fs.readFile(filelist, function(filelist_err, lines) {
    if(filelist_err) throw filelist_err;

    // HACK split adds empty line to array, use replace to fix
    var array = lines.toString().replace(/\n$/, '').split('\n');
    key_count = array.length;
    console.log('INIT lines['+key_count+']');

    async.eachLimit(array, 2, insertFile, function(err) { console.log('FAIL async err['+err+']');} );
    //array.forEach(function(data){insertFile(data);return;});
});

The result of test input using array.forEach:

INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
LOAD [files.txt]
PASS [files.little.txt]
PASS [files.big.txt]
PASS [files.txt]
DONE Shutting down client, no more keys

Test record output using async.eachLimit:

INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
PASS [files.little.txt]
PASS [files.big.txt]
... hang, never gets to 3...
+3
source share
1 answer

. insertFile. ! :

var fs = require('fs');
var couchbase = require('couchbase');
var async = require('async');

var filelist = 'testlist.txt';
var key_count = 0;
var cb_config = { host: 'localhost:8091', bucket: 'default'};
var db = new couchbase.Connection(cb_config, function(err) {
  if (err) {
    console.log('ERRR connect to couchbase at config['+cb_config+']');
    throw err;
  }
});

var insertFile=function(line, callback) {
    console.log('LOAD ['+line+']');

    fs.readFile(line, function(file_err, f_doc) {
        if(file_err) throw file_err;

        db.set(line, f_doc, function(db_err, db_res){
            if (db_err) {
                console.log('FAIL ['+line+'] err['+db_err+']');
                callback(db_err);
            } else {
                console.log('PASS ['+line+']');
                callback();
            }
        });
    });
}

// read list of files into data array from file filelist
fs.readFile(filelist, function(filelist_err, data) {
    if(filelist_err) throw filelist_err;

    // HACK stoopid bug split adds empty line to array, use replace to fix
    var array = data.toString().replace(/\n$/, '').split('\n');
    key_count = array.length;
    console.log('READ files['+key_count+']');

    async.eachLimit(array, 2, insertFile, function(err) { 
        if (err) console.log('LAST with async err['+err+']');

        console.log('DONE Shutting down client, no more keys');
        db.shutdown();
    });
});

:

$ node testcase.js
READ files[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
PASS [files.little.txt]
LOAD [files.txt]
PASS [files.big.txt]
PASS [files.txt]
DONE Shutting down client, no more keys
+4

All Articles