My problem seems to be the opposite of any node.js query :-) I have a simple forEach loop to read a list of files and insert them into the Couchbase database. This works fine, but never exits after reading all the lines. So I added a counter to disconnect the couchbase connection after all the inserts have completed. It works.
This process is designed to download hundreds of thousands of files, so I brought the asynchronization module into the mix for batch insertion into groups of 100. Async.eachLimit is used to iterate over the array and insert documents into packages. Now, the Orin problem has returned. No matter what magic is used by async.eachLimit to recognize the process, it does not.
I have passed javascript validation, callbacks, async, etc. Google searches affect keywords, but not this problem. I reduced the code to the next test test. To test, create three files and add their names to testlist.txt.
Async.eachLimit in place runs until it reaches the limit and then freezes. Comment on this and uncomment the array.forEach array and it works. Thanks in advance!
var fs = require('fs');
var couchbase = require('couchbase');
var async = require('async');
var filelist = 'testlist.txt';
var key_count = 0;
var cb_config = { host: 'localhost:8091', bucket: 'default'};
var db = new couchbase.Connection(cb_config, function(err) {
if (err) {
console.log('ERRR connect to couchbase at config['+cb_config+']');
throw err;
}
});
var insertFile=function(line) {
console.log('LOAD ['+line+']');
fs.readFile(line, function(file_err, f_doc) {
if(file_err) throw file_err;
db.set(line, f_doc, function(db_err, db_res){
if (db_err) {
console.log('FAIL ['+line+'] err['+db_err+']');
} else {
console.log('PASS ['+line+']');
}
key_count--;
if (key_count == 0) {
console.log('DONE Shutting down client, no more keys');
db.shutdown();
}
});
});
}
fs.readFile(filelist, function(filelist_err, lines) {
if(filelist_err) throw filelist_err;
var array = lines.toString().replace(/\n$/, '').split('\n');
key_count = array.length;
console.log('INIT lines['+key_count+']');
async.eachLimit(array, 2, insertFile, function(err) { console.log('FAIL async err['+err+']');} );
});
The result of test input using array.forEach:
INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
LOAD [files.txt]
PASS [files.little.txt]
PASS [files.big.txt]
PASS [files.txt]
DONE Shutting down client, no more keys
Test record output using async.eachLimit:
INIT lines[3]
LOAD [files.big.txt]
LOAD [files.little.txt]
PASS [files.little.txt]
PASS [files.big.txt]
... hang, never gets to 3...