RangeError: Invalid string length on very large json dataset

[jigsawmegs]

Environment

• Version of docxtemplater : 3.0.8
• Used docxtemplater-modules :
• Runner : NodeJs and iisnode

How to reproduce my problem :

My template is the following : template.zip
The data is of the following structure (only 1 record shown)
{ "data": [{"dru_id":"01568.1115950.00","drp_id":"0","letter_date":"23rd March, 2012","letter_date_p9d":"1st April, 2012","mail_name":"Mr X YZ & Mrs D YZ","mail_address1":"7 XXXXXX Place","mail_address2":"","mail_address3":"","mail_suburb":"SUBURBX","mail_postcode":"2000","mail_state":"NSW","mail_pafbarcode":"1301011130302012100102312123110133313","mail_pafbsp":"005","mail_pafdpid":0,"school_name_suburb":"Primary School, SuburbX","school_name":"Primary School","school_address1":"PO Box 1","school_address2":"","school_address3":"","school_state":"NSW","school_suburb":"SUBURBx","school_postcode":"2000","school_phone":"999 999","school_logo":"logox.jpg","school_main_contact":"Bill Anna","school_contact_name":"Bill Anna","contact_phone":"9999 99999","letter_signature":"Steve Anna","letter_signature_phone":"9999 99999","letter_signature_title":"Principal","bpay_code":"12345","ddf":"S2s4","ddf_crn":"99999","centrelink_crn":"XYX","overdue":832.0000,"total":2496.0000,"future":1664.0000,"deet":1568,"outstanding":832.0000,"bpay_ref":"3106 6069 7994 0","bpay_barcode":"3106 6069 7994 0","postbill_ref":"08323000000000031066069799","dd_reference":"3106606979949","ufp":370544,"first_record":true,"email":"","mobile":"","contact_method":0}], "templateName" : "Action1-ReminderStatement.docx", "hasFreeText" : false, "freeText" : "", "hasSummary" : false, "sortSummary" : [], "schoolSummary" : []}

The actual file has 7.5k records

With the following js file :

var express = require('express');
var bodyParser = require('body-parser');
var app = express();
var fs = require('fs');
var http = require('http');
var Docxtemplater = require('docxtemplater');
var ImageModule = require('docxtemplater-image-module');
var numeral = require('numeral');
var request = require('sync-request');
var cache = require('memory-cache');
var sizeOf = require('image-size');
var entities = require('entities');
var JSZip = require('jszip');

var code128 = {
    /**
             * Converts an input string to the equivilant string, that need to be produced using the 'Code 128' font.
             *
             * @static
             * @public
             * @this Demo.BarcodeConverter128
             * @memberof Demo.BarcodeConverter128
             * @param   {string}    value    String to be encoded
             * @return  {string}             Encoded string start/stop and checksum characters included
             */
            stringToBarcode: function (value) {
         //value = '08323000000000032943007019'
        // Parameters : a string
        // Return     : a string which give the bar code when it is dispayed with CODE128.TTF font
        //              : an empty string if the supplied parameter is no good

       var charPos, minCharPos;
  var currentChar, checksum;
  var isTableB = true, isValid = true;
  var returnValue = "";
  if (value.length > 0) {

    // Check for valid characters
    for (var charCount = 0; charCount < value.length; charCount++) {
      //currentChar = char.GetNumericValue(value, charPos);
      currentChar = value.charCodeAt(charCount);// value.substr(charCount, 1);
      if (!(currentChar >= 32 && currentChar <= 126)) {
        isValid = false;
        break;
      }
    }

    // Barcode is full of ascii characters, we can now process it
    if (isValid) {
      charPos = 0;
      var weight = 1;
      checksum = 0;
      while (charPos < value.length) {

        if (isTableB) {
          // See if interesting to switch to table C
          // yes for 4 digits at start or end, else if 6 digits
          if (charPos === 0 || charPos + 4 === value.length) {
            minCharPos = 4;
          }
          else {
            minCharPos = 6;
          }


          minCharPos = this.isNumber(value, charPos, minCharPos);

          if (minCharPos < 0) {
            // Choice table C
            if (charPos === 0) {
              // Starting with table C
              returnValue = String.fromCharCode(237); // char.ConvertFromUtf32(205);
            }
            else {
              // Switch to table C
              returnValue = returnValue + String.fromCharCode(199)
            }
            isTableB = false;
          }
          else {
            if (charPos === 0) {
              // Starting with table B
              returnValue = String.fromCharCode(232); // char.ConvertFromUtf32(204);
            }

          }
        }

        if (!isTableB) {
          // We are on table C, try to process 2 digits
          minCharPos = 2;
         
        
          minCharPos = this.isNumber(value, charPos, minCharPos);
          if (minCharPos < 0) {
            currentChar = parseInt(value.substr(charPos, 2));
            checksum = checksum + (currentChar * weight);
            weight=weight+1;
            if ((currentChar <= 94) && (currentChar >= 0)) {
              currentChar = currentChar + 32
            }
            else if ((currentChar <= 106) && (currentChar >= 95)) {
              currentChar = currentChar + 32 + 100
            }
            else {
              currentChar = -1
            }
            returnValue = returnValue + String.fromCharCode(currentChar);
            charPos += 2;
          }
          else {
            // We haven't 2 digits, switch to table B
            returnValue = returnValue + String.fromCharCode(231);
            isTableB = true;
          }
        }
        //if (isTableB) {
        //  // Process 1 digit with table B
        //  returnValue = returnValue + value.substr(charPos, 1);
        //  charPos++;
        //}
      }

      // Calculation of the checksum
      checksum = checksum + 105;
      //checksum = 0;
      //for (var loop = 0; loop < returnValue.length; loop++) {
      //  currentChar = returnValue.charCodeAt(loop);// returnValue.substr(loop, 1);
      //  currentChar = currentChar < 127 ? currentChar - 32 : currentChar - 100;
      //  if (loop === 0) {
      //    checksum = 105;
      //  }
      //  //else {
      //    checksum = (checksum + ((loop+1) * currentChar)) 
      //  //}
      //}

      var inputvalue = checksum % 103;
      if ((inputvalue <= 94) && (inputvalue >= 0)) {
        inputvalue = inputvalue + 32
      }
      else if ((inputvalue <= 106) && (inputvalue >= 95)) {
        inputvalue = inputvalue + 32+100
      }
      else {
        inputvalue = -1
      }
      // Add the checksum and the STOP
      returnValue = returnValue + String.fromCharCode(inputvalue) + String.fromCharCode(238);
      //var utf8_txt = cp1252_to_utf8(returnValue);
    }
  }
 
  return returnValue;
  },
    isNumber: function (InputValue, CharPos, MinCharPos) {
        // if the MinCharPos characters from CharPos are numeric, then MinCharPos = -1
        MinCharPos--;
        if (CharPos + MinCharPos < InputValue.length) {
            while (MinCharPos >= 0) {
                //if (parseInt(InputValue.substr(CharPos + MinCharPos, 1)) < 48 || parseInt(InputValue.substr(CharPos + MinCharPos, 1)) > 57) {
                if (InputValue.charCodeAt(CharPos + MinCharPos) < 48 || InputValue.charCodeAt(CharPos + MinCharPos) > 57) {
                    break;
                }
                MinCharPos--;
            }
        }
        return MinCharPos;
    }
};

app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));

var expressions = require('angular-expressions');
expressions.filters.num = function (input, digits) {
    if (input == null)
        return '0';
    else
        return input.toFixed(digits);
};
expressions.filters.drupart = function (input) {
    if (input == null) { return ''; }
    else
        if (input.length <= 14) {
            return input;
        }
        else {
            var array = input.split('.');
            var value = array[1];
            return value;
        }
}
expressions.filters.currency = function (input) {
    if (input == null)
        return '$0.00';
    else
    return numeral(input).format('$0, 0.00');
}
expressions.filters.billpay = function (input){
    if (input == null)
        return "";
    else {
        input = input.replace(/ /g, '');
        if (input.length == 13) //needs to be a valid bpay code
        {
            return '*71 230 ' + input.substr(0, 11) + ' ' + input.substr(11, 2);
        }
        else return input;
    }
}
expressions.filters.encodebarcode = function (barcode){
    if ((barcode == null) || (barcode == ''))
        return "";
    else {
       var result = code128.stringToBarcode(barcode);
	    result = entities.encodeXML(result);
         return '<w:p><w:r><w:rPr> <w:rFonts w:ascii="CCode128_S3" w:eastAsia="CCode128_S3" w:hAnsi="CCode128_S3"/><w:color w:val="000000"/><w:sz w:val="24"/></w:rPr><w:t>'+result+'</w:t></w:r></w:p>';

   }
}
expressions.filters.dmsbcode = function (barcode){
    if (barcode == "")
        return "";
    else
        return '<w:p><w:r><w:rPr><w:rFonts w:ascii="DMSBCode"/><w:sz w:val="56" /></w:rPr><w:t>' + barcode + '</w:t></w:r></w:p>';
}

angularParser = function (tag) {
    expr = expressions.compile(tag);
    return { get: expr };
}

//setup image options
var opts = {};
opts = { centered: false }
opts.getImage = function (imgData, tagName) {
    //var imgHttp = 'http://webapps.parra.catholic.edu.au/intweb2/crest-for-fas/bw/' + imgData;
     var imgHttp = 'http://webapps.parra.catholic.edu.au/intweb2/crest-for-fas/bw/' + imgData
    var _result = cache.get(imgData);
    if (_result == null) {
        _result = (request('GET', imgHttp).getBody());
        cache.put(imgData, _result);
    };
    return _result;
}
opts.getSize = function (img, tagValue, tag) {
    var _result = cache.get("s" + tagValue);
    if (_result == null) {
        _result = sizeOf(img);
        cache.put("s" + tagValue, _result);
    }
    return [_result.width, _result.height]; 
}

app.use(function (req, res, next) {
    res.header("Access-Control-Allow-Origin", "*");
    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
    next();
});

app.get('/node/matthew/', function (req, res) {
//app.get('/', function (req, res) {
    res.send('To merge your document make a POST request to /Merger');
});
app.post('/node/matthew/merger', function (req, res) {
//app.post('/merger', function (req, res) {
    var template = __dirname + "\\templates\\" + req.body.templateName;

    var content = fs.readFileSync(template, "binary");
    
    var imageModule = new ImageModule(opts);
    //var doc = new Docxtemplater(content);
    var zip = new JSZip(content);
    var doc = new Docxtemplater().loadZip(zip)
    
    //use angular expressions
    doc.setOptions({ parser: angularParser })
    
    doc.attachModule(imageModule);
    
    var merge = {
        letters: req.body.data, freeText: req.body.freeText, hasFreeText: req.body.hasFreeText, 
        hasSummary: req.body.hasSummary, sortSummary: req.body.sortSummary, schoolSummary: req.body.schoolSummary,
        newPage: '<w:p><w:br w:type="page" /></w:p>'
    };
    doc.setData(merge);

    //doc.render();
    
try {
	doc.render();
}
catch (error) {
	console.log(JSON.stringify({error: error}));
}
    
    
    var buf = doc.getZip()
                 .generate({ type: "nodebuffer", compression: 'DEFLATE' });
    
    res.json({ blob: buf.toString('base64') });
    //});
});

var server = app.listen(process.env.PORT, function () {
//var server = app.listen(1337, function () {
    var host = server.address().address;
    var port = server.address().port;
});

Error:

RangeError: Invalid string length
at Array.join (native)
at Object.render (C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\modules\loop.js:56:30)
at moduleRender (C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\render.js:10:28)
at C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\render.js:25:24
at Array.map (native)
at render (C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\render.js:24:26)
at XmlTemplater.render (C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\xml-templater.js:94:19)
at C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\docxtemplater.js:117:17
at Array.forEach (native)
at Docxtemplater.render (C:\Program Files\iisnode\www\matthew\node_modules\docxtemplater\js\docxtemplater.js:112:29)
Action1-ReminderStatement.zip

[edi9999]

I'm guessing that nodejs has a limit in the string length it can use. I will to reproduce your issue and see if I can think of a way to fix it (the first idea that comes to mind is to not use strings anymore when concatenating, but instead ArrayBuffers)

[jigsawmegs]

Thanks for the quick response. It's very strange. I found some references that speak about it really being a memory issue. If I reduce the number of records to 2.5k it works fine but slowly.

…

Sent from my iPad On 23 Mar 2017, at 7:13 pm, Edgar Hipp <[email protected]<mailto:[email protected]>> wrote: I'm guessing that nodejs has a limit in the string length it can use. I will to reproduce your issue and see if I can think of a way to fix it (the first idea that comes to mind is to not use strings anymore when concatenating, but instead ArrayBuffers) — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub<#291 (comment)>, or mute the thread<https://github.com/notifications/unsubscribe-auth/ABAY1OP9adH4faZJfGKUFCEqfoKWljeBks5roimngaJpZM4MmMvo>. Steve Mcgrath Office: (02) 9672 4222 Fax: (02) 9672 4211 [Jigsaw Technology]<http://www.jigsaw.com.au>

[edi9999]

From my experiments, the max size for string is 268434896 , which is 268 MB of string length. Do you think that you exceed that length ?

[edi9999]

This issue in nodejs reports a very similar number : nodejs/node#10738 (comment)

[edi9999]

Here is the patch I applied to test this :

diff --git a/es6/modules/loop.js b/es6/modules/loop.js
index cae4645..66a179b 100644
--- a/es6/modules/loop.js
+++ b/es6/modules/loop.js
@@ -53,7 +53,20 @@ const loopModule = {
 			));
 		}
 		options.scopeManager.loopOver(part.value, loopOver, part.inverted);
-		return {value: totalValue.join("")};
+		let result = "";
+		// console.log(JSON.stringify({"parts.length": parts.length}));
+		for (var i = 0, len = totalValue.length; i < len; i++) {
+			// if (result.length > 10000) {
+			// 	console.log(JSON.stringify({"result.length": result.length}));
+			// }
+			if (result.length > 1000000) {
+				console.log(JSON.stringify({"result.length": result.length}));
+
+			}
+			result+= totalValue[i];
+		}
+		return {value: result};
+		// return {value: totalValue.join("")};
 	},
 };
 
diff --git a/es6/tests/speed.js b/es6/tests/speed.js
index 9d69a8a..3b2c69b 100644
--- a/es6/tests/speed.js
+++ b/es6/tests/speed.js
@@ -62,4 +62,20 @@ describe("speed test", function () {
 			expect(duration).to.be.below(20000);
 		});
 	}
+
+	it.only("should be fast for many iterations", function () {
+		let innerContent = "";
+		for (let i = 1; i <= 1000; i++) {
+			innerContent+="a";
+		}
+		const content = `<w:t>{#users}${innerContent}{name}{/users}</w:t>`;
+		const users = [];
+		for (let i = 1; i <= 1000000; i++) {
+			users.push({name: "foo"});
+		}
+		const time = new Date();
+		testUtils.createXmlTemplaterDocx(content, {tags: {users}}).render();
+		const duration = new Date() - time;
+		expect(duration).to.be.below(600);
+	});
 });

[edi9999]

I'm closing this as we won't be able to fix it from docxtemplater code