From 7a1207e14112302ab92f964afe9c348f556dd4e5 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 03:04:11 +0100 Subject: [PATCH 01/11] POC fix IpcProvider chunk handling --- .../web3-providers/src/providers/IpcProvider.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index 11fe6629be4..cb2c576c0bb 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -74,13 +74,21 @@ export default class IpcProvider extends AbstractSocketProvider { const chunk = message.toString('utf8'); if (chunk.indexOf('\n') < 0) { - this.chunks += chunk; + this.chunks += chunk.substring(chunk.indexOf('\n') + 1); return; } - super.onMessage(this.chunks + chunk.substring(0, chunk.indexOf('\n'))); - this.chunks = chunk.substring(chunk.indexOf('\n') + 1); + this.chunks += chunk.substring(0, chunk.indexOf('\n')); + const parsedChunk = JSON.parse(this.chunks); + + if (isArray(parsedChunk)) { + parsedChunk.forEach(chunk => { + super.onMessage(chunk)); + }) + } + + super.onMessage(parsedChunk); } /** From 1a7484591bfb8a8908fd240de27772e50384ac68 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 03:18:09 +0100 Subject: [PATCH 02/11] onMessage method updated in IpcProvider --- packages/web3-providers/src/providers/IpcProvider.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index cb2c576c0bb..86b569e51bb 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -79,11 +79,17 @@ export default class IpcProvider extends AbstractSocketProvider { return; } - this.chunks += chunk.substring(0, chunk.indexOf('\n')); + if (this.chunks.length > 0) { + this.chunks += chunk.substring(0, chunk.indexOf('\n')); + } else { + this.chunks = chunk; + } + const parsedChunk = JSON.parse(this.chunks); + this.chunks = ''; if (isArray(parsedChunk)) { - parsedChunk.forEach(chunk => { + parsedChunk.forEach((chunk) => { super.onMessage(chunk)); }) } From 68fba67bff9db07f51480646e59e71e51328b0ae Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 11:00:31 +0100 Subject: [PATCH 03/11] onMessage method updated in IpcProvider --- packages/web3-providers/src/providers/IpcProvider.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index 86b569e51bb..f55e86a4ff1 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -72,15 +72,16 @@ export default class IpcProvider extends AbstractSocketProvider { */ onMessage(message) { const chunk = message.toString('utf8'); + const breakIndex = chunk.indexOf('\n'); - if (chunk.indexOf('\n') < 0) { - this.chunks += chunk.substring(chunk.indexOf('\n') + 1); + if (breakIndex < 0) { + this.chunks += chunk.substring(breakIndex + 1); return; } if (this.chunks.length > 0) { - this.chunks += chunk.substring(0, chunk.indexOf('\n')); + this.chunks += chunk.substring(0, breakIndex); } else { this.chunks = chunk; } @@ -91,7 +92,7 @@ export default class IpcProvider extends AbstractSocketProvider { if (isArray(parsedChunk)) { parsedChunk.forEach((chunk) => { super.onMessage(chunk)); - }) + }); } super.onMessage(parsedChunk); From 34b51393932e826dae16d21df729c81fc5cecdab Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 11:34:50 +0100 Subject: [PATCH 04/11] IpcProvider tested and fixed --- packages/web3-providers/src/providers/IpcProvider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index f55e86a4ff1..dd8ee354807 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -91,7 +91,7 @@ export default class IpcProvider extends AbstractSocketProvider { if (isArray(parsedChunk)) { parsedChunk.forEach((chunk) => { - super.onMessage(chunk)); + super.onMessage(chunk); }); } From b657f812a1b981a924fe933deff3a4aa113b4175 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 15:07:47 +0100 Subject: [PATCH 05/11] IpcProvider and related test updated --- .../src/providers/IpcProvider.js | 31 +++++++------- .../tests/src/providers/IpcProviderTest.js | 40 ++++++++++++++++++- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index dd8ee354807..f35959a86fe 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -35,7 +35,7 @@ export default class IpcProvider extends AbstractSocketProvider { constructor(connection, path) { super(connection, null); this.host = path; - this.chunks = ''; + this.chunks = []; } /** @@ -71,31 +71,28 @@ export default class IpcProvider extends AbstractSocketProvider { * @param {String|Buffer} message */ onMessage(message) { - const chunk = message.toString('utf8'); - const breakIndex = chunk.indexOf('\n'); + let chunk = message.toString('utf8'); + console.log(chunk); - if (breakIndex < 0) { - this.chunks += chunk.substring(breakIndex + 1); + if (chunk.indexOf('\n') < 0) { + this.chunks.push(chunk.replace('\n', '')); return; } - if (this.chunks.length > 0) { - this.chunks += chunk.substring(0, breakIndex); - } else { - this.chunks = chunk; - } - - const parsedChunk = JSON.parse(this.chunks); - this.chunks = ''; + this.chunks.push(chunk.replace('\n', '')); - if (isArray(parsedChunk)) { - parsedChunk.forEach((chunk) => { + if (this.chunks.length > 1) { + JSON.parse('[' + this.chunks.join() + ']').forEach((chunk) => { super.onMessage(chunk); }); + + this.chunks = []; + return; } - super.onMessage(parsedChunk); + super.onMessage(JSON.parse(this.chunks[0])); + this.chunks = []; } /** @@ -170,7 +167,7 @@ export default class IpcProvider extends AbstractSocketProvider { return; } - return reject(new Error("Connection error: Couldn't write on the socket with Socket.write(payload)")); + return reject(new Error('Connection error: Couldn\'t write on the socket with Socket.write(payload)')); }); } } diff --git a/packages/web3-providers/tests/src/providers/IpcProviderTest.js b/packages/web3-providers/tests/src/providers/IpcProviderTest.js index 402ccca800f..af010f5e691 100644 --- a/packages/web3-providers/tests/src/providers/IpcProviderTest.js +++ b/packages/web3-providers/tests/src/providers/IpcProviderTest.js @@ -66,16 +66,52 @@ describe('IpcProviderTest', () => { expect(socketMock.connect).toHaveBeenCalledWith({path: ipcProvider.path}); }); - it('calls onMessage', () => { + it('calls onMessage with one chunk', (done) => { const objWithToString = { toString: jest.fn(() => { return '{"id":"0x0"}'; }) }; + ipcProvider.on('0x0', (response) => { + expect(response).toEqual({id: '0x0'}); + + done(); + }); + ipcProvider.onMessage(objWithToString); - expect(objWithToString.toString).toHaveBeenCalled(); + expect(objWithToString.toString).toHaveBeenCalledWith('utf8'); + }); + + it('calls onMessage with more than one chunk', (done) => { + let callCount = 0; + const firstChunk = { + toString: jest.fn(() => { + return '\n[{"id":"0x0"}'; + }) + }; + const secondChunk = { + toString: jest.fn(() => { + return ',{"id":"0x0"}]'; + }) + }; + + ipcProvider.on('0x0', (response) => { + expect(response).toEqual({id: '0x0'}); + + if (callCount === 1) { + done(); + } + + callCount++; + }); + + ipcProvider.onMessage(firstChunk); + ipcProvider.onMessage(secondChunk); + + expect(firstChunk.toString).toHaveBeenCalledWith('utf8'); + expect(secondChunk.toString).toHaveBeenCalledWith('utf8'); }); it('calls registEventListener with a Socket object as connection', () => { From 6f2d300475487e0f5bdf7b0ad97713ea02b034f1 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Thu, 21 Mar 2019 16:49:55 +0100 Subject: [PATCH 06/11] handling of subscriptions and smaller requests fixed. There is still a problem with bigger responses (getCode()). --- .../src/providers/IpcProvider.js | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index f35959a86fe..831b7e0c61e 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -71,27 +71,24 @@ export default class IpcProvider extends AbstractSocketProvider { * @param {String|Buffer} message */ onMessage(message) { - let chunk = message.toString('utf8'); - console.log(chunk); + let chunk = message.toString().replace(new RegExp('\n', 'g'), ''); - if (chunk.indexOf('\n') < 0) { - this.chunks.push(chunk.replace('\n', '')); + if (!chunk.includes('}')) { + this.chunks.push(chunk); return; } - this.chunks.push(chunk.replace('\n', '')); - - if (this.chunks.length > 1) { - JSON.parse('[' + this.chunks.join() + ']').forEach((chunk) => { - super.onMessage(chunk); - }); - - this.chunks = []; - return; + if (chunk.includes('}{')) { + this.chunks.concat(chunk.replace(new RegExp('}{', 'g'), '}--{').split('--')); + } else { + this.chunks.push(chunk); } - super.onMessage(JSON.parse(this.chunks[0])); + JSON.parse('[' + this.chunks.join() + ']').forEach((chunk) => { + super.onMessage(chunk); + }); + this.chunks = []; } From 610194cd7505d798f78b537638c125030f17a820 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Fri, 22 Mar 2019 12:07:01 +0100 Subject: [PATCH 07/11] loading chunk handling added --- packages/web3-providers/src/providers/IpcProvider.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index 831b7e0c61e..a63b11a121f 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -36,6 +36,7 @@ export default class IpcProvider extends AbstractSocketProvider { super(connection, null); this.host = path; this.chunks = []; + this.loadingChunk = false; } /** @@ -74,7 +75,13 @@ export default class IpcProvider extends AbstractSocketProvider { let chunk = message.toString().replace(new RegExp('\n', 'g'), ''); if (!chunk.includes('}')) { - this.chunks.push(chunk); + this.loadingChunk = (this.chunks.push(chunk) - 1); + + return; + } + + if (!chunk.includes('{')) { + this.chunks[this.loadingChunk] = this.chunks[this.loadingChunk] + chunk; return; } @@ -90,6 +97,7 @@ export default class IpcProvider extends AbstractSocketProvider { }); this.chunks = []; + this.loadingChunk = false; } /** From d3300e55f98e1240baa249a3ef176eb5a85b85a1 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Fri, 22 Mar 2019 15:10:45 +0100 Subject: [PATCH 08/11] ipc connection more tested and onMessage method updated. --- packages/web3-providers/src/providers/IpcProvider.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index a63b11a121f..93dccef1b5c 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -73,15 +73,16 @@ export default class IpcProvider extends AbstractSocketProvider { */ onMessage(message) { let chunk = message.toString().replace(new RegExp('\n', 'g'), ''); + const JSONrpcValidStart = chunk.startsWith('{"jsonrpc"'); - if (!chunk.includes('}')) { + if (JSONrpcValidStart && !chunk.endsWith('}')) { this.loadingChunk = (this.chunks.push(chunk) - 1); return; } - if (!chunk.includes('{')) { - this.chunks[this.loadingChunk] = this.chunks[this.loadingChunk] + chunk; + if (!JSONrpcValidStart) { + this.chunks[this.loadingChunk] = this.chunks[this.loadingChunk].concat(chunk); return; } @@ -92,7 +93,7 @@ export default class IpcProvider extends AbstractSocketProvider { this.chunks.push(chunk); } - JSON.parse('[' + this.chunks.join() + ']').forEach((chunk) => { + JSON.parse('{"chunks": [' + this.chunks.join() + ']}').chunks.forEach((chunk) => { super.onMessage(chunk); }); From 546fb38f7c470b5d0d40029786d22fde8bf90e66 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Fri, 22 Mar 2019 18:13:02 +0100 Subject: [PATCH 09/11] stopped to create a cleaner chunks handling and the old parsing readded --- .../src/providers/IpcProvider.js | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index 93dccef1b5c..a76b846737a 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -35,8 +35,7 @@ export default class IpcProvider extends AbstractSocketProvider { constructor(connection, path) { super(connection, null); this.host = path; - this.chunks = []; - this.loadingChunk = false; + this.lastChunk = ''; } /** @@ -72,33 +71,36 @@ export default class IpcProvider extends AbstractSocketProvider { * @param {String|Buffer} message */ onMessage(message) { - let chunk = message.toString().replace(new RegExp('\n', 'g'), ''); - const JSONrpcValidStart = chunk.startsWith('{"jsonrpc"'); - - if (JSONrpcValidStart && !chunk.endsWith('}')) { - this.loadingChunk = (this.chunks.push(chunk) - 1); - - return; - } + let result = null; + let returnValues = []; + let dechunkedData = message.toString() + .replace(/\}[\n\r]?\{/g, '}|--|{') // }{ + .replace(/\}\][\n\r]?\[\{/g, '}]|--|[{') // }][{ + .replace(/\}[\n\r]?\[\{/g, '}|--|[{') // }[{ + .replace(/\}\][\n\r]?\{/g, '}]|--|{') // }]{ + .split('|--|'); + + dechunkedData.forEach((data) => { + result = null; + if (this.lastChunk) { + data = this.lastChunk + data; + } - if (!JSONrpcValidStart) { - this.chunks[this.loadingChunk] = this.chunks[this.loadingChunk].concat(chunk); + try { + result = JSON.parse(data); + } catch (e) { + this.lastChunk = data; - return; - } + return; + } - if (chunk.includes('}{')) { - this.chunks.concat(chunk.replace(new RegExp('}{', 'g'), '}--{').split('--')); - } else { - this.chunks.push(chunk); - } + this.lastChunk = null; + returnValues.push(result); + }); - JSON.parse('{"chunks": [' + this.chunks.join() + ']}').chunks.forEach((chunk) => { + returnValues.forEach((chunk) => { super.onMessage(chunk); }); - - this.chunks = []; - this.loadingChunk = false; } /** From 99fc31deffad8917b2fddfe39036c4ed7296995a Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Fri, 22 Mar 2019 18:21:16 +0100 Subject: [PATCH 10/11] test updated --- .../tests/src/providers/IpcProviderTest.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/web3-providers/tests/src/providers/IpcProviderTest.js b/packages/web3-providers/tests/src/providers/IpcProviderTest.js index af010f5e691..284044bd7c4 100644 --- a/packages/web3-providers/tests/src/providers/IpcProviderTest.js +++ b/packages/web3-providers/tests/src/providers/IpcProviderTest.js @@ -81,19 +81,24 @@ describe('IpcProviderTest', () => { ipcProvider.onMessage(objWithToString); - expect(objWithToString.toString).toHaveBeenCalledWith('utf8'); + expect(objWithToString.toString).toHaveBeenCalledWith(); }); it('calls onMessage with more than one chunk', (done) => { let callCount = 0; const firstChunk = { toString: jest.fn(() => { - return '\n[{"id":"0x0"}'; + return '{"id":"0x0"}{"id":"0x0"}'; }) }; const secondChunk = { toString: jest.fn(() => { - return ',{"id":"0x0"}]'; + return '{"id":"0x0"}{"id":"0x'; + }) + }; + const thirdChunk = { + toString: jest.fn(() => { + return '0"}'; }) }; @@ -109,9 +114,11 @@ describe('IpcProviderTest', () => { ipcProvider.onMessage(firstChunk); ipcProvider.onMessage(secondChunk); + ipcProvider.onMessage(thirdChunk); - expect(firstChunk.toString).toHaveBeenCalledWith('utf8'); - expect(secondChunk.toString).toHaveBeenCalledWith('utf8'); + expect(firstChunk.toString).toHaveBeenCalledWith(); + expect(secondChunk.toString).toHaveBeenCalledWith(); + expect(thirdChunk.toString).toHaveBeenCalledWith(); }); it('calls registEventListener with a Socket object as connection', () => { From 7de266c13ad3b700a23c98dc2ee5105a369d3cb1 Mon Sep 17 00:00:00 2001 From: Samuel Furter Date: Fri, 22 Mar 2019 19:35:33 +0100 Subject: [PATCH 11/11] eslint error fixed --- packages/web3-providers/src/providers/IpcProvider.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/web3-providers/src/providers/IpcProvider.js b/packages/web3-providers/src/providers/IpcProvider.js index a76b846737a..1be45b70476 100644 --- a/packages/web3-providers/src/providers/IpcProvider.js +++ b/packages/web3-providers/src/providers/IpcProvider.js @@ -73,7 +73,8 @@ export default class IpcProvider extends AbstractSocketProvider { onMessage(message) { let result = null; let returnValues = []; - let dechunkedData = message.toString() + let dechunkedData = message + .toString() .replace(/\}[\n\r]?\{/g, '}|--|{') // }{ .replace(/\}\][\n\r]?\[\{/g, '}]|--|[{') // }][{ .replace(/\}[\n\r]?\[\{/g, '}|--|[{') // }[{ @@ -88,7 +89,7 @@ export default class IpcProvider extends AbstractSocketProvider { try { result = JSON.parse(data); - } catch (e) { + } catch (error) { this.lastChunk = data; return; @@ -175,7 +176,7 @@ export default class IpcProvider extends AbstractSocketProvider { return; } - return reject(new Error('Connection error: Couldn\'t write on the socket with Socket.write(payload)')); + return reject(new Error("Connection error: Couldn't write on the socket with Socket.write(payload)")); }); } }