Skip to content

Commit 79a6547

Browse files
AmericasAmericas
committed
Add health monitor
1 parent e7f0e76 commit 79a6547

4 files changed

Lines changed: 341 additions & 1 deletion

File tree

‎src/health-monitor.js‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
'use strict';
2+
3+
/**
4+
* Module dependencies.
5+
*/
6+
7+
const log = require('debugnyan')('process-manager:health-monitor');
8+
9+
/**
10+
* `HealthMonitor`.
11+
*/
12+
13+
class HealthMonitor {
14+
/**
15+
* Constructor.
16+
*/
17+
18+
constructor() {
19+
this.checks = [];
20+
this.globalState = HealthMonitor.states.UNKNOWN;
21+
this.states = {};
22+
}
23+
24+
/**
25+
* Add health check.
26+
*/
27+
28+
addCheck({ handler, id, interval = 5000 }) {
29+
if (this.states[id]) {
30+
throw new Error('Cannot add handler since it would overwrite an existing one');
31+
}
32+
33+
this.states[id] = HealthMonitor.states.UNKNOWN;
34+
35+
const check = setInterval(async () => {
36+
let state;
37+
38+
try {
39+
state = await handler() ? HealthMonitor.states.HEALTHY : HealthMonitor.states.UNHEALTHY;
40+
} catch (e) {
41+
state = HealthMonitor.states.UNHEALTHY;
42+
}
43+
44+
this.updateState({ id, state });
45+
}, interval);
46+
47+
this.checks.push(check);
48+
}
49+
50+
/**
51+
* Cleanup health monitor by clearing all timers and resetting the internal state.
52+
*/
53+
54+
cleanup() {
55+
this.checks.forEach(clearInterval);
56+
57+
this.checks = [];
58+
this.globalState = HealthMonitor.states.UNKNOWN;
59+
this.states = {};
60+
}
61+
62+
/**
63+
* Handles state changes.
64+
*/
65+
66+
updateState({ id, state }) {
67+
if (this.states[id] === state) {
68+
return;
69+
}
70+
71+
log.info({ id, newState: state, oldState: this.states[id] }, 'Component health status has changed');
72+
73+
this.states[id] = state;
74+
75+
// The sorted states array makes it so that the state at the end of the array is the relevant one.
76+
// The global state is:
77+
// - UNKNOWN if one exists.
78+
// - UNHEALTHY if one exists and there are no UNKNOWN states.
79+
// - HEALTHY if there are no UNKNOWN and UNHEALTHY states.
80+
const globalState = Object.values(this.states).sort((left, right) => { return left < right ? 1 : -1; })[0];
81+
82+
if (this.globalState === globalState) {
83+
return;
84+
}
85+
86+
log.info({ newState: globalState, oldState: this.globalState }, 'Global health status has changed');
87+
88+
this.globalState = globalState;
89+
}
90+
}
91+
92+
/**
93+
* Health states.
94+
*/
95+
96+
HealthMonitor.states = {
97+
HEALTHY: 'healthy',
98+
UNHEALTHY: 'unhealthy',
99+
UNKNOWN: 'unknown'
100+
};
101+
102+
/**
103+
* Export `HealthMonitor` class.
104+
*/
105+
106+
module.exports = HealthMonitor;

‎src/index.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
* Module dependencies.
55
*/
66

7+
const HealthMonitor = require('./health-monitor');
78
const log = require('debugnyan')('process-manager');
89
const utils = require('./utils');
910

@@ -26,6 +27,7 @@ class ProcessManager {
2627
constructor() {
2728
this.errors = [];
2829
this.forceShutdown = utils.deferred();
30+
this.healthMonitor = new HealthMonitor();
2931
this.hooks = [];
3032
this.running = [];
3133
this.terminating = false;
@@ -183,6 +185,7 @@ class ProcessManager {
183185
.then(() => log.info('All running instances have stopped'))
184186
.then(() => this.hook('drain'))
185187
.then(() => log.info(`${this.hooks.filter(hook => hook.type === 'drain').length} server(s) drained`))
188+
.then(() => this.healthMonitor.cleanup())
186189
.then(() => this.hook('disconnect'))
187190
.then(() => log.info(`${this.hooks.filter(hook => hook.type === 'disconnect').length} service(s) disconnected`))
188191
.then(() => this.hook('exit', this.errors));

‎test/src/health-monitor.test.js‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
'use strict';
2+
3+
/**
4+
* Module Dependencies.
5+
*/
6+
7+
const HealthMonitor = require('../../src/health-monitor');
8+
const log = require('debugnyan')('process-manager:health-monitor');
9+
10+
/**
11+
* Instances.
12+
*/
13+
14+
const healthMonitor = new HealthMonitor();
15+
16+
/**
17+
* Test `HealthMonitor`.
18+
*/
19+
20+
describe('HealthMonitor', () => {
21+
afterEach(() => {
22+
healthMonitor.cleanup();
23+
});
24+
25+
describe('constructor()', () => {
26+
it('should set up the default values', () => {
27+
const healthMonitor = new HealthMonitor();
28+
29+
expect(healthMonitor.checks).toEqual([]);
30+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.UNKNOWN);
31+
expect(healthMonitor.states).toEqual({});
32+
});
33+
});
34+
35+
describe('addCheck()', () => {
36+
it('should throw an error if the `id` is already in use', () => {
37+
healthMonitor.addCheck({ handler: () => {}, id: 'foo' });
38+
39+
try {
40+
healthMonitor.addCheck({ handler: () => {}, id: 'foo' });
41+
42+
fail();
43+
} catch (e) {
44+
expect(e).toBeInstanceOf(Error);
45+
expect(e.message).toBe('Cannot add handler since it would overwrite an existing one');
46+
}
47+
});
48+
49+
it('should add the check and setup the state', () => {
50+
healthMonitor.addCheck({ handler: () => {}, id: 'foo' });
51+
52+
expect(healthMonitor.checks).toHaveLength(1);
53+
expect(healthMonitor.states.foo).toBe(HealthMonitor.states.UNKNOWN);
54+
});
55+
56+
describe('when running the check', () => {
57+
it('should call `healthMonitor.updateState()` with `UNHEALTHY` status if the check throws an error', done => {
58+
const handler = jest.fn(() => {
59+
throw new Error();
60+
});
61+
62+
jest.spyOn(healthMonitor, 'updateState').mockImplementation(({ id, state }) => {
63+
expect(handler).toHaveBeenCalledTimes(1);
64+
expect(id).toBe('foo');
65+
expect(state).toBe(HealthMonitor.states.UNHEALTHY);
66+
done();
67+
});
68+
69+
healthMonitor.addCheck({ handler, id: 'foo', interval: 0 });
70+
});
71+
72+
it('should call `healthMonitor.updateState()` with `UNHEALTHY` status if the check returns a falsy value', done => {
73+
const handler = jest.fn(() => {});
74+
75+
jest.spyOn(healthMonitor, 'updateState').mockImplementation(({ id, state }) => {
76+
expect(handler).toHaveBeenCalledTimes(1);
77+
expect(id).toBe('foo');
78+
expect(state).toBe(HealthMonitor.states.UNHEALTHY);
79+
done();
80+
});
81+
82+
healthMonitor.addCheck({ handler, id: 'foo', interval: 0 });
83+
});
84+
85+
it('should call `healthMonitor.updateState()` with `HEALTHY` status if the check returns a truthy value', done => {
86+
const handler = jest.fn(() => true);
87+
88+
jest.spyOn(healthMonitor, 'updateState').mockImplementation(({ id, state }) => {
89+
expect(handler).toHaveBeenCalledTimes(1);
90+
expect(id).toBe('foo');
91+
expect(state).toBe(HealthMonitor.states.HEALTHY);
92+
done();
93+
});
94+
95+
healthMonitor.addCheck({ handler, id: 'foo', interval: 0 });
96+
});
97+
98+
it('should call handle asynchronous checks', done => {
99+
const handler = jest.fn(() => Promise.resolve(true));
100+
101+
jest.spyOn(healthMonitor, 'updateState').mockImplementation(({ id, state }) => {
102+
expect(handler).toHaveBeenCalledTimes(1);
103+
expect(id).toBe('foo');
104+
expect(state).toBe(HealthMonitor.states.HEALTHY);
105+
done();
106+
});
107+
108+
healthMonitor.addCheck({ handler, id: 'foo', interval: 0 });
109+
});
110+
});
111+
});
112+
113+
describe('cleanup()', () => {
114+
it('should clear all checks and states currently running', async () => {
115+
healthMonitor.checks.push(setInterval(() => {}, 5000));
116+
healthMonitor.states.foo = HealthMonitor.states.HEALTHY;
117+
healthMonitor.globalState = HealthMonitor.states.HEALTHY;
118+
119+
healthMonitor.cleanup();
120+
121+
expect(healthMonitor.checks).toHaveLength(0);
122+
expect(healthMonitor.states).toEqual({});
123+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.UNKNOWN);
124+
});
125+
});
126+
127+
describe('updateState()', () => {
128+
it('should not update the component state if it has not changed', () => {
129+
healthMonitor.states.foo = HealthMonitor.states.HEALTHY;
130+
131+
jest.spyOn(log, 'info');
132+
133+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.HEALTHY });
134+
135+
expect(log.info).not.toHaveBeenCalled();
136+
137+
expect(healthMonitor.states.foo).toBe(HealthMonitor.states.HEALTHY);
138+
});
139+
140+
it('should update the component state if it has changed', () => {
141+
healthMonitor.states.foo = HealthMonitor.states.HEALTHY;
142+
healthMonitor.globalState = HealthMonitor.states.UNHEALTHY;
143+
144+
jest.spyOn(log, 'info');
145+
146+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.UNHEALTHY });
147+
148+
expect(log.info).toHaveBeenCalledTimes(1);
149+
expect(log.info).toHaveBeenCalledWith({
150+
id: 'foo',
151+
newState: HealthMonitor.states.UNHEALTHY,
152+
oldState: HealthMonitor.states.HEALTHY
153+
}, 'Component health status has changed');
154+
155+
expect(healthMonitor.states.foo).toBe(HealthMonitor.states.UNHEALTHY);
156+
});
157+
158+
it('should not update the global state if it has not changed', () => {
159+
healthMonitor.states.foo = HealthMonitor.states.UNHEALTHY;
160+
healthMonitor.globalState = HealthMonitor.states.HEALTHY;
161+
162+
jest.spyOn(log, 'info');
163+
164+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.HEALTHY });
165+
166+
expect(log.info).toHaveBeenCalledTimes(1);
167+
168+
expect(healthMonitor.states.foo).toBe(HealthMonitor.states.HEALTHY);
169+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.HEALTHY);
170+
});
171+
172+
it('should update the global state if it has changed', () => {
173+
healthMonitor.states.foo = HealthMonitor.states.HEALTHY;
174+
healthMonitor.globalState = HealthMonitor.states.HEALTHY;
175+
176+
jest.spyOn(log, 'info');
177+
178+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.UNHEALTHY });
179+
180+
expect(log.info).toHaveBeenCalledTimes(2);
181+
expect(log.info).toHaveBeenLastCalledWith({
182+
newState: HealthMonitor.states.UNHEALTHY,
183+
oldState: HealthMonitor.states.HEALTHY
184+
}, 'Global health status has changed');
185+
186+
expect(healthMonitor.states.foo).toBe(HealthMonitor.states.UNHEALTHY);
187+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.UNHEALTHY);
188+
});
189+
190+
describe('global state', () => {
191+
it('should be UNKNOWN if at least one of the components is in the UNKNOWN state', () => {
192+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.HEALTHY });
193+
healthMonitor.updateState({ id: 'bar', state: HealthMonitor.states.UNHEALTHY });
194+
healthMonitor.updateState({ id: 'biz', state: HealthMonitor.states.UNKNOWN });
195+
196+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.UNKNOWN);
197+
expect(healthMonitor.states).toEqual({
198+
bar: HealthMonitor.states.UNHEALTHY,
199+
biz: HealthMonitor.states.UNKNOWN,
200+
foo: HealthMonitor.states.HEALTHY
201+
});
202+
});
203+
204+
it('should be UNHEALTHY if no component is in the UNKNOWN state and at least one of the components is in the UNHEALTHY state', () => {
205+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.HEALTHY });
206+
healthMonitor.updateState({ id: 'bar', state: HealthMonitor.states.UNHEALTHY });
207+
healthMonitor.updateState({ id: 'biz', state: HealthMonitor.states.HEALTHY });
208+
209+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.UNHEALTHY);
210+
expect(healthMonitor.states).toEqual({
211+
bar: HealthMonitor.states.UNHEALTHY,
212+
biz: HealthMonitor.states.HEALTHY,
213+
foo: HealthMonitor.states.HEALTHY
214+
});
215+
});
216+
217+
it('should be HEALTHY if all components are in the HEALTHY state', () => {
218+
healthMonitor.updateState({ id: 'foo', state: HealthMonitor.states.HEALTHY });
219+
healthMonitor.updateState({ id: 'bar', state: HealthMonitor.states.HEALTHY });
220+
healthMonitor.updateState({ id: 'biz', state: HealthMonitor.states.HEALTHY });
221+
222+
expect(healthMonitor.globalState).toBe(HealthMonitor.states.HEALTHY);
223+
expect(healthMonitor.states).toEqual({
224+
bar: HealthMonitor.states.HEALTHY,
225+
biz: HealthMonitor.states.HEALTHY,
226+
foo: HealthMonitor.states.HEALTHY
227+
});
228+
});
229+
});
230+
});
231+
});

test/index.test.js renamed to test/src/index.test.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ describe('ProcessManager', () => {
1313
jest.spyOn(process, 'on').mockImplementation(() => {});
1414
jest.spyOn(console, 'error').mockImplementation(() => {});
1515

16-
processManager = require('../src');
16+
processManager = require('../../src');
1717
});
1818

1919
describe('constructor()', () => {

0 commit comments

Comments
 (0)