worker fixes (#4059)

Describe what your pull request does. If you can, add GIFs or images
showing the before and after of your change.

### Change type

- [x] `bugfix`
- [ ] `improvement`
- [ ] `feature`
- [ ] `api`
- [ ] `other`

### Test plan

1. Create a shape...
2.

- [ ] Unit tests
- [ ] End to end tests

### Release notes

- Fixed a bug with...

---------

Co-authored-by: alex <alex@dytry.ch>
This commit is contained in:
David Sheldrick 2024-07-02 11:53:27 +01:00 committed by GitHub
parent f66763371c
commit adb84d97e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 97 additions and 17 deletions

View file

@ -1,11 +1,20 @@
import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
import kleur from 'kleur'
import { lock } from 'proper-lockfile'
import stripAnsi from 'strip-ansi'
// at the time of writing, workerd will regularly crash with a segfault
// but the error is not caught by the process, so it will just hang
// this script wraps the process, tailing the logs and restarting the process
// if we encounter the string 'Segmentation fault'
const lockfileName = __dirname
/**
* a long time ago, workerd would regularly crash with a segfault but the error is not caught by the
* process, so it will just hang. this script wraps the process, tailing the logs and restarting the
* process if we encounter the string 'Segmentation fault'. we think this error is probably fixed
* now.
*
* there's a separate issue where spawning multiple workerd instances at once would cause them to
* fight and crash. we use a lockfile to start our wrokerd instances one at a time, waiting for the
* previous one to be ready before we start the next.
*/
class MiniflareMonitor {
private process: ChildProcessWithoutNullStreams | null = null
@ -14,9 +23,10 @@ class MiniflareMonitor {
private args: string[] = []
) {}
public start(): void {
this.stop() // Ensure any existing process is stopped
console.log(`Starting wrangler...`)
public async start(): Promise<void> {
await this.stop() // Ensure any existing process is stopped
await this.lock()
await console.log(`Starting wrangler...`)
this.process = spawn(this.command, this.args, {
env: {
NODE_ENV: 'development',
@ -35,6 +45,11 @@ class MiniflareMonitor {
private handleOutput(output: string, err = false): void {
if (!output) return
if (output.includes('Ready on') && this.isLocked()) {
this.release()
}
if (output.includes('Segmentation fault')) {
console.error('Segmentation fault detected. Restarting Miniflare...')
this.restart()
@ -43,18 +58,45 @@ class MiniflareMonitor {
}
}
private restart(): void {
private async restart(): Promise<void> {
console.log('Restarting wrangler...')
this.stop()
await this.stop()
setTimeout(() => this.start(), 3000) // Restart after a short delay
}
private stop(): void {
private async stop(): Promise<void> {
if (this.isLocked()) await this.release()
if (this.process) {
this.process.kill()
this.process = null
}
}
private _lockPromise?: Promise<() => Promise<void>>
private isLocked() {
return !!this._lockPromise
}
private async lock() {
if (this.isLocked()) throw new Error('Already locked')
console.log('Locking...')
this._lockPromise = lock(lockfileName, {
retries: {
minTimeout: 500,
retries: 10,
},
})
await this._lockPromise
console.log('Locked')
}
private async release() {
if (!this.isLocked()) throw new Error('Not locked')
console.log('Releasing...')
const lockPromise = this._lockPromise!
this._lockPromise = undefined
const release = await lockPromise
await release()
console.log('Released')
}
}
class SizeReporter {
@ -119,6 +161,7 @@ new MiniflareMonitor('wrangler', [
'info',
'--var',
'IS_LOCAL:true',
...process.argv.slice(2),
]).start()
new SizeReporter().start()