[infra] maybe fix canary publish (#1950)

This last day or two our npm publish script has been randomly failing
due to npm flakiness. I'm seeing the following error:

Failed to save packument. A common cause is if you try to publish a new
package before the previous package has been fully processed.

This doesn't seem to be our fault since we're publishing things in the
right order, the version numbers and package.json files are all correct,
and we're waiting for things to appear in the registry after publishing
before moving on to the next package.

So I'm thinking maybe npm is a little tired right now or something and
needs a little extra time to handle things.

So I've wrapped our publish command inside a retry block.

At the same time I noticed that the `--tolerate-republish` flag does not
seem to be working for canary version numbers, so I've added some extra
logic for that too. Hopefully this means if things fail due to
persistent npm flake we can just run the action again.

### Change Type

- [ ] `patch` — Bug fix
- [ ] `minor` — New feature
- [ ] `major` — Breaking change
- [ ] `dependencies` — Changes to package dependencies[^1]
- [ ] `documentation` — Changes to the documentation only[^2]
- [ ] `tests` — Changes to any test code only[^2]
- [x] `internal` — Any other changes that don't affect the published
package[^2]
- [ ] I don't know

[^1]: publishes a `patch` release, for devDependencies use `internal`
[^2]: will not publish a new version

scripts/lib/publishing.ts

@@ -1,8 +1,9 @@
 import { execSync } from 'child_process'
 import { fetch } from 'cross-fetch'
-import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'
+import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs'
 import path, { join } from 'path'
 import { compare, parse } from 'semver'
+import { exec } from './exec'
 import { BUBLIC_ROOT } from './file'
 import { nicelog } from './nicelog'
 
@@ -126,14 +127,50 @@ export async function publish() {
 			`Publishing ${packageDetails.name} with version ${packageDetails.version} under tag @${prereleaseTag}`
 		)
 
-		execSync(`yarn npm publish --tag ${prereleaseTag} --tolerate-republish --access public`, {
-			stdio: 'inherit',
-			cwd: packageDetails.dir,
-		})
-		let waitAttempts = 10
-
-		loop: while (waitAttempts > 0) {
+		await retry(
+			async () => {
+				let output = ''
 				try {
+					exec(
+						`yarn`,
+						[
+							'npm',
+							'publish',
+							'--tag',
+							String(prereleaseTag),
+							'--tolerate-republish',
+							'--access',
+							'public',
+						],
+						{
+							pwd: packageDetails.dir,
+							processStdoutLine: (line) => {
+								output += line + '\n'
+								nicelog(line)
+							},
+							processStderrLine: (line) => {
+								output += line + '\n'
+								nicelog(line)
+							},
+						}
+					)
+				} catch (e) {
+					if (output.includes('You cannot publish over the previously published versions')) {
+						// --tolerate-republish seems to not work for canary versions??? so let's just ignore this error
+						return
+					}
+					throw e
+				}
+			},
+			{
+				delay: 10_000,
+				numAttempts: 5,
+			}
+		)
+
+		await retry(
+			async ({ attempt, total }) => {
+				nicelog('Waiting for package to be published... attempt', attempt, 'of', total)
 				// fetch the new package directly from the npm registry
 				const newVersion = packageDetails.version
 				const unscopedName = packageDetails.name.replace('@tldraw/', '')
@@ -146,12 +183,36 @@ export async function publish() {
 				if (res.status >= 400) {
 					throw new Error(`Package not found: ${res.status}`)
 				}
-				break loop
-			} catch (e) {
-				nicelog('Waiting for package to be published... attemptsRemaining', waitAttempts)
-				waitAttempts--
-				await new Promise((resolve) => setTimeout(resolve, 3000))
+			},
+			{
+				delay: 3000,
+				numAttempts: 10,
+			}
+		)
 	}
 }
+
+function retry(
+	fn: (args: { attempt: number; remaining: number; total: number }) => Promise<void>,
+	opts: {
+		numAttempts: number
+		delay: number
 	}
+): Promise<void> {
+	return new Promise((resolve, reject) => {
+		let attempts = 0
+		function attempt() {
+			fn({ attempt: attempts, remaining: opts.numAttempts - attempts, total: opts.numAttempts })
+				.then(resolve)
+				.catch((err) => {
+					attempts++
+					if (attempts >= opts.numAttempts) {
+						reject(err)
+					} else {
+						setTimeout(attempt, opts.delay)
+					}
+				})
+		}
+		attempt()
+	})
 }