🐛 introduce retryability into the changing room notifier because infura sometimes breaks

functions/integrations/changingroom.js

@@ -3,6 +3,7 @@ const { db, dataFromSnap } = require( '../modules/firebase' )
 const { dev, log } = require( '../modules/helpers' )
 const { ask_signer_is_for_available_emails } = require( './signer_is' )
 const { send_outfit_available_email } = require( './ses' )
+const { throttle_and_retry } = require( '../modules/helpers' )
 
 // Web3 APIs
 const { getOwingAddressOfTokenId } = require( '../modules/contract' )
@@ -207,11 +208,13 @@ exports.setPrimaryOutfit = async function( req, res ) {
 // /////////////////////////////*/
 exports.notify_holders_of_changing_room_updates = async context => {
 
+	const newOutfitAllowedInterval = 1000 * 60 * 60 * 24 * 30
+
 	try {
 
 		// Get all Rocketeers with outfits available
 		const network = dev ? `rinkeby` : `mainnet`
-		const limit = dev ? 50 : 5000 // max supply 3475
+		const limit = dev ? 5000 : 5000 // max supply 3475
 		console.log( `Getting ${ limit } rocketeers on ${ network }` )
 		let all_rocketeers = await db.collection( `${ network }Rocketeers` ).limit( limit ).get().then( dataFromSnap )
 		console.log( `Got ${ all_rocketeers.length } Rocketeers` )
@@ -220,27 +223,39 @@ exports.notify_holders_of_changing_room_updates = async context => {
 		all_rocketeers = all_rocketeers.filter( ( {uid} ) => uid > 0 && uid <= 3475 )
 		console.log( `Proceeding with ${ all_rocketeers.length } valid Rocketeers` )
 		
+
 		// Check which rocketeers have outfits available
 		const has_outfit_available = all_rocketeers.filter( rocketeer => {
 
-			const { attributes } = rocketeer
+			const { value: last_outfit_change } = rocketeer.attributes.find( ( { trait_type } ) => trait_type === 'last outfit change' ) || { value: 0 }
-			const outfit_available = attributes.find( ( { trait_type } ) => trait_type == 'last outfit change' )
+			const timeUntilAllowedToChange = newOutfitAllowedInterval - ( Date.now() - last_outfit_change )
-			// If outfit available is in the past, keep it
+
-			if( outfit_available?.value < Date.now() ) return true
+			// Keep those with changes allowed
+			if( timeUntilAllowedToChange < 0 ) return true
 
 			// If outfit available in the future, discard
 			return false
 
 		} )
 
-
-
 		// Get the owning wallets of available outfits
-		const owners = await Promise.all( has_outfit_available.map( async ( { uid } ) => {
+		const owner_getting_queue = has_outfit_available.map( ( { uid } ) => async () => {
-			log( `Getting owner of `, uid )
+			log( `Getting owner of ${ uid }` )
 			const owning_address = await getOwingAddressOfTokenId( uid )
 			return { uid, owning_address }
-		} )  )
+		} )
+		let owners = await throttle_and_retry( owner_getting_queue, 10, `get owners`, 2, 5 )
+		console.log( `${ owners.length } Rocketeer owners found` )
+
+		// Get the owners we have already emailed recently
+		const owner_meta = await db.collection( `meta` ).get().then( dataFromSnap )
+		const owners_emailed_recently = owner_meta
+											.filter( ( { last_emailed_about_outfit } ) => last_emailed_about_outfit > ( Date.now() - newOutfitAllowedInterval ) )
+											.map( ( { uid } ) => uid )
+		
+		// Remove owners from list of they were emailed too recently
+		console.log( `${ owners_emailed_recently.length } owners emailed too recently` )
+		owners = owners.filter( address => !owners_emailed_recently.includes( address ) )
 
 		// Check which owners have signer.is emails
 		const owners_with_signer_email = await ask_signer_is_for_available_emails( owners.map( ( { owning_address } ) => owning_address ) )
@@ -265,13 +280,23 @@ exports.notify_holders_of_changing_room_updates = async context => {
 		// List the owning emails
 		const owners_to_email = Object.keys( rocketeers_by_address )
 
+		// Take note of who we emailed so as to not spam them
+		const meta_writing_queue = owners_to_email.map( ( address ) => () => {
+
+			return db.collection( `meta` ).doc( address ).set( { last_emailed_about_outfit: Date.now(), updated: Date.now(), updated_human: new Date().toString() }, { merge: true } )
+
+	} )
+		await throttle_and_retry( meta_writing_queue, 50, `keep track of who we emailed`, 2, 10 )
+
 		// Send emails to the relevant owners
-		await Promise.all( owners_to_email.map( async owning_address => {
+		console.log( `Sending email to ${ owners_to_email.length } addresses` )
+		const email_sending_queue = owners_to_email.map( ( owning_address ) => async () => {
 
 			const rocketeers = rocketeers_by_address[ owning_address ]
 			await send_outfit_available_email( rocketeers, `${ owning_address }@signer.is` )
 
-		} ) )
+		} )
+		await throttle_and_retry( email_sending_queue, 10, `send email`, 2, 10 )
 
 		// Log result
 		console.log( `Sent ${ owners_to_email.length } emails for ${ network } outfits` )

functions/modules/helpers.js

@@ -2,12 +2,14 @@
 // Helper functions
 // ///////////////////////////////
 exports.dev = !!process.env.development
-exports.log = ( ...messages ) => {
+const log = ( ...messages ) => {
     if( process.env.development ) console.log( ...messages )
 }
+exports.log = log
 
 // Wait in async
-exports.wait = timeInMs => new Promise( resolve => setTimeout( resolve ), timeInMs )
+const wait = timeInMs => new Promise( resolve => setTimeout( resolve ), timeInMs )
+exports.wait = wait
 
 // Pick random item from an array
 const pickRandomArrayEntry = array => array[ Math.floor( Math.random() * array.length ) ]
@@ -115,3 +117,78 @@ exports.globalAttributes = [
 exports.heavenlyBodies = [ "Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune", "Pluto", "the Moon", "the Sun" ]
 exports.web2domain = 'https://rocketeer.fans'
 exports.lorem = 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'
+
+/* ///////////////////////////////
+// Retryable & throttled async
+// /////////////////////////////*/
+const Throttle = require( 'promise-parallel-throttle' )
+const Retrier = require( 'promise-retry' )
+
+/**
+* Make async function (promise) retryable
+* @param { function } async_function The function to make retryable
+* @param { string } logging_label The label to add to the log entries
+* @param { number } retry_times The amount of times to retry before throwing
+* @param { number } cooldown_in_s The amount of seconds to wait between retries
+* @param { boolean } cooldown_entropy Whether to add entropy to the retry delay to prevent retries from clustering in time
+* @returns { function } An async function (promise) that will retry retry_times before throwing
+*/
+function make_retryable( async_function, logging_label='unlabeled retry', retry_times=5, cooldown_in_s=10, cooldown_entropy=true ) {
+
+	// Formulate retry logic
+	const retryable_function = () => Retrier( ( do_retry, retry_counter ) => {
+
+		// Failure handling
+		return async_function().catch( async e => {
+
+			// If retry attempts exhausted, throw out
+			if( retry_counter >= retry_times ) {
+				log( `♻️🚨 ${ logging_label } retry failed after ${ retry_counter } attempts` )
+				throw e
+			}
+
+			// If retries left, retry with a progressive delay
+			const entropy = !cooldown_entropy ? 0 : ( .1 + Math.random() )
+			const cooldown_in_ms = ( cooldown_in_s + entropy ) * 1000
+			const cooldown = cooldown_in_ms + ( cooldown_in_ms * ( retry_counter - 1 ) )
+			log( `♻️ ${ logging_label } retry failed ${ retry_counter }x, waiting for ${ cooldown / 1000 }s` )
+			await wait( cooldown )
+			log( `♻️ ${ logging_label } cooldown complete, continuing...` )
+			return do_retry()
+
+		} )
+
+	} )
+
+	return retryable_function
+
+}
+
+/**
+* Make async function (promise) retryable
+* @param { array } async_function_array Array of async functions (promises) to run in throttled parallel
+* @param { number } max_parallell The maximum amount of functions allowed to run at the same time
+* @param { string } logging_label The label to add to the log entries
+* @param { number } retry_times The amount of times to retry before throwing
+* @param { number } cooldown_in_s The amount of seconds to wait between retries
+* @returns { Promise } An async function (promise) that will retry retry_times before throwing
+*/
+async function throttle_and_retry( async_function_array=[], max_parallell=2, logging_label, retry_times, cooldown_in_s ) {
+
+	// Create array of retryable functions
+	const retryable_async_functions = async_function_array.map( async_function => {
+		const retryable_function = make_retryable( async_function, logging_label, retry_times, cooldown_in_s )
+		return retryable_function
+	} )
+
+	// Throttle configuration
+	const throttle_config = {
+		maxInProgress: max_parallell	
+	}
+
+	// Return throttler
+	return Throttle.all( retryable_async_functions, throttle_config )
+
+}
+
+exports.throttle_and_retry = throttle_and_retry

functions/package-lock.json

@@ -21,6 +21,7 @@
         "juice": "^8.0.0",
         "mailgun.js": "^4.1.4",
         "promise-parallel-throttle": "^3.3.0",
+        "promise-retry": "^2.0.1",
         "pug": "^3.0.2",
         "puppeteer": "^12.0.0",
         "puppeteer-extra": "^3.2.3",
@@ -3075,6 +3076,11 @@
         "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
+    "node_modules/err-code": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
+      "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
+    },
     "node_modules/es-abstract": {
       "version": "1.19.1",
       "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz",
@@ -6597,6 +6603,26 @@
       "resolved": "https://registry.npmjs.org/promise-parallel-throttle/-/promise-parallel-throttle-3.3.0.tgz",
       "integrity": "sha512-tThe11SfFXlGMhuO2D+Nba6L8FJFM17w2zwlMV1kqaLfuT2E8NMtMF1WhJBZaSpWz6V76pP/bGAj8BXTAMOncw=="
     },
+    "node_modules/promise-retry": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
+      "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
+      "dependencies": {
+        "err-code": "^2.0.2",
+        "retry": "^0.12.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/promise-retry/node_modules/retry": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
+      "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/proto3-json-serializer": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-0.1.7.tgz",
@@ -11732,6 +11758,11 @@
       "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
       "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
     },
+    "err-code": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
+      "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
+    },
     "es-abstract": {
       "version": "1.19.1",
       "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz",
@@ -14441,6 +14472,22 @@
       "resolved": "https://registry.npmjs.org/promise-parallel-throttle/-/promise-parallel-throttle-3.3.0.tgz",
       "integrity": "sha512-tThe11SfFXlGMhuO2D+Nba6L8FJFM17w2zwlMV1kqaLfuT2E8NMtMF1WhJBZaSpWz6V76pP/bGAj8BXTAMOncw=="
     },
+    "promise-retry": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
+      "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
+      "requires": {
+        "err-code": "^2.0.2",
+        "retry": "^0.12.0"
+      },
+      "dependencies": {
+        "retry": {
+          "version": "0.12.0",
+          "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
+          "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="
+        }
+      }
+    },
     "proto3-json-serializer": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-0.1.7.tgz",

functions/package.json

@@ -30,6 +30,7 @@
     "juice": "^8.0.0",
     "mailgun.js": "^4.1.4",
     "promise-parallel-throttle": "^3.3.0",
+    "promise-retry": "^2.0.1",
     "pug": "^3.0.2",
     "puppeteer": "^12.0.0",
     "puppeteer-extra": "^3.2.3",