From 8792113e081dd5fe72740ef343ea3e81b9febfc5 Mon Sep 17 00:00:00 2001 From: Marco van Dijk Date: Mon, 7 Mar 2022 23:07:53 +0100 Subject: [PATCH] Simple mode to only serve the API for coin and Livepeer data Added various other config variables to the backend --- backend/src/config.js | 10 ++- backend/src/routes/livepeer.js | 129 ++++++++++++++++++--------------- backend/src/server.js | 78 +++++++++++--------- 3 files changed, 119 insertions(+), 98 deletions(-) diff --git a/backend/src/config.js b/backend/src/config.js index 8d39e86..0973fb6 100644 --- a/backend/src/config.js +++ b/backend/src/config.js @@ -1,6 +1,6 @@ //Server configuration variables export const { - PORT = 42609, + NODE_PORT = 42609, NODE_ENV = 'local', MONGO_URI = "DB PRODUCTION EVIRONMENT MONGODB", MONGO_URI_DEV = 'DB TEST EVIRONMENT MONGODB', @@ -11,6 +11,10 @@ export const { API_CMC = "Coinmarketcap API key", API_L1_HTTP = "ETH L1 HTTP API KEY", API_L2_HTTP = "ETH L2 HTTP API KEY", - API_L2_WS = "ALCHEMY WSS API KEY", - CONF_DEFAULT_ORCH = "YOUR OWN ORCHESTRATORS PUBLIC ADDRESS" + API_L2_WS = "ETH L2 WSS API KEY", + CONF_DEFAULT_ORCH = "YOUR OWN ORCHESTRATORS PUBLIC ADDRESS", + CONF_SIMPLE_MODE = false, + CONF_TIMEOUT_CMC = 360000, + CONF_TIMEOUT_ALCHEMY = 2000, + CONF_TIMEOUT_LIVEPEER = 60000 } = process.env; diff --git a/backend/src/routes/livepeer.js b/backend/src/routes/livepeer.js index bce43f2..6d2f9d9 100644 --- a/backend/src/routes/livepeer.js +++ b/backend/src/routes/livepeer.js @@ -3,7 +3,9 @@ import Event from '../models/event'; import Block from '../models/block'; const apiRouter = express.Router(); import { - API_CMC, API_L1_HTTP, API_L2_HTTP, API_L2_WS, CONF_DEFAULT_ORCH + API_CMC, API_L1_HTTP, API_L2_HTTP, API_L2_WS, + CONF_DEFAULT_ORCH, CONF_SIMPLE_MODE, CONF_TIMEOUT_CMC, + CONF_TIMEOUT_ALCHEMY, CONF_TIMEOUT_LIVEPEER } from "../config"; // Do API requests to other API's const https = require('https'); @@ -19,11 +21,16 @@ const { createAlchemyWeb3 } = require("@alch/alchemy-web3"); // Gets gas prices const web3layer1 = createAlchemyWeb3(API_L1_HTTP); const web3layer2 = createAlchemyWeb3(API_L2_HTTP); + +let web3layer2WS; +// Skip if running in basic mode +if (!CONF_SIMPLE_MODE) { + web3layer2WS = createAlchemyWeb3(API_L2_WS); +} // For listening to blockchain events -const web3layer2WS = createAlchemyWeb3(API_L2_WS); // Update CoinMarketCap related api calls every 5 minutes -const timeoutCMC = 360000; +const timeoutCMC = CONF_TIMEOUT_CMC; let cmcPriceGet = 0; let ethPrice = 0; let lptPrice = 0; @@ -31,7 +38,7 @@ let cmcQuotes = {}; let cmcCache = {}; // Update Alchemy related API calls every 2 seconds -const timeoutAlchemy = 2000; +const timeoutAlchemy = CONF_TIMEOUT_ALCHEMY; let l2Gwei = 0; let l1Gwei = 0; let l2block = 0; @@ -60,7 +67,7 @@ let serviceUriFeeCostL1 = 0; let serviceUriFeeCostL2 = 0; // Update O info from thegraph every 1 minute -const timeoutTheGraph = 60000; +const timeoutTheGraph = CONF_TIMEOUT_LIVEPEER; // Will contain addr, lastGet, and obj of any requested O's let orchestratorCache = []; // Contains delegator addr and the address of the O they are bounded to @@ -71,13 +78,17 @@ let eventsCache = []; let latestMissedDuringSync = 0; let lastBlockDataAdded = 0; let syncCache = []; -// Set to true to drop the entire collection on boot and get all events -const fullSync = false; // https://arbiscan.io/address/0x35Bcf3c30594191d53231E4FF333E8A770453e40#events -const BondingManagerTargetJson = fs.readFileSync('src/abi/BondingManagerTarget.json'); -const BondingManagerTargetAbi = JSON.parse(BondingManagerTargetJson); -const BondingManagerProxyAddr = "0x35Bcf3c30594191d53231E4FF333E8A770453e40"; -const contractInstance = new web3layer2WS.eth.Contract(BondingManagerTargetAbi.abi, BondingManagerProxyAddr); +let BondingManagerTargetJson; +let BondingManagerTargetAbi; +let BondingManagerProxyAddr; +let contractInstance; +if (!CONF_SIMPLE_MODE) { + BondingManagerTargetJson = fs.readFileSync('src/abi/BondingManagerTarget.json'); + BondingManagerTargetAbi = JSON.parse(BondingManagerTargetJson); + BondingManagerProxyAddr = "0x35Bcf3c30594191d53231E4FF333E8A770453e40"; + contractInstance = new web3layer2WS.eth.Contract(BondingManagerTargetAbi.abi, BondingManagerProxyAddr); +} let blockCache = []; const getBlock = async function (blockNumber) { @@ -100,49 +111,47 @@ const getBlock = async function (blockNumber) { return thisBlock; } -// If fullsync: drop collection on DB -if (fullSync) { - console.log("dropping old data due to full synchronization"); - Event.collection.drop(); -} // Set special flag to make sure also get blocks that pass us by while we are syncing let isSyncing = true; let isSyncRunning = false; // Start Listening for live updates -var BondingManagerProxyListener = contractInstance.events.allEvents(async (error, event) => { - try { - if (error) { - throw error +var BondingManagerProxyListener; +if (!CONF_SIMPLE_MODE) { + BondingManagerProxyListener = contractInstance.events.allEvents(async (error, event) => { + try { + if (error) { + throw error + } + if (isSyncing) { + console.log('Received new Event on block ' + event.blockNumber + " during sync"); + } else { + console.log('Received new Event on block ' + event.blockNumber); + } + const thisBlock = await getBlock(event.blockNumber); + // Push obj of event to cache and create a new entry for it in the DB + const eventObj = { + address: event.address, + transactionHash: event.transactionHash, + transactionUrl: "https://arbiscan.io/tx/" + event.transactionHash, + name: event.event, + data: event.returnValues, + blockNumber: thisBlock.number, + blockTime: thisBlock.timestamp + } + if (!isSyncing) { + const dbObj = new Event(eventObj); + await dbObj.save(); + eventsCache.push(eventObj); + } else { + syncCache.push(eventObj); + } } - if (isSyncing) { - console.log('Received new Event on block ' + event.blockNumber + " during sync"); - } else { - console.log('Received new Event on block ' + event.blockNumber); + catch (err) { + console.log("FATAL ERROR: ", err); } - const thisBlock = await getBlock(event.blockNumber); - // Push obj of event to cache and create a new entry for it in the DB - const eventObj = { - address: event.address, - transactionHash: event.transactionHash, - transactionUrl: "https://arbiscan.io/tx/" + event.transactionHash, - name: event.event, - data: event.returnValues, - blockNumber: thisBlock.number, - blockTime: thisBlock.timestamp - } - if (!isSyncing) { - const dbObj = new Event(eventObj); - await dbObj.save(); - eventsCache.push(eventObj); - } else { - syncCache.push(eventObj); - } - } - catch (err) { - console.log("FATAL ERROR: ", err); - } -}); -console.log("Listening for events on " + BondingManagerProxyAddr); + }); + console.log("Listening for events on " + BondingManagerProxyAddr); +} // Does the syncing const doSync = function () { @@ -237,7 +246,7 @@ const handleSync = async function () { console.log('done syncing') isSyncing = false; }; -if (!isSyncRunning) { +if (!isSyncRunning && !CONF_SIMPLE_MODE) { handleSync(); } @@ -710,57 +719,57 @@ apiRouter.get("/prometheus/:orchAddr", async (req, res) => { let orchObj = {}; if (reqOrch && reqOrch !== "") { orchObj = await parseOrchestrator(reqOrch); - if (orchObj){ + if (orchObj) { // Add details on the rewards from the last round - if (orchObj.lastRewardRound){ - if (orchObj.lastRewardRound.volumeETH){ + if (orchObj.lastRewardRound) { + if (orchObj.lastRewardRound.volumeETH) { outputString += "# HELP last_round_reward_eth Total earned fees in Eth from the previous round.\n"; outputString += "# TYPE last_round_reward_eth gauge\nlast_round_reward_eth "; outputString += orchObj.lastRewardRound.volumeETH + "\n\n"; } - if (orchObj.lastRewardRound.volumeUSD){ + if (orchObj.lastRewardRound.volumeUSD) { outputString += "# HELP last_round_reward_usd Total earned fees in USD from the previous round.\n"; outputString += "# TYPE last_round_reward_usd gauge\nlast_round_reward_usd "; outputString += orchObj.lastRewardRound.volumeUSD + "\n\n"; } - if (orchObj.lastRewardRound.participationRate){ + if (orchObj.lastRewardRound.participationRate) { outputString += "# HELP last_round_participation Participation rate of the previous round.\n"; outputString += "# TYPE last_round_participation gauge\nlast_round_participation "; outputString += orchObj.lastRewardRound.participationRate + "\n\n"; } } // Add O reward cut - if (orchObj.rewardCut){ + if (orchObj.rewardCut) { outputString += "# HELP orchestrator_reward_commission Reward commission rate of this Orchestrator.\n"; outputString += "# TYPE orchestrator_reward_commission gauge\norchestrator_reward_commission "; outputString += (orchObj.rewardCut / 10000) + "\n\n"; } // Add O fee cut - if (orchObj.feeShare){ + if (orchObj.feeShare) { outputString += "# HELP orchestrator_fee_commission Transcoding fee commission rate of this Orchestrator.\n"; outputString += "# TYPE orchestrator_fee_commission gauge\norchestrator_fee_commission "; outputString += (100 - (orchObj.feeShare / 10000)) + "\n\n"; } // Add O total stake - if (orchObj.totalStake){ + if (orchObj.totalStake) { outputString += "# HELP orchestrator_total_stake Total stake of this Orchestrator.\n"; outputString += "# TYPE orchestrator_total_stake gauge\norchestrator_total_stake "; outputString += orchObj.totalStake + "\n\n"; } // Add O self stake - if (orchObj.delegator && orchObj.delegator.bondedAmount){ + if (orchObj.delegator && orchObj.delegator.bondedAmount) { outputString += "# HELP orchestrator_self_stake Self stake of this Orchestrator.\n"; outputString += "# TYPE orchestrator_self_stake gauge\norchestrator_self_stake "; outputString += orchObj.delegator.bondedAmount + "\n\n"; } // Add O total fees earned in eth - if (orchObj.totalVolumeETH){ + if (orchObj.totalVolumeETH) { outputString += "# HELP orchestrator_earned_fees_eth Total transcoding rewards of this Orchestrator in Eth.\n"; outputString += "# TYPE orchestrator_earned_fees_eth counter\norchestrator_earned_fees_eth "; outputString += orchObj.totalVolumeETH + "\n\n"; } // Add O total fees earned in usd - if (orchObj.totalVolumeUSD){ + if (orchObj.totalVolumeUSD) { outputString += "# HELP orchestrator_earned_fees_usd Total transcoding rewards of this Orchestrator in USD.\n"; outputString += "# TYPE orchestrator_earned_fees_usd counter\norchestrator_earned_fees_usd "; outputString += orchObj.totalVolumeUSD + "\n\n"; diff --git a/backend/src/server.js b/backend/src/server.js index 3f853c3..2be440c 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -5,54 +5,62 @@ import session from "express-session"; import connectStore from "connect-mongo"; import { userRouter, sessionRouter, livepeerRouter } from './routes/index'; import { - PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET, SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL + NODE_PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET, + SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL, CONF_SIMPLE_MODE } from "./config"; // Env variable which determines which DB to connect to const { NODE_ENV: mode } = process.env; (async () => { try { - // Make DB connection - if (mode == "production"){ - await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false}); - }else if (mode == "development"){ - await mongoose.connect(MONGO_URI_DEV, { useNewUrlParser: true, useFindAndModify: false}); - }else if (mode == "local"){ - await mongoose.connect(MONGO_URI_LOCAL, { useNewUrlParser: true, useFindAndModify: false}); + // Make DB connection if needed + if (!CONF_SIMPLE_MODE){ + if (mode == "production"){ + await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false}); + }else if (mode == "development"){ + await mongoose.connect(MONGO_URI_DEV, { useNewUrlParser: true, useFindAndModify: false}); + }else if (mode == "local"){ + await mongoose.connect(MONGO_URI_LOCAL, { useNewUrlParser: true, useFindAndModify: false}); + }else{ + await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false}); + } + console.log('MongoDB connected on ' + mode); }else{ - await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false}); + console.log('Running in basic mode' ); } - console.log('MongoDB connected on ' + mode); // Web application framework const app = express(); app.disable('x-powered-by'); // Parses and validates requests to make things harder for malicious actors app.use(express.urlencoded({ extended: true })); app.use(express.json()); - // Import session module - const MongoStore = connectStore(session); - // Declare session data - app.use(session({ - name: SESS_NAME, - //TODO: change secret in config file - secret: SESS_SECRET, - //define where to store them - store: new MongoStore({ - mongooseConnection: mongoose.connection, - collection: 'session', - ttl: parseInt(SESS_LIFETIME) / 1000, - }), - saveUninitialized: false, - proxy: NODE_ENV === "production", - resave: false, - //cookie to send to users - cookie: { - sameSite: true, - secure: NODE_ENV === 'production', - maxAge: parseInt(SESS_LIFETIME) - } - })); + let MongoStore; + if (!CONF_SIMPLE_MODE){ + // Import session module + MongoStore = connectStore(session); + // Declare session data + app.use(session({ + name: SESS_NAME, + //TODO: change secret in config file + secret: SESS_SECRET, + //define where to store them + store: new MongoStore({ + mongooseConnection: mongoose.connection, + collection: 'session', + ttl: parseInt(SESS_LIFETIME) / 1000, + }), + saveUninitialized: false, + proxy: NODE_ENV === "production", + resave: false, + //cookie to send to users + cookie: { + sameSite: true, + secure: NODE_ENV === 'production', + maxAge: parseInt(SESS_LIFETIME) + } + })); + } // Define endpoint paths const apiRouter = express.Router(); @@ -73,8 +81,8 @@ const { NODE_ENV: mode } = process.env; }); // Start listening on the defined port - app.listen(PORT, "0.0.0.0", function () { - console.log(`Listening on port ${PORT}`); + app.listen(NODE_PORT, "0.0.0.0", function () { + console.log(`Listening on port ${NODE_PORT}`); }); } catch (err) { console.log(err);