Simple mode to only serve the API for coin and Livepeer data

Added various other config variables to the backend

backend/src/config.js

@@ -1,6 +1,6 @@
 //Server configuration variables
 export const {
-    PORT = 42609,
+    NODE_PORT = 42609,
     NODE_ENV = 'local',
     MONGO_URI = "DB PRODUCTION EVIRONMENT MONGODB",
     MONGO_URI_DEV = 'DB TEST EVIRONMENT MONGODB',
@@ -11,6 +11,10 @@ export const {
     API_CMC = "Coinmarketcap API key",
     API_L1_HTTP = "ETH L1 HTTP API KEY",
     API_L2_HTTP = "ETH L2 HTTP API KEY",
-    API_L2_WS = "ALCHEMY WSS API KEY",
+    API_L2_WS = "ETH L2 WSS API KEY",
-    CONF_DEFAULT_ORCH = "YOUR OWN ORCHESTRATORS PUBLIC ADDRESS"
+    CONF_DEFAULT_ORCH = "YOUR OWN ORCHESTRATORS PUBLIC ADDRESS",
+    CONF_SIMPLE_MODE = false,
+    CONF_TIMEOUT_CMC = 360000,
+    CONF_TIMEOUT_ALCHEMY = 2000,
+    CONF_TIMEOUT_LIVEPEER = 60000
 } = process.env;

backend/src/routes/livepeer.js

@@ -3,7 +3,9 @@ import Event from '../models/event';
 import Block from '../models/block';
 const apiRouter = express.Router();
 import {
-  API_CMC, API_L1_HTTP, API_L2_HTTP, API_L2_WS, CONF_DEFAULT_ORCH
+  API_CMC, API_L1_HTTP, API_L2_HTTP, API_L2_WS,
+  CONF_DEFAULT_ORCH, CONF_SIMPLE_MODE, CONF_TIMEOUT_CMC,
+  CONF_TIMEOUT_ALCHEMY, CONF_TIMEOUT_LIVEPEER
 } from "../config";
 // Do API requests to other API's
 const https = require('https');
@@ -19,11 +21,16 @@ const { createAlchemyWeb3 } = require("@alch/alchemy-web3");
 // Gets gas prices
 const web3layer1 = createAlchemyWeb3(API_L1_HTTP);
 const web3layer2 = createAlchemyWeb3(API_L2_HTTP);
+
+let web3layer2WS;
+// Skip if running in basic mode
+if (!CONF_SIMPLE_MODE) {
+  web3layer2WS = createAlchemyWeb3(API_L2_WS);
+}
 // For listening to blockchain events
-const web3layer2WS = createAlchemyWeb3(API_L2_WS);
 
 // Update CoinMarketCap related api calls every 5 minutes
-const timeoutCMC = 360000;
+const timeoutCMC = CONF_TIMEOUT_CMC;
 let cmcPriceGet = 0;
 let ethPrice = 0;
 let lptPrice = 0;
@@ -31,7 +38,7 @@ let cmcQuotes = {};
 let cmcCache = {};
 
 // Update Alchemy related API calls every 2 seconds
-const timeoutAlchemy = 2000;
+const timeoutAlchemy = CONF_TIMEOUT_ALCHEMY;
 let l2Gwei = 0;
 let l1Gwei = 0;
 let l2block = 0;
@@ -60,7 +67,7 @@ let serviceUriFeeCostL1 = 0;
 let serviceUriFeeCostL2 = 0;
 
 // Update O info from thegraph every 1 minute
-const timeoutTheGraph = 60000;
+const timeoutTheGraph = CONF_TIMEOUT_LIVEPEER;
 // Will contain addr, lastGet, and obj of any requested O's
 let orchestratorCache = [];
 // Contains delegator addr and the address of the O they are bounded to
@@ -71,13 +78,17 @@ let eventsCache = [];
 let latestMissedDuringSync = 0;
 let lastBlockDataAdded = 0;
 let syncCache = [];
-// Set to true to drop the entire collection on boot and get all events
-const fullSync = false;
 // https://arbiscan.io/address/0x35Bcf3c30594191d53231E4FF333E8A770453e40#events
-const BondingManagerTargetJson = fs.readFileSync('src/abi/BondingManagerTarget.json');
+let BondingManagerTargetJson;
-const BondingManagerTargetAbi = JSON.parse(BondingManagerTargetJson);
+let BondingManagerTargetAbi;
-const BondingManagerProxyAddr = "0x35Bcf3c30594191d53231E4FF333E8A770453e40";
+let BondingManagerProxyAddr;
-const contractInstance = new web3layer2WS.eth.Contract(BondingManagerTargetAbi.abi, BondingManagerProxyAddr);
+let contractInstance;
+if (!CONF_SIMPLE_MODE) {
+  BondingManagerTargetJson = fs.readFileSync('src/abi/BondingManagerTarget.json');
+  BondingManagerTargetAbi = JSON.parse(BondingManagerTargetJson);
+  BondingManagerProxyAddr = "0x35Bcf3c30594191d53231E4FF333E8A770453e40";
+  contractInstance = new web3layer2WS.eth.Contract(BondingManagerTargetAbi.abi, BondingManagerProxyAddr);
+}
 
 let blockCache = [];
 const getBlock = async function (blockNumber) {
@@ -100,49 +111,47 @@ const getBlock = async function (blockNumber) {
   return thisBlock;
 }
 
-// If fullsync: drop collection on DB
-if (fullSync) {
-  console.log("dropping old data due to full synchronization");
-  Event.collection.drop();
-}
 // Set special flag to make sure also get blocks that pass us by while we are syncing
 let isSyncing = true;
 let isSyncRunning = false;
 // Start Listening for live updates
-var BondingManagerProxyListener = contractInstance.events.allEvents(async (error, event) => {
+var BondingManagerProxyListener;
-  try {
+if (!CONF_SIMPLE_MODE) {
-    if (error) {
+  BondingManagerProxyListener = contractInstance.events.allEvents(async (error, event) => {
-      throw error
+    try {
+      if (error) {
+        throw error
+      }
+      if (isSyncing) {
+        console.log('Received new Event on block ' + event.blockNumber + " during sync");
+      } else {
+        console.log('Received new Event on block ' + event.blockNumber);
+      }
+      const thisBlock = await getBlock(event.blockNumber);
+      // Push obj of event to cache and create a new entry for it in the DB
+      const eventObj = {
+        address: event.address,
+        transactionHash: event.transactionHash,
+        transactionUrl: "https://arbiscan.io/tx/" + event.transactionHash,
+        name: event.event,
+        data: event.returnValues,
+        blockNumber: thisBlock.number,
+        blockTime: thisBlock.timestamp
+      }
+      if (!isSyncing) {
+        const dbObj = new Event(eventObj);
+        await dbObj.save();
+        eventsCache.push(eventObj);
+      } else {
+        syncCache.push(eventObj);
+      }
     }
-    if (isSyncing) {
+    catch (err) {
-      console.log('Received new Event on block ' + event.blockNumber + " during sync");
+      console.log("FATAL ERROR: ", err);
-    } else {
-      console.log('Received new Event on block ' + event.blockNumber);
     }
-    const thisBlock = await getBlock(event.blockNumber);
+  });
-    // Push obj of event to cache and create a new entry for it in the DB
+  console.log("Listening for events on " + BondingManagerProxyAddr);
-    const eventObj = {
+}
-      address: event.address,
-      transactionHash: event.transactionHash,
-      transactionUrl: "https://arbiscan.io/tx/" + event.transactionHash,
-      name: event.event,
-      data: event.returnValues,
-      blockNumber: thisBlock.number,
-      blockTime: thisBlock.timestamp
-    }
-    if (!isSyncing) {
-      const dbObj = new Event(eventObj);
-      await dbObj.save();
-      eventsCache.push(eventObj);
-    } else {
-      syncCache.push(eventObj);
-    }
-  }
-  catch (err) {
-    console.log("FATAL ERROR: ", err);
-  }
-});
-console.log("Listening for events on " + BondingManagerProxyAddr);
 
 // Does the syncing
 const doSync = function () {
@@ -237,7 +246,7 @@ const handleSync = async function () {
   console.log('done syncing')
   isSyncing = false;
 };
-if (!isSyncRunning) {
+if (!isSyncRunning && !CONF_SIMPLE_MODE) {
   handleSync();
 }
 
@@ -710,57 +719,57 @@ apiRouter.get("/prometheus/:orchAddr", async (req, res) => {
     let orchObj = {};
     if (reqOrch && reqOrch !== "") {
       orchObj = await parseOrchestrator(reqOrch);
-      if (orchObj){
+      if (orchObj) {
         // Add details on the rewards from the last round
-        if (orchObj.lastRewardRound){
+        if (orchObj.lastRewardRound) {
-          if (orchObj.lastRewardRound.volumeETH){
+          if (orchObj.lastRewardRound.volumeETH) {
             outputString += "# HELP last_round_reward_eth Total earned fees in Eth from the previous round.\n";
             outputString += "# TYPE last_round_reward_eth gauge\nlast_round_reward_eth ";
             outputString += orchObj.lastRewardRound.volumeETH + "\n\n";
           }
-          if (orchObj.lastRewardRound.volumeUSD){
+          if (orchObj.lastRewardRound.volumeUSD) {
             outputString += "# HELP last_round_reward_usd Total earned fees in USD from the previous round.\n";
             outputString += "# TYPE last_round_reward_usd gauge\nlast_round_reward_usd ";
             outputString += orchObj.lastRewardRound.volumeUSD + "\n\n";
           }
-          if (orchObj.lastRewardRound.participationRate){
+          if (orchObj.lastRewardRound.participationRate) {
             outputString += "# HELP last_round_participation Participation rate of the previous round.\n";
             outputString += "# TYPE last_round_participation gauge\nlast_round_participation ";
             outputString += orchObj.lastRewardRound.participationRate + "\n\n";
           }
         }
         // Add O reward cut
-        if (orchObj.rewardCut){
+        if (orchObj.rewardCut) {
           outputString += "# HELP orchestrator_reward_commission Reward commission rate of this Orchestrator.\n";
           outputString += "# TYPE orchestrator_reward_commission gauge\norchestrator_reward_commission ";
           outputString += (orchObj.rewardCut / 10000) + "\n\n";
         }
         // Add O fee cut
-        if (orchObj.feeShare){
+        if (orchObj.feeShare) {
           outputString += "# HELP orchestrator_fee_commission Transcoding fee commission rate of this Orchestrator.\n";
           outputString += "# TYPE orchestrator_fee_commission gauge\norchestrator_fee_commission ";
           outputString += (100 - (orchObj.feeShare / 10000)) + "\n\n";
         }
         // Add O total stake
-        if (orchObj.totalStake){
+        if (orchObj.totalStake) {
           outputString += "# HELP orchestrator_total_stake Total stake of this Orchestrator.\n";
           outputString += "# TYPE orchestrator_total_stake gauge\norchestrator_total_stake ";
           outputString += orchObj.totalStake + "\n\n";
         }
         // Add O self stake
-        if (orchObj.delegator && orchObj.delegator.bondedAmount){
+        if (orchObj.delegator && orchObj.delegator.bondedAmount) {
           outputString += "# HELP orchestrator_self_stake Self stake of this Orchestrator.\n";
           outputString += "# TYPE orchestrator_self_stake gauge\norchestrator_self_stake ";
           outputString += orchObj.delegator.bondedAmount + "\n\n";
         }
         // Add O total fees earned in eth
-        if (orchObj.totalVolumeETH){
+        if (orchObj.totalVolumeETH) {
           outputString += "# HELP orchestrator_earned_fees_eth Total transcoding rewards of this Orchestrator in Eth.\n";
           outputString += "# TYPE orchestrator_earned_fees_eth counter\norchestrator_earned_fees_eth ";
           outputString += orchObj.totalVolumeETH + "\n\n";
         }
         // Add O total fees earned in usd
-        if (orchObj.totalVolumeUSD){
+        if (orchObj.totalVolumeUSD) {
           outputString += "# HELP orchestrator_earned_fees_usd Total transcoding rewards of this Orchestrator in USD.\n";
           outputString += "# TYPE orchestrator_earned_fees_usd counter\norchestrator_earned_fees_usd ";
           outputString += orchObj.totalVolumeUSD + "\n\n";

backend/src/server.js

@@ -5,54 +5,62 @@ import session from "express-session";
 import connectStore from "connect-mongo";
 import { userRouter, sessionRouter, livepeerRouter } from './routes/index';
 import {
-  PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET, SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL
+  NODE_PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET,
+  SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL, CONF_SIMPLE_MODE
 } from "./config";
 // Env variable which determines which DB to connect to
 const { NODE_ENV: mode } = process.env;
 
 (async () => {
   try {
-    // Make DB connection
+    // Make DB connection if needed
-    if (mode == "production"){
+    if (!CONF_SIMPLE_MODE){
-      await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false});
+      if (mode == "production"){
-    }else if (mode == "development"){
+        await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false});
-      await mongoose.connect(MONGO_URI_DEV, { useNewUrlParser: true, useFindAndModify: false});
+      }else if (mode == "development"){
-    }else if (mode == "local"){
+        await mongoose.connect(MONGO_URI_DEV, { useNewUrlParser: true, useFindAndModify: false});
-      await mongoose.connect(MONGO_URI_LOCAL, { useNewUrlParser: true, useFindAndModify: false});
+      }else if (mode == "local"){
+        await mongoose.connect(MONGO_URI_LOCAL, { useNewUrlParser: true, useFindAndModify: false});
+      }else{
+        await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false});
+      }
+      console.log('MongoDB connected on ' + mode);
     }else{
-      await mongoose.connect(MONGO_URI, { useNewUrlParser: true, useFindAndModify: false});
+      console.log('Running in basic mode' );
     }
-    console.log('MongoDB connected on ' + mode);
     // Web application framework
     const app = express();
     app.disable('x-powered-by');
     // Parses and validates requests to make things harder for malicious actors 
     app.use(express.urlencoded({ extended: true }));
     app.use(express.json());
-    // Import session module
-    const MongoStore = connectStore(session);
 
-    // Declare session data
+    let MongoStore;
-    app.use(session({
+    if (!CONF_SIMPLE_MODE){
-      name: SESS_NAME,
+      // Import session module
-      //TODO: change secret in config file
+      MongoStore = connectStore(session);
-      secret: SESS_SECRET,
+      // Declare session data
-      //define where to store them
+      app.use(session({
-      store: new MongoStore({
+        name: SESS_NAME,
-        mongooseConnection: mongoose.connection,
+        //TODO: change secret in config file
-        collection: 'session',
+        secret: SESS_SECRET,
-        ttl: parseInt(SESS_LIFETIME) / 1000,
+        //define where to store them
-      }),
+        store: new MongoStore({
-      saveUninitialized: false,
+          mongooseConnection: mongoose.connection,
-      proxy: NODE_ENV === "production",
+          collection: 'session',
-      resave: false,
+          ttl: parseInt(SESS_LIFETIME) / 1000,
-      //cookie to send to users
+        }),
-      cookie: {
+        saveUninitialized: false,
-        sameSite: true,
+        proxy: NODE_ENV === "production",
-        secure: NODE_ENV === 'production',
+        resave: false,
-        maxAge: parseInt(SESS_LIFETIME)
+        //cookie to send to users
-      }
+        cookie: {
-    }));
+          sameSite: true,
+          secure: NODE_ENV === 'production',
+          maxAge: parseInt(SESS_LIFETIME)
+        }
+      }));
+    }
 
     // Define endpoint paths
     const apiRouter = express.Router();
@@ -73,8 +81,8 @@ const { NODE_ENV: mode } = process.env;
     });
 
     // Start listening on the defined port
-    app.listen(PORT, "0.0.0.0", function () {
+    app.listen(NODE_PORT, "0.0.0.0", function () {
-      console.log(`Listening on port ${PORT}`);
+      console.log(`Listening on port ${NODE_PORT}`);
     });
   } catch (err) {
     console.log(err);