Fix backend

backend/package.json

@@ -3,7 +3,7 @@
   "version": "0.4.2",
   "description": "",
   "main": "./src/index.js",
-  "module": "./src/server.js",
+  "module": "./src/index.js",
   "scripts": {
     "prod": "NODE_ENV=production pm2 start ecosystem.config.js",
     "start": "NODE_ENV=production node ./src/index.js",
@@ -12,28 +12,29 @@
   },
   "keywords": [],
   "author": "Marco van Dijk",
+  "type": "module",
   "license": "WTFPL",
   "dependencies": {
     "@alch/alchemy-web3": "^1.4.7",
     "alchemy-api": "^1.3.3",
-    "alchemy-sdk": "^2.2.4",
+    "alchemy-sdk": "^2.12.0",
     "coinmarketcap-api": "^3.1.1",
-    "connect-mongo": "^4.6.0",
+    "connect-mongo": "^5.1.0",
-    "connect-mongodb-session": "^3.1.1",
+    "connect-mongodb-session": "^5.0.0",
-    "crypto-js": "^4.1.1",
+    "crypto-js": "^4.2.0",
     "esm": "^3.2.25",
     "ethers": "^5.7.2",
-    "express": "^4.17.1",
+    "express": "^4.19.2",
-    "express-session": "^1.17.0",
+    "express-session": "^1.18.0",
-    "graphql-request": "^4.0.0",
+    "graphql-request": "^4.3.0",
     "install": "^0.13.0",
     "joi": "^14.3.1",
-    "mongoose": "^6.8.0",
+    "mongoose": "^8.8.3",
-    "npm": "^8.5.2",
+    "npm": "^8.19.4",
-    "prom-client": "^14.1.0",
+    "prom-client": "^14.2.0",
-    "web3": "^1.8.1"
+    "web3": "^1.10.4"
   },
   "devDependencies": {
-    "nodemon": "^1.18.10"
+    "nodemon": "^1.19.4"
   }
 }

backend/src/index.js

@@ -1,2 +1,101 @@
-require = require("esm")(module)
+//Server logic. Imports all necessary routes, models, etc
-module.exports = require("./server.js")
+import express from 'express';
+import mongoose from 'mongoose';
+import session from "express-session";
+import MongoStore from "connect-mongo";
+import { userRouter, sessionRouter, livepeerRouter } from './routes/index.js';
+import {
+  NODE_PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET,
+  SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL, CONF_SIMPLE_MODE,
+  CONF_DISABLE_DB
+} from "./config.js";
+// Env variable which determines which DB to connect to
+const { NODE_ENV: mode } = process.env;
+
+mongoose.connection.on('connected', () => {
+  console.log('Mongoose connected to MongoDB');
+});
+
+mongoose.connection.on('error', (err) => {
+  console.error('Mongoose connection error:', err);
+});
+
+(async () => {
+  try {
+    // Make DB connection if needed
+    let clientP;
+    if (!CONF_SIMPLE_MODE && !CONF_DISABLE_DB){
+      if (mode == "production"){
+        clientP = mongoose.connect(MONGO_URI).then(m => m.connection.getClient());
+      }else if (mode == "development"){
+        clientP = mongoose.connect(MONGO_URI_DEV).then(m => m.connection.getClient());
+      }else if (mode == "local"){
+        clientP = mongoose.connect(MONGO_URI_LOCAL).then(m => m.connection.getClient());
+      }else{
+        clientP = mongoose.connect(MONGO_URI).then(m => m.connection.getClient());
+      }
+      console.log('MongoDB connected on ' + mode);
+    }else{
+      console.log('Running without a database connection' );
+    }
+    
+    // Web application framework
+    const app = express();
+    app.disable('x-powered-by');
+    // Parses and validates requests to make things harder for malicious actors 
+    app.use(express.urlencoded({ extended: true }));
+    app.use(express.json());
+
+    if (!CONF_SIMPLE_MODE && !CONF_DISABLE_DB){
+      // Declare session data
+      app.use(session({
+        name: SESS_NAME,
+        //TODO: change secret in config file
+        secret: SESS_SECRET,
+        //define where to store them
+        store: MongoStore.create({
+          clientPromise: clientP,
+          collectionName: 'session',
+          ttl: parseInt(SESS_LIFETIME) / 1000,
+        }),
+        saveUninitialized: false,
+        proxy: NODE_ENV === "production",
+        resave: false,
+        //cookie to send to users
+        cookie: {
+          sameSite: false,
+          secure: false,
+          maxAge: parseInt(SESS_LIFETIME)
+        }
+      }));
+    }
+
+    // Define endpoint paths
+    const apiRouter = express.Router();
+    // Catch any requests from /api/* and send it to the appropriate routes
+    app.use('/api', apiRouter);
+    apiRouter.use('/users', userRouter);
+    apiRouter.use('/session', sessionRouter);
+    apiRouter.use('/livepeer', livepeerRouter);
+
+    // Error handler
+    app.use(function(err, req, res, next) {
+      res.locals.message = err.message;
+      // Also log it to the console
+      console.log(`${err.status || 500} - ${err.message} - ${req.originalUrl} - ${req.method} - ${req.ip}`);
+      // Render the error page
+      res.status(err.status || 500);
+      res.json({
+        message: err.message,
+        error: err
+      });
+    });
+
+    // Start listening on the defined port
+    app.listen(NODE_PORT, "0.0.0.0", function () {
+      console.log(`Listening on port ${NODE_PORT}`);
+    });
+  } catch (err) {
+    console.log(err);
+  }
+})(); 

backend/src/routes/index.js

@@ -1,4 +1,4 @@
-import userRouter from './user';
+import userRouter from './user.js';
-import sessionRouter from './session';
+import sessionRouter from './session.js';
-import livepeerRouter from './livepeer';
+import livepeerRouter from './livepeer.js';
 export { userRouter, sessionRouter, livepeerRouter };

backend/src/routes/livepeer.js

@@ -1,21 +1,21 @@
 import express from "express";
-import Event from '../models/event';
+import Event from '../models/event.js';
-import Block from '../models/block';
+import Block from '../models/block.js';
-import Ticket from '../models/ticketEvent';
+import Ticket from '../models/ticketEvent.js';
 
-import ActivateEvent from "../models/ActivateEvent";
+import ActivateEvent from "../models/ActivateEvent.js";
-import ClaimEvent from "../models/ClaimEvent";
+import ClaimEvent from "../models/ClaimEvent.js";
-import RedeemEvent from "../models/RedeemEvent";
+import RedeemEvent from "../models/RedeemEvent.js";
-import RewardEvent from "../models/RewardEvent";
+import RewardEvent from "../models/RewardEvent.js";
-import StakeEvent from "../models/StakeEvent";
+import StakeEvent from "../models/StakeEvent.js";
-import TransferEvent from "../models/TransferEvent";
+import TransferEvent from "../models/TransferEvent.js";
-import UnbondEvent from "../models/UnbondEvent";
+import UnbondEvent from "../models/UnbondEvent.js";
-import UpdateEvent from "../models/UpdateEvent";
+import UpdateEvent from "../models/UpdateEvent.js";
-import WithdrawFeesEvent from "../models/WithdrawFeesEvent";
+import WithdrawFeesEvent from "../models/WithdrawFeesEvent.js";
-import WithdrawStakeEvent from "../models/WithdrawStakeEvent";
+import WithdrawStakeEvent from "../models/WithdrawStakeEvent.js";
-import MonthlyStat from "../models/monthlyStat";
+import MonthlyStat from "../models/monthlyStat.js";
-import CommissionDataPoint from "../models/CommissionDataPoint";
+import CommissionDataPoint from "../models/CommissionDataPoint.js";
-import TotalStakeDataPoint from "../models/TotalStakeDataPoint";
+import TotalStakeDataPoint from "../models/TotalStakeDataPoint.js";
 
 const apiRouter = express.Router();
 import {
@@ -24,7 +24,9 @@ import {
   CONF_TIMEOUT_ALCHEMY, CONF_TIMEOUT_LIVEPEER,
   CONF_DISABLE_DB, CONF_DISABLE_CMC, CONF_TIMEOUT_ENS_DOMAIN,
   CONF_TIMEOUT_ENS_INFO
-} from "../config";
+} from "../config.js";
+
+const graphUri = "FILL THIS IN"
 
 /*
 
@@ -33,25 +35,26 @@ imported modules
 
 */
 
+const maxCacheEntries = 10000;
+
 // Do API requests to other API's
-const https = require('https');
+import https from "https";
 
 // Read ABI files
-const fs = require('fs');
+import fs from "fs";
 
 // Used for the livepeer thegraph API
 import { request, gql } from 'graphql-request';
-import Round from "../models/Round";
+import Round from "../models/Round.js";
 
 // Gets ETH, LPT and other coin info
-let CoinMarketCap = require('coinmarketcap-api');
+import CoinMarketCap from 'coinmarketcap-api';
-let cmcClient = new CoinMarketCap(API_CMC);
+let cmcClient;
 let cmcEnabled = false;
 if (!CONF_DISABLE_CMC) {
   if (API_CMC == "") {
     console.log("Please provide a CMC api key");
   } else {
-    CoinMarketCap = require('coinmarketcap-api');
     cmcClient = new CoinMarketCap(API_CMC);
     cmcEnabled = true;
   }
@@ -62,7 +65,7 @@ if (!CONF_DISABLE_CMC) {
 // Gets blockchain data
 
 // ENS stuff TODO: CONF_DISABLE_ENS
-const { ethers } = require("ethers");
+import { ethers } from 'ethers';
 const l1provider = new ethers.providers.JsonRpcProvider(API_L1_HTTP + API_L1_KEY);
 // const l1provider = new ethers.providers.AlchemyProvider("mainnet", API_L1_KEY);
 
@@ -72,7 +75,7 @@ const web3layer1 = new Alchemy({apiKey: API_L1_KEY, network: Network.ETH_MAINNET
 const web3layer2 = new Alchemy({ apiKey: API_L2_KEY, network: Network.ARB_MAINNET });
 
 // Smart contract event stuff
-const { createAlchemyWeb3 } = require("@alch/alchemy-web3");
+import { createAlchemyWeb3 } from '@alch/alchemy-web3';
 const deprWeb3main = createAlchemyWeb3(API_L1_HTTP + API_L1_KEY);
 const deprWeb3 = createAlchemyWeb3(API_L2_HTTP + API_L2_KEY);
 let BondingManagerTargetJson;
@@ -1490,7 +1493,7 @@ const getRoundInfo = async function (roundNumber) {
     }
   }
   `;
-  const roundObj = await request("https://api.thegraph.com/subgraphs/name/livepeer/arbitrum-one", roundQuery);
+  const roundObj = await request(graphUri, roundQuery);
   // Not found
   if (!roundObj || !roundObj.rounds || !roundObj.rounds.length) {
     console.log("No round found with number " + roundNumber);
@@ -1791,12 +1794,12 @@ const syncRounds = function (toBlock) {
 // Retrieves stuff from DB on first boot
 const initSync = async function () {
   startedInitSync = true;
+  console.log("Start initial sync...");
   // First collection -> cache
+  const count = await Block.countDocuments();
+  console.log(`Documents in Block collection: ${count}`);
   // Get all parsed blocks
-  blockCache = await Block.find({}, {
+  const blockCache = await Block.find().sort({ _id: -1 }).limit(maxCacheEntries);
-    blockNumber: 1,
-    blockTime: 1
-  });
   console.log("Retrieved existing Blocks of size " + blockCache.length);
   // Get all parsed Events
   eventsCache = await Event.find({}, {
@@ -1808,7 +1811,7 @@ const initSync = async function () {
     blockNumber: 1,
     blockTime: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   console.log("Retrieved existing raw Events of size " + eventsCache.length);
   // Get all parsedTickets
   ticketsCache = await Ticket.find({}, {
@@ -1820,7 +1823,7 @@ const initSync = async function () {
     blockNumber: 1,
     blockTime: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   console.log("Retrieved existing raw Tickets of size " + ticketsCache.length);
   // Then determine latest block number parsed based on collection
   for (var idx = 0; idx < eventsCache.length; idx++) {
@@ -1848,7 +1851,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed reward events and cache them
   rewardEventCache = await RewardEvent.find({}, {
     address: 1,
@@ -1858,7 +1861,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed claim events and cache them
   claimEventCache = await ClaimEvent.find({}, {
     address: 1,
@@ -1871,7 +1874,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed withdraw fees events and cache them
   withdrawFeesEventCache = await WithdrawFeesEvent.find({}, {
     address: 1,
@@ -1881,7 +1884,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed withdraw stake events and cache them
   withdrawStakeEventCache = await WithdrawStakeEvent.find({}, {
     address: 1,
@@ -1892,7 +1895,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed transfer winning ticket events and cache them
   transferTicketEventCache = await TransferEvent.find({}, {
     address: 1,
@@ -1903,7 +1906,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed redeem winning ticket events and cache them
   redeemTicketEventCache = await RedeemEvent.find({}, {
     address: 1,
@@ -1913,7 +1916,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   for (const winner of redeemTicketEventCache) {
     winningTicketCache.push({
       address: winner.address,
@@ -1933,7 +1936,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed unbond events and cache them
   unbondEventCache = await UnbondEvent.find({}, {
     address: 1,
@@ -1945,7 +1948,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed stake events and cache them
   stakeEventCache = await StakeEvent.find({}, {
     address: 1,
@@ -1957,7 +1960,7 @@ const initSync = async function () {
     blockTime: 1,
     blockRound: 1,
     _id: 0
-  });
+  }).sort({ _id: -1 }).limit(maxCacheEntries);
   // Get all parsed monthly stats and cache them
   monthlyStatCache = await MonthlyStat.find({}, {
     year: 1,
@@ -2024,7 +2027,7 @@ const initSync = async function () {
     movedStake: 1,
     newStake: 1,
     _id: 0
-  })
+  }).sort({ _id: -1 }).limit(maxCacheEntries)
   console.log("Retrieved existing rounds of size " + roundCache.length);
   // Then determine latest block number parsed based on collection
   for (var idx = 0; idx < roundCache.length; idx++) {
@@ -2615,7 +2618,7 @@ const parseOrchestrator = async function (reqAddr) {
       }
     }
   `;
-      orchestratorObj = await request("https://api.thegraph.com/subgraphs/name/livepeer/arbitrum-one", orchQuery);
+      orchestratorObj = await request(graphUri, orchQuery);
       orchestratorObj = orchestratorObj.transcoder;
       // Not found
       if (!orchestratorObj) {
@@ -2766,7 +2769,7 @@ const parseDelegator = async function (reqAddr) {
       }
     }
   `;
-    delegatorObj = await request("https://api.thegraph.com/subgraphs/name/livepeer/arbitrum-one", delegatorQuery);
+    delegatorObj = await request(graphUri, delegatorQuery);
     delegatorObj = delegatorObj.delegators[0];
     // Not found
     if (!delegatorObj) {

backend/src/routes/session.js

@@ -1,6 +1,6 @@
 import express from "express";
-import User from "../models/user";
+import User from "../models/user.js";
-import { SESS_NAME } from "../config";
+import { SESS_NAME } from "../config.js";
 const sessionRouter = express.Router();
 
 sessionRouter.post("", async (req, res) => {

backend/src/routes/user.js

@@ -1,5 +1,5 @@
 import express from 'express';
-import User from '../models/user';
+import User from '../models/user.js';
 const userRouter = express.Router();
 
 userRouter.post("/getVisitorStats", async (req, res) => {

backend/src/server.js

@@ -1,93 +0,0 @@
-//Server logic. Imports all necessary routes, models, etc
-import express from 'express';
-import mongoose from 'mongoose';
-import session from "express-session";
-import MongoStore from "connect-mongo";
-import { userRouter, sessionRouter, livepeerRouter } from './routes/index';
-import {
-  NODE_PORT, NODE_ENV, MONGO_URI, SESS_NAME, SESS_SECRET,
-  SESS_LIFETIME , MONGO_URI_DEV, MONGO_URI_LOCAL, CONF_SIMPLE_MODE,
-  CONF_DISABLE_DB
-} from "./config";
-// Env variable which determines which DB to connect to
-const { NODE_ENV: mode } = process.env;
-
-(async () => {
-  try {
-    // Make DB connection if needed
-    let clientP;
-    if (!CONF_SIMPLE_MODE && !CONF_DISABLE_DB){
-      if (mode == "production"){
-        clientP = mongoose.connect(MONGO_URI, { useNewUrlParser: true}).then(m => m.connection.getClient());
-      }else if (mode == "development"){
-        clientP = mongoose.connect(MONGO_URI_DEV, { useNewUrlParser: true}).then(m => m.connection.getClient());
-      }else if (mode == "local"){
-        clientP = mongoose.connect(MONGO_URI_LOCAL, { useNewUrlParser: true}).then(m => m.connection.getClient());
-      }else{
-        clientP = mongoose.connect(MONGO_URI, { useNewUrlParser: true}).then(m => m.connection.getClient());
-      }
-      console.log('MongoDB connected on ' + mode);
-    }else{
-      console.log('Running without a database connection' );
-    }
-    
-    // Web application framework
-    const app = express();
-    app.disable('x-powered-by');
-    // Parses and validates requests to make things harder for malicious actors 
-    app.use(express.urlencoded({ extended: true }));
-    app.use(express.json());
-
-    if (!CONF_SIMPLE_MODE && !CONF_DISABLE_DB){
-      // Declare session data
-      app.use(session({
-        name: SESS_NAME,
-        //TODO: change secret in config file
-        secret: SESS_SECRET,
-        //define where to store them
-        store: MongoStore.create({
-          clientPromise: clientP,
-          collectionName: 'session',
-          ttl: parseInt(SESS_LIFETIME) / 1000,
-        }),
-        saveUninitialized: false,
-        proxy: NODE_ENV === "production",
-        resave: false,
-        //cookie to send to users
-        cookie: {
-          sameSite: false,
-          secure: false,
-          maxAge: parseInt(SESS_LIFETIME)
-        }
-      }));
-    }
-
-    // Define endpoint paths
-    const apiRouter = express.Router();
-    // Catch any requests from /api/* and send it to the appropriate routes
-    app.use('/api', apiRouter);
-    apiRouter.use('/users', userRouter);
-    apiRouter.use('/session', sessionRouter);
-    apiRouter.use('/livepeer', livepeerRouter);
-
-    // Error handler
-    app.use(function(err, req, res, next) {
-      res.locals.message = err.message;
-      // Also log it to the console
-      console.log(`${err.status || 500} - ${err.message} - ${req.originalUrl} - ${req.method} - ${req.ip}`);
-      // Render the error page
-      res.status(err.status || 500);
-      res.json({
-        message: err.message,
-        error: err
-      });
-    });
-
-    // Start listening on the defined port
-    app.listen(NODE_PORT, "0.0.0.0", function () {
-      console.log(`Listening on port ${NODE_PORT}`);
-    });
-  } catch (err) {
-    console.log(err);
-  }
-})();