Move hard-coded values to config (#16872)

.github/assets.config.yaml

@@ -106,4 +106,27 @@ validators_settings:
         description: Tokens that are deflationary or use mechanism to burn a token on transfer/swap.
       - id: memes
         name: Memes
-        description: A cryptocurrency that is associated with some theme, often as a joke rather than a serious product.
+        description: A cryptocurrency that is associated with some theme, often as a joke rather than a serious product.
+
+trading_pair_settings:
+  uniswap:
+    url: https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2
+
+    primary_tokens: "WETH,ETH"
+    force_include_list: "TUSD,STAKE,YFI,BAT,MANA,1INCH,REP,KP3R,UNI,WBTC,HEX,CREAM,SLP,REN,XOR,Link,sUSD,HEGIC,RLC,DAI,SUSHI,FYZ,DYT,AAVE,LEND,UBT,DIA,RSR,SXP,OCEAN,MKR,USDC,CEL,BAL,BAND,COMP,SNX,OMG,AMPL,USDT,KNC,ZRX,AXS,ENJ,STMX,DPX,FTT,DPI,PAX"
+    force_exclude_list: "STARL,UFO"
+
+    min_liquidity: 2000000
+    min_vol_24: 1000000
+    min_tx_count_24: 480
+
+  pancakeswap:
+    url: https://api.bscgraph.org/subgraphs/name/cakeswap
+
+    primary_tokens: "WBNB,BNB"
+    force_include_list: "Cake,DAI,ETH,TWT,VAI,USDT,BLINK,BTCB,ALPHA,INJ,CTK,UNI,XVS,BUSD,HARD,BIFI,FRONT"
+    force_exclude_list:
+
+    min_liquidity: 1000000
+    min_vol_24: 500000
+    min_tx_count_24: 288

README.md

@@ -72,7 +72,7 @@ There are similar check logic implemented:
 Info on supported trading pairs are stored in `tokenlist.json` files.
 Trading pairs can be updated --
 from Uniswap/Ethereum and PancakeSwap/Smartchain -- using update script (and checking in changes).
-Minimal limit values for trading pair inclusion are set in the `config.ts` file.
+Minimal limit values for trading pair inclusion are set in the [config file](https://github.com/trustwallet/assets/blob/master/.github/assets.config.yaml).
 There are also options for force-include and force-exclude in the config.
 
 ## Disclaimer

internal/config/config.go

@@ -8,10 +8,11 @@ import (
 
 type (
 	Config struct {
-		App                App                `mapstructure:"app"`
+		App                 App                 `mapstructure:"app"`
-		ClientURLs         ClientsURLs        `mapstructure:"client_urls"`
+		ClientURLs          ClientsURLs         `mapstructure:"client_urls"`
-		URLs               URLs               `mapstructure:"urls"`
+		URLs                URLs                `mapstructure:"urls"`
-		ValidatorsSettings ValidatorsSettings `mapstructure:"validators_settings"`
+		ValidatorsSettings  ValidatorsSettings  `mapstructure:"validators_settings"`
+		TradingPairSettings TradingPairSettings `mapstructure:"trading_pair_settings"`
 	}
 
 	App struct {
@@ -39,6 +40,21 @@ type (
 		DappsFolder                DappsFolder                `mapstructure:"dapps_folder"`
 		CoinInfoFile               CoinInfoFile               `mapstructure:"coin_info_file"`
 	}
+
+	TradingPairSettings struct {
+		Uniswap     TradingPairSetting `mapstructure:"uniswap"`
+		Pancakeswap TradingPairSetting `mapstructure:"pancakeswap"`
+	}
+
+	TradingPairSetting struct {
+		URL              string `mapstructure:"url"`
+		PrimaryTokens    string `mapstructure:"primary_tokens"`
+		ForceIncludeList string `mapstructure:"force_include_list"`
+		ForceExcludeList string `mapstructure:"force_exclude_list"`
+		MinLiquidity     int    `mapstructure:"min_liquidity"`
+		MinVol24         int    `mapstructure:"min_vol_24"`
+		MinTxCount24     int    `mapstructure:"min_tx_count_24"`
+	}
 )
 
 // Default is a configuration instance.

internal/processor/service.go

@@ -137,19 +137,12 @@ func (s *Service) GetUpdatersAuto() []Updater {
 	}
 }
 
-// nolint:godot
 func (s *Service) GetUpdatersManual() []Updater {
 	return []Updater{
 		{
 			Name: "Update tokenlist.json for Ethereum",
 			Run:  s.UpdateEthereumTokenlist,
 		},
-		// TODO: Add a special unmarshalling for `fetchTradingPairs`.
-		// In addition, https://graphql.bitquery.io/ has new restrictions for API access, so this updater doesn't work.
-		// {
-		// 	Name: "Update tokenlist.json for Polygon",
-		// 	Run:  s.UpdatePolygonTokenlist,
-		// },
 		{
 			Name: "Update tokenlist.json for Smartchain",
 			Run:  s.UpdateSmartchainTokenlist,

internal/processor/updaters_manual.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -19,25 +18,6 @@ import (
 	"github.com/trustwallet/go-primitives/types"
 )
 
-var (
-	UniswapForceInclude = []string{
-		"TUSD", "STAKE", "YFI", "BAT", "MANA", "1INCH", "REP", "KP3R", "UNI", "WBTC", "HEX", "CREAM", "SLP",
-		"REN", "XOR", "Link", "sUSD", "HEGIC", "RLC", "DAI", "SUSHI", "FYZ", "DYT", "AAVE", "LEND", "UBT",
-		"DIA", "RSR", "SXP", "OCEAN", "MKR", "USDC", "CEL", "BAL", "BAND", "COMP", "SNX", "OMG", "AMPL",
-		"USDT", "KNC", "ZRX", "AXS", "ENJ", "STMX", "DPX", "FTT", "DPI", "PAX",
-	}
-	UniswapForceExclude = []string{"STARL", "UFO"}
-
-	PolygonSwapForceInclude = []string{}
-	PolygonSwapForceExclude = []string{}
-
-	PancakeSwapForceInclude = []string{
-		"Cake", "DAI", "ETH", "TWT", "VAI", "USDT", "BLINK", "BTCB", "ALPHA", "INJ", "CTK", "UNI", "XVS",
-		"BUSD", "HARD", "BIFI", "FRONT",
-	}
-	PancakeSwapForceExclude = []string{}
-)
-
 var (
 	UniswapTradingPairsQuery = map[string]string{
 		"operationName": "pairs",
@@ -56,21 +36,6 @@ var (
 		`,
 	}
 
-	PolygonSwapTradingPairsQuery = map[string]string{
-		"query": `
-			{
-				ethereum(network: matic) {
-					dexTrades(date: {is: "$DATE$"}) {
-						sellCurrency {address symbol name decimals}
-						buyCurrency {address symbol name decimals}
-						trade: count
-						tradeAmount(in: USD)
-					}
-				}
-			}
-		`,
-	}
-
 	PancakeSwapTradingPairsQuery = map[string]string{
 		"operationName": "pairs",
 		"query": `
@@ -87,42 +52,23 @@ var (
 			}
 		`,
 	}
-
-	UniswapTradingPairsUrl     = "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2"
-	PolygonSwapTradingPairsUrl = "https://graphql.bitquery.io"
-	PancakeSwapTradingPairsUrl = "https://api.bscgraph.org/subgraphs/name/cakeswap"
-)
-
-const (
-	UniswapMinLiquidity = 2000000
-	UniswapMinVol24     = 1000000
-	UniswapMinTxCount24 = 480
-
-	PolygonSwapMinLiquidity = 0
-	PolygonSwapMinVol24     = 500000
-	PolygonSwapMinTxCount24 = 288
-
-	PancakeSwapMinLiquidity = 1000000
-	PancakeSwapMinVol24     = 500000
-	PancakeSwapMinTxCount24 = 288
-)
-
-var (
-	PrimaryTokensEthereum   = []string{"WETH", "ETH"}
-	PrimaryTokensPolygon    = []string{"WMATIC", "MATIC", "WETH", "USDC", "USDT", "DAI"}
-	PrimaryTokensSmartChain = []string{"WBNB", "BNB"}
 )
 
 // nolint:dupl
 func (s *Service) UpdateEthereumTokenlist() error {
 	log.WithFields(log.Fields{
-		"limit_liquidity": UniswapMinLiquidity,
+		"limit_liquidity": config.Default.TradingPairSettings.Uniswap.MinLiquidity,
-		"volume":          UniswapMinVol24,
+		"volume":          config.Default.TradingPairSettings.Uniswap.MinVol24,
-		"tx_count":        UniswapMinTxCount24,
+		"tx_count":        config.Default.TradingPairSettings.Uniswap.MinTxCount24,
 	}).Debug("Retrieving pairs from Uniswap")
 
-	tradingPairs, err := retrievePairs(UniswapTradingPairsUrl, UniswapTradingPairsQuery,
+	forceInclude := strings.Split(config.Default.TradingPairSettings.Uniswap.ForceIncludeList, ",")
-		UniswapMinLiquidity, UniswapMinVol24, UniswapMinTxCount24, UniswapForceInclude, PrimaryTokensEthereum)
+	forceExclude := strings.Split(config.Default.TradingPairSettings.Uniswap.ForceExcludeList, ",")
+	primaryTokens := strings.Split(config.Default.TradingPairSettings.Uniswap.PrimaryTokens, ",")
+
+	tradingPairs, err := retrievePairs(config.Default.TradingPairSettings.Uniswap.URL, UniswapTradingPairsQuery,
+		config.Default.TradingPairSettings.Uniswap.MinLiquidity, config.Default.TradingPairSettings.Uniswap.MinVol24,
+		config.Default.TradingPairSettings.Uniswap.MinTxCount24, forceInclude, primaryTokens)
 	if err != nil {
 		return err
 	}
@@ -142,70 +88,32 @@ func (s *Service) UpdateEthereumTokenlist() error {
 			return err
 		}
 
-		if !isTokenPrimary(tradingPair.Token0, PrimaryTokensEthereum) {
+		if !isTokenPrimary(tradingPair.Token0, primaryTokens) {
 			tokenItem0, tokenItem1 = tokenItem1, tokenItem0
 		}
 
 		pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1})
 	}
 
-	return rebuildTokenList(chain, pairs, UniswapForceExclude)
+	return rebuildTokenList(chain, pairs, forceExclude)
-}
-
-func (s *Service) UpdatePolygonTokenlist() error {
-	log.WithFields(log.Fields{
-		"limit_liquidity": PolygonSwapMinLiquidity,
-		"volume":          PolygonSwapMinVol24,
-		"tx_count":        PolygonSwapMinTxCount24,
-	}).Debug("Retrieving pairs from PolygonSwap")
-
-	yesterdayDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
-
-	PolygonSwapTradingPairsQuery["query"] = strings.ReplaceAll(PolygonSwapTradingPairsQuery["query"],
-		"$DATE$", yesterdayDate)
-
-	tradingPairs, err := retrievePairs(PolygonSwapTradingPairsUrl, PolygonSwapTradingPairsQuery, PolygonSwapMinLiquidity,
-		PolygonSwapMinVol24, PolygonSwapMinTxCount24, PolygonSwapForceInclude, PrimaryTokensPolygon)
-	if err != nil {
-		return err
-	}
-
-	pairs := make([][]TokenItem, 0)
-
-	chain := coin.Coins[coin.POLYGON]
-
-	for _, tradingPair := range tradingPairs {
-		tokenItem0, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token0)
-		if err != nil {
-			return err
-		}
-
-		tokenItem1, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token1)
-		if err != nil {
-			return err
-		}
-
-		if !isTokenPrimary(tradingPair.Token0, PrimaryTokensEthereum) {
-			tokenItem0, tokenItem1 = tokenItem1, tokenItem0
-		}
-
-		pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1})
-	}
-
-	return rebuildTokenList(chain, pairs, PolygonSwapForceExclude)
 }
 
 // nolint:dupl
 func (s *Service) UpdateSmartchainTokenlist() error {
 	log.WithFields(log.Fields{
-		"limit_liquidity": PancakeSwapMinLiquidity,
+		"limit_liquidity": config.Default.TradingPairSettings.Pancakeswap.MinLiquidity,
-		"volume":          PancakeSwapMinVol24,
+		"volume":          config.Default.TradingPairSettings.Pancakeswap.MinVol24,
-		"tx_count":        PancakeSwapMinTxCount24,
+		"tx_count":        config.Default.TradingPairSettings.Pancakeswap.MinTxCount24,
 	}).Debug("Retrieving pairs from PancakeSwap")
 
-	tradingPairs, err := retrievePairs(PancakeSwapTradingPairsUrl, PancakeSwapTradingPairsQuery,
+	forceInclude := strings.Split(config.Default.TradingPairSettings.Pancakeswap.ForceIncludeList, ",")
-		PancakeSwapMinLiquidity, PancakeSwapMinVol24, PancakeSwapMinTxCount24,
+	forceExclude := strings.Split(config.Default.TradingPairSettings.Pancakeswap.ForceExcludeList, ",")
-		PancakeSwapForceInclude, PrimaryTokensSmartChain)
+	primaryTokens := strings.Split(config.Default.TradingPairSettings.Pancakeswap.PrimaryTokens, ",")
+
+	tradingPairs, err := retrievePairs(config.Default.TradingPairSettings.Pancakeswap.URL,
+		PancakeSwapTradingPairsQuery, config.Default.TradingPairSettings.Pancakeswap.MinLiquidity,
+		config.Default.TradingPairSettings.Pancakeswap.MinVol24,
+		config.Default.TradingPairSettings.Pancakeswap.MinTxCount24, forceInclude, primaryTokens)
 	if err != nil {
 		return err
 	}
@@ -225,14 +133,14 @@ func (s *Service) UpdateSmartchainTokenlist() error {
 			return err
 		}
 
-		if !isTokenPrimary(tradingPair.Token0, PrimaryTokensSmartChain) {
+		if !isTokenPrimary(tradingPair.Token0, primaryTokens) {
 			tokenItem0, tokenItem1 = tokenItem1, tokenItem0
 		}
 
 		pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1})
 	}
 
-	return rebuildTokenList(chain, pairs, PancakeSwapForceExclude)
+	return rebuildTokenList(chain, pairs, forceExclude)
 }
 
 func retrievePairs(url string, query map[string]string, minLiquidity, minVol24, minTxCount24 int,