From 72d5b319d43913d6d781727909e246fd87a6d374 Mon Sep 17 00:00:00 2001 From: Daniel <24758309+leedaniil@users.noreply.github.com> Date: Sun, 16 Jan 2022 17:13:01 +0300 Subject: [PATCH] Add address checksum validation for tokenlist files (#17416) * Add address checksum validation for tokenlist files * Remove updaters_manual.go, move tokenlist validation to assets-go-libs --- README.md | 1 - go.mod | 2 +- go.sum | 4 +- internal/manager/commands.go | 9 - internal/processor/model.go | 34 -- internal/processor/service.go | 7 - internal/processor/updaters_auto.go | 31 +- internal/processor/updaters_manual.go | 532 -------------------------- internal/processor/validators.go | 101 +---- internal/service/service.go | 5 - 10 files changed, 22 insertions(+), 704 deletions(-) delete mode 100644 internal/processor/updaters_manual.go diff --git a/README.md b/README.md index b6163e62e..f1668eb5d 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,6 @@ There are several scripts available for maintainers: - `make check` -- Execute validation checks; also used in continuous integration. - `make fix` -- Perform automatic fixes where possible - `make update-auto` -- Run automatic updates from external sources, executed regularly (GitHub action) -- `make update-manual` -- Run manual updates from external sources, for manual use. - `make add-token token=c60_t0x4Fabb145d64652a948d72533023f6E7A623C7C53` -- Create `info.json` file as asset template. ## On Checks diff --git a/go.mod b/go.mod index 5056b80ae..5a55a2342 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.3.0 - github.com/trustwallet/assets-go-libs v0.0.25 + github.com/trustwallet/assets-go-libs v0.0.26 github.com/trustwallet/go-libs v0.2.23 github.com/trustwallet/go-primitives v0.0.20 ) diff --git a/go.sum b/go.sum index 641f957cc..c835ce360 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/trustwallet/assets-go-libs v0.0.25 h1:JoBZnKOHpHp1pmFjLqWcdlGE1xgGDQUYgHmYIOCZGnY= -github.com/trustwallet/assets-go-libs v0.0.25/go.mod h1:UKacopV6UfT2JKRO4mkNSOVij9ChOO/NOXLscd9VhLk= +github.com/trustwallet/assets-go-libs v0.0.26 h1:CtqSfhu/sNV/q256977UF6niIabmEvzrJVUwD4GJlDg= +github.com/trustwallet/assets-go-libs v0.0.26/go.mod h1:UKacopV6UfT2JKRO4mkNSOVij9ChOO/NOXLscd9VhLk= github.com/trustwallet/go-libs v0.2.23 h1:CUB2OubedAoUbgPcraIC7wqpJHKQ5o3NJdINzH418WQ= github.com/trustwallet/go-libs v0.2.23/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc= github.com/trustwallet/go-primitives v0.0.20 h1:srXOScQzub2Usrj5mRq5d2/3+0Y3TBXh2wD6JXYH1No= diff --git a/internal/manager/commands.go b/internal/manager/commands.go index 4f18cf0b5..47d211f5e 100644 --- a/internal/manager/commands.go +++ b/internal/manager/commands.go @@ -23,7 +23,6 @@ func InitCommands() { rootCmd.AddCommand(checkCmd) rootCmd.AddCommand(fixCmd) rootCmd.AddCommand(updateAutoCmd) - rootCmd.AddCommand(updateManualCmd) rootCmd.AddCommand(addTokenCmd) } @@ -58,14 +57,6 @@ var ( assetsService.RunUpdateAuto() }, } - updateManualCmd = &cobra.Command{ - Use: "update-manual", - Short: "Run manual updates from external sources", - Run: func(cmd *cobra.Command, args []string) { - assetsService := InitAssetsService() - assetsService.RunUpdateManual() - }, - } addTokenCmd = &cobra.Command{ Use: "add-token", diff --git a/internal/processor/model.go b/internal/processor/model.go index 91f01a121..2c2d267bc 100644 --- a/internal/processor/model.go +++ b/internal/processor/model.go @@ -2,7 +2,6 @@ package processor import ( "github.com/trustwallet/assets/internal/file" - "github.com/trustwallet/go-primitives/types" ) type ( @@ -22,39 +21,6 @@ type ( } ) -type ( - TokenList struct { - Name string `json:"name"` - LogoURI string `json:"logoURI"` - Timestamp string `json:"timestamp"` - Tokens []TokenItem `json:"tokens"` - Version Version `json:"version"` - } - - TokenItem struct { - Asset string `json:"asset"` - Type types.TokenType `json:"type"` - Address string `json:"address"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Decimals uint `json:"decimals"` - LogoURI string `json:"logoURI"` - Pairs []Pair `json:"pairs"` - } - - Pair struct { - Base string `json:"base"` - LotSize string `json:"lotSize,omitempty"` - TickSize string `json:"tickSize,omitempty"` - } - - Version struct { - Major int `json:"major"` - Minor int `json:"minor"` - Patch int `json:"patch"` - } -) - type ( ForceListPair struct { Token0 string diff --git a/internal/processor/service.go b/internal/processor/service.go index 259a0a607..82f32def9 100644 --- a/internal/processor/service.go +++ b/internal/processor/service.go @@ -108,10 +108,3 @@ func (s *Service) GetUpdatersAuto() []Updater { {Name: "Retrieving missing token images, creating binance token list.", Run: s.UpdateBinanceTokens}, } } - -func (s *Service) GetUpdatersManual() []Updater { - return []Updater{ - {Name: "Update tokenlist.json for Ethereum", Run: s.UpdateEthereumTokenlist}, - {Name: "Update tokenlist.json for Smartchain", Run: s.UpdateSmartchainTokenlist}, - } -} diff --git a/internal/processor/updaters_auto.go b/internal/processor/updaters_auto.go index b6edca16d..91a4e3dd6 100644 --- a/internal/processor/updaters_auto.go +++ b/internal/processor/updaters_auto.go @@ -16,6 +16,7 @@ import ( "github.com/trustwallet/assets-go-libs/image" "github.com/trustwallet/assets-go-libs/path" "github.com/trustwallet/assets-go-libs/validation/info" + "github.com/trustwallet/assets-go-libs/validation/tokenlist" "github.com/trustwallet/assets/internal/config" "github.com/trustwallet/go-libs/blockchain/binance" "github.com/trustwallet/go-libs/blockchain/binance/explorer" @@ -137,10 +138,10 @@ func createInfoJSON(chain coin.Coin, a explorer.Bep2Asset) error { return fileLib.CreateJSONFile(assetInfoPath, &assetInfo) } -func createTokenListJSON(chain coin.Coin, tokens []TokenItem) error { +func createTokenListJSON(chain coin.Coin, tokens []tokenlist.Token) error { tokenListPath := path.GetTokenListPath(chain.Handle) - var oldTokenList TokenList + var oldTokenList tokenlist.Model err := fileLib.ReadJSONFile(tokenListPath, &oldTokenList) if err != nil { return nil @@ -157,16 +158,16 @@ func createTokenListJSON(chain coin.Coin, tokens []TokenItem) error { log.Debugf("Tokenlist: list with %d tokens and %d pairs written to %s.", len(tokens), countTotalPairs(tokens), tokenListPath) - return fileLib.CreateJSONFile(tokenListPath, &TokenList{ + return fileLib.CreateJSONFile(tokenListPath, &tokenlist.Model{ Name: fmt.Sprintf("Trust Wallet: %s", coin.Coins[chain.ID].Name), LogoURI: twLogoURL, Timestamp: time.Now().Format(timestampFormat), Tokens: tokens, - Version: Version{Major: oldTokenList.Version.Major + 1}, + Version: tokenlist.Version{Major: oldTokenList.Version.Major + 1}, }) } -func countTotalPairs(tokens []TokenItem) int { +func countTotalPairs(tokens []tokenlist.Token) int { var counter int for _, token := range tokens { counter += len(token.Pairs) @@ -175,7 +176,7 @@ func countTotalPairs(tokens []TokenItem) int { return counter } -func sortTokens(tokens []TokenItem) { +func sortTokens(tokens []tokenlist.Token) { sort.Slice(tokens, func(i, j int) bool { if len(tokens[i].Pairs) != len(tokens[j].Pairs) { return len(tokens[i].Pairs) > len(tokens[j].Pairs) @@ -191,7 +192,7 @@ func sortTokens(tokens []TokenItem) { } } -func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]TokenItem, error) { +func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]tokenlist.Token, error) { if len(marketPairs) < 5 { return nil, fmt.Errorf("no markets info is returned from Binance DEX: %d", len(marketPairs)) } @@ -200,7 +201,7 @@ func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Token return nil, fmt.Errorf("no tokens info is returned from Binance DEX: %d", len(tokenList)) } - pairsMap := make(map[string][]Pair) + pairsMap := make(map[string][]tokenlist.Pair) pairsList := make(map[string]struct{}) tokensMap := make(map[string]binance.Token) @@ -219,25 +220,25 @@ func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Token val = append(val, getPair(marketPair)) pairsMap[tokenSymbol] = val } else { - pairsMap[tokenSymbol] = []Pair{getPair(marketPair)} + pairsMap[tokenSymbol] = []tokenlist.Pair{getPair(marketPair)} } pairsList[marketPair.BaseAssetSymbol] = struct{}{} pairsList[marketPair.QuoteAssetSymbol] = struct{}{} } - tokenItems := make([]TokenItem, 0, len(pairsList)) + tokenItems := make([]tokenlist.Token, 0, len(pairsList)) for pair := range pairsList { token := tokensMap[pair] - var pairs []Pair + var pairs []tokenlist.Pair pairs, exists := pairsMap[token.Symbol] if !exists { - pairs = make([]Pair, 0) + pairs = make([]tokenlist.Pair, 0) } - tokenItems = append(tokenItems, TokenItem{ + tokenItems = append(tokenItems, tokenlist.Token{ Asset: getAssetIDSymbol(token.Symbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE), Type: getTokenType(token.Symbol, coin.Coins[coin.BINANCE].Symbol, types.BEP2), Address: token.Symbol, @@ -288,8 +289,8 @@ func isTokenExistOrActive(symbol string) bool { return true } -func getPair(marketPair binance.MarketPair) Pair { - return Pair{ +func getPair(marketPair binance.MarketPair) tokenlist.Pair { + return tokenlist.Pair{ Base: getAssetIDSymbol(marketPair.BaseAssetSymbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE), LotSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.LotSize), 10), TickSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.TickSize), 10), diff --git a/internal/processor/updaters_manual.go b/internal/processor/updaters_manual.go deleted file mode 100644 index 410c0bf2c..000000000 --- a/internal/processor/updaters_manual.go +++ /dev/null @@ -1,532 +0,0 @@ -package processor - -import ( - "encoding/json" - "fmt" - "strconv" - "strings" - - log "github.com/sirupsen/logrus" - - fileLib "github.com/trustwallet/assets-go-libs/file" - "github.com/trustwallet/assets-go-libs/http" - "github.com/trustwallet/assets-go-libs/path" - "github.com/trustwallet/assets/internal/config" - "github.com/trustwallet/go-libs/client/api/backend" - "github.com/trustwallet/go-primitives/address" - "github.com/trustwallet/go-primitives/coin" - "github.com/trustwallet/go-primitives/types" -) - -var ( - UniswapTradingPairsQuery = map[string]string{ - "operationName": "pairs", - "query": ` - query pairs { - pairs(first: 800, orderBy: reserveUSD, orderDirection: desc) { - id reserveUSD trackedReserveETH volumeUSD txCount untrackedVolumeUSD __typename - token0 { - id symbol name decimals __typename - } - token1 { - id symbol name decimals __typename - } - } - } - `, - } - - PancakeSwapTradingPairsQuery = map[string]string{ - "operationName": "pairs", - "query": ` - query pairs { - pairs(first: 300, orderBy: reserveUSD, orderDirection: desc) { - id reserveUSD volumeUSD txCount __typename - token0 { - id symbol name decimals __typename - } - token1 { - id symbol name decimals __typename - } - } - } - `, - } -) - -// nolint:dupl -func (s *Service) UpdateEthereumTokenlist() error { - log.WithFields(log.Fields{ - "limit_liquidity": config.Default.TradingPairSettings.Uniswap.MinLiquidity, - "volume": config.Default.TradingPairSettings.Uniswap.MinVol24, - "tx_count": config.Default.TradingPairSettings.Uniswap.MinTxCount24, - }).Debug("Retrieving pairs from Uniswap") - - forceInclude := strings.Split(config.Default.TradingPairSettings.Uniswap.ForceIncludeList, ",") - forceExclude := strings.Split(config.Default.TradingPairSettings.Uniswap.ForceExcludeList, ",") - primaryTokens := strings.Split(config.Default.TradingPairSettings.Uniswap.PrimaryTokens, ",") - - tradingPairs, err := retrievePairs(config.Default.TradingPairSettings.Uniswap.URL, UniswapTradingPairsQuery, - config.Default.TradingPairSettings.Uniswap.MinLiquidity, config.Default.TradingPairSettings.Uniswap.MinVol24, - config.Default.TradingPairSettings.Uniswap.MinTxCount24, forceInclude, primaryTokens) - if err != nil { - return err - } - - pairs := make([][]TokenItem, 0) - - chain := coin.Coins[coin.ETHEREUM] - - for _, tradingPair := range tradingPairs { - tokenItem0, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token0) - if err != nil { - return err - } - - tokenItem1, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token1) - if err != nil { - return err - } - - if !isTokenPrimary(tradingPair.Token0, primaryTokens) { - tokenItem0, tokenItem1 = tokenItem1, tokenItem0 - } - - pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1}) - } - - return rebuildTokenList(chain, pairs, forceExclude) -} - -// nolint:dupl -func (s *Service) UpdateSmartchainTokenlist() error { - log.WithFields(log.Fields{ - "limit_liquidity": config.Default.TradingPairSettings.Pancakeswap.MinLiquidity, - "volume": config.Default.TradingPairSettings.Pancakeswap.MinVol24, - "tx_count": config.Default.TradingPairSettings.Pancakeswap.MinTxCount24, - }).Debug("Retrieving pairs from PancakeSwap") - - forceInclude := strings.Split(config.Default.TradingPairSettings.Pancakeswap.ForceIncludeList, ",") - forceExclude := strings.Split(config.Default.TradingPairSettings.Pancakeswap.ForceExcludeList, ",") - primaryTokens := strings.Split(config.Default.TradingPairSettings.Pancakeswap.PrimaryTokens, ",") - - tradingPairs, err := retrievePairs(config.Default.TradingPairSettings.Pancakeswap.URL, - PancakeSwapTradingPairsQuery, config.Default.TradingPairSettings.Pancakeswap.MinLiquidity, - config.Default.TradingPairSettings.Pancakeswap.MinVol24, - config.Default.TradingPairSettings.Pancakeswap.MinTxCount24, forceInclude, primaryTokens) - if err != nil { - return err - } - - pairs := make([][]TokenItem, 0) - - chain := coin.Coins[coin.SMARTCHAIN] - - for _, tradingPair := range tradingPairs { - tokenItem0, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token0) - if err != nil { - return err - } - - tokenItem1, err := getTokenInfoFromSubgraphToken(chain, tradingPair.Token1) - if err != nil { - return err - } - - if !isTokenPrimary(tradingPair.Token0, primaryTokens) { - tokenItem0, tokenItem1 = tokenItem1, tokenItem0 - } - - pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1}) - } - - return rebuildTokenList(chain, pairs, forceExclude) -} - -func retrievePairs(url string, query map[string]string, minLiquidity, minVol24, minTxCount24 int, - forceIncludeList []string, primaryTokens []string) ([]TradingPair, error) { - includeList := parseForceList(forceIncludeList) - - pairs, err := fetchTradingPairs(url, query) - if err != nil { - return nil, err - } - - filtered := make([]TradingPair, 0) - - for _, pair := range pairs.Data.Pairs { - ok, err := checkTradingPairOK(pair, minLiquidity, minVol24, minTxCount24, primaryTokens, includeList) - if err != nil { - log.Debug(err) - } - if ok { - filtered = append(filtered, pair) - } - } - - log.Debugf("Retrieved & filtered: %d", len(filtered)) - - return filtered, nil -} - -func parseForceList(forceList []string) []ForceListPair { - result := make([]ForceListPair, 0, len(forceList)) - - for _, item := range forceList { - tokens := strings.Split(item, "-") - pair := ForceListPair{ - Token0: tokens[0], - } - - if len(tokens) >= 2 { - pair.Token1 = tokens[1] - } - - result = append(result, pair) - } - - return result -} - -func fetchTradingPairs(url string, query map[string]string) (*TradingPairs, error) { - jsonValue, err := json.Marshal(query) - if err != nil { - return nil, err - } - - log.WithField("url", url).Debug("Retrieving trading pair infos") - - var result TradingPairs - err = http.PostHTTPResponse(url, jsonValue, &result) - if err != nil { - return nil, err - } - - log.Debugf("Retrieved %d trading pair infos", len(result.Data.Pairs)) - - return &result, nil -} - -func checkTradingPairOK(pair TradingPair, minLiquidity, minVol24, minTxCount24 int, primaryTokens []string, - forceIncludeList []ForceListPair) (bool, error) { - if pair.ID == "" || pair.ReserveUSD == "" || pair.VolumeUSD == "" || pair.TxCount == "" || - pair.Token0 == nil || pair.Token1 == nil { - - return false, nil - } - - if !(isTokenPrimary(pair.Token0, primaryTokens) || isTokenPrimary(pair.Token1, primaryTokens)) { - log.Debugf("pair with no primary coin: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) - return false, nil - } - - if isPairMatchedToForceList(getTokenItemFromInfo(pair.Token0), getTokenItemFromInfo(pair.Token1), forceIncludeList) { - log.Debugf("pair included due to FORCE INCLUDE: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) - return true, nil - } - - reserveUSD, err := strconv.ParseFloat(pair.ReserveUSD, 64) - if err != nil { - return false, err - } - if int(reserveUSD) < minLiquidity { - log.Debugf("pair with low liquidity: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) - return false, nil - } - - volumeUSD, err := strconv.ParseFloat(pair.VolumeUSD, 64) - if err != nil { - return false, err - } - if int(volumeUSD) < minVol24 { - log.Debugf("pair with low volume: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) - return false, nil - } - - txCount, err := strconv.ParseFloat(pair.TxCount, 64) - if err != nil { - return false, err - } - if int(txCount) < minTxCount24 { - log.Debugf("pair with low tx count: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) - return false, nil - } - - return true, nil -} - -func getTokenItemFromInfo(tokenInfo *TokenInfo) *TokenItem { - decimals, err := strconv.Atoi(tokenInfo.Decimals) - if err != nil { - return nil - } - - return &TokenItem{ - Asset: tokenInfo.ID, - Address: tokenInfo.ID, - Name: tokenInfo.Name, - Symbol: tokenInfo.Symbol, - Decimals: uint(decimals), - } -} - -func getTokenInfoFromSubgraphToken(chain coin.Coin, token *TokenInfo) (*TokenItem, error) { - checksum, err := address.EIP55Checksum(token.ID) - if err != nil { - return nil, err - } - - decimals, err := strconv.Atoi(token.Decimals) - if err != nil { - return nil, err - } - - tokenType, ok := types.GetTokenType(chain.ID, token.ID) - if !ok { - return nil, fmt.Errorf("failed to get a token type for %s %s", chain.Symbol, token.ID) - } - - return &TokenItem{ - Asset: getAssetIDSymbol(checksum, chain.Symbol, chain.ID), - Type: types.TokenType(tokenType), - Address: checksum, - Name: token.Name, - Symbol: token.Symbol, - Decimals: uint(decimals), - LogoURI: getLogoURI(token.Symbol, chain.Handle, chain.Symbol), - Pairs: make([]Pair, 0), - }, nil -} - -func isTokenPrimary(token *TokenInfo, primaryTokens []string) bool { - if token == nil { - return false - } - - for _, primaryToken := range primaryTokens { - if strings.EqualFold(primaryToken, token.Symbol) { - return true - } - } - - return false -} - -func isPairMatchedToForceList(token0, token1 *TokenItem, forceIncludeList []ForceListPair) bool { - var matched bool - - for _, forcePair := range forceIncludeList { - if matchPairToForceListEntry(token0, token1, forcePair) { - matched = true - } - } - - return matched -} - -func matchPairToForceListEntry(token0, token1 *TokenItem, forceListEntry ForceListPair) bool { - if forceListEntry.Token1 == "" { - if matchTokenToForceListEntry(token0, forceListEntry.Token0) || - (token1 != nil && matchTokenToForceListEntry(token1, forceListEntry.Token0)) { - - return true - } - - return false - } - - if token1 == nil { - return false - } - - if matchTokenToForceListEntry(token0, forceListEntry.Token0) && - matchTokenToForceListEntry(token0, forceListEntry.Token1) { - - return true - } - - if matchTokenToForceListEntry(token0, forceListEntry.Token1) && - matchTokenToForceListEntry(token1, forceListEntry.Token0) { - - return true - } - - return false -} - -func matchTokenToForceListEntry(token *TokenItem, forceListEntry string) bool { - if strings.EqualFold(forceListEntry, token.Symbol) || - strings.EqualFold(forceListEntry, token.Asset) || - strings.EqualFold(forceListEntry, token.Name) { - - return true - } - - return false -} - -func matchPairToForceList(token0, token1 *TokenItem, forceList []ForceListPair) bool { - var matched bool - for _, forcePair := range forceList { - if matchPairToForceListEntry(token0, token1, forcePair) { - matched = true - } - } - - return matched -} - -func rebuildTokenList(chain coin.Coin, pairs [][]TokenItem, forceExcludeList []string) error { - if pairs == nil || len(pairs) < 5 { - return nil - } - - excludeList := parseForceList(forceExcludeList) - - pairs2 := make([][]TokenItem, 0) - - for _, pair := range pairs { - if !checkTokenExists(chain.Handle, pair[0].Address) { - log.Debugf("pair with unsupported 1st coin: %s-%s", pair[0].Symbol, pair[1].Symbol) - continue - } - - if !checkTokenExists(chain.Handle, pair[1].Address) { - log.Debugf("pair with unsupported 2nd coin: %s-%s", pair[0].Symbol, pair[1].Symbol) - continue - } - - if matchPairToForceList(&pair[0], &pair[1], excludeList) { - log.Debugf("pair excluded due to FORCE EXCLUDE: %s-%s", pair[0].Symbol, pair[1].Symbol) - continue - } - - pairs2 = append(pairs2, pair) - } - - filteredCount := len(pairs) - len(pairs2) - - log.Debugf("%d unsupported tokens filtered out, %d pairs", filteredCount, len(pairs2)) - - tokenListPath := path.GetTokenListPath(chain.Handle) - - var list TokenList - err := fileLib.ReadJSONFile(tokenListPath, &list) - if err != nil { - return nil - } - - log.Debugf("Tokenlist original: %d tokens", len(list.Tokens)) - - removeAllPairs(list.Tokens) - - for _, pair := range pairs2 { - err = addPairIfNeeded(&pair[0], &pair[1], &list) - if err != nil { - return err - } - } - - log.Debugf("Tokenlist updated: %d tokens", len(list.Tokens)) - - sortTokens(list.Tokens) - - return createTokenListJSON(chain, list.Tokens) -} - -func checkTokenExists(chain, tokenID string) bool { - logoPath := path.GetAssetLogoPath(chain, tokenID) - - return fileLib.FileExists(logoPath) -} - -func removeAllPairs(tokens []TokenItem) { - for i := range tokens { - tokens[i].Pairs = make([]Pair, 0) - } -} - -func addPairIfNeeded(token0, token1 *TokenItem, list *TokenList) error { - err := addTokenIfNeeded(token0, list) - if err != nil { - return err - } - - err = addTokenIfNeeded(token1, list) - if err != nil { - return err - } - - addPairToToken(token1, token0, list) - - return nil -} - -func addTokenIfNeeded(token *TokenItem, list *TokenList) error { - for _, t := range list.Tokens { - if strings.EqualFold(t.Address, token.Address) { - return nil - } - } - - err := updateTokenInfo(token) - if err != nil { - return err - } - - list.Tokens = append(list.Tokens, *token) - - return nil -} - -func updateTokenInfo(token *TokenItem) error { - backendClient := backend.InitClient(config.Default.ClientURLs.BackendAPI, nil) - assetInfo, err := backendClient.GetAssetInfo(token.Asset) - if err != nil { - return fmt.Errorf("failed to get asset info for '%s': %w", token.Address, err) - } - - if token.Name != assetInfo.Name { - log.Debugf("Token name adjusted: '%s' -> '%s'", token.Name, assetInfo.Name) - token.Name = assetInfo.Name - } - - if token.Symbol != assetInfo.Symbol { - log.Debugf("Token symbol adjusted: '%s' -> '%s'", token.Symbol, assetInfo.Symbol) - token.Symbol = assetInfo.Symbol - } - - if token.Decimals != uint(assetInfo.Decimals) { - log.Debugf("Token decimals adjusted: '%d' -> '%d'", token.Decimals, assetInfo.Decimals) - token.Decimals = uint(assetInfo.Decimals) - } - - return nil -} - -func addPairToToken(pairToken, token *TokenItem, list *TokenList) { - var tokenInListIndex = -1 - - for i, t := range list.Tokens { - if t.Address == token.Address { - tokenInListIndex = i - break - } - } - - if tokenInListIndex == -1 { - return - } - - if list.Tokens[tokenInListIndex].Pairs == nil { - list.Tokens[tokenInListIndex].Pairs = make([]Pair, 0) - } - - for _, pair := range list.Tokens[tokenInListIndex].Pairs { - if pair.Base == pairToken.Asset { - return - } - } - - list.Tokens[tokenInListIndex].Pairs = append(list.Tokens[tokenInListIndex].Pairs, Pair{Base: pairToken.Asset}) -} diff --git a/internal/processor/validators.go b/internal/processor/validators.go index d80592342..15596598e 100644 --- a/internal/processor/validators.go +++ b/internal/processor/validators.go @@ -11,10 +11,10 @@ import ( "github.com/trustwallet/assets-go-libs/validation" "github.com/trustwallet/assets-go-libs/validation/info" "github.com/trustwallet/assets-go-libs/validation/list" + "github.com/trustwallet/assets-go-libs/validation/tokenlist" "github.com/trustwallet/assets/internal/config" "github.com/trustwallet/assets/internal/file" "github.com/trustwallet/go-primitives/coin" - "github.com/trustwallet/go-primitives/types" ) func (s *Service) ValidateJSON(f *file.AssetFile) error { @@ -354,112 +354,17 @@ func (s *Service) ValidateTokenListFile(f *file.AssetFile) error { return err } - var model TokenList + var model tokenlist.Model err = json.Unmarshal(buf.Bytes(), &model) if err != nil { return err } - err = checkTokenListAssets(model, f) + err = tokenlist.ValidateTokenList(model, f.Chain(), f.Path()) if err != nil { return err } - err = checkTokenListPairs(model) - if err != nil { - return err - } - - return nil -} - -func checkTokenListAssets(model TokenList, f *file.AssetFile) error { - compErr := validation.NewErrComposite() - - for _, token := range model.Tokens { - var assetPath string - - if token.Type == types.Coin { - assetPath = path.GetChainInfoPath(f.Chain().Handle) - } else { - assetPath = path.GetAssetInfoPath(f.Chain().Handle, token.Address) - } - - infoFile, err := os.Open(assetPath) - if err != nil { - return err - } - - buf := bytes.NewBuffer(nil) - if _, err = buf.ReadFrom(infoFile); err != nil { - return err - } - - infoFile.Close() - - var infoAsset info.AssetModel - err = json.Unmarshal(buf.Bytes(), &infoAsset) - if err != nil { - return err - } - - if string(token.Type) != *infoAsset.Type { - compErr.Append(fmt.Errorf("field type - '%s' differs from '%s' in %s", - token.Type, *infoAsset.Type, assetPath)) - } - - if token.Symbol != *infoAsset.Symbol { - compErr.Append(fmt.Errorf("field symbol - '%s' differs from '%s' in %s", - token.Symbol, *infoAsset.Symbol, assetPath)) - } - - if token.Decimals != uint(*infoAsset.Decimals) { - compErr.Append(fmt.Errorf("field decimals - '%d' differs from '%d' in %s", - token.Decimals, *infoAsset.Decimals, assetPath)) - } - - if token.Name != *infoAsset.Name { - compErr.Append(fmt.Errorf("field name - '%s' differs from '%s' in %s", - token.Name, *infoAsset.Name, assetPath)) - } - - if infoAsset.GetStatus() != activeStatus { - compErr.Append(fmt.Errorf("token '%s' is not active, remove it from %s", token.Address, f.Path())) - } - } - - if compErr.Len() > 0 { - return compErr - } - - return nil -} - -func checkTokenListPairs(model TokenList) error { - compErr := validation.NewErrComposite() - - tokensMap := make(map[string]struct{}) - for _, t := range model.Tokens { - tokensMap[t.Asset] = struct{}{} - } - - pairs := make(map[string]string) - for _, t := range model.Tokens { - for _, pair := range t.Pairs { - pairs[pair.Base] = t.Address - } - } - - for pairToken, token := range pairs { - if _, exists := tokensMap[pairToken]; !exists { - compErr.Append(fmt.Errorf("token '%s' contains non-existing pair token '%s'", token, pairToken)) - } - } - - if compErr.Len() > 0 { - return compErr - } - return nil } diff --git a/internal/service/service.go b/internal/service/service.go index 91edcac39..e2c68686a 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -65,11 +65,6 @@ func (s *Service) RunUpdateAuto() { s.runUpdaters(updaters) } -func (s *Service) RunUpdateManual() { - updaters := s.processorService.GetUpdatersManual() - s.runUpdaters(updaters) -} - func (s *Service) runUpdaters(updaters []processor.Updater) { for _, updater := range updaters { err := updater.Run()