Implement tokenlist/tokenlist-extended duplicate validaton, refactor code (#17782)

Co-authored-by: Daniil Lee <24758309+leedaniil@users.noreply.github.com>
This commit is contained in:
daniel 2022-01-31 17:11:27 +03:00 committed by GitHub
parent 7597a43c99
commit 72854d0c40
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 69 additions and 178 deletions

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "A blockchain-based middleware, acting as a bridge between cryptocurrency smart contracts, data feeds, APIs and traditional bank account payments.",
"website": "https://chain.link/",
"explorer": "https://optimistic.etherscan.io/address/0x350a791Bfc2C21F9Ed5d10980Dad2e2638ffa7f6",
"explorer": "https://optimistic.etherscan.io/token/0x350a791Bfc2C21F9Ed5d10980Dad2e2638ffa7f6",
"status": "active",
"id": "0x350a791Bfc2C21F9Ed5d10980Dad2e2638ffa7f6",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "wETH is wrapped ETH",
"website": "https://weth.io/",
"explorer": "https://optimistic.etherscan.io/address/0x4200000000000000000000000000000000000006",
"explorer": "https://optimistic.etherscan.io/token/0x4200000000000000000000000000000000000006",
"status": "active",
"id": "0x4200000000000000000000000000000000000006",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "BitANT is a governance token of BitBTC Protocol.10 billion BitANT have been minted at genesis, the team does not own any tokens: 80% liquidity is locked in DEX for 8 years; 15%,community airdrops; 5%,community mining. The exchange fee of the platform is mainly used to repurchase and burn BitANT.",
"website": "https://bitbtc.money/",
"explorer": "https://optimistic.etherscan.io/address/0x5029C236320b8f15eF0a657054B84d90bfBEDED3",
"explorer": "https://optimistic.etherscan.io/token/0x5029C236320b8f15eF0a657054B84d90bfBEDED3",
"status": "active",
"id": "0x5029C236320b8f15eF0a657054B84d90bfBEDED3",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "Lyra is an options trading protocol accessing the scalability of Layer 2 Ethereum to provide a robust, lightning-fast and reliable trading experience.",
"website": "https://www.lyra.finance",
"explorer": "https://optimistic.etherscan.io/address/0x50c5725949A6F0c72E6C4a641F24049A917DB0Cb",
"explorer": "https://optimistic.etherscan.io/token/0x50c5725949A6F0c72E6C4a641F24049A917DB0Cb",
"status": "active",
"id": "0x50c5725949A6F0c72E6C4a641F24049A917DB0Cb",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "Aelin is a fundraising protocol built on Ethereum and launched on Optimism",
"website": "https://aelin.xyz",
"explorer": "https://optimistic.etherscan.io/address/0x61BAADcF22d2565B0F471b291C475db5555e0b76",
"explorer": "https://optimistic.etherscan.io/token/0x61BAADcF22d2565B0F471b291C475db5555e0b76",
"status": "active",
"id": "0x61BAADcF22d2565B0F471b291C475db5555e0b76",
"links": [

View File

@ -1,12 +1,12 @@
{
"name": "Synthetix Network",
"website": "https://synthetix.io",
"description": "Synthetix is a derivatives liquidity protocol on Ethereum that enables the issuance and trading of synthetic assets.",
"explorer": "https://optimistic.etherscan.io/address/0x8700daec35af8ff88c16bdf0418774cb3d7599b4",
"research": "https://research.binance.com/en/projects/synthetix",
"type": "OPTIMISM",
"symbol": "SNX",
"type": "OPTIMISM",
"decimals": 18,
"description": "Synthetix is a derivatives liquidity protocol on Ethereum that enables the issuance and trading of synthetic assets.",
"website": "https://synthetix.io",
"explorer": "https://optimistic.etherscan.io/token/0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4",
"research": "https://research.binance.com/en/projects/synthetix",
"status": "active",
"id": "0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "Rocket Pool is the first truly decentralised Ethereum staking pool.",
"website": "https://rocketpool.net",
"explorer": "https://optimistic.etherscan.io/address/0x9Bcef72be871e61ED4fBbc7630889beE758eb81D",
"explorer": "https://optimistic.etherscan.io/token/0x9Bcef72be871e61ED4fBbc7630889beE758eb81D",
"status": "active",
"id": "0x9Bcef72be871e61ED4fBbc7630889beE758eb81D",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "On-chain derivative DEX with liquidity provisioning and up to 10x leverage for makers and takers.",
"website": "https://www.perp.com/",
"explorer": "https://optimistic.etherscan.io/address/0x9e1028F5F1D5eDE59748FFceE5532509976840E0",
"explorer": "https://optimistic.etherscan.io/token/0x9e1028F5F1D5eDE59748FFceE5532509976840E0",
"status": "active",
"id": "0x9e1028F5F1D5eDE59748FFceE5532509976840E0",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "Non-mintable ERC20 token deployed on Ethereum, secured by OpenZepellin and audited by MythX with a total supply of 100 million.",
"website": "https://kromatika.finance",
"explorer": "https://optimistic.etherscan.io/address/0xF98dCd95217E15E05d8638da4c91125E59590B07",
"explorer": "https://optimistic.etherscan.io/token/0xF98dCd95217E15E05d8638da4c91125E59590B07",
"status": "active",
"id": "0xF98dCd95217E15E05d8638da4c91125E59590B07",
"links": [

View File

@ -5,7 +5,7 @@
"decimals": 18,
"description": "The Rubicon Protocol is a decentralized exchange protocol on the Ethereum blockchain.",
"website": "https://www.rubicon.finance",
"explorer": "https://optimistic.etherscan.io/address/0xffD27b572246f35c992EfB28DD8cB730b93Ed301",
"explorer": "https://optimistic.etherscan.io/token/0xffD27b572246f35c992EfB28DD8cB730b93Ed301",
"status": "active",
"id": "0xffD27b572246f35c992EfB28DD8cB730b93Ed301",
"links": [

View File

@ -2,8 +2,7 @@
"name": "Trust Wallet: Polygon Extended List",
"logoURI": "https://trustwallet.com/assets/images/favicon.png",
"timestamp": "2022-01-27T14:33:26.183301",
"tokens": [
],
"tokens": [],
"version": {
"major": 1,
"minor": 0,

View File

@ -2,9 +2,7 @@
"name": "Trust Wallet: SmartChain Extended List",
"logoURI": "https://trustwallet.com/assets/images/favicon.png",
"timestamp": "2022-01-27T14:33:26.183301",
"tokens": [
],
"tokens": [],
"version": {
"major": 1,
"minor": 0,

4
go.mod
View File

@ -5,9 +5,9 @@ go 1.17
require (
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.3.0
github.com/trustwallet/assets-go-libs v0.0.35
github.com/trustwallet/assets-go-libs v0.0.37
github.com/trustwallet/go-libs v0.2.26
github.com/trustwallet/go-primitives v0.0.24
github.com/trustwallet/go-primitives v0.0.28
)
require (

8
go.sum
View File

@ -340,12 +340,12 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/trustwallet/assets-go-libs v0.0.35 h1:iOTjuFsXkmCAYhGxMu17zfL2pYUbsNac/zfwA+xGi24=
github.com/trustwallet/assets-go-libs v0.0.35/go.mod h1:O0lGt8p+ffhEbNDMLSuuYBBAnNUI7nSQIcHRjWBxbgo=
github.com/trustwallet/assets-go-libs v0.0.37 h1:qddLNa0CfnDPf5LYjZsjhVLxfPhzJLjh77VG3ZxFcX0=
github.com/trustwallet/assets-go-libs v0.0.37/go.mod h1:1h30Q29B61gfrkR3Q5rS582dtyB67ozPnWy07BGf2ik=
github.com/trustwallet/go-libs v0.2.26 h1:WpDc7X23EQwdrMRZ1JqXWXUk15+8pfej4pTt/3hBIJo=
github.com/trustwallet/go-libs v0.2.26/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
github.com/trustwallet/go-primitives v0.0.24 h1:C1QGcX3dRL7gS2miJvSrRT732zRjob1Gd4D4dJvVwq8=
github.com/trustwallet/go-primitives v0.0.24/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
github.com/trustwallet/go-primitives v0.0.28 h1:aJGjxggsxFH9GCXyqlVxv9nqmUe6lxvAZncwp1MCLVw=
github.com/trustwallet/go-primitives v0.0.28/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

View File

@ -101,7 +101,7 @@ func calculateTargetDimension(width, height int) (targetW, targetH int) {
}
func (s *Service) FixChainInfoJSON(f *file.AssetFile) error {
chainInfo := info.CoinModel{}
var chainInfo info.CoinModel
err := fileLib.ReadJSONFile(f.Path(), &chainInfo)
if err != nil {
@ -119,7 +119,7 @@ func (s *Service) FixChainInfoJSON(f *file.AssetFile) error {
}
func (s *Service) FixAssetInfo(f *file.AssetFile) error {
assetInfo := info.AssetModel{}
var assetInfo info.AssetModel
err := fileLib.ReadJSONFile(f.Path(), &assetInfo)
if err != nil {

View File

@ -1,10 +1,7 @@
package processor
import (
"bytes"
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strconv"
@ -257,24 +254,9 @@ func isTokenExistOrActive(symbol string) bool {
assetPath := path.GetAssetInfoPath(coin.Coins[coin.BINANCE].Handle, symbol)
infoFile, err := os.Open(assetPath)
if err != nil {
log.Debugf("asset file open error: %s", err.Error())
return false
}
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(infoFile); err != nil {
log.Debugf("buffer read error: %s", err.Error())
return false
}
infoFile.Close()
var infoAsset info.AssetModel
err = json.Unmarshal(buf.Bytes(), &infoAsset)
if err != nil {
log.Debugf("json unmarshalling error: %s", err.Error())
if err := fileLib.ReadJSONFile(assetPath, infoAsset); err != nil {
log.Debug(err)
return false
}

View File

@ -2,11 +2,10 @@ package processor
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
filelib "github.com/trustwallet/assets-go-libs/file"
"github.com/trustwallet/assets-go-libs/path"
"github.com/trustwallet/assets-go-libs/validation"
"github.com/trustwallet/assets-go-libs/validation/info"
@ -39,13 +38,7 @@ func (s *Service) ValidateJSON(f *file.AssetFile) error {
}
func (s *Service) ValidateRootFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
dirFiles, err := filelib.ReadDir(f.Path())
if err != nil {
return err
}
@ -117,13 +110,7 @@ func (s *Service) ValidateImage(f *file.AssetFile) error {
}
func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
dirFiles, err := filelib.ReadDir(f.Path())
if err != nil {
return err
}
@ -144,25 +131,10 @@ func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
errLogo := validation.ValidateHasFiles(dirFiles, []string{"logo.png"})
if errLogo != nil || errInfo != nil {
file2, err := os.Open(path.GetAssetInfoPath(f.Chain().Handle, f.Asset()))
if err != nil {
return err
}
defer file2.Close()
_, err = file2.Seek(0, io.SeekStart)
if err != nil {
return err
}
b, err := io.ReadAll(file2)
if err != nil {
return err
}
assetInfoPath := path.GetAssetInfoPath(f.Chain().Handle, f.Asset())
var infoJson info.AssetModel
err = json.Unmarshal(b, &infoJson)
if err != nil {
if err = filelib.ReadJSONFile(assetInfoPath, &infoJson); err != nil {
return err
}
@ -179,18 +151,13 @@ func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
}
func (s *Service) ValidateDappsFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
dirFiles, err := filelib.ReadDir(f.Path())
if err != nil {
return err
}
var compErr = validation.NewErrComposite()
for _, dirFile := range dirFiles {
err = validation.ValidateExtension(dirFile.Name(), config.Default.ValidatorsSettings.DappsFolder.Ext)
if err != nil {
@ -211,28 +178,10 @@ func (s *Service) ValidateDappsFolder(f *file.AssetFile) error {
}
func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
var coinInfo info.CoinModel
if err := filelib.ReadJSONFile(f.Path(), &coinInfo); err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
_, err = buf.ReadFrom(file)
if err != nil {
return err
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJSON)
}
var payload info.CoinModel
err = json.Unmarshal(buf.Bytes(), &payload)
if err != nil {
return fmt.Errorf("%w: failed to decode", err)
}
receivedTags, err := s.assetsManager.GetTagValues()
if err != nil {
@ -244,7 +193,7 @@ func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
tags[i] = t.ID
}
err = info.ValidateCoin(payload, tags)
err = info.ValidateCoin(coinInfo, tags)
if err != nil {
return err
}
@ -253,29 +202,12 @@ func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
}
func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
var assetInfo info.AssetModel
if err := filelib.ReadJSONFile(f.Path(), &assetInfo); err != nil {
return err
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJSON)
}
var payload info.AssetModel
err = json.Unmarshal(buf.Bytes(), &payload)
if err != nil {
return fmt.Errorf("%w: failed to decode", err)
}
err = info.ValidateAsset(payload, f.Chain(), f.Asset())
err := info.ValidateAsset(assetInfo, f.Chain(), f.Asset())
if err != nil {
return err
}
@ -284,28 +216,16 @@ func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error {
}
func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
if !isStackingChain(f.Chain()) {
return nil
}
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
return err
}
var model []list.Model
err = json.Unmarshal(buf.Bytes(), &model)
if err != nil {
if err := filelib.ReadJSONFile(f.Path(), &model); err != nil {
return err
}
err = list.ValidateList(model)
err := list.ValidateList(model)
if err != nil {
return err
}
@ -318,18 +238,12 @@ func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error {
assetsPath := path.GetValidatorAssetsPath(f.Chain().Handle)
assetFolder := s.fileService.GetAssetFile(assetsPath)
file2, err := os.Open(assetFolder.Path())
if err != nil {
return err
}
defer file2.Close()
dirAssetFolderFiles, err := file2.ReadDir(0)
dirFiles, err := filelib.ReadDir(assetFolder.Path())
if err != nil {
return err
}
err = validation.ValidateAllowedFiles(dirAssetFolderFiles, listIDs)
err = validation.ValidateAllowedFiles(dirFiles, listIDs)
if err != nil {
return err
}
@ -348,24 +262,34 @@ func isStackingChain(c coin.Coin) bool {
}
func (s *Service) ValidateTokenListFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
return err
}
var model tokenlist.Model
err = json.Unmarshal(buf.Bytes(), &model)
var tokenList tokenlist.Model
err := filelib.ReadJSONFile(f.Path(), &tokenList)
if err != nil {
return err
}
err = tokenlist.ValidateTokenList(model, f.Chain(), f.Path())
tokenListExtendedPath := path.GetTokenListPath(f.Chain().Handle, path.TokenlistExtended)
if filelib.Exists(tokenListExtendedPath) {
var tokenListExtended tokenlist.Model
err = filelib.ReadJSONFile(tokenListExtendedPath, &tokenListExtended)
if err != nil {
return err
}
tokensMap := make(map[string]bool)
for _, token := range tokenListExtended.Tokens {
tokensMap[token.Asset] = true
}
for _, token := range tokenList.Tokens {
if _, exists := tokensMap[token.Asset]; exists {
return fmt.Errorf("duplicate asset: %s from %s, already exist in %s",
token.Asset, f.Path(), tokenListExtendedPath)
}
}
}
err = tokenlist.ValidateTokenList(tokenList, f.Chain(), f.Path())
if err != nil {
return err
}
@ -374,13 +298,7 @@ func (s *Service) ValidateTokenListFile(f *file.AssetFile) error {
}
func (s *Service) ValidateInfoFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
dirFiles, err := filelib.ReadDir(f.Path())
if err != nil {
return err
}
@ -394,13 +312,7 @@ func (s *Service) ValidateInfoFolder(f *file.AssetFile) error {
}
func (s *Service) ValidateValidatorsAssetFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
dirFiles, err := filelib.ReadDir(f.Path())
if err != nil {
return err
}