Move CI scripts logic from assets-go-libs (#16680)

* Move CI scripts logic from assets-go-libs

* Add Makefile, .golangci.yml and lint jobs to workflow

* Fix
This commit is contained in:
Daniel 2021-12-20 00:39:53 +03:00 committed by GitHub
parent 526d283090
commit d547fb1473
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 2422 additions and 35 deletions

View File

@ -5,6 +5,7 @@ client_urls:
binance:
dex: https://dex.binance.org
explorer: https://explorer.binance.org
backend_api: https://api.trustwallet.com
urls:
tw_assets_app: https://assets.trustwalletapp.com
@ -31,8 +32,11 @@ validators_settings:
- ".eslintignore"
- ".eslintrc.js"
- "cmd"
- "internal"
- "go.mod"
- "go.sum"
- ".golangci.yml"
- "Makefile"
skip_files:
- "node_modules"

View File

@ -7,7 +7,7 @@ jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
@ -18,3 +18,9 @@ jobs:
- name: Run check
run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker
# - name: Unit Test
# run: make test
# - name: Lint
# run: make lint

View File

@ -7,7 +7,7 @@ jobs:
fix-dryrun:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17

View File

@ -27,7 +27,7 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ github.ref }}
- name: Set up Go 1.17
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17

View File

@ -13,7 +13,7 @@ jobs:
token: ${{ secrets.COMMIT_TOKEN }}
ref: ${{ github.head_ref }}
- name: Set up Go 1.17
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17

View File

@ -11,7 +11,7 @@ jobs:
pull_request_ci:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
@ -22,3 +22,9 @@ jobs:
- name: Run check
run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker
# - name: Unit Test
# run: make test
# - name: Lint
# run: make lint

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ node_modules/
*.txt
.env
.env.test
bin/

250
.golangci.yml Normal file
View File

@ -0,0 +1,250 @@
linters-settings:
cyclop:
max-complexity: 20
# the maximal average package complexity. If it's higher than 0.0 (float) the check is enabled (default 0.0)
package-average: 0.0
skip-tests: false
dogsled:
# checks assignments with too many blank identifiers; default is 2
max-blank-identifiers: 2
dupl:
threshold: 100
errcheck:
check-type-assertions: false
check-blank: false
errorlint:
errorf: false
asserts: true
comparison: true
forbidigo:
# Forbid the following identifiers (identifiers are written using regexp):
forbid:
- ^print.*$
- 'fmt\.Print.*'
exclude_godoc_examples: true
funlen:
lines: 60
statements: 60
gocognit:
min-complexity: 35
goconst:
min-len: 3
min-occurrences: 3
ignore-tests: true
match-constant: true
numbers: false
min: 3
max: 3
ignore-calls: true
gocritic:
# Which checks should be disabled; can't be combined with 'enabled-checks'
disabled-checks:
- regexpMust
gocyclo:
min-complexity: 20
godot:
# comments to be checked: `declarations`, `toplevel`, or `all`
scope: all
capital: false
godox:
# report any comments starting with keywords, this is useful for TODO or FIXME comments that
# might be left in the code accidentally and should be resolved before merging
keywords:
- TODO
- BUG
- FIXME
- NOTE
- OPTIMIZE # marks code that should be optimized before merging
- HACK # marks hack-arounds that should be removed before merging
gofmt:
# simplify code: gofmt with `-s` option
simplify: true
gofumpt:
# Select the Go version to target
lang-version: "1.17"
extra-rules: false
goimports:
# put imports beginning with prefix after 3rd-party packages;
# it's a comma-separated list of prefixes
local-prefixes: github.com/trustwallet
gomnd:
settings:
mnd:
# the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description.
checks:
- argument
- case
- condition
- operation
- return
- assign
ignored-numbers: 10,1000
gomoddirectives:
replace-local: false
gosec:
# To select a subset of rules to run.
# Available rules: https://github.com/securego/gosec#available-rules
includes:
- G401
- G306
- G101
gosimple:
# Select the Go version to target
go: "1.17"
# https://staticcheck.io/docs/options#checks
checks: [ "all" ]
govet:
check-shadowing: true
ifshort:
# Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax.
# Has higher priority than max-decl-chars.
max-decl-lines: 1
# Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
max-decl-chars: 30
importas:
# if set to `true`, force to use alias.
no-unaliased: true
# List of aliases
alias:
- pkg: github:com/trustwallet/go-libs/gin
alias: golibsGin
lll:
# max line length
line-length: 120
tab-width: 1
misspell:
locale: US
# ignore-words:
nolintlint:
# Enable to ensure that nolint directives are all used
allow-unused: false
# Disable to ensure that nolint directives don't have a leading space
allow-leading-space: true
# Exclude following linters from requiring an explanation
allow-no-explanation: []
# Enable to require an explanation of nonzero length after each nolint directive
require-explanation: false
# Enable to require nolint directives to mention the specific linter being suppressed
require-specific: true
revive:
# see https://github.com/mgechev/revive#available-rules for details.
ignore-generated-header: true
severity: warning
rules:
- name: indent-error-flow
severity: warning
- name: time-naming
severity: warn
- name: errorf
severity: warn
- name: blank-imports
sevetiry: warn
# The error return parameter should be last
- name: error-return
severity: error
# Redundant if when returning an error
- name: if-return
severity: warn
# Warns when there are heading or trailing newlines in a block
- name: empty-lines
severity: error
staticcheck:
# Select the Go version to target
go: "1.17"
# https://staticcheck.io/docs/options#checks
checks: [ "all" ]
stylecheck:
# Select the Go version to target
go: "1.17"
# https://staticcheck.io/docs/options#checks
checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ]
# https://staticcheck.io/docs/options#dot_import_whitelist
dot-import-whitelist:
- fmt
# https://staticcheck.io/docs/options#initialisms
initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ]
# https://staticcheck.io/docs/options#http_status_code_whitelist
http-status-code-whitelist: [ "200", "400", "404", "500" ]
unused:
# Select the Go version to target
go: "1.17"
whitespace:
multi-if: true # Enforces newlines (or comments) after every multi-line if statement
multi-func: false # Enforces newlines (or comments) after every multi-line function signature
wsl:
# See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for
# documentation of available settings
allow-assign-and-anything: false
allow-assign-and-call: true
allow-cuddle-declarations: false
allow-multiline-assign: true
allow-separated-leading-comment: false
allow-trailing-comment: false
force-case-trailing-whitespace: 0
force-err-cuddling: false
force-short-decl-cuddling: false
strict-append: true
linters:
disable-all: true
enable:
- bodyclose
- deadcode
- depguard
- dogsled
- dupl
- errcheck
- exportloopref
- exhaustive
- funlen
- gochecknoinits
- goconst
- gocritic
- gocyclo
- gofmt
- goimports
# - gomnd
- goprintffuncname
- gosec
- gosimple
- govet
- ineffassign
- lll
- misspell
- nakedret
- noctx
- nolintlint
- rowserrcheck
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
- asciicheck
# - gochecknoglobals
- gocognit
- godot
# - godox
# - goerr113
# - nestif
- prealloc
- testpackage
# - revive
# - wsl
# don't enable:
# - interfacer
# - maligned
# - scopelint
output:
format: colored-line-number
print-issued-lines: true
print-linter-name: true
uniq-by-line: true
path-prefix: ""
sort-results: true

28
Makefile Normal file
View File

@ -0,0 +1,28 @@
#! /usr/bin/make -f
# Go related variables.
GOBASE := $(shell pwd)
GOBIN := $(GOBASE)/bin
# Go files.
GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
all: fmt lint test
test:
@echo " > Running unit tests"
GOBIN=$(GOBIN) go test -cover -race -coverprofile=coverage.txt -covermode=atomic -v ./...
fmt:
@echo " > Format all go files"
GOBIN=$(GOBIN) gofmt -w ${GOFMT_FILES}
lint-install:
ifeq (,$(wildcard test -f bin/golangci-lint))
@echo " > Installing golint"
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s
endif
lint: lint-install
@echo " > Running golint"
bin/golangci-lint run --timeout=2m

View File

@ -5,10 +5,10 @@ import (
log "github.com/sirupsen/logrus"
"github.com/trustwallet/assets-go-libs/pkg/file"
"github.com/trustwallet/assets-go-libs/src/config"
"github.com/trustwallet/assets-go-libs/src/core"
"github.com/trustwallet/assets-go-libs/src/processor"
"github.com/trustwallet/assets/internal/config"
"github.com/trustwallet/assets/internal/file"
"github.com/trustwallet/assets/internal/processor"
"github.com/trustwallet/assets/internal/service"
)
var (
@ -20,42 +20,41 @@ func main() {
paths, err := file.ReadLocalFileStructure(root, config.Default.ValidatorsSettings.RootFolder.SkipFiles)
if err != nil {
log.WithError(err).Fatal("failed to load file structure")
log.WithError(err).Fatal("Failed to load file structure.")
}
fileStorage := file.NewService(paths...)
validatorsService := core.NewService(fileStorage)
assetfsProcessor := processor.NewService(fileStorage, validatorsService)
validatorsService := processor.NewService(fileStorage)
assetfsProcessor := service.NewService(fileStorage, validatorsService)
switch script {
case "checker":
err = assetfsProcessor.RunJob(paths, assetfsProcessor.Check)
assetfsProcessor.RunJob(paths, assetfsProcessor.Check)
case "fixer":
err = assetfsProcessor.RunJob(paths, assetfsProcessor.Fix)
assetfsProcessor.RunJob(paths, assetfsProcessor.Fix)
case "updater-auto":
err = assetfsProcessor.RunUpdateAuto()
assetfsProcessor.RunUpdateAuto()
case "updater-manual":
assetfsProcessor.RunUpdateManual()
default:
log.Info("Nothing to launch. Use --script flag to choose a script to run.")
}
if err != nil {
log.WithError(err).Fatal("Script failed")
}
}
func setup() {
flag.StringVar(&configPath, "config", "", "path to config file")
flag.StringVar(&configPath, "config", "./.github/assets.config.yaml", "path to config file")
flag.StringVar(&root, "root", "./", "path to the root of the dir")
flag.StringVar(&script, "script", "", "script type to run")
flag.Parse()
if err := config.SetConfig(configPath); err != nil {
log.WithError(err).Fatal("failed to set config")
log.WithError(err).Fatal("Failed to set config.")
}
logLevel, err := log.ParseLevel(config.Default.App.LogLevel)
if err != nil {
log.WithError(err).Fatal("failed to parse log level")
log.WithError(err).Fatal("Failed to parse log level.")
}
log.SetLevel(logLevel)

6
go.mod
View File

@ -4,7 +4,9 @@ go 1.17
require (
github.com/sirupsen/logrus v1.8.1
github.com/trustwallet/assets-go-libs v0.0.15
github.com/trustwallet/assets-go-libs v0.0.16
github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793
github.com/trustwallet/go-primitives v0.0.17
)
require (
@ -22,8 +24,6 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.10.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/trustwallet/go-libs v0.2.20 // indirect
github.com/trustwallet/go-primitives v0.0.17 // indirect
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827 // indirect

12
go.sum
View File

@ -40,14 +40,10 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d h1:L2WPNWlTMoBfVsHDqw73dH3fBZYdh0xdn3cmXGs7IX8=
github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
github.com/trustwallet/assets-go-libs v0.0.14 h1:7fs6tPFf8XF1svj1fksJ+PN4ukT3v3sRfz/z4W86JAA=
github.com/trustwallet/assets-go-libs v0.0.14/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
github.com/trustwallet/assets-go-libs v0.0.15 h1:0q0xUR0FATqqdLr9OShRwYK0r3KDqGLWrxpOOc4lJ4o=
github.com/trustwallet/assets-go-libs v0.0.15/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
github.com/trustwallet/go-libs v0.2.20 h1:pYstFNgsc7CVyVeYt5GHsMa0JNQHJVRvPQqMvXMpCtY=
github.com/trustwallet/go-libs v0.2.20/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
github.com/trustwallet/assets-go-libs v0.0.16 h1:nbKrf/pHKQCut4Q5Mdg5BinLdFxOqvzEeAgIgzX0/P8=
github.com/trustwallet/assets-go-libs v0.0.16/go.mod h1:agKWTQ9ECSzQ++7P/9viSxJnG1Kp+WhfDyI3pAmtnVM=
github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793 h1:KFtyLpBPbMyUdeCth/Zcej/SSgAFIo6fxdS2eEPEg3I=
github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
github.com/trustwallet/go-primitives v0.0.17 h1:1fBxZMKGCHdHtgdUzsqdFlD21+1GneIk/sxN6jxYBds=
github.com/trustwallet/go-primitives v0.0.17/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

57
internal/config/config.go Normal file
View File

@ -0,0 +1,57 @@
package config
import (
"path/filepath"
"github.com/trustwallet/go-libs/config/viper"
)
type (
Config struct {
App App `mapstructure:"app"`
ClientURLs ClientsURLs `mapstructure:"client_urls"`
URLs URLs `mapstructure:"urls"`
ValidatorsSettings ValidatorsSettings `mapstructure:"validators_settings"`
}
App struct {
LogLevel string `mapstructure:"log_level"`
}
ClientsURLs struct {
Binance struct {
Dex string `mapstructure:"dex"`
Explorer string `mapstructure:"explorer"`
} `mapstructure:"binance"`
BackendAPI string `mapstructure:"backend_api"`
}
URLs struct {
TWAssetsApp string `mapstructure:"tw_assets_app"`
}
ValidatorsSettings struct {
RootFolder RootFolder `mapstructure:"root_folder"`
ChainFolder ChainFolder `mapstructure:"chain_folder"`
AssetFolder AssetFolder `mapstructure:"asset_folder"`
ChainInfoFolder ChainInfoFolder `mapstructure:"chain_info_folder"`
ChainValidatorsAssetFolder ChainValidatorsAssetFolder `mapstructure:"chain_validators_asset_folder"`
DappsFolder DappsFolder `mapstructure:"dapps_folder"`
CoinInfoFile CoinInfoFile `mapstructure:"coin_info_file"`
}
)
// Default is a configuration instance.
var Default = Config{} // nolint:gochecknoglobals // config must be global
// SetConfig reads a config file and returs an initialized config instance.
func SetConfig(confPath string) error {
confPath, err := filepath.Abs(confPath)
if err != nil {
return err
}
viper.Load(confPath, &Default)
return nil
}

View File

@ -0,0 +1,36 @@
package config
type RootFolder struct {
AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
SkipFiles []string `mapstructure:"skip_files,omitempty"`
}
type ChainFolder struct {
AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
}
type AssetFolder struct {
AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
}
type ChainInfoFolder struct {
HasFiles []string `mapstructure:"has_files,omitempty"`
}
type ChainValidatorsAssetFolder struct {
HasFiles []string `mapstructure:"has_files,omitempty"`
}
type DappsFolder struct {
Ext string `mapstructure:"ext,omitempty"`
}
type CoinInfoFile struct {
Tags []Tag `mapstructure:"tags,omitempty"`
}
type Tag struct {
ID string `mapstructure:"id,omitempty"`
Name string `mapstructure:"name,omitempty"`
Description string `mapstructure:"description,omitempty"`
}

15
internal/config/values.go Normal file
View File

@ -0,0 +1,15 @@
package config
import "github.com/trustwallet/go-primitives/coin"
// TODO: Move to go-libs.
var StackingChains = []coin.Coin{
coin.Tezos(),
coin.Cosmos(),
coin.Iotex(),
coin.Tron(),
coin.Waves(),
coin.Kava(),
coin.Terra(),
coin.Binance(),
}

58
internal/file/cache.go Normal file
View File

@ -0,0 +1,58 @@
package file
import (
"path/filepath"
"strings"
"sync"
)
type Service struct {
mu *sync.RWMutex
cache map[string]*AssetFile
}
func NewService(filePaths ...string) *Service {
var filesMap = make(map[string]*AssetFile)
for _, path := range filePaths {
assetFile := NewAssetFile(path)
filesMap[path] = assetFile
}
return &Service{
mu: &sync.RWMutex{},
cache: filesMap,
}
}
func (f *Service) GetAssetFile(path string) *AssetFile {
f.mu.RLock()
defer f.mu.RUnlock()
return f.getFile(path)
}
func (f *Service) UpdateFile(file *AssetFile, newFileBaseName string) {
f.mu.RLock()
defer f.mu.RUnlock()
oldFileBaseName := filepath.Base(file.Path())
for path := range f.cache {
if strings.Contains(path, oldFileBaseName) {
newPath := strings.ReplaceAll(path, oldFileBaseName, newFileBaseName)
f.cache[path] = NewAssetFile(newPath)
}
}
}
func (f *Service) getFile(path string) *AssetFile {
if file, exists := f.cache[path]; exists {
return file
}
assetF := NewAssetFile(path)
f.cache[path] = assetF
return assetF
}

29
internal/file/file.go Normal file
View File

@ -0,0 +1,29 @@
package file
import (
"github.com/trustwallet/go-primitives/coin"
)
type AssetFile struct {
path *Path
}
func NewAssetFile(path string) *AssetFile {
return &AssetFile{path: NewPath(path)}
}
func (i *AssetFile) Path() string {
return i.path.String()
}
func (i *AssetFile) Type() string {
return i.path.fileType
}
func (i *AssetFile) Chain() coin.Coin {
return i.path.chain
}
func (i *AssetFile) Asset() string {
return i.path.asset
}

161
internal/file/path.go Normal file
View File

@ -0,0 +1,161 @@
package file
import (
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/trustwallet/assets-go-libs/pkg"
"github.com/trustwallet/go-primitives/coin"
)
var (
regexAssetInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/info.json$`)
regexAssetLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/logo.png$`)
regexChainInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/info.json$`)
regexChainLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/logo.png$`)
regexTokenListFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/tokenlist.json$`)
regexValidatorsAssetLogo = regexp.MustCompile(
`./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)/logo.png$`)
regexValidatorsList = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/list.json$`)
regexDappsLogo = regexp.MustCompile(`./dapps/[a-zA-Z-.]+\.png$`)
)
var (
regexAssetFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)$`)
regexAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets$`)
regexValidatorsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators$`)
regexValidatorsAssetFolder = regexp.MustCompile(
`./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)$`)
regexValidatorsAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/assets$`)
regexChainFolder = regexp.MustCompile(`./blockchains/(\w+[^/])$`)
regexChainInfoFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info$`)
regexChainsFolder = regexp.MustCompile(`./blockchains$`)
regexDappsFolder = regexp.MustCompile(`./dapps$`)
regexRoot = regexp.MustCompile(`./$`)
)
var regexes = map[string]*regexp.Regexp{
TypeAssetInfoFile: regexAssetInfoFile,
TypeAssetLogoFile: regexAssetLogoFile,
TypeChainInfoFile: regexChainInfoFile,
TypeChainLogoFile: regexChainLogoFile,
TypeTokenListFile: regexTokenListFile,
TypeValidatorsListFile: regexValidatorsList,
TypeValidatorsLogoFile: regexValidatorsAssetLogo,
TypeDappsLogoFile: regexDappsLogo,
TypeAssetFolder: regexAssetFolder,
TypeAssetsFolder: regexAssetsFolder,
TypeChainFolder: regexChainFolder,
TypeChainsFolder: regexChainsFolder,
TypeChainInfoFolder: regexChainInfoFolder,
TypeDappsFolder: regexDappsFolder,
TypeRootFolder: regexRoot,
TypeValidatorsFolder: regexValidatorsFolder,
TypeValidatorsAssetsFolder: regexValidatorsAssetsFolder,
TypeValidatorsAssetFolder: regexValidatorsAssetFolder,
}
type Path struct {
path string
chain coin.Coin
asset string
fileType string
}
func NewPath(path string) *Path {
p := Path{path: path}
fileType, reg := defineFileType(path)
if reg == nil {
p.fileType = TypeUnknown
return &p
}
match := reg.FindStringSubmatch(path)
if fileType != TypeUnknown {
p.fileType = fileType
}
if len(match) >= 2 {
chain, err := coin.GetCoinForId(match[1])
if err != nil {
p.chain = coin.Coin{Handle: match[1]}
} else {
p.chain = chain
}
}
if len(match) == 3 {
p.asset = match[2]
}
return &p
}
func (p Path) Type() string {
return p.fileType
}
func (p Path) String() string {
return p.path
}
func (p Path) Chain() coin.Coin {
return p.chain
}
func (p Path) Asset() string {
return p.asset
}
func defineFileType(p string) (string, *regexp.Regexp) {
for t, r := range regexes {
if r.MatchString(p) {
return t, r
}
}
return TypeUnknown, nil
}
func ReadLocalFileStructure(root string, filesToSkip []string) ([]string, error) {
var paths = []string{"./"}
err := filepath.Walk(root,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if pkg.Contains(path, filesToSkip) {
return nil
}
paths = append(paths, fmt.Sprintf("./%s", path))
return nil
})
if err != nil {
return nil, err
}
return paths, nil
}

25
internal/file/types.go Normal file
View File

@ -0,0 +1,25 @@
package file
const (
TypeAssetInfoFile = "asset_info"
TypeAssetLogoFile = "asset_logo"
TypeChainInfoFile = "chain_info"
TypeChainLogoFile = "chain_logo"
TypeTokenListFile = "chain_token_list"
TypeValidatorsListFile = "validators_list"
TypeValidatorsLogoFile = "validators_logo"
TypeDappsLogoFile = "dapps_logo"
TypeAssetFolder = "asset"
TypeAssetsFolder = "assets"
TypeChainFolder = "chain"
TypeChainsFolder = "chains"
TypeDappsFolder = "dapps"
TypeRootFolder = "root"
TypeChainInfoFolder = "chain_info_folder"
TypeValidatorsFolder = "validators"
TypeValidatorsAssetFolder = "validators_asset"
TypeValidatorsAssetsFolder = "validators_assets"
TypeUnknown = "unknown"
)

View File

@ -0,0 +1,171 @@
package processor
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/trustwallet/assets-go-libs/pkg"
"github.com/trustwallet/assets-go-libs/pkg/validation"
"github.com/trustwallet/assets-go-libs/pkg/validation/info"
"github.com/trustwallet/assets/internal/file"
"github.com/trustwallet/go-primitives/address"
"github.com/trustwallet/go-primitives/coin"
"github.com/trustwallet/go-primitives/types"
log "github.com/sirupsen/logrus"
)
func (s *Service) FixJSON(f *file.AssetFile) error {
return pkg.FormatJSONFile(f.Path())
}
func (s *Service) FixETHAddressChecksum(f *file.AssetFile) error {
if !coin.IsEVM(f.Chain().ID) {
return nil
}
assetDir := filepath.Base(f.Path())
err := validation.ValidateETHForkAddress(f.Chain(), assetDir)
if err != nil {
checksum, e := address.EIP55Checksum(assetDir)
if e != nil {
return fmt.Errorf("failed to get checksum: %s", e)
}
newName := fmt.Sprintf("blockchains/%s/assets/%s", f.Chain().Handle, checksum)
if e = os.Rename(f.Path(), newName); e != nil {
return fmt.Errorf("failed to rename dir: %s", e)
}
s.fileService.UpdateFile(f, checksum)
log.WithField("from", assetDir).
WithField("to", checksum).
Debug("Renamed asset")
}
return nil
}
func (s *Service) FixLogo(f *file.AssetFile) error {
width, height, err := pkg.GetPNGImageDimensions(f.Path())
if err != nil {
return err
}
var isLogoTooLarge bool
if width > validation.MaxW || height > validation.MaxH {
isLogoTooLarge = true
}
if isLogoTooLarge {
log.WithField("path", f.Path()).Debug("Fixing too large image")
targetW, targetH := calculateTargetDimension(width, height)
err = pkg.ResizePNG(f.Path(), targetW, targetH)
if err != nil {
return err
}
}
err = validation.ValidateLogoFileSize(f.Path())
if err != nil { // nolint:staticcheck
// TODO: Compress images.
}
return nil
}
func calculateTargetDimension(width, height int) (targetW, targetH int) {
widthFloat := float32(width)
heightFloat := float32(height)
maxEdge := widthFloat
if heightFloat > widthFloat {
maxEdge = heightFloat
}
ratio := validation.MaxW / maxEdge
targetW = int(widthFloat * ratio)
targetH = int(heightFloat * ratio)
return targetW, targetH
}
func (s *Service) FixChainInfoJSON(f *file.AssetFile) error {
chainInfo := info.CoinModel{}
err := pkg.ReadJSONFile(f.Path(), &chainInfo)
if err != nil {
return err
}
expectedType := string(types.Coin)
if chainInfo.Type == nil || *chainInfo.Type != expectedType {
chainInfo.Type = &expectedType
return pkg.CreateJSONFile(f.Path(), &chainInfo)
}
return nil
}
func (s *Service) FixAssetInfoJSON(file *file.AssetFile) error {
assetInfo := info.AssetModel{}
err := pkg.ReadJSONFile(file.Path(), &assetInfo)
if err != nil {
return err
}
var isModified bool
// Fix asset type.
var assetType string
if assetInfo.Type != nil {
assetType = *assetInfo.Type
}
// We need to skip error check to fix asset type if it's incorrect or empty.
chain, _ := types.GetChainFromAssetType(assetType)
expectedTokenType, ok := types.GetTokenType(file.Chain().ID, file.Asset())
if !ok {
expectedTokenType = strings.ToUpper(assetType)
}
if chain.ID != file.Chain().ID || !strings.EqualFold(assetType, expectedTokenType) {
assetInfo.Type = &expectedTokenType
isModified = true
}
// Fix asset id.
assetID := file.Asset()
if assetInfo.ID == nil || *assetInfo.ID != assetID {
assetInfo.ID = &assetID
isModified = true
}
expectedExplorerURL, err := coin.GetCoinExploreURL(file.Chain(), file.Asset())
if err != nil {
return err
}
// Fix asset explorer url.
if assetInfo.Explorer == nil || !strings.EqualFold(expectedExplorerURL, *assetInfo.Explorer) {
assetInfo.Explorer = &expectedExplorerURL
isModified = true
}
if isModified {
return pkg.CreateJSONFile(file.Path(), &assetInfo)
}
return nil
}

View File

@ -0,0 +1,82 @@
package processor
import "github.com/trustwallet/assets/internal/file"
type (
Validator struct {
Name string
Run func(f *file.AssetFile) error
}
Fixer struct {
Name string
Run func(f *file.AssetFile) error
}
Updater struct {
Name string
Run func() error
}
)
type (
TokenList struct {
Name string `json:"name"`
LogoURI string `json:"logoURI"`
Timestamp string `json:"timestamp"`
Tokens []TokenItem `json:"tokens"`
Version Version `json:"version"`
}
TokenItem struct {
Asset string `json:"asset"`
Type string `json:"type"`
Address string `json:"address"`
Name string `json:"name"`
Symbol string `json:"symbol"`
Decimals uint `json:"decimals"`
LogoURI string `json:"logoURI"`
Pairs []Pair `json:"pairs"`
}
Pair struct {
Base string `json:"base"`
LotSize string `json:"lotSize,omitempty"`
TickSize string `json:"tickSize,omitempty"`
}
Version struct {
Major int `json:"major"`
Minor int `json:"minor"`
Patch int `json:"patch"`
}
)
type (
ForceListPair struct {
Token0 string
Token1 string
}
TradingPairs struct {
Data struct {
Pairs []TradingPair `json:"pairs"`
} `json:"data"`
}
TradingPair struct {
ID string `json:"id"`
ReserveUSD string `json:"reserveUSD"`
VolumeUSD string `json:"volumeUSD"`
TxCount string `json:"txCount"`
Token0 *TokenInfo `json:"token0"`
Token1 *TokenInfo `json:"token1"`
}
TokenInfo struct {
ID string `json:"id"`
Symbol string `json:"symbol"`
Name string `json:"name"`
Decimals string `json:"decimals"`
}
)

View File

@ -0,0 +1,156 @@
package processor
import (
"github.com/trustwallet/assets/internal/file"
)
type Service struct {
fileService *file.Service
}
func NewService(fileProvider *file.Service) *Service {
return &Service{fileService: fileProvider}
}
// nolint:funlen
func (s *Service) GetValidator(f *file.AssetFile) *Validator {
switch f.Type() {
case file.TypeRootFolder:
return &Validator{
Name: "Root folder contains only allowed files",
Run: s.ValidateRootFolder,
}
case file.TypeChainFolder:
return &Validator{
Name: "Chain folders are lowercase and contains only allowed files",
Run: s.ValidateChainFolder,
}
case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile:
return &Validator{
Name: "Logos (size, dimension)",
Run: s.ValidateImage,
}
case file.TypeAssetFolder:
return &Validator{
Name: "Each asset folder has valid asset address and contains logo/info",
Run: s.ValidateAssetFolder,
}
case file.TypeDappsFolder:
return &Validator{
Name: "Dapps folder (allowed only png files, lowercase)",
Run: s.ValidateDappsFolder,
}
case file.TypeAssetInfoFile:
return &Validator{
Name: "Asset info (is valid json, fields)",
Run: s.ValidateAssetInfoFile,
}
case file.TypeChainInfoFile:
return &Validator{
Name: "Chain Info (is valid json, fields)",
Run: s.ValidateChainInfoFile,
}
case file.TypeValidatorsListFile:
return &Validator{
Name: "Validators list file",
Run: s.ValidateValidatorsListFile,
}
case file.TypeTokenListFile:
return &Validator{
Name: "Token list (if assets from list present in chain)",
Run: s.ValidateTokenListFile,
}
case file.TypeChainInfoFolder:
return &Validator{
Name: "Chain Info Folder (has files)",
Run: s.ValidateInfoFolder,
}
case file.TypeValidatorsAssetFolder:
return &Validator{
Name: "Validators asset folder (has logo, valid asset address)",
Run: s.ValidateValidatorsAssetFolder,
}
}
return nil
}
func (s *Service) GetFixers(f *file.AssetFile) []Fixer {
infoFixer := Fixer{
Name: "Formatting all info.json files",
Run: s.FixJSON,
}
ethAssetFixer := Fixer{
Name: "Renaming EVM's asset folder to valid address checksum",
Run: s.FixETHAddressChecksum,
}
logoFixer := Fixer{
Name: "Resizing and compressing logo images",
Run: s.FixLogo,
}
chainInfoFixer := Fixer{
Name: "Fixing chain info.json files",
Run: s.FixChainInfoJSON,
}
assetInfoFixer := Fixer{
Name: "Fixing asset info.json files",
Run: s.FixAssetInfoJSON,
}
switch f.Type() {
case file.TypeChainInfoFile:
return []Fixer{
infoFixer,
chainInfoFixer,
}
case file.TypeAssetInfoFile:
return []Fixer{
infoFixer,
assetInfoFixer,
}
case file.TypeValidatorsListFile:
return []Fixer{
infoFixer,
}
case file.TypeAssetFolder:
return []Fixer{
ethAssetFixer,
}
case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile:
return []Fixer{
logoFixer,
}
}
return nil
}
func (s *Service) GetUpdatersAuto() []Updater {
return []Updater{
{
Name: "Retrieving missing token images, creating binance token list.",
Run: s.UpdateBinanceTokens,
},
}
}
func (s *Service) GetUpdatersManual() []Updater {
return []Updater{
{
Name: "Update tokenlist.json for Ethereum",
Run: s.UpdateEthereumTokenlist,
},
{
Name: "Update tokenlist.json for Polygon",
Run: s.UpdatePolygonTokenlist,
},
{
Name: "Update tokenlist.json for Smartchain",
Run: s.UpdateSmartchainTokenlist,
},
}
}

View File

@ -0,0 +1,261 @@
package processor
import (
"fmt"
"reflect"
"sort"
"strconv"
"time"
"github.com/trustwallet/assets-go-libs/pkg"
"github.com/trustwallet/assets-go-libs/pkg/asset"
"github.com/trustwallet/assets-go-libs/pkg/validation/info"
"github.com/trustwallet/assets/internal/config"
"github.com/trustwallet/go-libs/blockchain/binance"
"github.com/trustwallet/go-libs/blockchain/binance/explorer"
assetlib "github.com/trustwallet/go-primitives/asset"
"github.com/trustwallet/go-primitives/coin"
"github.com/trustwallet/go-primitives/numbers"
"github.com/trustwallet/go-primitives/types"
)
const (
assetsPage = 1
assetsRows = 1000
marketPairsLimit = 1000
tokensListLimit = 10000
twLogoURL = "https://trustwallet.com/assets/images/favicon.png"
timestampFormat = "2006-01-02T15:04:05.000000"
)
func (s *Service) UpdateBinanceTokens() error {
explorerClient := explorer.InitClient(config.Default.ClientURLs.Binance.Explorer, nil)
bep2AssetList, err := explorerClient.FetchBep2Assets(assetsPage, assetsRows)
if err != nil {
return err
}
dexClient := binance.InitClient(config.Default.ClientURLs.Binance.Dex, "", nil)
marketPairs, err := dexClient.FetchMarketPairs(marketPairsLimit)
if err != nil {
return err
}
tokensList, err := dexClient.FetchTokens(tokensListLimit)
if err != nil {
return err
}
chain, err := types.GetChainFromAssetType(string(types.BEP2))
if err != nil {
return err
}
err = fetchMissingAssets(chain, bep2AssetList.AssetInfoList)
if err != nil {
return err
}
return createTokenListJSON(chain, marketPairs, tokensList)
}
func fetchMissingAssets(chain coin.Coin, assets []explorer.Bep2Asset) error {
for _, a := range assets {
if a.AssetImg == "" || a.Decimals == 0 {
continue
}
assetLogoPath := asset.GetAssetLogoPath(chain.Handle, a.Asset)
if pkg.FileExists(assetLogoPath) {
continue
}
if err := createLogo(assetLogoPath, a); err != nil {
return err
}
if err := createInfoJSON(chain, a); err != nil {
return err
}
}
return nil
}
func createLogo(assetLogoPath string, a explorer.Bep2Asset) error {
err := pkg.CreateDirPath(assetLogoPath)
if err != nil {
return err
}
return pkg.CreatePNGFromURL(a.AssetImg, assetLogoPath)
}
func createInfoJSON(chain coin.Coin, a explorer.Bep2Asset) error {
explorerURL, err := coin.GetCoinExploreURL(chain, a.Asset)
if err != nil {
return err
}
assetType := string(types.BEP2)
website := ""
description := "-"
status := "active"
assetInfo := info.AssetModel{
Name: &a.Name,
Type: &assetType,
Symbol: &a.MappedAsset,
Decimals: &a.Decimals,
Website: &website,
Description: &description,
Explorer: &explorerURL,
Status: &status,
ID: &a.Asset,
}
assetInfoPath := asset.GetAssetInfoPath(chain.Handle, a.Asset)
return pkg.CreateJSONFile(assetInfoPath, &assetInfo)
}
func createTokenListJSON(chain coin.Coin, marketPairs []binance.MarketPair, tokenList binance.Tokens) error {
tokens, err := generateTokenList(marketPairs, tokenList)
if err != nil {
return nil
}
tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle)
var oldTokenList TokenList
err = pkg.ReadJSONFile(tokenListPath, &oldTokenList)
if err != nil {
return nil
}
sortTokens(tokens)
if reflect.DeepEqual(oldTokenList.Tokens, tokens) {
return nil
}
if len(tokens) > 0 {
return pkg.CreateJSONFile(tokenListPath, &TokenList{
Name: fmt.Sprintf("Trust Wallet: %s", coin.Coins[coin.BINANCE].Symbol),
LogoURI: twLogoURL,
Timestamp: time.Now().Format(timestampFormat),
Tokens: tokens,
Version: Version{Major: oldTokenList.Version.Major + 1},
})
}
return nil
}
func sortTokens(tokens []TokenItem) {
sort.Slice(tokens, func(i, j int) bool {
if len(tokens[i].Pairs) != len(tokens[j].Pairs) {
return len(tokens[i].Pairs) > len(tokens[j].Pairs)
}
return tokens[i].Address < tokens[j].Address
})
for _, token := range tokens {
sort.Slice(token.Pairs, func(i, j int) bool {
return token.Pairs[i].Base < token.Pairs[j].Base
})
}
}
func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]TokenItem, error) {
if len(marketPairs) < 5 {
return nil, fmt.Errorf("no markets info is returned from Binance DEX: %d", len(marketPairs))
}
if len(tokenList) < 5 {
return nil, fmt.Errorf("no tokens info is returned from Binance DEX: %d", len(tokenList))
}
pairsMap := make(map[string][]Pair)
pairsList := make(map[string]struct{})
tokensMap := make(map[string]binance.Token)
for _, token := range tokenList {
tokensMap[token.Symbol] = token
}
for _, marketPair := range marketPairs {
key := marketPair.QuoteAssetSymbol
if val, exists := pairsMap[key]; exists {
val = append(val, getPair(marketPair))
pairsMap[key] = val
} else {
pairsMap[key] = []Pair{getPair(marketPair)}
}
pairsList[marketPair.BaseAssetSymbol] = struct{}{}
pairsList[marketPair.QuoteAssetSymbol] = struct{}{}
}
tokenItems := make([]TokenItem, 0, len(pairsList))
for pair := range pairsList {
token := tokensMap[pair]
var pairs []Pair
pairs, exists := pairsMap[token.Symbol]
if !exists {
pairs = make([]Pair, 0)
}
tokenItems = append(tokenItems, TokenItem{
Asset: getAssetIDSymbol(token.Symbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
Type: getTokenType(token.Symbol, coin.Coins[coin.BINANCE].Symbol, string(types.BEP2)),
Address: token.Symbol,
Name: token.Name,
Symbol: token.OriginalSymbol,
Decimals: coin.Coins[coin.BINANCE].Decimals,
LogoURI: getLogoURI(token.Symbol, coin.Coins[coin.BINANCE].Handle, coin.Coins[coin.BINANCE].Symbol),
Pairs: pairs,
})
}
return tokenItems, nil
}
func getPair(marketPair binance.MarketPair) Pair {
return Pair{
Base: getAssetIDSymbol(marketPair.BaseAssetSymbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
LotSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.LotSize), 10),
TickSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.TickSize), 10),
}
}
func getAssetIDSymbol(tokenID string, nativeCoinID string, coinType uint) string {
if tokenID == nativeCoinID {
return assetlib.BuildID(coinType, "")
}
return assetlib.BuildID(coinType, tokenID)
}
func getTokenType(symbol string, nativeCoinSymbol string, tokenType string) string {
if symbol == nativeCoinSymbol {
return "coin"
}
return tokenType
}
func getLogoURI(id, githubChainFolder, nativeCoinSymbol string) string {
if id == nativeCoinSymbol {
return fmt.Sprintf("%s/blockchains/%s/info/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder)
}
return fmt.Sprintf("%s/blockchains/%s/assets/%s/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder, id)
}

View File

@ -0,0 +1,537 @@
package processor
import (
"encoding/json"
"fmt"
"strconv"
"strings"
log "github.com/sirupsen/logrus"
"github.com/trustwallet/assets-go-libs/pkg"
"github.com/trustwallet/assets-go-libs/pkg/asset"
"github.com/trustwallet/assets/internal/config"
"github.com/trustwallet/go-libs/client/api/backend"
"github.com/trustwallet/go-primitives/address"
"github.com/trustwallet/go-primitives/coin"
"github.com/trustwallet/go-primitives/types"
)
var (
UniswapForceInclude = []string{
"TUSD", "STAKE", "YFI", "BAT", "MANA", "1INCH", "REP", "KP3R", "UNI", "WBTC", "HEX", "CREAM", "SLP",
"REN", "XOR", "Link", "sUSD", "HEGIC", "RLC", "DAI", "SUSHI", "FYZ", "DYT", "AAVE", "LEND", "UBT",
"DIA", "RSR", "SXP", "OCEAN", "MKR", "USDC", "CEL", "BAL", "BAND", "COMP", "SNX", "OMG", "AMPL",
"USDT", "KNC", "ZRX", "AXS", "ENJ", "STMX", "DPX", "FTT", "DPI", "PAX",
}
UniswapForceExclude = []string{"STARL", "UFO"}
PolygonSwapForceInclude = []string{}
PolygonSwapForceExclude = []string{}
PancakeSwapForceInclude = []string{
"Cake", "DAI", "ETH", "TWT", "VAI", "USDT", "BLINK", "BTCB", "ALPHA", "INJ", "CTK", "UNI", "XVS",
"BUSD", "HARD", "BIFI", "FRONT",
}
PancakeSwapForceExclude = []string{}
)
var (
UniswapTradingPairsQuery = map[string]string{
"operationName": "pairs",
"query": `
query pairs {
pairs(first: 800, orderBy: reserveUSD, orderDirection: desc) {
id reserveUSD trackedReserveETH volumeUSD txCount untrackedVolumeUSD __typename
token0 {
id symbol name decimals __typename
}
token1 {
id symbol name decimals __typename
}
}
}
`,
}
PolygonSwap_TradingPairsQuery = map[string]string{
"operationName": "pairs",
"query": `
{
ethereum(network: matic) {
dexTrades(date: {is: "$DATE$"}) {
sellCurrency {address symbol name decimals}
buyCurrency {address symbol name decimals}
trade: count
tradeAmount(in: USD)
}
}
}
`,
}
PancakeSwap_TradingPairsQuery = map[string]string{
"operationName": "pairs",
"query": `
query pairs {
pairs(first: 300, orderBy: reserveUSD, orderDirection: desc) {
id reserveUSD volumeUSD txCount __typename
token0 {
id symbol name decimals __typename
}
token1 {
id symbol name decimals __typename
}
}
}
`,
}
UniswapTradingPairsUrl = "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2"
PolygonSwapTradingPairsUrl = "https://graphql.bitquery.io"
PancakeSwapradingPairsUrl = "https://api.bscgraph.org/subgraphs/name/cakeswap"
)
const (
UniswapMinLiquidity = 2000000
UniswapMinVol24 = 1000000
UniswapMinTxCount24 = 480
PolygonSwapMinVol24 = 500000
PolygonSwapMinTxCount24 = 288
PancakeSwapMinLiquidity = 1000000
PancakeSwapMinVol24 = 500000
PancakeSwapMinTxCount24 = 288
)
var (
PrimaryTokensETH = []string{"WETH", "ETH"}
)
func (s *Service) UpdateEthereumTokenlist() error {
log.WithFields(log.Fields{
"limit_liquidity": UniswapMinLiquidity,
"volume": UniswapMinVol24,
"tx_count": UniswapMinTxCount24,
}).Debug("Retrieving pairs from Uniswap")
tradingPairs, err := retrieveUniswapPairs(UniswapTradingPairsUrl, UniswapTradingPairsQuery,
UniswapMinLiquidity, UniswapMinVol24, UniswapMinTxCount24, UniswapForceInclude, PrimaryTokensETH)
if err != nil {
return err
}
pairs := make([][]TokenItem, 0)
for _, tradingPair := range tradingPairs {
tokenItem0, err := getTokenInfoFromSubgraphToken(tradingPair.Token0)
if err != nil {
return err
}
tokenItem1, err := getTokenInfoFromSubgraphToken(tradingPair.Token1)
if err != nil {
return err
}
if !isTokenPrimary(tradingPair.Token0, PrimaryTokensETH) {
tokenItem0, tokenItem1 = tokenItem1, tokenItem0
}
pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1})
}
return rebuildTokenList(coin.Coins[coin.ETHEREUM], pairs, UniswapForceExclude)
}
func (s *Service) UpdatePolygonTokenlist() error {
return nil
}
func (s *Service) UpdateSmartchainTokenlist() error {
return nil
}
func retrieveUniswapPairs(url string, query map[string]string, minLiquidity, minVol24, minTxCount24 int,
forceIncludeList []string, primaryTokens []string) ([]TradingPair, error) {
includeList := parseForceList(forceIncludeList)
pairs, err := fetchTradingPairs(url, query)
if err != nil {
return nil, err
}
filtered := make([]TradingPair, 0)
for _, pair := range pairs.Data.Pairs {
ok, err := checkTradingPairOK(pair, minLiquidity, minVol24, minTxCount24, primaryTokens, includeList)
if err != nil {
log.Debug(err)
}
if ok {
filtered = append(filtered, pair)
}
}
return filtered, nil
}
func parseForceList(forceList []string) []ForceListPair {
result := make([]ForceListPair, 0, len(forceList))
for _, item := range forceList {
tokens := strings.Split(item, "-")
pair := ForceListPair{
Token0: tokens[0],
}
if len(tokens) >= 2 {
pair.Token1 = tokens[1]
}
result = append(result, pair)
}
return result
}
func fetchTradingPairs(url string, query map[string]string) (*TradingPairs, error) {
jsonValue, err := json.Marshal(query)
if err != nil {
return nil, err
}
log.WithField("url", url).Debug("Retrieving trading pair infos")
var result TradingPairs
err = pkg.PostHTTPResponse(url, jsonValue, &result)
if err != nil {
return nil, err
}
log.Debugf("Retrieved %d trading pair infos", len(result.Data.Pairs))
return &result, nil
}
func checkTradingPairOK(pair TradingPair, minLiquidity, minVol24, minTxCount24 int, primaryTokens []string,
forceIncludeList []ForceListPair) (bool, error) {
if pair.ID == "" || pair.ReserveUSD == "" || pair.VolumeUSD == "" || pair.TxCount == "" ||
pair.Token0 == nil || pair.Token1 == nil {
return false, nil
}
if !(isTokenPrimary(pair.Token0, primaryTokens) || isTokenPrimary(pair.Token1, primaryTokens)) {
log.Debugf("pair with no primary coin: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
return false, nil
}
if isPairMatchedToForceList(getTokenItemFromInfo(pair.Token0), getTokenItemFromInfo(pair.Token1), forceIncludeList) {
log.Debugf("pair included due to FORCE INCLUDE: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
return true, nil
}
reserveUSD, err := strconv.ParseFloat(pair.ReserveUSD, 64)
if err != nil {
return false, err
}
if int(reserveUSD) < minLiquidity {
log.Debugf("pair with low liquidity: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
return false, nil
}
volumeUSD, err := strconv.ParseFloat(pair.VolumeUSD, 64)
if err != nil {
return false, err
}
if int(volumeUSD) < minVol24 {
log.Debugf("pair with low volume: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
return false, nil
}
txCount, err := strconv.ParseFloat(pair.TxCount, 64)
if err != nil {
return false, err
}
if int(txCount) < minTxCount24 {
log.Debugf("pair with low tx count: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
return false, nil
}
return true, nil
}
func getTokenItemFromInfo(tokenInfo *TokenInfo) *TokenItem {
decimals, err := strconv.Atoi(tokenInfo.Decimals)
if err != nil {
return nil
}
return &TokenItem{
Asset: tokenInfo.ID,
Address: tokenInfo.ID,
Name: tokenInfo.Name,
Symbol: tokenInfo.Symbol,
Decimals: uint(decimals),
}
}
func getTokenInfoFromSubgraphToken(token *TokenInfo) (*TokenItem, error) {
checksum, err := address.EIP55Checksum(token.ID)
if err != nil {
return nil, err
}
decimals, err := strconv.Atoi(token.Decimals)
if err != nil {
return nil, err
}
return &TokenItem{
Asset: getAssetIDSymbol(checksum, coin.Coins[coin.ETHEREUM].Symbol, coin.ETHEREUM),
Type: string(types.ERC20),
Address: checksum,
Name: token.Name,
Symbol: token.Symbol,
Decimals: uint(decimals),
LogoURI: getLogoURI(token.Symbol, coin.Coins[coin.ETHEREUM].Handle, coin.Coins[coin.ETHEREUM].Symbol),
Pairs: make([]Pair, 0),
}, nil
}
func isTokenPrimary(token *TokenInfo, primaryTokens []string) bool {
if token == nil {
return false
}
for _, primaryToken := range primaryTokens {
if strings.EqualFold(primaryToken, token.Symbol) {
return true
}
}
return false
}
func isPairMatchedToForceList(token0, token1 *TokenItem, forceIncludeList []ForceListPair) bool {
var matched bool
for _, forcePair := range forceIncludeList {
if matchPairToForceListEntry(token0, token1, forcePair) {
matched = true
}
}
return matched
}
func matchPairToForceListEntry(token0, token1 *TokenItem, forceListEntry ForceListPair) bool {
if forceListEntry.Token1 == "" {
if matchTokenToForceListEntry(token0, forceListEntry.Token0) ||
(token1 != nil && matchTokenToForceListEntry(token1, forceListEntry.Token0)) {
return true
}
return false
}
if token1 == nil {
return false
}
if matchTokenToForceListEntry(token0, forceListEntry.Token0) &&
matchTokenToForceListEntry(token0, forceListEntry.Token1) {
return true
}
if matchTokenToForceListEntry(token0, forceListEntry.Token1) &&
matchTokenToForceListEntry(token1, forceListEntry.Token0) {
return true
}
return false
}
func matchTokenToForceListEntry(token *TokenItem, forceListEntry string) bool {
if strings.EqualFold(forceListEntry, token.Symbol) ||
strings.EqualFold(forceListEntry, token.Asset) ||
strings.EqualFold(forceListEntry, token.Name) {
return true
}
return false
}
func matchPairToForceList(token0, token1 *TokenItem, forceList []ForceListPair) bool {
var matched bool
for _, forcePair := range forceList {
if matchPairToForceListEntry(token0, token1, forcePair) {
matched = true
}
}
return matched
}
func rebuildTokenList(chain coin.Coin, pairs [][]TokenItem, forceExcludeList []string) error {
if pairs == nil || len(pairs) < 5 {
return nil
}
excludeList := parseForceList(forceExcludeList)
pairs2 := make([][]TokenItem, 0)
for _, pair := range pairs {
if !checkTokenExists(chain.Handle, pair[0].Address) {
log.Debugf("pair with unsupported 1st coin: %s-%s", pair[0].Symbol, pair[1].Symbol)
continue
}
if !checkTokenExists(chain.Handle, pair[1].Address) {
log.Debugf("pair with unsupported 2nd coin: %s-%s", pair[0].Symbol, pair[1].Symbol)
continue
}
if matchPairToForceList(&pair[0], &pair[1], excludeList) {
log.Debugf("pair excluded due to FORCE EXCLUDE: %s-%s", pair[0].Symbol, pair[1].Symbol)
continue
}
pairs2 = append(pairs2, pair)
}
filteredCount := len(pairs) - len(pairs2)
log.Debugf("%d unsupported tokens filtered out, %d pairs", filteredCount, len(pairs2))
tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle)
var list TokenList
err := pkg.ReadJSONFile(tokenListPath, &list)
if err != nil {
return nil
}
removeAllPairs(&list)
for _, pair := range pairs2 {
err = addPairIfNeeded(&pair[0], &pair[1], &list)
if err != nil {
return err
}
}
log.Debugf("Tokenlist updated: %d tokens", len(list.Tokens))
var totalPairs int
for _, item := range list.Tokens {
totalPairs += len(item.Pairs)
}
log.Debugf("Tokenlist: list with %d tokens and %d pairs written to %s.",
len(list.Tokens), totalPairs, tokenListPath)
return pkg.CreateJSONFile(tokenListPath, list)
}
func checkTokenExists(chain, tokenID string) bool {
logoPath := asset.GetAssetLogoPath(chain, tokenID)
return pkg.FileExists(logoPath)
}
func removeAllPairs(list *TokenList) {
for _, token := range list.Tokens {
token.Pairs = make([]Pair, 0)
}
}
func addPairIfNeeded(token0, token1 *TokenItem, list *TokenList) error {
err := addTokenIfNeeded(token0, list)
if err != nil {
return err
}
err = addTokenIfNeeded(token1, list)
if err != nil {
return err
}
addPairToToken(token1, token0, list)
return nil
}
func addTokenIfNeeded(token *TokenItem, list *TokenList) error {
for _, t := range list.Tokens {
if strings.EqualFold(t.Address, token.Address) {
return nil
}
}
err := updateTokenInfo(token)
if err != nil {
return err
}
list.Tokens = append(list.Tokens, *token)
return nil
}
func updateTokenInfo(token *TokenItem) error {
backendClient := backend.InitClient(config.Default.ClientURLs.BackendAPI, nil)
assetInfo, err := backendClient.GetAssetInfo(token.Asset)
if err != nil {
return err
}
if token.Name != assetInfo.Name {
token.Name = assetInfo.Name
}
if token.Symbol != assetInfo.Symbol {
token.Symbol = assetInfo.Symbol
}
if token.Decimals != uint(assetInfo.Decimals) {
token.Decimals = uint(assetInfo.Decimals)
}
return nil
}
func addPairToToken(pairToken, token *TokenItem, list *TokenList) {
var tokenInListIndex = -1
for i, t := range list.Tokens {
if t.Address == token.Address {
tokenInListIndex = i
break
}
}
if tokenInListIndex == -1 {
return
}
if list.Tokens[tokenInListIndex].Pairs == nil {
list.Tokens[tokenInListIndex].Pairs = make([]Pair, 0)
}
for _, pair := range list.Tokens[tokenInListIndex].Pairs {
if pair.Base == pairToken.Asset {
return
}
}
list.Tokens[tokenInListIndex].Pairs = append(list.Tokens[tokenInListIndex].Pairs, Pair{Base: pairToken.Asset})
}

View File

@ -0,0 +1,407 @@
package processor
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"github.com/trustwallet/assets-go-libs/pkg/validation"
"github.com/trustwallet/assets-go-libs/pkg/validation/info"
"github.com/trustwallet/assets-go-libs/pkg/validation/list"
"github.com/trustwallet/assets/internal/config"
"github.com/trustwallet/assets/internal/file"
"github.com/trustwallet/go-primitives/coin"
)
func (s *Service) ValidateRootFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.RootFolder.AllowedFiles)
if err != nil {
return err
}
return nil
}
func (s *Service) ValidateChainFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
var compErr = validation.NewErrComposite()
err = validation.ValidateLowercase(fileInfo.Name())
if err != nil {
compErr.Append(err)
}
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.ChainFolder.AllowedFiles)
if err != nil {
compErr.Append(err)
}
if compErr.Len() > 0 {
return compErr
}
return nil
}
func (s *Service) ValidateImage(f *file.AssetFile) error {
var compErr = validation.NewErrComposite()
err := validation.ValidateLogoFileSize(f.Path())
if err != nil {
compErr.Append(err)
}
// TODO: Replace it with validation.ValidatePngImageDimension when "assets" repo is fixed.
// Read comments inValidatePngImageDimensionForCI.
err = validation.ValidatePngImageDimensionForCI(f.Path())
if err != nil {
compErr.Append(err)
}
if compErr.Len() > 0 {
return compErr
}
return nil
}
func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
var compErr = validation.NewErrComposite()
err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.AssetFolder.AllowedFiles)
if err != nil {
compErr.Append(err)
}
err = validation.ValidateAssetAddress(f.Chain(), f.Asset())
if err != nil {
compErr.Append(err)
}
errInfo := validation.ValidateHasFiles(dirFiles, []string{"info.json"})
errLogo := validation.ValidateHasFiles(dirFiles, []string{"logo.png"})
if errLogo != nil || errInfo != nil {
infoFile := s.fileService.GetAssetFile(fmt.Sprintf("%s/info.json", f.Path()))
file2, err := os.Open(infoFile.Path())
if err != nil {
return err
}
defer file2.Close()
_, err = file2.Seek(0, io.SeekStart)
if err != nil {
return err
}
b, err := io.ReadAll(file2)
if err != nil {
return err
}
var infoJson info.AssetModel
err = json.Unmarshal(b, &infoJson)
if err != nil {
return err
}
if infoJson.GetStatus() != "spam" && infoJson.GetStatus() != "abandoned" {
compErr.Append(fmt.Errorf("%w: logo.png for non-spam assest", validation.ErrMissingFile))
}
}
if compErr.Len() > 0 {
return compErr
}
return nil
}
func (s *Service) ValidateDappsFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
var compErr = validation.NewErrComposite()
for _, dirFile := range dirFiles {
err = validation.ValidateExtension(dirFile.Name(), config.Default.ValidatorsSettings.DappsFolder.Ext)
if err != nil {
compErr.Append(err)
}
err = validation.ValidateLowercase(dirFile.Name())
if err != nil {
compErr.Append(err)
}
}
if compErr.Len() > 0 {
return compErr
}
return nil
}
func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
_, err = buf.ReadFrom(file)
if err != nil {
return err
}
err = validation.ValidateJson(buf.Bytes())
if err != nil {
return err
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson)
}
var payload info.CoinModel
err = json.Unmarshal(buf.Bytes(), &payload)
if err != nil {
return fmt.Errorf("%w: failed to decode", err)
}
tags := make([]string, len(config.Default.ValidatorsSettings.CoinInfoFile.Tags))
for i, t := range config.Default.ValidatorsSettings.CoinInfoFile.Tags {
tags[i] = t.ID
}
err = info.ValidateCoin(payload, f.Chain(), f.Asset(), tags)
if err != nil {
return err
}
return nil
}
func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
return err
}
err = validation.ValidateJson(buf.Bytes())
if err != nil {
return err
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson)
}
var payload info.AssetModel
err = json.Unmarshal(buf.Bytes(), &payload)
if err != nil {
return fmt.Errorf("%w: failed to decode", err)
}
err = info.ValidateAsset(payload, f.Chain(), f.Asset())
if err != nil {
return err
}
return nil
}
func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
if !isStackingChain(f.Chain()) {
return nil
}
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
return err
}
err = validation.ValidateJson(buf.Bytes())
if err != nil {
return err
}
var model []list.Model
err = json.Unmarshal(buf.Bytes(), &model)
if err != nil {
return err
}
err = list.ValidateList(model)
if err != nil {
return err
}
listIDs := make([]string, len(model))
for i, listItem := range model {
listIDs[i] = *listItem.ID
}
assetsPath := fmt.Sprintf("blockchains/%s/validators/assets", f.Chain().Handle)
assetFolder := s.fileService.GetAssetFile(assetsPath)
file2, err := os.Open(assetFolder.Path())
if err != nil {
return err
}
defer file2.Close()
dirAssetFolderFiles, err := file2.ReadDir(0)
if err != nil {
return err
}
err = validation.ValidateAllowedFiles(dirAssetFolderFiles, listIDs)
if err != nil {
return err
}
return nil
}
func isStackingChain(c coin.Coin) bool {
for _, stackingChain := range config.StackingChains {
if c.ID == stackingChain.ID {
return true
}
}
return false
}
func (s *Service) ValidateTokenListFile(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
buf := bytes.NewBuffer(nil)
if _, err = buf.ReadFrom(file); err != nil {
return err
}
err = validation.ValidateJson(buf.Bytes())
if err != nil {
return err
}
return nil
}
func (s *Service) ValidateInfoFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainInfoFolder.HasFiles)
if err != nil {
return err
}
return nil
}
func (s *Service) ValidateValidatorsAssetFolder(f *file.AssetFile) error {
file, err := os.Open(f.Path())
if err != nil {
return err
}
defer file.Close()
dirFiles, err := file.ReadDir(0)
if err != nil {
return err
}
compErr := validation.NewErrComposite()
err = validation.ValidateValidatorsAddress(f.Chain(), f.Asset())
if err != nil {
compErr.Append(err)
}
err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainValidatorsAssetFolder.HasFiles)
if err != nil {
compErr.Append(err)
}
if compErr.Len() > 0 {
return compErr
}
return nil
}

102
internal/service/service.go Normal file
View File

@ -0,0 +1,102 @@
package service
import (
"github.com/trustwallet/assets-go-libs/pkg/validation"
"github.com/trustwallet/assets/internal/file"
"github.com/trustwallet/assets/internal/processor"
log "github.com/sirupsen/logrus"
)
type Service struct {
fileService *file.Service
processorService *processor.Service
}
func NewService(fs *file.Service, cs *processor.Service) *Service {
return &Service{
fileService: fs,
processorService: cs,
}
}
func (s *Service) RunJob(paths []string, job func(*file.AssetFile)) {
for _, path := range paths {
f := s.fileService.GetAssetFile(path)
job(f)
}
}
func (s *Service) Check(f *file.AssetFile) {
validator := s.processorService.GetValidator(f)
if validator != nil {
if err := validator.Run(f); err != nil {
// TODO: somehow return an error from Check if there are any errors.
HandleError(err, f, validator.Name)
}
}
}
func (s *Service) Fix(f *file.AssetFile) {
fixers := s.processorService.GetFixers(f)
for _, fixer := range fixers {
if err := fixer.Run(f); err != nil {
HandleError(err, f, fixer.Name)
}
}
}
func (s *Service) RunUpdateAuto() {
updaters := s.processorService.GetUpdatersAuto()
s.runUpdaters(updaters)
}
func (s *Service) RunUpdateManual() {
updaters := s.processorService.GetUpdatersManual()
s.runUpdaters(updaters)
}
func (s *Service) runUpdaters(updaters []processor.Updater) {
for _, updater := range updaters {
err := updater.Run()
if err != nil {
log.WithError(err).Error()
}
}
}
func HandleError(err error, info *file.AssetFile, valName string) {
errors := UnwrapComposite(err)
for _, err := range errors {
logFields := log.Fields{
"type": info.Type(),
"chain": info.Chain().Handle,
"asset": info.Asset(),
"path": info.Path(),
"validation": valName,
}
if warn, ok := err.(*validation.Warning); ok {
log.WithFields(logFields).Warning(warn)
} else {
log.WithFields(logFields).Error(err)
}
}
}
func UnwrapComposite(err error) []error {
compErr, ok := err.(*validation.ErrComposite)
if !ok {
return []error{err}
}
var errors []error
for _, e := range compErr.GetErrors() {
errors = append(errors, UnwrapComposite(e)...)
}
return errors
}