diff --git a/.github/assets.config.yaml b/.github/assets.config.yaml index 849d31a8a..0f1ee4260 100644 --- a/.github/assets.config.yaml +++ b/.github/assets.config.yaml @@ -5,6 +5,7 @@ client_urls: binance: dex: https://dex.binance.org explorer: https://explorer.binance.org + backend_api: https://api.trustwallet.com urls: tw_assets_app: https://assets.trustwalletapp.com @@ -31,8 +32,11 @@ validators_settings: - ".eslintignore" - ".eslintrc.js" - "cmd" + - "internal" - "go.mod" - "go.sum" + - ".golangci.yml" + - "Makefile" skip_files: - "node_modules" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a06fed555..a74789d3c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,7 +7,7 @@ jobs: check: runs-on: ubuntu-latest steps: - - name: Set up Go 1.17 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 @@ -17,4 +17,10 @@ jobs: uses: actions/checkout@v2 - name: Run check - run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker \ No newline at end of file + run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker + + # - name: Unit Test + # run: make test + + # - name: Lint + # run: make lint \ No newline at end of file diff --git a/.github/workflows/fix-dryrun.yml b/.github/workflows/fix-dryrun.yml index ba57fae16..2be933153 100644 --- a/.github/workflows/fix-dryrun.yml +++ b/.github/workflows/fix-dryrun.yml @@ -7,7 +7,7 @@ jobs: fix-dryrun: runs-on: ubuntu-latest steps: - - name: Set up Go 1.17 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 diff --git a/.github/workflows/fix.yml b/.github/workflows/fix.yml index 28ddcf884..c225cac53 100644 --- a/.github/workflows/fix.yml +++ b/.github/workflows/fix.yml @@ -27,7 +27,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.ref }} - - name: Set up Go 1.17 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 diff --git a/.github/workflows/periodic-update.yml b/.github/workflows/periodic-update.yml index 17e0fd1bb..2e68d4494 100644 --- a/.github/workflows/periodic-update.yml +++ b/.github/workflows/periodic-update.yml @@ -13,7 +13,7 @@ jobs: token: ${{ secrets.COMMIT_TOKEN }} ref: ${{ github.head_ref }} - - name: Set up Go 1.17 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 77dd09f8f..07fc10cc4 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -11,7 +11,7 @@ jobs: pull_request_ci: runs-on: ubuntu-latest steps: - - name: Set up Go 1.17 + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 @@ -21,4 +21,10 @@ jobs: uses: actions/checkout@v2 - name: Run check - run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker \ No newline at end of file + run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker + + # - name: Unit Test + # run: make test + + # - name: Lint + # run: make lint \ No newline at end of file diff --git a/.gitignore b/.gitignore index d12bd9311..ba2901bac 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ node_modules/ *.txt .env .env.test +bin/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..12e2ce5b3 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,250 @@ +linters-settings: + cyclop: + max-complexity: 20 + # the maximal average package complexity. If it's higher than 0.0 (float) the check is enabled (default 0.0) + package-average: 0.0 + skip-tests: false + dogsled: + # checks assignments with too many blank identifiers; default is 2 + max-blank-identifiers: 2 + dupl: + threshold: 100 + errcheck: + check-type-assertions: false + check-blank: false + errorlint: + errorf: false + asserts: true + comparison: true + forbidigo: + # Forbid the following identifiers (identifiers are written using regexp): + forbid: + - ^print.*$ + - 'fmt\.Print.*' + exclude_godoc_examples: true + funlen: + lines: 60 + statements: 60 + gocognit: + min-complexity: 35 + goconst: + min-len: 3 + min-occurrences: 3 + ignore-tests: true + match-constant: true + numbers: false + min: 3 + max: 3 + ignore-calls: true + gocritic: + # Which checks should be disabled; can't be combined with 'enabled-checks' + disabled-checks: + - regexpMust + gocyclo: + min-complexity: 20 + godot: + # comments to be checked: `declarations`, `toplevel`, or `all` + scope: all + capital: false + godox: + # report any comments starting with keywords, this is useful for TODO or FIXME comments that + # might be left in the code accidentally and should be resolved before merging + keywords: + - TODO + - BUG + - FIXME + - NOTE + - OPTIMIZE # marks code that should be optimized before merging + - HACK # marks hack-arounds that should be removed before merging + gofmt: + # simplify code: gofmt with `-s` option + simplify: true + gofumpt: + # Select the Go version to target + lang-version: "1.17" + extra-rules: false + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: github.com/trustwallet + gomnd: + settings: + mnd: + # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. + checks: + - argument + - case + - condition + - operation + - return + - assign + ignored-numbers: 10,1000 + gomoddirectives: + replace-local: false + gosec: + # To select a subset of rules to run. + # Available rules: https://github.com/securego/gosec#available-rules + includes: + - G401 + - G306 + - G101 + gosimple: + # Select the Go version to target + go: "1.17" + # https://staticcheck.io/docs/options#checks + checks: [ "all" ] + govet: + check-shadowing: true + ifshort: + # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax. + # Has higher priority than max-decl-chars. + max-decl-lines: 1 + # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. + max-decl-chars: 30 + importas: + # if set to `true`, force to use alias. + no-unaliased: true + # List of aliases + alias: + - pkg: github:com/trustwallet/go-libs/gin + alias: golibsGin + lll: + # max line length + line-length: 120 + tab-width: 1 + misspell: + locale: US + # ignore-words: + nolintlint: + # Enable to ensure that nolint directives are all used + allow-unused: false + # Disable to ensure that nolint directives don't have a leading space + allow-leading-space: true + # Exclude following linters from requiring an explanation + allow-no-explanation: [] + # Enable to require an explanation of nonzero length after each nolint directive + require-explanation: false + # Enable to require nolint directives to mention the specific linter being suppressed + require-specific: true + revive: + # see https://github.com/mgechev/revive#available-rules for details. + ignore-generated-header: true + severity: warning + rules: + - name: indent-error-flow + severity: warning + - name: time-naming + severity: warn + - name: errorf + severity: warn + - name: blank-imports + sevetiry: warn + # The error return parameter should be last + - name: error-return + severity: error + # Redundant if when returning an error + - name: if-return + severity: warn + # Warns when there are heading or trailing newlines in a block + - name: empty-lines + severity: error + staticcheck: + # Select the Go version to target + go: "1.17" + # https://staticcheck.io/docs/options#checks + checks: [ "all" ] + stylecheck: + # Select the Go version to target + go: "1.17" + # https://staticcheck.io/docs/options#checks + checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ] + # https://staticcheck.io/docs/options#dot_import_whitelist + dot-import-whitelist: + - fmt + # https://staticcheck.io/docs/options#initialisms + initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ] + # https://staticcheck.io/docs/options#http_status_code_whitelist + http-status-code-whitelist: [ "200", "400", "404", "500" ] + unused: + # Select the Go version to target + go: "1.17" + whitespace: + multi-if: true # Enforces newlines (or comments) after every multi-line if statement + multi-func: false # Enforces newlines (or comments) after every multi-line function signature + wsl: + # See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for + # documentation of available settings + allow-assign-and-anything: false + allow-assign-and-call: true + allow-cuddle-declarations: false + allow-multiline-assign: true + allow-separated-leading-comment: false + allow-trailing-comment: false + force-case-trailing-whitespace: 0 + force-err-cuddling: false + force-short-decl-cuddling: false + strict-append: true + +linters: + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - exportloopref + - exhaustive + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + # - gomnd + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - noctx + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace + - asciicheck + # - gochecknoglobals + - gocognit + - godot + # - godox + # - goerr113 + # - nestif + - prealloc + - testpackage + # - revive + # - wsl + + # don't enable: + # - interfacer + # - maligned + # - scopelint + +output: + format: colored-line-number + print-issued-lines: true + print-linter-name: true + uniq-by-line: true + path-prefix: "" + sort-results: true \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a17159f55 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +#! /usr/bin/make -f + +# Go related variables. +GOBASE := $(shell pwd) +GOBIN := $(GOBASE)/bin + +# Go files. +GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor) + +all: fmt lint test + +test: + @echo " > Running unit tests" + GOBIN=$(GOBIN) go test -cover -race -coverprofile=coverage.txt -covermode=atomic -v ./... + +fmt: + @echo " > Format all go files" + GOBIN=$(GOBIN) gofmt -w ${GOFMT_FILES} + +lint-install: +ifeq (,$(wildcard test -f bin/golangci-lint)) + @echo " > Installing golint" + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s +endif + +lint: lint-install + @echo " > Running golint" + bin/golangci-lint run --timeout=2m \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 6c0eb1f2a..ebd11e8f0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -5,10 +5,10 @@ import ( log "github.com/sirupsen/logrus" - "github.com/trustwallet/assets-go-libs/pkg/file" - "github.com/trustwallet/assets-go-libs/src/config" - "github.com/trustwallet/assets-go-libs/src/core" - "github.com/trustwallet/assets-go-libs/src/processor" + "github.com/trustwallet/assets/internal/config" + "github.com/trustwallet/assets/internal/file" + "github.com/trustwallet/assets/internal/processor" + "github.com/trustwallet/assets/internal/service" ) var ( @@ -20,42 +20,41 @@ func main() { paths, err := file.ReadLocalFileStructure(root, config.Default.ValidatorsSettings.RootFolder.SkipFiles) if err != nil { - log.WithError(err).Fatal("failed to load file structure") + log.WithError(err).Fatal("Failed to load file structure.") } fileStorage := file.NewService(paths...) - validatorsService := core.NewService(fileStorage) - assetfsProcessor := processor.NewService(fileStorage, validatorsService) + validatorsService := processor.NewService(fileStorage) + assetfsProcessor := service.NewService(fileStorage, validatorsService) switch script { case "checker": - err = assetfsProcessor.RunJob(paths, assetfsProcessor.Check) + assetfsProcessor.RunJob(paths, assetfsProcessor.Check) case "fixer": - err = assetfsProcessor.RunJob(paths, assetfsProcessor.Fix) + assetfsProcessor.RunJob(paths, assetfsProcessor.Fix) case "updater-auto": - err = assetfsProcessor.RunUpdateAuto() + assetfsProcessor.RunUpdateAuto() + case "updater-manual": + assetfsProcessor.RunUpdateManual() default: log.Info("Nothing to launch. Use --script flag to choose a script to run.") } - - if err != nil { - log.WithError(err).Fatal("Script failed") - } } func setup() { - flag.StringVar(&configPath, "config", "", "path to config file") + flag.StringVar(&configPath, "config", "./.github/assets.config.yaml", "path to config file") flag.StringVar(&root, "root", "./", "path to the root of the dir") flag.StringVar(&script, "script", "", "script type to run") + flag.Parse() if err := config.SetConfig(configPath); err != nil { - log.WithError(err).Fatal("failed to set config") + log.WithError(err).Fatal("Failed to set config.") } logLevel, err := log.ParseLevel(config.Default.App.LogLevel) if err != nil { - log.WithError(err).Fatal("failed to parse log level") + log.WithError(err).Fatal("Failed to parse log level.") } log.SetLevel(logLevel) diff --git a/go.mod b/go.mod index 21dd6633d..32bb67edd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.17 require ( github.com/sirupsen/logrus v1.8.1 - github.com/trustwallet/assets-go-libs v0.0.15 + github.com/trustwallet/assets-go-libs v0.0.16 + github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793 + github.com/trustwallet/go-primitives v0.0.17 ) require ( @@ -22,8 +24,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.10.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/trustwallet/go-libs v0.2.20 // indirect - github.com/trustwallet/go-primitives v0.0.17 // indirect golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827 // indirect diff --git a/go.sum b/go.sum index 3a5059347..ee2543191 100644 --- a/go.sum +++ b/go.sum @@ -40,14 +40,10 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d h1:L2WPNWlTMoBfVsHDqw73dH3fBZYdh0xdn3cmXGs7IX8= -github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI= -github.com/trustwallet/assets-go-libs v0.0.14 h1:7fs6tPFf8XF1svj1fksJ+PN4ukT3v3sRfz/z4W86JAA= -github.com/trustwallet/assets-go-libs v0.0.14/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI= -github.com/trustwallet/assets-go-libs v0.0.15 h1:0q0xUR0FATqqdLr9OShRwYK0r3KDqGLWrxpOOc4lJ4o= -github.com/trustwallet/assets-go-libs v0.0.15/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI= -github.com/trustwallet/go-libs v0.2.20 h1:pYstFNgsc7CVyVeYt5GHsMa0JNQHJVRvPQqMvXMpCtY= -github.com/trustwallet/go-libs v0.2.20/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc= +github.com/trustwallet/assets-go-libs v0.0.16 h1:nbKrf/pHKQCut4Q5Mdg5BinLdFxOqvzEeAgIgzX0/P8= +github.com/trustwallet/assets-go-libs v0.0.16/go.mod h1:agKWTQ9ECSzQ++7P/9viSxJnG1Kp+WhfDyI3pAmtnVM= +github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793 h1:KFtyLpBPbMyUdeCth/Zcej/SSgAFIo6fxdS2eEPEg3I= +github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc= github.com/trustwallet/go-primitives v0.0.17 h1:1fBxZMKGCHdHtgdUzsqdFlD21+1GneIk/sxN6jxYBds= github.com/trustwallet/go-primitives v0.0.17/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 000000000..031cf2c90 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,57 @@ +package config + +import ( + "path/filepath" + + "github.com/trustwallet/go-libs/config/viper" +) + +type ( + Config struct { + App App `mapstructure:"app"` + ClientURLs ClientsURLs `mapstructure:"client_urls"` + URLs URLs `mapstructure:"urls"` + ValidatorsSettings ValidatorsSettings `mapstructure:"validators_settings"` + } + + App struct { + LogLevel string `mapstructure:"log_level"` + } + + ClientsURLs struct { + Binance struct { + Dex string `mapstructure:"dex"` + Explorer string `mapstructure:"explorer"` + } `mapstructure:"binance"` + BackendAPI string `mapstructure:"backend_api"` + } + + URLs struct { + TWAssetsApp string `mapstructure:"tw_assets_app"` + } + + ValidatorsSettings struct { + RootFolder RootFolder `mapstructure:"root_folder"` + ChainFolder ChainFolder `mapstructure:"chain_folder"` + AssetFolder AssetFolder `mapstructure:"asset_folder"` + ChainInfoFolder ChainInfoFolder `mapstructure:"chain_info_folder"` + ChainValidatorsAssetFolder ChainValidatorsAssetFolder `mapstructure:"chain_validators_asset_folder"` + DappsFolder DappsFolder `mapstructure:"dapps_folder"` + CoinInfoFile CoinInfoFile `mapstructure:"coin_info_file"` + } +) + +// Default is a configuration instance. +var Default = Config{} // nolint:gochecknoglobals // config must be global + +// SetConfig reads a config file and returs an initialized config instance. +func SetConfig(confPath string) error { + confPath, err := filepath.Abs(confPath) + if err != nil { + return err + } + + viper.Load(confPath, &Default) + + return nil +} diff --git a/internal/config/validators.go b/internal/config/validators.go new file mode 100644 index 000000000..6f1eb6268 --- /dev/null +++ b/internal/config/validators.go @@ -0,0 +1,36 @@ +package config + +type RootFolder struct { + AllowedFiles []string `mapstructure:"allowed_files,omitempty"` + SkipFiles []string `mapstructure:"skip_files,omitempty"` +} + +type ChainFolder struct { + AllowedFiles []string `mapstructure:"allowed_files,omitempty"` +} + +type AssetFolder struct { + AllowedFiles []string `mapstructure:"allowed_files,omitempty"` +} + +type ChainInfoFolder struct { + HasFiles []string `mapstructure:"has_files,omitempty"` +} + +type ChainValidatorsAssetFolder struct { + HasFiles []string `mapstructure:"has_files,omitempty"` +} + +type DappsFolder struct { + Ext string `mapstructure:"ext,omitempty"` +} + +type CoinInfoFile struct { + Tags []Tag `mapstructure:"tags,omitempty"` +} + +type Tag struct { + ID string `mapstructure:"id,omitempty"` + Name string `mapstructure:"name,omitempty"` + Description string `mapstructure:"description,omitempty"` +} diff --git a/internal/config/values.go b/internal/config/values.go new file mode 100644 index 000000000..aef4b7523 --- /dev/null +++ b/internal/config/values.go @@ -0,0 +1,15 @@ +package config + +import "github.com/trustwallet/go-primitives/coin" + +// TODO: Move to go-libs. +var StackingChains = []coin.Coin{ + coin.Tezos(), + coin.Cosmos(), + coin.Iotex(), + coin.Tron(), + coin.Waves(), + coin.Kava(), + coin.Terra(), + coin.Binance(), +} diff --git a/internal/file/cache.go b/internal/file/cache.go new file mode 100644 index 000000000..277db0a7f --- /dev/null +++ b/internal/file/cache.go @@ -0,0 +1,58 @@ +package file + +import ( + "path/filepath" + "strings" + "sync" +) + +type Service struct { + mu *sync.RWMutex + cache map[string]*AssetFile +} + +func NewService(filePaths ...string) *Service { + var filesMap = make(map[string]*AssetFile) + + for _, path := range filePaths { + assetFile := NewAssetFile(path) + filesMap[path] = assetFile + } + + return &Service{ + mu: &sync.RWMutex{}, + cache: filesMap, + } +} + +func (f *Service) GetAssetFile(path string) *AssetFile { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.getFile(path) +} + +func (f *Service) UpdateFile(file *AssetFile, newFileBaseName string) { + f.mu.RLock() + defer f.mu.RUnlock() + + oldFileBaseName := filepath.Base(file.Path()) + + for path := range f.cache { + if strings.Contains(path, oldFileBaseName) { + newPath := strings.ReplaceAll(path, oldFileBaseName, newFileBaseName) + f.cache[path] = NewAssetFile(newPath) + } + } +} + +func (f *Service) getFile(path string) *AssetFile { + if file, exists := f.cache[path]; exists { + return file + } + + assetF := NewAssetFile(path) + f.cache[path] = assetF + + return assetF +} diff --git a/internal/file/file.go b/internal/file/file.go new file mode 100644 index 000000000..a274e2028 --- /dev/null +++ b/internal/file/file.go @@ -0,0 +1,29 @@ +package file + +import ( + "github.com/trustwallet/go-primitives/coin" +) + +type AssetFile struct { + path *Path +} + +func NewAssetFile(path string) *AssetFile { + return &AssetFile{path: NewPath(path)} +} + +func (i *AssetFile) Path() string { + return i.path.String() +} + +func (i *AssetFile) Type() string { + return i.path.fileType +} + +func (i *AssetFile) Chain() coin.Coin { + return i.path.chain +} + +func (i *AssetFile) Asset() string { + return i.path.asset +} diff --git a/internal/file/path.go b/internal/file/path.go new file mode 100644 index 000000000..16d576f85 --- /dev/null +++ b/internal/file/path.go @@ -0,0 +1,161 @@ +package file + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/trustwallet/assets-go-libs/pkg" + "github.com/trustwallet/go-primitives/coin" +) + +var ( + regexAssetInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/info.json$`) + regexAssetLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/logo.png$`) + + regexChainInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/info.json$`) + regexChainLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/logo.png$`) + + regexTokenListFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/tokenlist.json$`) + + regexValidatorsAssetLogo = regexp.MustCompile( + `./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)/logo.png$`) + regexValidatorsList = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/list.json$`) + + regexDappsLogo = regexp.MustCompile(`./dapps/[a-zA-Z-.]+\.png$`) +) + +var ( + regexAssetFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)$`) + regexAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets$`) + + regexValidatorsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators$`) + regexValidatorsAssetFolder = regexp.MustCompile( + `./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)$`) + regexValidatorsAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/assets$`) + + regexChainFolder = regexp.MustCompile(`./blockchains/(\w+[^/])$`) + regexChainInfoFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info$`) + regexChainsFolder = regexp.MustCompile(`./blockchains$`) + + regexDappsFolder = regexp.MustCompile(`./dapps$`) + regexRoot = regexp.MustCompile(`./$`) +) + +var regexes = map[string]*regexp.Regexp{ + TypeAssetInfoFile: regexAssetInfoFile, + TypeAssetLogoFile: regexAssetLogoFile, + + TypeChainInfoFile: regexChainInfoFile, + TypeChainLogoFile: regexChainLogoFile, + + TypeTokenListFile: regexTokenListFile, + + TypeValidatorsListFile: regexValidatorsList, + TypeValidatorsLogoFile: regexValidatorsAssetLogo, + + TypeDappsLogoFile: regexDappsLogo, + + TypeAssetFolder: regexAssetFolder, + TypeAssetsFolder: regexAssetsFolder, + + TypeChainFolder: regexChainFolder, + TypeChainsFolder: regexChainsFolder, + TypeChainInfoFolder: regexChainInfoFolder, + + TypeDappsFolder: regexDappsFolder, + TypeRootFolder: regexRoot, + + TypeValidatorsFolder: regexValidatorsFolder, + TypeValidatorsAssetsFolder: regexValidatorsAssetsFolder, + TypeValidatorsAssetFolder: regexValidatorsAssetFolder, +} + +type Path struct { + path string + chain coin.Coin + asset string + fileType string +} + +func NewPath(path string) *Path { + p := Path{path: path} + + fileType, reg := defineFileType(path) + if reg == nil { + p.fileType = TypeUnknown + + return &p + } + + match := reg.FindStringSubmatch(path) + if fileType != TypeUnknown { + p.fileType = fileType + } + + if len(match) >= 2 { + chain, err := coin.GetCoinForId(match[1]) + if err != nil { + p.chain = coin.Coin{Handle: match[1]} + } else { + p.chain = chain + } + } + + if len(match) == 3 { + p.asset = match[2] + } + + return &p +} + +func (p Path) Type() string { + return p.fileType +} + +func (p Path) String() string { + return p.path +} + +func (p Path) Chain() coin.Coin { + return p.chain +} + +func (p Path) Asset() string { + return p.asset +} + +func defineFileType(p string) (string, *regexp.Regexp) { + for t, r := range regexes { + if r.MatchString(p) { + return t, r + } + } + + return TypeUnknown, nil +} + +func ReadLocalFileStructure(root string, filesToSkip []string) ([]string, error) { + var paths = []string{"./"} + err := filepath.Walk(root, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if pkg.Contains(path, filesToSkip) { + return nil + } + + paths = append(paths, fmt.Sprintf("./%s", path)) + + return nil + }) + + if err != nil { + return nil, err + } + + return paths, nil +} diff --git a/internal/file/types.go b/internal/file/types.go new file mode 100644 index 000000000..90943468b --- /dev/null +++ b/internal/file/types.go @@ -0,0 +1,25 @@ +package file + +const ( + TypeAssetInfoFile = "asset_info" + TypeAssetLogoFile = "asset_logo" + TypeChainInfoFile = "chain_info" + TypeChainLogoFile = "chain_logo" + TypeTokenListFile = "chain_token_list" + TypeValidatorsListFile = "validators_list" + TypeValidatorsLogoFile = "validators_logo" + TypeDappsLogoFile = "dapps_logo" + + TypeAssetFolder = "asset" + TypeAssetsFolder = "assets" + TypeChainFolder = "chain" + TypeChainsFolder = "chains" + TypeDappsFolder = "dapps" + TypeRootFolder = "root" + TypeChainInfoFolder = "chain_info_folder" + TypeValidatorsFolder = "validators" + TypeValidatorsAssetFolder = "validators_asset" + TypeValidatorsAssetsFolder = "validators_assets" + + TypeUnknown = "unknown" +) diff --git a/internal/processor/fixers.go b/internal/processor/fixers.go new file mode 100644 index 000000000..eefe51aa4 --- /dev/null +++ b/internal/processor/fixers.go @@ -0,0 +1,171 @@ +package processor + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/trustwallet/assets-go-libs/pkg" + "github.com/trustwallet/assets-go-libs/pkg/validation" + "github.com/trustwallet/assets-go-libs/pkg/validation/info" + "github.com/trustwallet/assets/internal/file" + "github.com/trustwallet/go-primitives/address" + "github.com/trustwallet/go-primitives/coin" + "github.com/trustwallet/go-primitives/types" + + log "github.com/sirupsen/logrus" +) + +func (s *Service) FixJSON(f *file.AssetFile) error { + return pkg.FormatJSONFile(f.Path()) +} + +func (s *Service) FixETHAddressChecksum(f *file.AssetFile) error { + if !coin.IsEVM(f.Chain().ID) { + return nil + } + + assetDir := filepath.Base(f.Path()) + + err := validation.ValidateETHForkAddress(f.Chain(), assetDir) + if err != nil { + checksum, e := address.EIP55Checksum(assetDir) + if e != nil { + return fmt.Errorf("failed to get checksum: %s", e) + } + + newName := fmt.Sprintf("blockchains/%s/assets/%s", f.Chain().Handle, checksum) + + if e = os.Rename(f.Path(), newName); e != nil { + return fmt.Errorf("failed to rename dir: %s", e) + } + + s.fileService.UpdateFile(f, checksum) + + log.WithField("from", assetDir). + WithField("to", checksum). + Debug("Renamed asset") + } + + return nil +} + +func (s *Service) FixLogo(f *file.AssetFile) error { + width, height, err := pkg.GetPNGImageDimensions(f.Path()) + if err != nil { + return err + } + + var isLogoTooLarge bool + if width > validation.MaxW || height > validation.MaxH { + isLogoTooLarge = true + } + + if isLogoTooLarge { + log.WithField("path", f.Path()).Debug("Fixing too large image") + + targetW, targetH := calculateTargetDimension(width, height) + + err = pkg.ResizePNG(f.Path(), targetW, targetH) + if err != nil { + return err + } + } + + err = validation.ValidateLogoFileSize(f.Path()) + if err != nil { // nolint:staticcheck + // TODO: Compress images. + } + + return nil +} + +func calculateTargetDimension(width, height int) (targetW, targetH int) { + widthFloat := float32(width) + heightFloat := float32(height) + + maxEdge := widthFloat + if heightFloat > widthFloat { + maxEdge = heightFloat + } + + ratio := validation.MaxW / maxEdge + + targetW = int(widthFloat * ratio) + targetH = int(heightFloat * ratio) + + return targetW, targetH +} + +func (s *Service) FixChainInfoJSON(f *file.AssetFile) error { + chainInfo := info.CoinModel{} + + err := pkg.ReadJSONFile(f.Path(), &chainInfo) + if err != nil { + return err + } + + expectedType := string(types.Coin) + if chainInfo.Type == nil || *chainInfo.Type != expectedType { + chainInfo.Type = &expectedType + + return pkg.CreateJSONFile(f.Path(), &chainInfo) + } + + return nil +} + +func (s *Service) FixAssetInfoJSON(file *file.AssetFile) error { + assetInfo := info.AssetModel{} + + err := pkg.ReadJSONFile(file.Path(), &assetInfo) + if err != nil { + return err + } + + var isModified bool + + // Fix asset type. + var assetType string + if assetInfo.Type != nil { + assetType = *assetInfo.Type + } + + // We need to skip error check to fix asset type if it's incorrect or empty. + chain, _ := types.GetChainFromAssetType(assetType) + + expectedTokenType, ok := types.GetTokenType(file.Chain().ID, file.Asset()) + if !ok { + expectedTokenType = strings.ToUpper(assetType) + } + + if chain.ID != file.Chain().ID || !strings.EqualFold(assetType, expectedTokenType) { + assetInfo.Type = &expectedTokenType + isModified = true + } + + // Fix asset id. + assetID := file.Asset() + if assetInfo.ID == nil || *assetInfo.ID != assetID { + assetInfo.ID = &assetID + isModified = true + } + + expectedExplorerURL, err := coin.GetCoinExploreURL(file.Chain(), file.Asset()) + if err != nil { + return err + } + + // Fix asset explorer url. + if assetInfo.Explorer == nil || !strings.EqualFold(expectedExplorerURL, *assetInfo.Explorer) { + assetInfo.Explorer = &expectedExplorerURL + isModified = true + } + + if isModified { + return pkg.CreateJSONFile(file.Path(), &assetInfo) + } + + return nil +} diff --git a/internal/processor/model.go b/internal/processor/model.go new file mode 100644 index 000000000..0e65aa3af --- /dev/null +++ b/internal/processor/model.go @@ -0,0 +1,82 @@ +package processor + +import "github.com/trustwallet/assets/internal/file" + +type ( + Validator struct { + Name string + Run func(f *file.AssetFile) error + } + + Fixer struct { + Name string + Run func(f *file.AssetFile) error + } + + Updater struct { + Name string + Run func() error + } +) + +type ( + TokenList struct { + Name string `json:"name"` + LogoURI string `json:"logoURI"` + Timestamp string `json:"timestamp"` + Tokens []TokenItem `json:"tokens"` + Version Version `json:"version"` + } + + TokenItem struct { + Asset string `json:"asset"` + Type string `json:"type"` + Address string `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals uint `json:"decimals"` + LogoURI string `json:"logoURI"` + Pairs []Pair `json:"pairs"` + } + + Pair struct { + Base string `json:"base"` + LotSize string `json:"lotSize,omitempty"` + TickSize string `json:"tickSize,omitempty"` + } + + Version struct { + Major int `json:"major"` + Minor int `json:"minor"` + Patch int `json:"patch"` + } +) + +type ( + ForceListPair struct { + Token0 string + Token1 string + } + + TradingPairs struct { + Data struct { + Pairs []TradingPair `json:"pairs"` + } `json:"data"` + } + + TradingPair struct { + ID string `json:"id"` + ReserveUSD string `json:"reserveUSD"` + VolumeUSD string `json:"volumeUSD"` + TxCount string `json:"txCount"` + Token0 *TokenInfo `json:"token0"` + Token1 *TokenInfo `json:"token1"` + } + + TokenInfo struct { + ID string `json:"id"` + Symbol string `json:"symbol"` + Name string `json:"name"` + Decimals string `json:"decimals"` + } +) diff --git a/internal/processor/processor.go b/internal/processor/processor.go new file mode 100644 index 000000000..5e2bf98ce --- /dev/null +++ b/internal/processor/processor.go @@ -0,0 +1,156 @@ +package processor + +import ( + "github.com/trustwallet/assets/internal/file" +) + +type Service struct { + fileService *file.Service +} + +func NewService(fileProvider *file.Service) *Service { + return &Service{fileService: fileProvider} +} + +// nolint:funlen +func (s *Service) GetValidator(f *file.AssetFile) *Validator { + switch f.Type() { + case file.TypeRootFolder: + return &Validator{ + Name: "Root folder contains only allowed files", + Run: s.ValidateRootFolder, + } + case file.TypeChainFolder: + return &Validator{ + Name: "Chain folders are lowercase and contains only allowed files", + Run: s.ValidateChainFolder, + } + case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile: + return &Validator{ + Name: "Logos (size, dimension)", + Run: s.ValidateImage, + } + case file.TypeAssetFolder: + return &Validator{ + Name: "Each asset folder has valid asset address and contains logo/info", + Run: s.ValidateAssetFolder, + } + case file.TypeDappsFolder: + return &Validator{ + Name: "Dapps folder (allowed only png files, lowercase)", + Run: s.ValidateDappsFolder, + } + case file.TypeAssetInfoFile: + return &Validator{ + Name: "Asset info (is valid json, fields)", + Run: s.ValidateAssetInfoFile, + } + case file.TypeChainInfoFile: + return &Validator{ + Name: "Chain Info (is valid json, fields)", + Run: s.ValidateChainInfoFile, + } + case file.TypeValidatorsListFile: + return &Validator{ + Name: "Validators list file", + Run: s.ValidateValidatorsListFile, + } + case file.TypeTokenListFile: + return &Validator{ + Name: "Token list (if assets from list present in chain)", + Run: s.ValidateTokenListFile, + } + case file.TypeChainInfoFolder: + return &Validator{ + Name: "Chain Info Folder (has files)", + Run: s.ValidateInfoFolder, + } + case file.TypeValidatorsAssetFolder: + return &Validator{ + Name: "Validators asset folder (has logo, valid asset address)", + Run: s.ValidateValidatorsAssetFolder, + } + } + + return nil +} + +func (s *Service) GetFixers(f *file.AssetFile) []Fixer { + infoFixer := Fixer{ + Name: "Formatting all info.json files", + Run: s.FixJSON, + } + + ethAssetFixer := Fixer{ + Name: "Renaming EVM's asset folder to valid address checksum", + Run: s.FixETHAddressChecksum, + } + + logoFixer := Fixer{ + Name: "Resizing and compressing logo images", + Run: s.FixLogo, + } + + chainInfoFixer := Fixer{ + Name: "Fixing chain info.json files", + Run: s.FixChainInfoJSON, + } + + assetInfoFixer := Fixer{ + Name: "Fixing asset info.json files", + Run: s.FixAssetInfoJSON, + } + + switch f.Type() { + case file.TypeChainInfoFile: + return []Fixer{ + infoFixer, + chainInfoFixer, + } + case file.TypeAssetInfoFile: + return []Fixer{ + infoFixer, + assetInfoFixer, + } + case file.TypeValidatorsListFile: + return []Fixer{ + infoFixer, + } + case file.TypeAssetFolder: + return []Fixer{ + ethAssetFixer, + } + case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile: + return []Fixer{ + logoFixer, + } + } + + return nil +} + +func (s *Service) GetUpdatersAuto() []Updater { + return []Updater{ + { + Name: "Retrieving missing token images, creating binance token list.", + Run: s.UpdateBinanceTokens, + }, + } +} + +func (s *Service) GetUpdatersManual() []Updater { + return []Updater{ + { + Name: "Update tokenlist.json for Ethereum", + Run: s.UpdateEthereumTokenlist, + }, + { + Name: "Update tokenlist.json for Polygon", + Run: s.UpdatePolygonTokenlist, + }, + { + Name: "Update tokenlist.json for Smartchain", + Run: s.UpdateSmartchainTokenlist, + }, + } +} diff --git a/internal/processor/updaters_auto.go b/internal/processor/updaters_auto.go new file mode 100644 index 000000000..d605b98ac --- /dev/null +++ b/internal/processor/updaters_auto.go @@ -0,0 +1,261 @@ +package processor + +import ( + "fmt" + "reflect" + "sort" + "strconv" + "time" + + "github.com/trustwallet/assets-go-libs/pkg" + "github.com/trustwallet/assets-go-libs/pkg/asset" + "github.com/trustwallet/assets-go-libs/pkg/validation/info" + "github.com/trustwallet/assets/internal/config" + "github.com/trustwallet/go-libs/blockchain/binance" + "github.com/trustwallet/go-libs/blockchain/binance/explorer" + assetlib "github.com/trustwallet/go-primitives/asset" + "github.com/trustwallet/go-primitives/coin" + "github.com/trustwallet/go-primitives/numbers" + "github.com/trustwallet/go-primitives/types" +) + +const ( + assetsPage = 1 + assetsRows = 1000 + marketPairsLimit = 1000 + tokensListLimit = 10000 + + twLogoURL = "https://trustwallet.com/assets/images/favicon.png" + timestampFormat = "2006-01-02T15:04:05.000000" +) + +func (s *Service) UpdateBinanceTokens() error { + explorerClient := explorer.InitClient(config.Default.ClientURLs.Binance.Explorer, nil) + + bep2AssetList, err := explorerClient.FetchBep2Assets(assetsPage, assetsRows) + if err != nil { + return err + } + + dexClient := binance.InitClient(config.Default.ClientURLs.Binance.Dex, "", nil) + + marketPairs, err := dexClient.FetchMarketPairs(marketPairsLimit) + if err != nil { + return err + } + + tokensList, err := dexClient.FetchTokens(tokensListLimit) + if err != nil { + return err + } + + chain, err := types.GetChainFromAssetType(string(types.BEP2)) + if err != nil { + return err + } + + err = fetchMissingAssets(chain, bep2AssetList.AssetInfoList) + if err != nil { + return err + } + + return createTokenListJSON(chain, marketPairs, tokensList) +} + +func fetchMissingAssets(chain coin.Coin, assets []explorer.Bep2Asset) error { + for _, a := range assets { + if a.AssetImg == "" || a.Decimals == 0 { + continue + } + + assetLogoPath := asset.GetAssetLogoPath(chain.Handle, a.Asset) + if pkg.FileExists(assetLogoPath) { + continue + } + + if err := createLogo(assetLogoPath, a); err != nil { + return err + } + + if err := createInfoJSON(chain, a); err != nil { + return err + } + } + + return nil +} + +func createLogo(assetLogoPath string, a explorer.Bep2Asset) error { + err := pkg.CreateDirPath(assetLogoPath) + if err != nil { + return err + } + + return pkg.CreatePNGFromURL(a.AssetImg, assetLogoPath) +} + +func createInfoJSON(chain coin.Coin, a explorer.Bep2Asset) error { + explorerURL, err := coin.GetCoinExploreURL(chain, a.Asset) + if err != nil { + return err + } + + assetType := string(types.BEP2) + website := "" + description := "-" + status := "active" + + assetInfo := info.AssetModel{ + Name: &a.Name, + Type: &assetType, + Symbol: &a.MappedAsset, + Decimals: &a.Decimals, + Website: &website, + Description: &description, + Explorer: &explorerURL, + Status: &status, + ID: &a.Asset, + } + + assetInfoPath := asset.GetAssetInfoPath(chain.Handle, a.Asset) + + return pkg.CreateJSONFile(assetInfoPath, &assetInfo) +} + +func createTokenListJSON(chain coin.Coin, marketPairs []binance.MarketPair, tokenList binance.Tokens) error { + tokens, err := generateTokenList(marketPairs, tokenList) + if err != nil { + return nil + } + + tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle) + + var oldTokenList TokenList + err = pkg.ReadJSONFile(tokenListPath, &oldTokenList) + if err != nil { + return nil + } + + sortTokens(tokens) + + if reflect.DeepEqual(oldTokenList.Tokens, tokens) { + return nil + } + + if len(tokens) > 0 { + return pkg.CreateJSONFile(tokenListPath, &TokenList{ + Name: fmt.Sprintf("Trust Wallet: %s", coin.Coins[coin.BINANCE].Symbol), + LogoURI: twLogoURL, + Timestamp: time.Now().Format(timestampFormat), + Tokens: tokens, + Version: Version{Major: oldTokenList.Version.Major + 1}, + }) + } + + return nil +} + +func sortTokens(tokens []TokenItem) { + sort.Slice(tokens, func(i, j int) bool { + if len(tokens[i].Pairs) != len(tokens[j].Pairs) { + return len(tokens[i].Pairs) > len(tokens[j].Pairs) + } + + return tokens[i].Address < tokens[j].Address + }) + + for _, token := range tokens { + sort.Slice(token.Pairs, func(i, j int) bool { + return token.Pairs[i].Base < token.Pairs[j].Base + }) + } +} + +func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]TokenItem, error) { + if len(marketPairs) < 5 { + return nil, fmt.Errorf("no markets info is returned from Binance DEX: %d", len(marketPairs)) + } + + if len(tokenList) < 5 { + return nil, fmt.Errorf("no tokens info is returned from Binance DEX: %d", len(tokenList)) + } + + pairsMap := make(map[string][]Pair) + pairsList := make(map[string]struct{}) + tokensMap := make(map[string]binance.Token) + + for _, token := range tokenList { + tokensMap[token.Symbol] = token + } + + for _, marketPair := range marketPairs { + key := marketPair.QuoteAssetSymbol + + if val, exists := pairsMap[key]; exists { + val = append(val, getPair(marketPair)) + pairsMap[key] = val + } else { + pairsMap[key] = []Pair{getPair(marketPair)} + } + + pairsList[marketPair.BaseAssetSymbol] = struct{}{} + pairsList[marketPair.QuoteAssetSymbol] = struct{}{} + } + + tokenItems := make([]TokenItem, 0, len(pairsList)) + + for pair := range pairsList { + token := tokensMap[pair] + + var pairs []Pair + pairs, exists := pairsMap[token.Symbol] + if !exists { + pairs = make([]Pair, 0) + } + + tokenItems = append(tokenItems, TokenItem{ + Asset: getAssetIDSymbol(token.Symbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE), + Type: getTokenType(token.Symbol, coin.Coins[coin.BINANCE].Symbol, string(types.BEP2)), + Address: token.Symbol, + Name: token.Name, + Symbol: token.OriginalSymbol, + Decimals: coin.Coins[coin.BINANCE].Decimals, + LogoURI: getLogoURI(token.Symbol, coin.Coins[coin.BINANCE].Handle, coin.Coins[coin.BINANCE].Symbol), + Pairs: pairs, + }) + } + + return tokenItems, nil +} + +func getPair(marketPair binance.MarketPair) Pair { + return Pair{ + Base: getAssetIDSymbol(marketPair.BaseAssetSymbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE), + LotSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.LotSize), 10), + TickSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.TickSize), 10), + } +} + +func getAssetIDSymbol(tokenID string, nativeCoinID string, coinType uint) string { + if tokenID == nativeCoinID { + return assetlib.BuildID(coinType, "") + } + + return assetlib.BuildID(coinType, tokenID) +} + +func getTokenType(symbol string, nativeCoinSymbol string, tokenType string) string { + if symbol == nativeCoinSymbol { + return "coin" + } + + return tokenType +} + +func getLogoURI(id, githubChainFolder, nativeCoinSymbol string) string { + if id == nativeCoinSymbol { + return fmt.Sprintf("%s/blockchains/%s/info/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder) + } + + return fmt.Sprintf("%s/blockchains/%s/assets/%s/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder, id) +} diff --git a/internal/processor/updaters_manual.go b/internal/processor/updaters_manual.go new file mode 100644 index 000000000..f1c49c6f3 --- /dev/null +++ b/internal/processor/updaters_manual.go @@ -0,0 +1,537 @@ +package processor + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/trustwallet/assets-go-libs/pkg" + "github.com/trustwallet/assets-go-libs/pkg/asset" + "github.com/trustwallet/assets/internal/config" + "github.com/trustwallet/go-libs/client/api/backend" + "github.com/trustwallet/go-primitives/address" + "github.com/trustwallet/go-primitives/coin" + "github.com/trustwallet/go-primitives/types" +) + +var ( + UniswapForceInclude = []string{ + "TUSD", "STAKE", "YFI", "BAT", "MANA", "1INCH", "REP", "KP3R", "UNI", "WBTC", "HEX", "CREAM", "SLP", + "REN", "XOR", "Link", "sUSD", "HEGIC", "RLC", "DAI", "SUSHI", "FYZ", "DYT", "AAVE", "LEND", "UBT", + "DIA", "RSR", "SXP", "OCEAN", "MKR", "USDC", "CEL", "BAL", "BAND", "COMP", "SNX", "OMG", "AMPL", + "USDT", "KNC", "ZRX", "AXS", "ENJ", "STMX", "DPX", "FTT", "DPI", "PAX", + } + UniswapForceExclude = []string{"STARL", "UFO"} + + PolygonSwapForceInclude = []string{} + PolygonSwapForceExclude = []string{} + + PancakeSwapForceInclude = []string{ + "Cake", "DAI", "ETH", "TWT", "VAI", "USDT", "BLINK", "BTCB", "ALPHA", "INJ", "CTK", "UNI", "XVS", + "BUSD", "HARD", "BIFI", "FRONT", + } + PancakeSwapForceExclude = []string{} +) + +var ( + UniswapTradingPairsQuery = map[string]string{ + "operationName": "pairs", + "query": ` + query pairs { + pairs(first: 800, orderBy: reserveUSD, orderDirection: desc) { + id reserveUSD trackedReserveETH volumeUSD txCount untrackedVolumeUSD __typename + token0 { + id symbol name decimals __typename + } + token1 { + id symbol name decimals __typename + } + } + } + `, + } + + PolygonSwap_TradingPairsQuery = map[string]string{ + "operationName": "pairs", + "query": ` + { + ethereum(network: matic) { + dexTrades(date: {is: "$DATE$"}) { + sellCurrency {address symbol name decimals} + buyCurrency {address symbol name decimals} + trade: count + tradeAmount(in: USD) + } + } + } + `, + } + + PancakeSwap_TradingPairsQuery = map[string]string{ + "operationName": "pairs", + "query": ` + query pairs { + pairs(first: 300, orderBy: reserveUSD, orderDirection: desc) { + id reserveUSD volumeUSD txCount __typename + token0 { + id symbol name decimals __typename + } + token1 { + id symbol name decimals __typename + } + } + } + `, + } + + UniswapTradingPairsUrl = "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2" + PolygonSwapTradingPairsUrl = "https://graphql.bitquery.io" + PancakeSwapradingPairsUrl = "https://api.bscgraph.org/subgraphs/name/cakeswap" +) + +const ( + UniswapMinLiquidity = 2000000 + UniswapMinVol24 = 1000000 + UniswapMinTxCount24 = 480 + + PolygonSwapMinVol24 = 500000 + PolygonSwapMinTxCount24 = 288 + + PancakeSwapMinLiquidity = 1000000 + PancakeSwapMinVol24 = 500000 + PancakeSwapMinTxCount24 = 288 +) + +var ( + PrimaryTokensETH = []string{"WETH", "ETH"} +) + +func (s *Service) UpdateEthereumTokenlist() error { + log.WithFields(log.Fields{ + "limit_liquidity": UniswapMinLiquidity, + "volume": UniswapMinVol24, + "tx_count": UniswapMinTxCount24, + }).Debug("Retrieving pairs from Uniswap") + + tradingPairs, err := retrieveUniswapPairs(UniswapTradingPairsUrl, UniswapTradingPairsQuery, + UniswapMinLiquidity, UniswapMinVol24, UniswapMinTxCount24, UniswapForceInclude, PrimaryTokensETH) + if err != nil { + return err + } + + pairs := make([][]TokenItem, 0) + + for _, tradingPair := range tradingPairs { + tokenItem0, err := getTokenInfoFromSubgraphToken(tradingPair.Token0) + if err != nil { + return err + } + + tokenItem1, err := getTokenInfoFromSubgraphToken(tradingPair.Token1) + if err != nil { + return err + } + + if !isTokenPrimary(tradingPair.Token0, PrimaryTokensETH) { + tokenItem0, tokenItem1 = tokenItem1, tokenItem0 + } + + pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1}) + } + + return rebuildTokenList(coin.Coins[coin.ETHEREUM], pairs, UniswapForceExclude) +} + +func (s *Service) UpdatePolygonTokenlist() error { + return nil +} + +func (s *Service) UpdateSmartchainTokenlist() error { + return nil +} + +func retrieveUniswapPairs(url string, query map[string]string, minLiquidity, minVol24, minTxCount24 int, + forceIncludeList []string, primaryTokens []string) ([]TradingPair, error) { + includeList := parseForceList(forceIncludeList) + + pairs, err := fetchTradingPairs(url, query) + if err != nil { + return nil, err + } + + filtered := make([]TradingPair, 0) + + for _, pair := range pairs.Data.Pairs { + ok, err := checkTradingPairOK(pair, minLiquidity, minVol24, minTxCount24, primaryTokens, includeList) + if err != nil { + log.Debug(err) + } + if ok { + filtered = append(filtered, pair) + } + } + + return filtered, nil +} + +func parseForceList(forceList []string) []ForceListPair { + result := make([]ForceListPair, 0, len(forceList)) + + for _, item := range forceList { + tokens := strings.Split(item, "-") + pair := ForceListPair{ + Token0: tokens[0], + } + + if len(tokens) >= 2 { + pair.Token1 = tokens[1] + } + + result = append(result, pair) + } + + return result +} + +func fetchTradingPairs(url string, query map[string]string) (*TradingPairs, error) { + jsonValue, err := json.Marshal(query) + if err != nil { + return nil, err + } + + log.WithField("url", url).Debug("Retrieving trading pair infos") + + var result TradingPairs + err = pkg.PostHTTPResponse(url, jsonValue, &result) + if err != nil { + return nil, err + } + + log.Debugf("Retrieved %d trading pair infos", len(result.Data.Pairs)) + + return &result, nil +} + +func checkTradingPairOK(pair TradingPair, minLiquidity, minVol24, minTxCount24 int, primaryTokens []string, + forceIncludeList []ForceListPair) (bool, error) { + if pair.ID == "" || pair.ReserveUSD == "" || pair.VolumeUSD == "" || pair.TxCount == "" || + pair.Token0 == nil || pair.Token1 == nil { + + return false, nil + } + + if !(isTokenPrimary(pair.Token0, primaryTokens) || isTokenPrimary(pair.Token1, primaryTokens)) { + log.Debugf("pair with no primary coin: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) + return false, nil + } + + if isPairMatchedToForceList(getTokenItemFromInfo(pair.Token0), getTokenItemFromInfo(pair.Token1), forceIncludeList) { + log.Debugf("pair included due to FORCE INCLUDE: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) + return true, nil + } + + reserveUSD, err := strconv.ParseFloat(pair.ReserveUSD, 64) + if err != nil { + return false, err + } + if int(reserveUSD) < minLiquidity { + log.Debugf("pair with low liquidity: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) + return false, nil + } + + volumeUSD, err := strconv.ParseFloat(pair.VolumeUSD, 64) + if err != nil { + return false, err + } + if int(volumeUSD) < minVol24 { + log.Debugf("pair with low volume: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) + return false, nil + } + + txCount, err := strconv.ParseFloat(pair.TxCount, 64) + if err != nil { + return false, err + } + if int(txCount) < minTxCount24 { + log.Debugf("pair with low tx count: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol) + return false, nil + } + + return true, nil +} + +func getTokenItemFromInfo(tokenInfo *TokenInfo) *TokenItem { + decimals, err := strconv.Atoi(tokenInfo.Decimals) + if err != nil { + return nil + } + + return &TokenItem{ + Asset: tokenInfo.ID, + Address: tokenInfo.ID, + Name: tokenInfo.Name, + Symbol: tokenInfo.Symbol, + Decimals: uint(decimals), + } +} + +func getTokenInfoFromSubgraphToken(token *TokenInfo) (*TokenItem, error) { + checksum, err := address.EIP55Checksum(token.ID) + if err != nil { + return nil, err + } + + decimals, err := strconv.Atoi(token.Decimals) + if err != nil { + return nil, err + } + + return &TokenItem{ + Asset: getAssetIDSymbol(checksum, coin.Coins[coin.ETHEREUM].Symbol, coin.ETHEREUM), + Type: string(types.ERC20), + Address: checksum, + Name: token.Name, + Symbol: token.Symbol, + Decimals: uint(decimals), + LogoURI: getLogoURI(token.Symbol, coin.Coins[coin.ETHEREUM].Handle, coin.Coins[coin.ETHEREUM].Symbol), + Pairs: make([]Pair, 0), + }, nil +} + +func isTokenPrimary(token *TokenInfo, primaryTokens []string) bool { + if token == nil { + return false + } + + for _, primaryToken := range primaryTokens { + if strings.EqualFold(primaryToken, token.Symbol) { + return true + } + } + + return false +} + +func isPairMatchedToForceList(token0, token1 *TokenItem, forceIncludeList []ForceListPair) bool { + var matched bool + + for _, forcePair := range forceIncludeList { + if matchPairToForceListEntry(token0, token1, forcePair) { + matched = true + } + } + + return matched +} + +func matchPairToForceListEntry(token0, token1 *TokenItem, forceListEntry ForceListPair) bool { + if forceListEntry.Token1 == "" { + if matchTokenToForceListEntry(token0, forceListEntry.Token0) || + (token1 != nil && matchTokenToForceListEntry(token1, forceListEntry.Token0)) { + + return true + } + + return false + } + + if token1 == nil { + return false + } + + if matchTokenToForceListEntry(token0, forceListEntry.Token0) && + matchTokenToForceListEntry(token0, forceListEntry.Token1) { + + return true + } + + if matchTokenToForceListEntry(token0, forceListEntry.Token1) && + matchTokenToForceListEntry(token1, forceListEntry.Token0) { + + return true + } + + return false +} + +func matchTokenToForceListEntry(token *TokenItem, forceListEntry string) bool { + if strings.EqualFold(forceListEntry, token.Symbol) || + strings.EqualFold(forceListEntry, token.Asset) || + strings.EqualFold(forceListEntry, token.Name) { + + return true + } + + return false +} + +func matchPairToForceList(token0, token1 *TokenItem, forceList []ForceListPair) bool { + var matched bool + for _, forcePair := range forceList { + if matchPairToForceListEntry(token0, token1, forcePair) { + matched = true + } + } + + return matched +} + +func rebuildTokenList(chain coin.Coin, pairs [][]TokenItem, forceExcludeList []string) error { + if pairs == nil || len(pairs) < 5 { + return nil + } + + excludeList := parseForceList(forceExcludeList) + + pairs2 := make([][]TokenItem, 0) + + for _, pair := range pairs { + if !checkTokenExists(chain.Handle, pair[0].Address) { + log.Debugf("pair with unsupported 1st coin: %s-%s", pair[0].Symbol, pair[1].Symbol) + continue + } + + if !checkTokenExists(chain.Handle, pair[1].Address) { + log.Debugf("pair with unsupported 2nd coin: %s-%s", pair[0].Symbol, pair[1].Symbol) + continue + } + + if matchPairToForceList(&pair[0], &pair[1], excludeList) { + log.Debugf("pair excluded due to FORCE EXCLUDE: %s-%s", pair[0].Symbol, pair[1].Symbol) + continue + } + + pairs2 = append(pairs2, pair) + } + + filteredCount := len(pairs) - len(pairs2) + + log.Debugf("%d unsupported tokens filtered out, %d pairs", filteredCount, len(pairs2)) + + tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle) + + var list TokenList + err := pkg.ReadJSONFile(tokenListPath, &list) + if err != nil { + return nil + } + + removeAllPairs(&list) + + for _, pair := range pairs2 { + err = addPairIfNeeded(&pair[0], &pair[1], &list) + if err != nil { + return err + } + } + + log.Debugf("Tokenlist updated: %d tokens", len(list.Tokens)) + + var totalPairs int + + for _, item := range list.Tokens { + totalPairs += len(item.Pairs) + } + + log.Debugf("Tokenlist: list with %d tokens and %d pairs written to %s.", + len(list.Tokens), totalPairs, tokenListPath) + + return pkg.CreateJSONFile(tokenListPath, list) +} + +func checkTokenExists(chain, tokenID string) bool { + logoPath := asset.GetAssetLogoPath(chain, tokenID) + + return pkg.FileExists(logoPath) +} + +func removeAllPairs(list *TokenList) { + for _, token := range list.Tokens { + token.Pairs = make([]Pair, 0) + } +} + +func addPairIfNeeded(token0, token1 *TokenItem, list *TokenList) error { + err := addTokenIfNeeded(token0, list) + if err != nil { + return err + } + + err = addTokenIfNeeded(token1, list) + if err != nil { + return err + } + + addPairToToken(token1, token0, list) + + return nil +} + +func addTokenIfNeeded(token *TokenItem, list *TokenList) error { + for _, t := range list.Tokens { + if strings.EqualFold(t.Address, token.Address) { + return nil + } + } + + err := updateTokenInfo(token) + if err != nil { + return err + } + + list.Tokens = append(list.Tokens, *token) + + return nil +} + +func updateTokenInfo(token *TokenItem) error { + backendClient := backend.InitClient(config.Default.ClientURLs.BackendAPI, nil) + assetInfo, err := backendClient.GetAssetInfo(token.Asset) + if err != nil { + return err + } + + if token.Name != assetInfo.Name { + token.Name = assetInfo.Name + } + + if token.Symbol != assetInfo.Symbol { + token.Symbol = assetInfo.Symbol + } + + if token.Decimals != uint(assetInfo.Decimals) { + token.Decimals = uint(assetInfo.Decimals) + } + + return nil +} + +func addPairToToken(pairToken, token *TokenItem, list *TokenList) { + var tokenInListIndex = -1 + + for i, t := range list.Tokens { + if t.Address == token.Address { + tokenInListIndex = i + break + } + } + + if tokenInListIndex == -1 { + return + } + + if list.Tokens[tokenInListIndex].Pairs == nil { + list.Tokens[tokenInListIndex].Pairs = make([]Pair, 0) + } + + for _, pair := range list.Tokens[tokenInListIndex].Pairs { + if pair.Base == pairToken.Asset { + return + } + } + + list.Tokens[tokenInListIndex].Pairs = append(list.Tokens[tokenInListIndex].Pairs, Pair{Base: pairToken.Asset}) +} diff --git a/internal/processor/validators.go b/internal/processor/validators.go new file mode 100644 index 000000000..642848d99 --- /dev/null +++ b/internal/processor/validators.go @@ -0,0 +1,407 @@ +package processor + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/trustwallet/assets-go-libs/pkg/validation" + "github.com/trustwallet/assets-go-libs/pkg/validation/info" + "github.com/trustwallet/assets-go-libs/pkg/validation/list" + "github.com/trustwallet/assets/internal/config" + "github.com/trustwallet/assets/internal/file" + "github.com/trustwallet/go-primitives/coin" +) + +func (s *Service) ValidateRootFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.RootFolder.AllowedFiles) + if err != nil { + return err + } + + return nil +} + +func (s *Service) ValidateChainFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return err + } + + var compErr = validation.NewErrComposite() + + err = validation.ValidateLowercase(fileInfo.Name()) + if err != nil { + compErr.Append(err) + } + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.ChainFolder.AllowedFiles) + if err != nil { + compErr.Append(err) + } + + if compErr.Len() > 0 { + return compErr + } + + return nil +} + +func (s *Service) ValidateImage(f *file.AssetFile) error { + var compErr = validation.NewErrComposite() + + err := validation.ValidateLogoFileSize(f.Path()) + if err != nil { + compErr.Append(err) + } + + // TODO: Replace it with validation.ValidatePngImageDimension when "assets" repo is fixed. + // Read comments inValidatePngImageDimensionForCI. + err = validation.ValidatePngImageDimensionForCI(f.Path()) + if err != nil { + compErr.Append(err) + } + + if compErr.Len() > 0 { + return compErr + } + + return nil +} + +func (s *Service) ValidateAssetFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + var compErr = validation.NewErrComposite() + + err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.AssetFolder.AllowedFiles) + if err != nil { + compErr.Append(err) + } + + err = validation.ValidateAssetAddress(f.Chain(), f.Asset()) + if err != nil { + compErr.Append(err) + } + + errInfo := validation.ValidateHasFiles(dirFiles, []string{"info.json"}) + errLogo := validation.ValidateHasFiles(dirFiles, []string{"logo.png"}) + + if errLogo != nil || errInfo != nil { + infoFile := s.fileService.GetAssetFile(fmt.Sprintf("%s/info.json", f.Path())) + + file2, err := os.Open(infoFile.Path()) + if err != nil { + return err + } + defer file2.Close() + + _, err = file2.Seek(0, io.SeekStart) + if err != nil { + return err + } + + b, err := io.ReadAll(file2) + if err != nil { + return err + } + + var infoJson info.AssetModel + err = json.Unmarshal(b, &infoJson) + if err != nil { + return err + } + + if infoJson.GetStatus() != "spam" && infoJson.GetStatus() != "abandoned" { + compErr.Append(fmt.Errorf("%w: logo.png for non-spam assest", validation.ErrMissingFile)) + } + } + + if compErr.Len() > 0 { + return compErr + } + + return nil +} + +func (s *Service) ValidateDappsFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + var compErr = validation.NewErrComposite() + for _, dirFile := range dirFiles { + err = validation.ValidateExtension(dirFile.Name(), config.Default.ValidatorsSettings.DappsFolder.Ext) + if err != nil { + compErr.Append(err) + } + + err = validation.ValidateLowercase(dirFile.Name()) + if err != nil { + compErr.Append(err) + } + } + + if compErr.Len() > 0 { + return compErr + } + + return nil +} + +func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + buf := bytes.NewBuffer(nil) + _, err = buf.ReadFrom(file) + if err != nil { + return err + } + + err = validation.ValidateJson(buf.Bytes()) + if err != nil { + return err + } + + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson) + } + + var payload info.CoinModel + err = json.Unmarshal(buf.Bytes(), &payload) + if err != nil { + return fmt.Errorf("%w: failed to decode", err) + } + + tags := make([]string, len(config.Default.ValidatorsSettings.CoinInfoFile.Tags)) + for i, t := range config.Default.ValidatorsSettings.CoinInfoFile.Tags { + tags[i] = t.ID + } + + err = info.ValidateCoin(payload, f.Chain(), f.Asset(), tags) + if err != nil { + return err + } + + return nil +} + +func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + buf := bytes.NewBuffer(nil) + if _, err = buf.ReadFrom(file); err != nil { + return err + } + + err = validation.ValidateJson(buf.Bytes()) + if err != nil { + return err + } + + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson) + } + + var payload info.AssetModel + err = json.Unmarshal(buf.Bytes(), &payload) + if err != nil { + return fmt.Errorf("%w: failed to decode", err) + } + + err = info.ValidateAsset(payload, f.Chain(), f.Asset()) + if err != nil { + return err + } + + return nil +} + +func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + if !isStackingChain(f.Chain()) { + return nil + } + + buf := bytes.NewBuffer(nil) + if _, err = buf.ReadFrom(file); err != nil { + return err + } + + err = validation.ValidateJson(buf.Bytes()) + if err != nil { + return err + } + + var model []list.Model + err = json.Unmarshal(buf.Bytes(), &model) + if err != nil { + return err + } + + err = list.ValidateList(model) + if err != nil { + return err + } + + listIDs := make([]string, len(model)) + for i, listItem := range model { + listIDs[i] = *listItem.ID + } + + assetsPath := fmt.Sprintf("blockchains/%s/validators/assets", f.Chain().Handle) + assetFolder := s.fileService.GetAssetFile(assetsPath) + + file2, err := os.Open(assetFolder.Path()) + if err != nil { + return err + } + defer file2.Close() + + dirAssetFolderFiles, err := file2.ReadDir(0) + if err != nil { + return err + } + + err = validation.ValidateAllowedFiles(dirAssetFolderFiles, listIDs) + if err != nil { + return err + } + + return nil +} + +func isStackingChain(c coin.Coin) bool { + for _, stackingChain := range config.StackingChains { + if c.ID == stackingChain.ID { + return true + } + } + + return false +} + +func (s *Service) ValidateTokenListFile(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + buf := bytes.NewBuffer(nil) + if _, err = buf.ReadFrom(file); err != nil { + return err + } + + err = validation.ValidateJson(buf.Bytes()) + if err != nil { + return err + } + + return nil +} + +func (s *Service) ValidateInfoFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainInfoFolder.HasFiles) + if err != nil { + return err + } + + return nil +} + +func (s *Service) ValidateValidatorsAssetFolder(f *file.AssetFile) error { + file, err := os.Open(f.Path()) + if err != nil { + return err + } + defer file.Close() + + dirFiles, err := file.ReadDir(0) + if err != nil { + return err + } + + compErr := validation.NewErrComposite() + err = validation.ValidateValidatorsAddress(f.Chain(), f.Asset()) + if err != nil { + compErr.Append(err) + } + + err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainValidatorsAssetFolder.HasFiles) + if err != nil { + compErr.Append(err) + } + + if compErr.Len() > 0 { + return compErr + } + + return nil +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 000000000..233df6a87 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,102 @@ +package service + +import ( + "github.com/trustwallet/assets-go-libs/pkg/validation" + "github.com/trustwallet/assets/internal/file" + "github.com/trustwallet/assets/internal/processor" + + log "github.com/sirupsen/logrus" +) + +type Service struct { + fileService *file.Service + processorService *processor.Service +} + +func NewService(fs *file.Service, cs *processor.Service) *Service { + return &Service{ + fileService: fs, + processorService: cs, + } +} + +func (s *Service) RunJob(paths []string, job func(*file.AssetFile)) { + for _, path := range paths { + f := s.fileService.GetAssetFile(path) + job(f) + } +} + +func (s *Service) Check(f *file.AssetFile) { + validator := s.processorService.GetValidator(f) + + if validator != nil { + if err := validator.Run(f); err != nil { + // TODO: somehow return an error from Check if there are any errors. + HandleError(err, f, validator.Name) + } + } +} + +func (s *Service) Fix(f *file.AssetFile) { + fixers := s.processorService.GetFixers(f) + + for _, fixer := range fixers { + if err := fixer.Run(f); err != nil { + HandleError(err, f, fixer.Name) + } + } +} + +func (s *Service) RunUpdateAuto() { + updaters := s.processorService.GetUpdatersAuto() + s.runUpdaters(updaters) +} + +func (s *Service) RunUpdateManual() { + updaters := s.processorService.GetUpdatersManual() + s.runUpdaters(updaters) +} + +func (s *Service) runUpdaters(updaters []processor.Updater) { + for _, updater := range updaters { + err := updater.Run() + if err != nil { + log.WithError(err).Error() + } + } +} + +func HandleError(err error, info *file.AssetFile, valName string) { + errors := UnwrapComposite(err) + + for _, err := range errors { + logFields := log.Fields{ + "type": info.Type(), + "chain": info.Chain().Handle, + "asset": info.Asset(), + "path": info.Path(), + "validation": valName, + } + + if warn, ok := err.(*validation.Warning); ok { + log.WithFields(logFields).Warning(warn) + } else { + log.WithFields(logFields).Error(err) + } + } +} + +func UnwrapComposite(err error) []error { + compErr, ok := err.(*validation.ErrComposite) + if !ok { + return []error{err} + } + + var errors []error + for _, e := range compErr.GetErrors() { + errors = append(errors, UnwrapComposite(e)...) + } + + return errors +}