Move CI scripts logic from assets-go-libs (#16680)

* Move CI scripts logic from assets-go-libs

* Add Makefile, .golangci.yml and lint jobs to workflow

* Fix

.github/assets.config.yaml

@@ -5,6 +5,7 @@ client_urls:
   binance:
     dex: https://dex.binance.org
     explorer: https://explorer.binance.org
+  backend_api: https://api.trustwallet.com 
 
 urls:
   tw_assets_app: https://assets.trustwalletapp.com
@@ -31,8 +32,11 @@ validators_settings:
       - ".eslintignore"
       - ".eslintrc.js"
       - "cmd"
+      - "internal"
       - "go.mod"
       - "go.sum"
+      - ".golangci.yml"
+      - "Makefile"
     skip_files:
       - "node_modules"
 

.github/workflows/check.yml

@@ -7,7 +7,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
@@ -18,3 +18,9 @@ jobs:
 
       - name: Run check
         run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker
+
+      # - name: Unit Test
+      #   run: make test
+
+      # - name: Lint
+      #   run: make lint

.github/workflows/fix-dryrun.yml

@@ -7,7 +7,7 @@ jobs:
   fix-dryrun:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17

.github/workflows/fix.yml

@@ -27,7 +27,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.ref }}
 
-      - name: Set up Go 1.17
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17

.github/workflows/periodic-update.yml

@@ -13,7 +13,7 @@ jobs:
           token: ${{ secrets.COMMIT_TOKEN }}
           ref: ${{ github.head_ref }} 
 
-      - name: Set up Go 1.17
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17

.github/workflows/pr-ci.yml

@@ -11,7 +11,7 @@ jobs:
   pull_request_ci:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
@@ -22,3 +22,9 @@ jobs:
 
       - name: Run check
         run: go run ./cmd/main.go --config=./.github/assets.config.yaml --script=checker
+
+      # - name: Unit Test
+      #   run: make test
+
+      # - name: Lint
+      #   run: make lint

.gitignore

@@ -6,3 +6,4 @@ node_modules/
 *.txt
 .env
 .env.test
+bin/

.golangci.yml

@@ -0,0 +1,250 @@
+linters-settings:
+  cyclop:
+    max-complexity: 20
+    # the maximal average package complexity. If it's higher than 0.0 (float) the check is enabled (default 0.0)
+    package-average: 0.0
+    skip-tests: false
+  dogsled:
+    # checks assignments with too many blank identifiers; default is 2
+    max-blank-identifiers: 2
+  dupl:
+    threshold: 100
+  errcheck:
+    check-type-assertions: false
+    check-blank: false
+  errorlint:
+    errorf: false
+    asserts: true
+    comparison: true
+  forbidigo:
+    # Forbid the following identifiers (identifiers are written using regexp):
+    forbid:
+      - ^print.*$
+      - 'fmt\.Print.*'
+    exclude_godoc_examples: true
+  funlen:
+    lines: 60
+    statements: 60
+  gocognit:
+    min-complexity: 35
+  goconst:
+    min-len: 3
+    min-occurrences: 3
+    ignore-tests: true
+    match-constant: true
+    numbers: false
+    min: 3
+    max: 3
+    ignore-calls: true
+  gocritic:
+    # Which checks should be disabled; can't be combined with 'enabled-checks'
+    disabled-checks:
+      - regexpMust
+  gocyclo:
+    min-complexity: 20
+  godot:
+    # comments to be checked: `declarations`, `toplevel`, or `all`
+    scope: all
+    capital: false
+  godox:
+    # report any comments starting with keywords, this is useful for TODO or FIXME comments that
+    # might be left in the code accidentally and should be resolved before merging
+    keywords:
+      - TODO
+      - BUG
+      - FIXME
+      - NOTE
+      - OPTIMIZE # marks code that should be optimized before merging
+      - HACK # marks hack-arounds that should be removed before merging
+  gofmt:
+    # simplify code: gofmt with `-s` option
+    simplify: true
+  gofumpt:
+    # Select the Go version to target
+    lang-version: "1.17"
+    extra-rules: false
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: github.com/trustwallet
+  gomnd:
+    settings:
+      mnd:
+        # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description.
+        checks:
+        - argument
+        - case
+        - condition
+        - operation
+        - return
+        - assign
+        ignored-numbers: 10,1000
+  gomoddirectives:
+    replace-local: false
+  gosec:
+    # To select a subset of rules to run.
+    # Available rules: https://github.com/securego/gosec#available-rules
+    includes:
+      - G401
+      - G306
+      - G101
+  gosimple:
+    # Select the Go version to target
+    go: "1.17"
+    # https://staticcheck.io/docs/options#checks
+    checks: [ "all" ]
+  govet:
+    check-shadowing: true
+  ifshort:
+    # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax.
+    # Has higher priority than max-decl-chars.
+    max-decl-lines: 1
+    # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
+    max-decl-chars: 30
+  importas:
+    # if set to `true`, force to use alias.
+    no-unaliased: true
+    # List of aliases
+    alias:
+      - pkg: github:com/trustwallet/go-libs/gin
+        alias: golibsGin
+  lll:
+    # max line length
+    line-length: 120
+    tab-width: 1
+  misspell:
+    locale: US
+  #  ignore-words:
+  nolintlint:
+    # Enable to ensure that nolint directives are all used
+    allow-unused: false
+    # Disable to ensure that nolint directives don't have a leading space
+    allow-leading-space: true
+    # Exclude following linters from requiring an explanation
+    allow-no-explanation: []
+    # Enable to require an explanation of nonzero length after each nolint directive
+    require-explanation: false
+    # Enable to require nolint directives to mention the specific linter being suppressed
+    require-specific: true
+  revive:
+    # see https://github.com/mgechev/revive#available-rules for details.
+    ignore-generated-header: true
+    severity: warning
+    rules:
+      - name: indent-error-flow
+        severity: warning
+      - name: time-naming
+        severity: warn
+      - name: errorf
+        severity: warn
+      - name: blank-imports
+        sevetiry: warn
+      # The error return parameter should be last
+      - name: error-return
+        severity: error
+      # Redundant if when returning an error
+      - name: if-return
+        severity: warn
+      # Warns when there are heading or trailing newlines in a block
+      - name: empty-lines
+        severity: error
+  staticcheck:
+    # Select the Go version to target
+    go: "1.17"
+    # https://staticcheck.io/docs/options#checks
+    checks: [ "all" ]
+  stylecheck:
+    # Select the Go version to target
+    go: "1.17"
+    # https://staticcheck.io/docs/options#checks
+    checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ]
+    # https://staticcheck.io/docs/options#dot_import_whitelist
+    dot-import-whitelist:
+      - fmt
+    # https://staticcheck.io/docs/options#initialisms
+    initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ]
+    # https://staticcheck.io/docs/options#http_status_code_whitelist
+    http-status-code-whitelist: [ "200", "400", "404", "500" ]
+  unused:
+    # Select the Go version to target
+    go: "1.17"
+  whitespace:
+    multi-if: true   # Enforces newlines (or comments) after every multi-line if statement
+    multi-func: false # Enforces newlines (or comments) after every multi-line function signature
+  wsl:
+    # See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for
+    # documentation of available settings
+    allow-assign-and-anything: false
+    allow-assign-and-call: true
+    allow-cuddle-declarations: false
+    allow-multiline-assign: true
+    allow-separated-leading-comment: false
+    allow-trailing-comment: false
+    force-case-trailing-whitespace: 0
+    force-err-cuddling: false
+    force-short-decl-cuddling: false
+    strict-append: true
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    - dupl
+    - errcheck
+    - exportloopref
+    - exhaustive
+    - funlen
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    # - gomnd
+    - goprintffuncname
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - lll
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+    - asciicheck
+    # - gochecknoglobals
+    - gocognit
+    - godot
+    # - godox
+    # - goerr113
+    # - nestif
+    - prealloc
+    - testpackage
+    # - revive
+    # - wsl
+
+  # don't enable:
+  # - interfacer
+  # - maligned
+  # - scopelint
+
+output:
+  format: colored-line-number
+  print-issued-lines: true
+  print-linter-name: true
+  uniq-by-line: true
+  path-prefix: ""
+  sort-results: true

Makefile

@@ -0,0 +1,28 @@
+#! /usr/bin/make -f
+
+# Go related variables.
+GOBASE := $(shell pwd)
+GOBIN := $(GOBASE)/bin
+
+# Go files.
+GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
+
+all: fmt lint test
+
+test:
+	@echo "  >  Running unit tests"
+	GOBIN=$(GOBIN) go test -cover -race -coverprofile=coverage.txt -covermode=atomic -v ./...
+
+fmt:
+	@echo "  >  Format all go files"
+	GOBIN=$(GOBIN) gofmt -w ${GOFMT_FILES}
+
+lint-install:
+ifeq (,$(wildcard test -f bin/golangci-lint))
+	@echo "  >  Installing golint"
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s
+endif
+
+lint: lint-install
+	@echo "  >  Running golint"
+	bin/golangci-lint run --timeout=2m

cmd/main.go

@@ -5,10 +5,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/trustwallet/assets-go-libs/pkg/file"
+	"github.com/trustwallet/assets/internal/config"
-	"github.com/trustwallet/assets-go-libs/src/config"
+	"github.com/trustwallet/assets/internal/file"
-	"github.com/trustwallet/assets-go-libs/src/core"
+	"github.com/trustwallet/assets/internal/processor"
-	"github.com/trustwallet/assets-go-libs/src/processor"
+	"github.com/trustwallet/assets/internal/service"
 )
 
 var (
@@ -20,42 +20,41 @@ func main() {
 
 	paths, err := file.ReadLocalFileStructure(root, config.Default.ValidatorsSettings.RootFolder.SkipFiles)
 	if err != nil {
-		log.WithError(err).Fatal("failed to load file structure")
+		log.WithError(err).Fatal("Failed to load file structure.")
 	}
 
 	fileStorage := file.NewService(paths...)
-	validatorsService := core.NewService(fileStorage)
+	validatorsService := processor.NewService(fileStorage)
-	assetfsProcessor := processor.NewService(fileStorage, validatorsService)
+	assetfsProcessor := service.NewService(fileStorage, validatorsService)
 
 	switch script {
 	case "checker":
-		err = assetfsProcessor.RunJob(paths, assetfsProcessor.Check)
+		assetfsProcessor.RunJob(paths, assetfsProcessor.Check)
 	case "fixer":
-		err = assetfsProcessor.RunJob(paths, assetfsProcessor.Fix)
+		assetfsProcessor.RunJob(paths, assetfsProcessor.Fix)
 	case "updater-auto":
-		err = assetfsProcessor.RunUpdateAuto()
+		assetfsProcessor.RunUpdateAuto()
+	case "updater-manual":
+		assetfsProcessor.RunUpdateManual()
 	default:
 		log.Info("Nothing to launch. Use --script flag to choose a script to run.")
 	}
-
-	if err != nil {
-		log.WithError(err).Fatal("Script failed")
-	}
 }
 
 func setup() {
-	flag.StringVar(&configPath, "config", "", "path to config file")
+	flag.StringVar(&configPath, "config", "./.github/assets.config.yaml", "path to config file")
 	flag.StringVar(&root, "root", "./", "path to the root of the dir")
 	flag.StringVar(&script, "script", "", "script type to run")
+
 	flag.Parse()
 
 	if err := config.SetConfig(configPath); err != nil {
-		log.WithError(err).Fatal("failed to set config")
+		log.WithError(err).Fatal("Failed to set config.")
 	}
 
 	logLevel, err := log.ParseLevel(config.Default.App.LogLevel)
 	if err != nil {
-		log.WithError(err).Fatal("failed to parse log level")
+		log.WithError(err).Fatal("Failed to parse log level.")
 	}
 
 	log.SetLevel(logLevel)

go.mod

@@ -4,7 +4,9 @@ go 1.17
 
 require (
 	github.com/sirupsen/logrus v1.8.1
-	github.com/trustwallet/assets-go-libs v0.0.15
+	github.com/trustwallet/assets-go-libs v0.0.16
+	github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793
+	github.com/trustwallet/go-primitives v0.0.17
 )
 
 require (
@@ -22,8 +24,6 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.10.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/trustwallet/go-libs v0.2.20 // indirect
-	github.com/trustwallet/go-primitives v0.0.17 // indirect
 	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
 	golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827 // indirect

go.sum

@@ -40,14 +40,10 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d h1:L2WPNWlTMoBfVsHDqw73dH3fBZYdh0xdn3cmXGs7IX8=
+github.com/trustwallet/assets-go-libs v0.0.16 h1:nbKrf/pHKQCut4Q5Mdg5BinLdFxOqvzEeAgIgzX0/P8=
-github.com/trustwallet/assets-go-libs v0.0.14-0.20211215100024-87d144f2d64d/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
+github.com/trustwallet/assets-go-libs v0.0.16/go.mod h1:agKWTQ9ECSzQ++7P/9viSxJnG1Kp+WhfDyI3pAmtnVM=
-github.com/trustwallet/assets-go-libs v0.0.14 h1:7fs6tPFf8XF1svj1fksJ+PN4ukT3v3sRfz/z4W86JAA=
+github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793 h1:KFtyLpBPbMyUdeCth/Zcej/SSgAFIo6fxdS2eEPEg3I=
-github.com/trustwallet/assets-go-libs v0.0.14/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
+github.com/trustwallet/go-libs v0.2.21-0.20211217144209-59d4828f9793/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
-github.com/trustwallet/assets-go-libs v0.0.15 h1:0q0xUR0FATqqdLr9OShRwYK0r3KDqGLWrxpOOc4lJ4o=
-github.com/trustwallet/assets-go-libs v0.0.15/go.mod h1:feaOqyxqy7alNo6t9iVmtaDHYeCIM15JlkH3x4ozpoI=
-github.com/trustwallet/go-libs v0.2.20 h1:pYstFNgsc7CVyVeYt5GHsMa0JNQHJVRvPQqMvXMpCtY=
-github.com/trustwallet/go-libs v0.2.20/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
 github.com/trustwallet/go-primitives v0.0.17 h1:1fBxZMKGCHdHtgdUzsqdFlD21+1GneIk/sxN6jxYBds=
 github.com/trustwallet/go-primitives v0.0.17/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

internal/config/config.go

@@ -0,0 +1,57 @@
+package config
+
+import (
+	"path/filepath"
+
+	"github.com/trustwallet/go-libs/config/viper"
+)
+
+type (
+	Config struct {
+		App                App                `mapstructure:"app"`
+		ClientURLs         ClientsURLs        `mapstructure:"client_urls"`
+		URLs               URLs               `mapstructure:"urls"`
+		ValidatorsSettings ValidatorsSettings `mapstructure:"validators_settings"`
+	}
+
+	App struct {
+		LogLevel string `mapstructure:"log_level"`
+	}
+
+	ClientsURLs struct {
+		Binance struct {
+			Dex      string `mapstructure:"dex"`
+			Explorer string `mapstructure:"explorer"`
+		} `mapstructure:"binance"`
+		BackendAPI string `mapstructure:"backend_api"`
+	}
+
+	URLs struct {
+		TWAssetsApp string `mapstructure:"tw_assets_app"`
+	}
+
+	ValidatorsSettings struct {
+		RootFolder                 RootFolder                 `mapstructure:"root_folder"`
+		ChainFolder                ChainFolder                `mapstructure:"chain_folder"`
+		AssetFolder                AssetFolder                `mapstructure:"asset_folder"`
+		ChainInfoFolder            ChainInfoFolder            `mapstructure:"chain_info_folder"`
+		ChainValidatorsAssetFolder ChainValidatorsAssetFolder `mapstructure:"chain_validators_asset_folder"`
+		DappsFolder                DappsFolder                `mapstructure:"dapps_folder"`
+		CoinInfoFile               CoinInfoFile               `mapstructure:"coin_info_file"`
+	}
+)
+
+// Default is a configuration instance.
+var Default = Config{} // nolint:gochecknoglobals // config must be global
+
+// SetConfig reads a config file and returs an initialized config instance.
+func SetConfig(confPath string) error {
+	confPath, err := filepath.Abs(confPath)
+	if err != nil {
+		return err
+	}
+
+	viper.Load(confPath, &Default)
+
+	return nil
+}

internal/config/validators.go

@@ -0,0 +1,36 @@
+package config
+
+type RootFolder struct {
+	AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
+	SkipFiles    []string `mapstructure:"skip_files,omitempty"`
+}
+
+type ChainFolder struct {
+	AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
+}
+
+type AssetFolder struct {
+	AllowedFiles []string `mapstructure:"allowed_files,omitempty"`
+}
+
+type ChainInfoFolder struct {
+	HasFiles []string `mapstructure:"has_files,omitempty"`
+}
+
+type ChainValidatorsAssetFolder struct {
+	HasFiles []string `mapstructure:"has_files,omitempty"`
+}
+
+type DappsFolder struct {
+	Ext string `mapstructure:"ext,omitempty"`
+}
+
+type CoinInfoFile struct {
+	Tags []Tag `mapstructure:"tags,omitempty"`
+}
+
+type Tag struct {
+	ID          string `mapstructure:"id,omitempty"`
+	Name        string `mapstructure:"name,omitempty"`
+	Description string `mapstructure:"description,omitempty"`
+}

internal/config/values.go

@@ -0,0 +1,15 @@
+package config
+
+import "github.com/trustwallet/go-primitives/coin"
+
+// TODO: Move to go-libs.
+var StackingChains = []coin.Coin{
+	coin.Tezos(),
+	coin.Cosmos(),
+	coin.Iotex(),
+	coin.Tron(),
+	coin.Waves(),
+	coin.Kava(),
+	coin.Terra(),
+	coin.Binance(),
+}

internal/file/cache.go

@@ -0,0 +1,58 @@
+package file
+
+import (
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+type Service struct {
+	mu    *sync.RWMutex
+	cache map[string]*AssetFile
+}
+
+func NewService(filePaths ...string) *Service {
+	var filesMap = make(map[string]*AssetFile)
+
+	for _, path := range filePaths {
+		assetFile := NewAssetFile(path)
+		filesMap[path] = assetFile
+	}
+
+	return &Service{
+		mu:    &sync.RWMutex{},
+		cache: filesMap,
+	}
+}
+
+func (f *Service) GetAssetFile(path string) *AssetFile {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+
+	return f.getFile(path)
+}
+
+func (f *Service) UpdateFile(file *AssetFile, newFileBaseName string) {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+
+	oldFileBaseName := filepath.Base(file.Path())
+
+	for path := range f.cache {
+		if strings.Contains(path, oldFileBaseName) {
+			newPath := strings.ReplaceAll(path, oldFileBaseName, newFileBaseName)
+			f.cache[path] = NewAssetFile(newPath)
+		}
+	}
+}
+
+func (f *Service) getFile(path string) *AssetFile {
+	if file, exists := f.cache[path]; exists {
+		return file
+	}
+
+	assetF := NewAssetFile(path)
+	f.cache[path] = assetF
+
+	return assetF
+}

internal/file/file.go

@@ -0,0 +1,29 @@
+package file
+
+import (
+	"github.com/trustwallet/go-primitives/coin"
+)
+
+type AssetFile struct {
+	path *Path
+}
+
+func NewAssetFile(path string) *AssetFile {
+	return &AssetFile{path: NewPath(path)}
+}
+
+func (i *AssetFile) Path() string {
+	return i.path.String()
+}
+
+func (i *AssetFile) Type() string {
+	return i.path.fileType
+}
+
+func (i *AssetFile) Chain() coin.Coin {
+	return i.path.chain
+}
+
+func (i *AssetFile) Asset() string {
+	return i.path.asset
+}

internal/file/path.go

@@ -0,0 +1,161 @@
+package file
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/trustwallet/assets-go-libs/pkg"
+	"github.com/trustwallet/go-primitives/coin"
+)
+
+var (
+	regexAssetInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/info.json$`)
+	regexAssetLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)/logo.png$`)
+
+	regexChainInfoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/info.json$`)
+	regexChainLogoFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info/logo.png$`)
+
+	regexTokenListFile = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/tokenlist.json$`)
+
+	regexValidatorsAssetLogo = regexp.MustCompile(
+		`./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)/logo.png$`)
+	regexValidatorsList = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/list.json$`)
+
+	regexDappsLogo = regexp.MustCompile(`./dapps/[a-zA-Z-.]+\.png$`)
+)
+
+var (
+	regexAssetFolder  = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets/(\w+[\-]\w+|\w+)$`)
+	regexAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/assets$`)
+
+	regexValidatorsFolder      = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators$`)
+	regexValidatorsAssetFolder = regexp.MustCompile(
+		`./blockchains/(\w+[\-]\w+|\w+)/validators/assets/(\w+[\-]\w+|\w+)$`)
+	regexValidatorsAssetsFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/validators/assets$`)
+
+	regexChainFolder     = regexp.MustCompile(`./blockchains/(\w+[^/])$`)
+	regexChainInfoFolder = regexp.MustCompile(`./blockchains/(\w+[\-]\w+|\w+)/info$`)
+	regexChainsFolder    = regexp.MustCompile(`./blockchains$`)
+
+	regexDappsFolder = regexp.MustCompile(`./dapps$`)
+	regexRoot        = regexp.MustCompile(`./$`)
+)
+
+var regexes = map[string]*regexp.Regexp{
+	TypeAssetInfoFile: regexAssetInfoFile,
+	TypeAssetLogoFile: regexAssetLogoFile,
+
+	TypeChainInfoFile: regexChainInfoFile,
+	TypeChainLogoFile: regexChainLogoFile,
+
+	TypeTokenListFile: regexTokenListFile,
+
+	TypeValidatorsListFile: regexValidatorsList,
+	TypeValidatorsLogoFile: regexValidatorsAssetLogo,
+
+	TypeDappsLogoFile: regexDappsLogo,
+
+	TypeAssetFolder:  regexAssetFolder,
+	TypeAssetsFolder: regexAssetsFolder,
+
+	TypeChainFolder:     regexChainFolder,
+	TypeChainsFolder:    regexChainsFolder,
+	TypeChainInfoFolder: regexChainInfoFolder,
+
+	TypeDappsFolder: regexDappsFolder,
+	TypeRootFolder:  regexRoot,
+
+	TypeValidatorsFolder:       regexValidatorsFolder,
+	TypeValidatorsAssetsFolder: regexValidatorsAssetsFolder,
+	TypeValidatorsAssetFolder:  regexValidatorsAssetFolder,
+}
+
+type Path struct {
+	path     string
+	chain    coin.Coin
+	asset    string
+	fileType string
+}
+
+func NewPath(path string) *Path {
+	p := Path{path: path}
+
+	fileType, reg := defineFileType(path)
+	if reg == nil {
+		p.fileType = TypeUnknown
+
+		return &p
+	}
+
+	match := reg.FindStringSubmatch(path)
+	if fileType != TypeUnknown {
+		p.fileType = fileType
+	}
+
+	if len(match) >= 2 {
+		chain, err := coin.GetCoinForId(match[1])
+		if err != nil {
+			p.chain = coin.Coin{Handle: match[1]}
+		} else {
+			p.chain = chain
+		}
+	}
+
+	if len(match) == 3 {
+		p.asset = match[2]
+	}
+
+	return &p
+}
+
+func (p Path) Type() string {
+	return p.fileType
+}
+
+func (p Path) String() string {
+	return p.path
+}
+
+func (p Path) Chain() coin.Coin {
+	return p.chain
+}
+
+func (p Path) Asset() string {
+	return p.asset
+}
+
+func defineFileType(p string) (string, *regexp.Regexp) {
+	for t, r := range regexes {
+		if r.MatchString(p) {
+			return t, r
+		}
+	}
+
+	return TypeUnknown, nil
+}
+
+func ReadLocalFileStructure(root string, filesToSkip []string) ([]string, error) {
+	var paths = []string{"./"}
+	err := filepath.Walk(root,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if pkg.Contains(path, filesToSkip) {
+				return nil
+			}
+
+			paths = append(paths, fmt.Sprintf("./%s", path))
+
+			return nil
+		})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return paths, nil
+}

internal/file/types.go

@@ -0,0 +1,25 @@
+package file
+
+const (
+	TypeAssetInfoFile      = "asset_info"
+	TypeAssetLogoFile      = "asset_logo"
+	TypeChainInfoFile      = "chain_info"
+	TypeChainLogoFile      = "chain_logo"
+	TypeTokenListFile      = "chain_token_list"
+	TypeValidatorsListFile = "validators_list"
+	TypeValidatorsLogoFile = "validators_logo"
+	TypeDappsLogoFile      = "dapps_logo"
+
+	TypeAssetFolder            = "asset"
+	TypeAssetsFolder           = "assets"
+	TypeChainFolder            = "chain"
+	TypeChainsFolder           = "chains"
+	TypeDappsFolder            = "dapps"
+	TypeRootFolder             = "root"
+	TypeChainInfoFolder        = "chain_info_folder"
+	TypeValidatorsFolder       = "validators"
+	TypeValidatorsAssetFolder  = "validators_asset"
+	TypeValidatorsAssetsFolder = "validators_assets"
+
+	TypeUnknown = "unknown"
+)

internal/processor/fixers.go

@@ -0,0 +1,171 @@
+package processor
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/trustwallet/assets-go-libs/pkg"
+	"github.com/trustwallet/assets-go-libs/pkg/validation"
+	"github.com/trustwallet/assets-go-libs/pkg/validation/info"
+	"github.com/trustwallet/assets/internal/file"
+	"github.com/trustwallet/go-primitives/address"
+	"github.com/trustwallet/go-primitives/coin"
+	"github.com/trustwallet/go-primitives/types"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func (s *Service) FixJSON(f *file.AssetFile) error {
+	return pkg.FormatJSONFile(f.Path())
+}
+
+func (s *Service) FixETHAddressChecksum(f *file.AssetFile) error {
+	if !coin.IsEVM(f.Chain().ID) {
+		return nil
+	}
+
+	assetDir := filepath.Base(f.Path())
+
+	err := validation.ValidateETHForkAddress(f.Chain(), assetDir)
+	if err != nil {
+		checksum, e := address.EIP55Checksum(assetDir)
+		if e != nil {
+			return fmt.Errorf("failed to get checksum: %s", e)
+		}
+
+		newName := fmt.Sprintf("blockchains/%s/assets/%s", f.Chain().Handle, checksum)
+
+		if e = os.Rename(f.Path(), newName); e != nil {
+			return fmt.Errorf("failed to rename dir: %s", e)
+		}
+
+		s.fileService.UpdateFile(f, checksum)
+
+		log.WithField("from", assetDir).
+			WithField("to", checksum).
+			Debug("Renamed asset")
+	}
+
+	return nil
+}
+
+func (s *Service) FixLogo(f *file.AssetFile) error {
+	width, height, err := pkg.GetPNGImageDimensions(f.Path())
+	if err != nil {
+		return err
+	}
+
+	var isLogoTooLarge bool
+	if width > validation.MaxW || height > validation.MaxH {
+		isLogoTooLarge = true
+	}
+
+	if isLogoTooLarge {
+		log.WithField("path", f.Path()).Debug("Fixing too large image")
+
+		targetW, targetH := calculateTargetDimension(width, height)
+
+		err = pkg.ResizePNG(f.Path(), targetW, targetH)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = validation.ValidateLogoFileSize(f.Path())
+	if err != nil { // nolint:staticcheck
+		// TODO: Compress images.
+	}
+
+	return nil
+}
+
+func calculateTargetDimension(width, height int) (targetW, targetH int) {
+	widthFloat := float32(width)
+	heightFloat := float32(height)
+
+	maxEdge := widthFloat
+	if heightFloat > widthFloat {
+		maxEdge = heightFloat
+	}
+
+	ratio := validation.MaxW / maxEdge
+
+	targetW = int(widthFloat * ratio)
+	targetH = int(heightFloat * ratio)
+
+	return targetW, targetH
+}
+
+func (s *Service) FixChainInfoJSON(f *file.AssetFile) error {
+	chainInfo := info.CoinModel{}
+
+	err := pkg.ReadJSONFile(f.Path(), &chainInfo)
+	if err != nil {
+		return err
+	}
+
+	expectedType := string(types.Coin)
+	if chainInfo.Type == nil || *chainInfo.Type != expectedType {
+		chainInfo.Type = &expectedType
+
+		return pkg.CreateJSONFile(f.Path(), &chainInfo)
+	}
+
+	return nil
+}
+
+func (s *Service) FixAssetInfoJSON(file *file.AssetFile) error {
+	assetInfo := info.AssetModel{}
+
+	err := pkg.ReadJSONFile(file.Path(), &assetInfo)
+	if err != nil {
+		return err
+	}
+
+	var isModified bool
+
+	// Fix asset type.
+	var assetType string
+	if assetInfo.Type != nil {
+		assetType = *assetInfo.Type
+	}
+
+	// We need to skip error check to fix asset type if it's incorrect or empty.
+	chain, _ := types.GetChainFromAssetType(assetType)
+
+	expectedTokenType, ok := types.GetTokenType(file.Chain().ID, file.Asset())
+	if !ok {
+		expectedTokenType = strings.ToUpper(assetType)
+	}
+
+	if chain.ID != file.Chain().ID || !strings.EqualFold(assetType, expectedTokenType) {
+		assetInfo.Type = &expectedTokenType
+		isModified = true
+	}
+
+	// Fix asset id.
+	assetID := file.Asset()
+	if assetInfo.ID == nil || *assetInfo.ID != assetID {
+		assetInfo.ID = &assetID
+		isModified = true
+	}
+
+	expectedExplorerURL, err := coin.GetCoinExploreURL(file.Chain(), file.Asset())
+	if err != nil {
+		return err
+	}
+
+	// Fix asset explorer url.
+	if assetInfo.Explorer == nil || !strings.EqualFold(expectedExplorerURL, *assetInfo.Explorer) {
+		assetInfo.Explorer = &expectedExplorerURL
+		isModified = true
+	}
+
+	if isModified {
+		return pkg.CreateJSONFile(file.Path(), &assetInfo)
+	}
+
+	return nil
+}

internal/processor/model.go

@@ -0,0 +1,82 @@
+package processor
+
+import "github.com/trustwallet/assets/internal/file"
+
+type (
+	Validator struct {
+		Name string
+		Run  func(f *file.AssetFile) error
+	}
+
+	Fixer struct {
+		Name string
+		Run  func(f *file.AssetFile) error
+	}
+
+	Updater struct {
+		Name string
+		Run  func() error
+	}
+)
+
+type (
+	TokenList struct {
+		Name      string      `json:"name"`
+		LogoURI   string      `json:"logoURI"`
+		Timestamp string      `json:"timestamp"`
+		Tokens    []TokenItem `json:"tokens"`
+		Version   Version     `json:"version"`
+	}
+
+	TokenItem struct {
+		Asset    string `json:"asset"`
+		Type     string `json:"type"`
+		Address  string `json:"address"`
+		Name     string `json:"name"`
+		Symbol   string `json:"symbol"`
+		Decimals uint   `json:"decimals"`
+		LogoURI  string `json:"logoURI"`
+		Pairs    []Pair `json:"pairs"`
+	}
+
+	Pair struct {
+		Base     string `json:"base"`
+		LotSize  string `json:"lotSize,omitempty"`
+		TickSize string `json:"tickSize,omitempty"`
+	}
+
+	Version struct {
+		Major int `json:"major"`
+		Minor int `json:"minor"`
+		Patch int `json:"patch"`
+	}
+)
+
+type (
+	ForceListPair struct {
+		Token0 string
+		Token1 string
+	}
+
+	TradingPairs struct {
+		Data struct {
+			Pairs []TradingPair `json:"pairs"`
+		} `json:"data"`
+	}
+
+	TradingPair struct {
+		ID         string     `json:"id"`
+		ReserveUSD string     `json:"reserveUSD"`
+		VolumeUSD  string     `json:"volumeUSD"`
+		TxCount    string     `json:"txCount"`
+		Token0     *TokenInfo `json:"token0"`
+		Token1     *TokenInfo `json:"token1"`
+	}
+
+	TokenInfo struct {
+		ID       string `json:"id"`
+		Symbol   string `json:"symbol"`
+		Name     string `json:"name"`
+		Decimals string `json:"decimals"`
+	}
+)

internal/processor/processor.go

@@ -0,0 +1,156 @@
+package processor
+
+import (
+	"github.com/trustwallet/assets/internal/file"
+)
+
+type Service struct {
+	fileService *file.Service
+}
+
+func NewService(fileProvider *file.Service) *Service {
+	return &Service{fileService: fileProvider}
+}
+
+// nolint:funlen
+func (s *Service) GetValidator(f *file.AssetFile) *Validator {
+	switch f.Type() {
+	case file.TypeRootFolder:
+		return &Validator{
+			Name: "Root folder contains only allowed files",
+			Run:  s.ValidateRootFolder,
+		}
+	case file.TypeChainFolder:
+		return &Validator{
+			Name: "Chain folders are lowercase and contains only allowed files",
+			Run:  s.ValidateChainFolder,
+		}
+	case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile:
+		return &Validator{
+			Name: "Logos (size, dimension)",
+			Run:  s.ValidateImage,
+		}
+	case file.TypeAssetFolder:
+		return &Validator{
+			Name: "Each asset folder has valid asset address and contains logo/info",
+			Run:  s.ValidateAssetFolder,
+		}
+	case file.TypeDappsFolder:
+		return &Validator{
+			Name: "Dapps folder (allowed only png files, lowercase)",
+			Run:  s.ValidateDappsFolder,
+		}
+	case file.TypeAssetInfoFile:
+		return &Validator{
+			Name: "Asset info (is valid json, fields)",
+			Run:  s.ValidateAssetInfoFile,
+		}
+	case file.TypeChainInfoFile:
+		return &Validator{
+			Name: "Chain Info (is valid json, fields)",
+			Run:  s.ValidateChainInfoFile,
+		}
+	case file.TypeValidatorsListFile:
+		return &Validator{
+			Name: "Validators list file",
+			Run:  s.ValidateValidatorsListFile,
+		}
+	case file.TypeTokenListFile:
+		return &Validator{
+			Name: "Token list (if assets from list present in chain)",
+			Run:  s.ValidateTokenListFile,
+		}
+	case file.TypeChainInfoFolder:
+		return &Validator{
+			Name: "Chain Info Folder (has files)",
+			Run:  s.ValidateInfoFolder,
+		}
+	case file.TypeValidatorsAssetFolder:
+		return &Validator{
+			Name: "Validators asset folder (has logo, valid asset address)",
+			Run:  s.ValidateValidatorsAssetFolder,
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) GetFixers(f *file.AssetFile) []Fixer {
+	infoFixer := Fixer{
+		Name: "Formatting all info.json files",
+		Run:  s.FixJSON,
+	}
+
+	ethAssetFixer := Fixer{
+		Name: "Renaming EVM's asset folder to valid address checksum",
+		Run:  s.FixETHAddressChecksum,
+	}
+
+	logoFixer := Fixer{
+		Name: "Resizing and compressing logo images",
+		Run:  s.FixLogo,
+	}
+
+	chainInfoFixer := Fixer{
+		Name: "Fixing chain info.json files",
+		Run:  s.FixChainInfoJSON,
+	}
+
+	assetInfoFixer := Fixer{
+		Name: "Fixing asset info.json files",
+		Run:  s.FixAssetInfoJSON,
+	}
+
+	switch f.Type() {
+	case file.TypeChainInfoFile:
+		return []Fixer{
+			infoFixer,
+			chainInfoFixer,
+		}
+	case file.TypeAssetInfoFile:
+		return []Fixer{
+			infoFixer,
+			assetInfoFixer,
+		}
+	case file.TypeValidatorsListFile:
+		return []Fixer{
+			infoFixer,
+		}
+	case file.TypeAssetFolder:
+		return []Fixer{
+			ethAssetFixer,
+		}
+	case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile:
+		return []Fixer{
+			logoFixer,
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) GetUpdatersAuto() []Updater {
+	return []Updater{
+		{
+			Name: "Retrieving missing token images, creating binance token list.",
+			Run:  s.UpdateBinanceTokens,
+		},
+	}
+}
+
+func (s *Service) GetUpdatersManual() []Updater {
+	return []Updater{
+		{
+			Name: "Update tokenlist.json for Ethereum",
+			Run:  s.UpdateEthereumTokenlist,
+		},
+		{
+			Name: "Update tokenlist.json for Polygon",
+			Run:  s.UpdatePolygonTokenlist,
+		},
+		{
+			Name: "Update tokenlist.json for Smartchain",
+			Run:  s.UpdateSmartchainTokenlist,
+		},
+	}
+}

internal/processor/updaters_auto.go

@@ -0,0 +1,261 @@
+package processor
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/trustwallet/assets-go-libs/pkg"
+	"github.com/trustwallet/assets-go-libs/pkg/asset"
+	"github.com/trustwallet/assets-go-libs/pkg/validation/info"
+	"github.com/trustwallet/assets/internal/config"
+	"github.com/trustwallet/go-libs/blockchain/binance"
+	"github.com/trustwallet/go-libs/blockchain/binance/explorer"
+	assetlib "github.com/trustwallet/go-primitives/asset"
+	"github.com/trustwallet/go-primitives/coin"
+	"github.com/trustwallet/go-primitives/numbers"
+	"github.com/trustwallet/go-primitives/types"
+)
+
+const (
+	assetsPage       = 1
+	assetsRows       = 1000
+	marketPairsLimit = 1000
+	tokensListLimit  = 10000
+
+	twLogoURL       = "https://trustwallet.com/assets/images/favicon.png"
+	timestampFormat = "2006-01-02T15:04:05.000000"
+)
+
+func (s *Service) UpdateBinanceTokens() error {
+	explorerClient := explorer.InitClient(config.Default.ClientURLs.Binance.Explorer, nil)
+
+	bep2AssetList, err := explorerClient.FetchBep2Assets(assetsPage, assetsRows)
+	if err != nil {
+		return err
+	}
+
+	dexClient := binance.InitClient(config.Default.ClientURLs.Binance.Dex, "", nil)
+
+	marketPairs, err := dexClient.FetchMarketPairs(marketPairsLimit)
+	if err != nil {
+		return err
+	}
+
+	tokensList, err := dexClient.FetchTokens(tokensListLimit)
+	if err != nil {
+		return err
+	}
+
+	chain, err := types.GetChainFromAssetType(string(types.BEP2))
+	if err != nil {
+		return err
+	}
+
+	err = fetchMissingAssets(chain, bep2AssetList.AssetInfoList)
+	if err != nil {
+		return err
+	}
+
+	return createTokenListJSON(chain, marketPairs, tokensList)
+}
+
+func fetchMissingAssets(chain coin.Coin, assets []explorer.Bep2Asset) error {
+	for _, a := range assets {
+		if a.AssetImg == "" || a.Decimals == 0 {
+			continue
+		}
+
+		assetLogoPath := asset.GetAssetLogoPath(chain.Handle, a.Asset)
+		if pkg.FileExists(assetLogoPath) {
+			continue
+		}
+
+		if err := createLogo(assetLogoPath, a); err != nil {
+			return err
+		}
+
+		if err := createInfoJSON(chain, a); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func createLogo(assetLogoPath string, a explorer.Bep2Asset) error {
+	err := pkg.CreateDirPath(assetLogoPath)
+	if err != nil {
+		return err
+	}
+
+	return pkg.CreatePNGFromURL(a.AssetImg, assetLogoPath)
+}
+
+func createInfoJSON(chain coin.Coin, a explorer.Bep2Asset) error {
+	explorerURL, err := coin.GetCoinExploreURL(chain, a.Asset)
+	if err != nil {
+		return err
+	}
+
+	assetType := string(types.BEP2)
+	website := ""
+	description := "-"
+	status := "active"
+
+	assetInfo := info.AssetModel{
+		Name:        &a.Name,
+		Type:        &assetType,
+		Symbol:      &a.MappedAsset,
+		Decimals:    &a.Decimals,
+		Website:     &website,
+		Description: &description,
+		Explorer:    &explorerURL,
+		Status:      &status,
+		ID:          &a.Asset,
+	}
+
+	assetInfoPath := asset.GetAssetInfoPath(chain.Handle, a.Asset)
+
+	return pkg.CreateJSONFile(assetInfoPath, &assetInfo)
+}
+
+func createTokenListJSON(chain coin.Coin, marketPairs []binance.MarketPair, tokenList binance.Tokens) error {
+	tokens, err := generateTokenList(marketPairs, tokenList)
+	if err != nil {
+		return nil
+	}
+
+	tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle)
+
+	var oldTokenList TokenList
+	err = pkg.ReadJSONFile(tokenListPath, &oldTokenList)
+	if err != nil {
+		return nil
+	}
+
+	sortTokens(tokens)
+
+	if reflect.DeepEqual(oldTokenList.Tokens, tokens) {
+		return nil
+	}
+
+	if len(tokens) > 0 {
+		return pkg.CreateJSONFile(tokenListPath, &TokenList{
+			Name:      fmt.Sprintf("Trust Wallet: %s", coin.Coins[coin.BINANCE].Symbol),
+			LogoURI:   twLogoURL,
+			Timestamp: time.Now().Format(timestampFormat),
+			Tokens:    tokens,
+			Version:   Version{Major: oldTokenList.Version.Major + 1},
+		})
+	}
+
+	return nil
+}
+
+func sortTokens(tokens []TokenItem) {
+	sort.Slice(tokens, func(i, j int) bool {
+		if len(tokens[i].Pairs) != len(tokens[j].Pairs) {
+			return len(tokens[i].Pairs) > len(tokens[j].Pairs)
+		}
+
+		return tokens[i].Address < tokens[j].Address
+	})
+
+	for _, token := range tokens {
+		sort.Slice(token.Pairs, func(i, j int) bool {
+			return token.Pairs[i].Base < token.Pairs[j].Base
+		})
+	}
+}
+
+func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]TokenItem, error) {
+	if len(marketPairs) < 5 {
+		return nil, fmt.Errorf("no markets info is returned from Binance DEX: %d", len(marketPairs))
+	}
+
+	if len(tokenList) < 5 {
+		return nil, fmt.Errorf("no tokens info is returned from Binance DEX: %d", len(tokenList))
+	}
+
+	pairsMap := make(map[string][]Pair)
+	pairsList := make(map[string]struct{})
+	tokensMap := make(map[string]binance.Token)
+
+	for _, token := range tokenList {
+		tokensMap[token.Symbol] = token
+	}
+
+	for _, marketPair := range marketPairs {
+		key := marketPair.QuoteAssetSymbol
+
+		if val, exists := pairsMap[key]; exists {
+			val = append(val, getPair(marketPair))
+			pairsMap[key] = val
+		} else {
+			pairsMap[key] = []Pair{getPair(marketPair)}
+		}
+
+		pairsList[marketPair.BaseAssetSymbol] = struct{}{}
+		pairsList[marketPair.QuoteAssetSymbol] = struct{}{}
+	}
+
+	tokenItems := make([]TokenItem, 0, len(pairsList))
+
+	for pair := range pairsList {
+		token := tokensMap[pair]
+
+		var pairs []Pair
+		pairs, exists := pairsMap[token.Symbol]
+		if !exists {
+			pairs = make([]Pair, 0)
+		}
+
+		tokenItems = append(tokenItems, TokenItem{
+			Asset:    getAssetIDSymbol(token.Symbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
+			Type:     getTokenType(token.Symbol, coin.Coins[coin.BINANCE].Symbol, string(types.BEP2)),
+			Address:  token.Symbol,
+			Name:     token.Name,
+			Symbol:   token.OriginalSymbol,
+			Decimals: coin.Coins[coin.BINANCE].Decimals,
+			LogoURI:  getLogoURI(token.Symbol, coin.Coins[coin.BINANCE].Handle, coin.Coins[coin.BINANCE].Symbol),
+			Pairs:    pairs,
+		})
+	}
+
+	return tokenItems, nil
+}
+
+func getPair(marketPair binance.MarketPair) Pair {
+	return Pair{
+		Base:     getAssetIDSymbol(marketPair.BaseAssetSymbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
+		LotSize:  strconv.FormatInt(numbers.ToSatoshi(marketPair.LotSize), 10),
+		TickSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.TickSize), 10),
+	}
+}
+
+func getAssetIDSymbol(tokenID string, nativeCoinID string, coinType uint) string {
+	if tokenID == nativeCoinID {
+		return assetlib.BuildID(coinType, "")
+	}
+
+	return assetlib.BuildID(coinType, tokenID)
+}
+
+func getTokenType(symbol string, nativeCoinSymbol string, tokenType string) string {
+	if symbol == nativeCoinSymbol {
+		return "coin"
+	}
+
+	return tokenType
+}
+
+func getLogoURI(id, githubChainFolder, nativeCoinSymbol string) string {
+	if id == nativeCoinSymbol {
+		return fmt.Sprintf("%s/blockchains/%s/info/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder)
+	}
+
+	return fmt.Sprintf("%s/blockchains/%s/assets/%s/logo.png", config.Default.URLs.TWAssetsApp, githubChainFolder, id)
+}

internal/processor/updaters_manual.go

@@ -0,0 +1,537 @@
+package processor
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/trustwallet/assets-go-libs/pkg"
+	"github.com/trustwallet/assets-go-libs/pkg/asset"
+	"github.com/trustwallet/assets/internal/config"
+	"github.com/trustwallet/go-libs/client/api/backend"
+	"github.com/trustwallet/go-primitives/address"
+	"github.com/trustwallet/go-primitives/coin"
+	"github.com/trustwallet/go-primitives/types"
+)
+
+var (
+	UniswapForceInclude = []string{
+		"TUSD", "STAKE", "YFI", "BAT", "MANA", "1INCH", "REP", "KP3R", "UNI", "WBTC", "HEX", "CREAM", "SLP",
+		"REN", "XOR", "Link", "sUSD", "HEGIC", "RLC", "DAI", "SUSHI", "FYZ", "DYT", "AAVE", "LEND", "UBT",
+		"DIA", "RSR", "SXP", "OCEAN", "MKR", "USDC", "CEL", "BAL", "BAND", "COMP", "SNX", "OMG", "AMPL",
+		"USDT", "KNC", "ZRX", "AXS", "ENJ", "STMX", "DPX", "FTT", "DPI", "PAX",
+	}
+	UniswapForceExclude = []string{"STARL", "UFO"}
+
+	PolygonSwapForceInclude = []string{}
+	PolygonSwapForceExclude = []string{}
+
+	PancakeSwapForceInclude = []string{
+		"Cake", "DAI", "ETH", "TWT", "VAI", "USDT", "BLINK", "BTCB", "ALPHA", "INJ", "CTK", "UNI", "XVS",
+		"BUSD", "HARD", "BIFI", "FRONT",
+	}
+	PancakeSwapForceExclude = []string{}
+)
+
+var (
+	UniswapTradingPairsQuery = map[string]string{
+		"operationName": "pairs",
+		"query": `
+			query pairs {
+				pairs(first: 800, orderBy: reserveUSD, orderDirection: desc) {
+					id reserveUSD trackedReserveETH volumeUSD txCount untrackedVolumeUSD __typename
+					token0 {
+						id symbol name decimals __typename
+					}
+					token1 {
+						id symbol name decimals __typename
+					}
+				}
+			}
+		`,
+	}
+
+	PolygonSwap_TradingPairsQuery = map[string]string{
+		"operationName": "pairs",
+		"query": `
+			{
+				ethereum(network: matic) {
+					dexTrades(date: {is: "$DATE$"}) {
+						sellCurrency {address symbol name decimals}
+						buyCurrency {address symbol name decimals}
+						trade: count
+						tradeAmount(in: USD)
+					}
+				}
+			}
+		`,
+	}
+
+	PancakeSwap_TradingPairsQuery = map[string]string{
+		"operationName": "pairs",
+		"query": `
+			query pairs {
+				pairs(first: 300, orderBy: reserveUSD, orderDirection: desc) {
+					id reserveUSD volumeUSD txCount __typename
+					token0 {
+						id symbol name decimals __typename
+					}
+					token1 {
+						id symbol name decimals __typename
+					}
+				}
+			}
+		`,
+	}
+
+	UniswapTradingPairsUrl     = "https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2"
+	PolygonSwapTradingPairsUrl = "https://graphql.bitquery.io"
+	PancakeSwapradingPairsUrl  = "https://api.bscgraph.org/subgraphs/name/cakeswap"
+)
+
+const (
+	UniswapMinLiquidity = 2000000
+	UniswapMinVol24     = 1000000
+	UniswapMinTxCount24 = 480
+
+	PolygonSwapMinVol24     = 500000
+	PolygonSwapMinTxCount24 = 288
+
+	PancakeSwapMinLiquidity = 1000000
+	PancakeSwapMinVol24     = 500000
+	PancakeSwapMinTxCount24 = 288
+)
+
+var (
+	PrimaryTokensETH = []string{"WETH", "ETH"}
+)
+
+func (s *Service) UpdateEthereumTokenlist() error {
+	log.WithFields(log.Fields{
+		"limit_liquidity": UniswapMinLiquidity,
+		"volume":          UniswapMinVol24,
+		"tx_count":        UniswapMinTxCount24,
+	}).Debug("Retrieving pairs from Uniswap")
+
+	tradingPairs, err := retrieveUniswapPairs(UniswapTradingPairsUrl, UniswapTradingPairsQuery,
+		UniswapMinLiquidity, UniswapMinVol24, UniswapMinTxCount24, UniswapForceInclude, PrimaryTokensETH)
+	if err != nil {
+		return err
+	}
+
+	pairs := make([][]TokenItem, 0)
+
+	for _, tradingPair := range tradingPairs {
+		tokenItem0, err := getTokenInfoFromSubgraphToken(tradingPair.Token0)
+		if err != nil {
+			return err
+		}
+
+		tokenItem1, err := getTokenInfoFromSubgraphToken(tradingPair.Token1)
+		if err != nil {
+			return err
+		}
+
+		if !isTokenPrimary(tradingPair.Token0, PrimaryTokensETH) {
+			tokenItem0, tokenItem1 = tokenItem1, tokenItem0
+		}
+
+		pairs = append(pairs, []TokenItem{*tokenItem0, *tokenItem1})
+	}
+
+	return rebuildTokenList(coin.Coins[coin.ETHEREUM], pairs, UniswapForceExclude)
+}
+
+func (s *Service) UpdatePolygonTokenlist() error {
+	return nil
+}
+
+func (s *Service) UpdateSmartchainTokenlist() error {
+	return nil
+}
+
+func retrieveUniswapPairs(url string, query map[string]string, minLiquidity, minVol24, minTxCount24 int,
+	forceIncludeList []string, primaryTokens []string) ([]TradingPair, error) {
+	includeList := parseForceList(forceIncludeList)
+
+	pairs, err := fetchTradingPairs(url, query)
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := make([]TradingPair, 0)
+
+	for _, pair := range pairs.Data.Pairs {
+		ok, err := checkTradingPairOK(pair, minLiquidity, minVol24, minTxCount24, primaryTokens, includeList)
+		if err != nil {
+			log.Debug(err)
+		}
+		if ok {
+			filtered = append(filtered, pair)
+		}
+	}
+
+	return filtered, nil
+}
+
+func parseForceList(forceList []string) []ForceListPair {
+	result := make([]ForceListPair, 0, len(forceList))
+
+	for _, item := range forceList {
+		tokens := strings.Split(item, "-")
+		pair := ForceListPair{
+			Token0: tokens[0],
+		}
+
+		if len(tokens) >= 2 {
+			pair.Token1 = tokens[1]
+		}
+
+		result = append(result, pair)
+	}
+
+	return result
+}
+
+func fetchTradingPairs(url string, query map[string]string) (*TradingPairs, error) {
+	jsonValue, err := json.Marshal(query)
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithField("url", url).Debug("Retrieving trading pair infos")
+
+	var result TradingPairs
+	err = pkg.PostHTTPResponse(url, jsonValue, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf("Retrieved %d trading pair infos", len(result.Data.Pairs))
+
+	return &result, nil
+}
+
+func checkTradingPairOK(pair TradingPair, minLiquidity, minVol24, minTxCount24 int, primaryTokens []string,
+	forceIncludeList []ForceListPair) (bool, error) {
+	if pair.ID == "" || pair.ReserveUSD == "" || pair.VolumeUSD == "" || pair.TxCount == "" ||
+		pair.Token0 == nil || pair.Token1 == nil {
+
+		return false, nil
+	}
+
+	if !(isTokenPrimary(pair.Token0, primaryTokens) || isTokenPrimary(pair.Token1, primaryTokens)) {
+		log.Debugf("pair with no primary coin: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
+		return false, nil
+	}
+
+	if isPairMatchedToForceList(getTokenItemFromInfo(pair.Token0), getTokenItemFromInfo(pair.Token1), forceIncludeList) {
+		log.Debugf("pair included due to FORCE INCLUDE: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
+		return true, nil
+	}
+
+	reserveUSD, err := strconv.ParseFloat(pair.ReserveUSD, 64)
+	if err != nil {
+		return false, err
+	}
+	if int(reserveUSD) < minLiquidity {
+		log.Debugf("pair with low liquidity: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
+		return false, nil
+	}
+
+	volumeUSD, err := strconv.ParseFloat(pair.VolumeUSD, 64)
+	if err != nil {
+		return false, err
+	}
+	if int(volumeUSD) < minVol24 {
+		log.Debugf("pair with low volume: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
+		return false, nil
+	}
+
+	txCount, err := strconv.ParseFloat(pair.TxCount, 64)
+	if err != nil {
+		return false, err
+	}
+	if int(txCount) < minTxCount24 {
+		log.Debugf("pair with low tx count: %s -- %s", pair.Token0.Symbol, pair.Token1.Symbol)
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func getTokenItemFromInfo(tokenInfo *TokenInfo) *TokenItem {
+	decimals, err := strconv.Atoi(tokenInfo.Decimals)
+	if err != nil {
+		return nil
+	}
+
+	return &TokenItem{
+		Asset:    tokenInfo.ID,
+		Address:  tokenInfo.ID,
+		Name:     tokenInfo.Name,
+		Symbol:   tokenInfo.Symbol,
+		Decimals: uint(decimals),
+	}
+}
+
+func getTokenInfoFromSubgraphToken(token *TokenInfo) (*TokenItem, error) {
+	checksum, err := address.EIP55Checksum(token.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	decimals, err := strconv.Atoi(token.Decimals)
+	if err != nil {
+		return nil, err
+	}
+
+	return &TokenItem{
+		Asset:    getAssetIDSymbol(checksum, coin.Coins[coin.ETHEREUM].Symbol, coin.ETHEREUM),
+		Type:     string(types.ERC20),
+		Address:  checksum,
+		Name:     token.Name,
+		Symbol:   token.Symbol,
+		Decimals: uint(decimals),
+		LogoURI:  getLogoURI(token.Symbol, coin.Coins[coin.ETHEREUM].Handle, coin.Coins[coin.ETHEREUM].Symbol),
+		Pairs:    make([]Pair, 0),
+	}, nil
+}
+
+func isTokenPrimary(token *TokenInfo, primaryTokens []string) bool {
+	if token == nil {
+		return false
+	}
+
+	for _, primaryToken := range primaryTokens {
+		if strings.EqualFold(primaryToken, token.Symbol) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isPairMatchedToForceList(token0, token1 *TokenItem, forceIncludeList []ForceListPair) bool {
+	var matched bool
+
+	for _, forcePair := range forceIncludeList {
+		if matchPairToForceListEntry(token0, token1, forcePair) {
+			matched = true
+		}
+	}
+
+	return matched
+}
+
+func matchPairToForceListEntry(token0, token1 *TokenItem, forceListEntry ForceListPair) bool {
+	if forceListEntry.Token1 == "" {
+		if matchTokenToForceListEntry(token0, forceListEntry.Token0) ||
+			(token1 != nil && matchTokenToForceListEntry(token1, forceListEntry.Token0)) {
+
+			return true
+		}
+
+		return false
+	}
+
+	if token1 == nil {
+		return false
+	}
+
+	if matchTokenToForceListEntry(token0, forceListEntry.Token0) &&
+		matchTokenToForceListEntry(token0, forceListEntry.Token1) {
+
+		return true
+	}
+
+	if matchTokenToForceListEntry(token0, forceListEntry.Token1) &&
+		matchTokenToForceListEntry(token1, forceListEntry.Token0) {
+
+		return true
+	}
+
+	return false
+}
+
+func matchTokenToForceListEntry(token *TokenItem, forceListEntry string) bool {
+	if strings.EqualFold(forceListEntry, token.Symbol) ||
+		strings.EqualFold(forceListEntry, token.Asset) ||
+		strings.EqualFold(forceListEntry, token.Name) {
+
+		return true
+	}
+
+	return false
+}
+
+func matchPairToForceList(token0, token1 *TokenItem, forceList []ForceListPair) bool {
+	var matched bool
+	for _, forcePair := range forceList {
+		if matchPairToForceListEntry(token0, token1, forcePair) {
+			matched = true
+		}
+	}
+
+	return matched
+}
+
+func rebuildTokenList(chain coin.Coin, pairs [][]TokenItem, forceExcludeList []string) error {
+	if pairs == nil || len(pairs) < 5 {
+		return nil
+	}
+
+	excludeList := parseForceList(forceExcludeList)
+
+	pairs2 := make([][]TokenItem, 0)
+
+	for _, pair := range pairs {
+		if !checkTokenExists(chain.Handle, pair[0].Address) {
+			log.Debugf("pair with unsupported 1st coin: %s-%s", pair[0].Symbol, pair[1].Symbol)
+			continue
+		}
+
+		if !checkTokenExists(chain.Handle, pair[1].Address) {
+			log.Debugf("pair with unsupported 2nd coin: %s-%s", pair[0].Symbol, pair[1].Symbol)
+			continue
+		}
+
+		if matchPairToForceList(&pair[0], &pair[1], excludeList) {
+			log.Debugf("pair excluded due to FORCE EXCLUDE: %s-%s", pair[0].Symbol, pair[1].Symbol)
+			continue
+		}
+
+		pairs2 = append(pairs2, pair)
+	}
+
+	filteredCount := len(pairs) - len(pairs2)
+
+	log.Debugf("%d unsupported tokens filtered out, %d pairs", filteredCount, len(pairs2))
+
+	tokenListPath := fmt.Sprintf("blockchains/%s/tokenlist.json", chain.Handle)
+
+	var list TokenList
+	err := pkg.ReadJSONFile(tokenListPath, &list)
+	if err != nil {
+		return nil
+	}
+
+	removeAllPairs(&list)
+
+	for _, pair := range pairs2 {
+		err = addPairIfNeeded(&pair[0], &pair[1], &list)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Debugf("Tokenlist updated: %d tokens", len(list.Tokens))
+
+	var totalPairs int
+
+	for _, item := range list.Tokens {
+		totalPairs += len(item.Pairs)
+	}
+
+	log.Debugf("Tokenlist: list with %d tokens and %d pairs written to %s.",
+		len(list.Tokens), totalPairs, tokenListPath)
+
+	return pkg.CreateJSONFile(tokenListPath, list)
+}
+
+func checkTokenExists(chain, tokenID string) bool {
+	logoPath := asset.GetAssetLogoPath(chain, tokenID)
+
+	return pkg.FileExists(logoPath)
+}
+
+func removeAllPairs(list *TokenList) {
+	for _, token := range list.Tokens {
+		token.Pairs = make([]Pair, 0)
+	}
+}
+
+func addPairIfNeeded(token0, token1 *TokenItem, list *TokenList) error {
+	err := addTokenIfNeeded(token0, list)
+	if err != nil {
+		return err
+	}
+
+	err = addTokenIfNeeded(token1, list)
+	if err != nil {
+		return err
+	}
+
+	addPairToToken(token1, token0, list)
+
+	return nil
+}
+
+func addTokenIfNeeded(token *TokenItem, list *TokenList) error {
+	for _, t := range list.Tokens {
+		if strings.EqualFold(t.Address, token.Address) {
+			return nil
+		}
+	}
+
+	err := updateTokenInfo(token)
+	if err != nil {
+		return err
+	}
+
+	list.Tokens = append(list.Tokens, *token)
+
+	return nil
+}
+
+func updateTokenInfo(token *TokenItem) error {
+	backendClient := backend.InitClient(config.Default.ClientURLs.BackendAPI, nil)
+	assetInfo, err := backendClient.GetAssetInfo(token.Asset)
+	if err != nil {
+		return err
+	}
+
+	if token.Name != assetInfo.Name {
+		token.Name = assetInfo.Name
+	}
+
+	if token.Symbol != assetInfo.Symbol {
+		token.Symbol = assetInfo.Symbol
+	}
+
+	if token.Decimals != uint(assetInfo.Decimals) {
+		token.Decimals = uint(assetInfo.Decimals)
+	}
+
+	return nil
+}
+
+func addPairToToken(pairToken, token *TokenItem, list *TokenList) {
+	var tokenInListIndex = -1
+
+	for i, t := range list.Tokens {
+		if t.Address == token.Address {
+			tokenInListIndex = i
+			break
+		}
+	}
+
+	if tokenInListIndex == -1 {
+		return
+	}
+
+	if list.Tokens[tokenInListIndex].Pairs == nil {
+		list.Tokens[tokenInListIndex].Pairs = make([]Pair, 0)
+	}
+
+	for _, pair := range list.Tokens[tokenInListIndex].Pairs {
+		if pair.Base == pairToken.Asset {
+			return
+		}
+	}
+
+	list.Tokens[tokenInListIndex].Pairs = append(list.Tokens[tokenInListIndex].Pairs, Pair{Base: pairToken.Asset})
+}

internal/processor/validators.go

@@ -0,0 +1,407 @@
+package processor
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/trustwallet/assets-go-libs/pkg/validation"
+	"github.com/trustwallet/assets-go-libs/pkg/validation/info"
+	"github.com/trustwallet/assets-go-libs/pkg/validation/list"
+	"github.com/trustwallet/assets/internal/config"
+	"github.com/trustwallet/assets/internal/file"
+	"github.com/trustwallet/go-primitives/coin"
+)
+
+func (s *Service) ValidateRootFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.RootFolder.AllowedFiles)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateChainFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+
+	err = validation.ValidateLowercase(fileInfo.Name())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.ChainFolder.AllowedFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateImage(f *file.AssetFile) error {
+	var compErr = validation.NewErrComposite()
+
+	err := validation.ValidateLogoFileSize(f.Path())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	// TODO: Replace it with validation.ValidatePngImageDimension when "assets" repo is fixed.
+	// Read comments inValidatePngImageDimensionForCI.
+	err = validation.ValidatePngImageDimensionForCI(f.Path())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.AssetFolder.AllowedFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	err = validation.ValidateAssetAddress(f.Chain(), f.Asset())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	errInfo := validation.ValidateHasFiles(dirFiles, []string{"info.json"})
+	errLogo := validation.ValidateHasFiles(dirFiles, []string{"logo.png"})
+
+	if errLogo != nil || errInfo != nil {
+		infoFile := s.fileService.GetAssetFile(fmt.Sprintf("%s/info.json", f.Path()))
+
+		file2, err := os.Open(infoFile.Path())
+		if err != nil {
+			return err
+		}
+		defer file2.Close()
+
+		_, err = file2.Seek(0, io.SeekStart)
+		if err != nil {
+			return err
+		}
+
+		b, err := io.ReadAll(file2)
+		if err != nil {
+			return err
+		}
+
+		var infoJson info.AssetModel
+		err = json.Unmarshal(b, &infoJson)
+		if err != nil {
+			return err
+		}
+
+		if infoJson.GetStatus() != "spam" && infoJson.GetStatus() != "abandoned" {
+			compErr.Append(fmt.Errorf("%w: logo.png for non-spam assest", validation.ErrMissingFile))
+		}
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateDappsFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+	for _, dirFile := range dirFiles {
+		err = validation.ValidateExtension(dirFile.Name(), config.Default.ValidatorsSettings.DappsFolder.Ext)
+		if err != nil {
+			compErr.Append(err)
+		}
+
+		err = validation.ValidateLowercase(dirFile.Name())
+		if err != nil {
+			compErr.Append(err)
+		}
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	buf := bytes.NewBuffer(nil)
+	_, err = buf.ReadFrom(file)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateJson(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson)
+	}
+
+	var payload info.CoinModel
+	err = json.Unmarshal(buf.Bytes(), &payload)
+	if err != nil {
+		return fmt.Errorf("%w: failed to decode", err)
+	}
+
+	tags := make([]string, len(config.Default.ValidatorsSettings.CoinInfoFile.Tags))
+	for i, t := range config.Default.ValidatorsSettings.CoinInfoFile.Tags {
+		tags[i] = t.ID
+	}
+
+	err = info.ValidateCoin(payload, f.Chain(), f.Asset(), tags)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	buf := bytes.NewBuffer(nil)
+	if _, err = buf.ReadFrom(file); err != nil {
+		return err
+	}
+
+	err = validation.ValidateJson(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("%w: failed to seek reader", validation.ErrInvalidJson)
+	}
+
+	var payload info.AssetModel
+	err = json.Unmarshal(buf.Bytes(), &payload)
+	if err != nil {
+		return fmt.Errorf("%w: failed to decode", err)
+	}
+
+	err = info.ValidateAsset(payload, f.Chain(), f.Asset())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	if !isStackingChain(f.Chain()) {
+		return nil
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if _, err = buf.ReadFrom(file); err != nil {
+		return err
+	}
+
+	err = validation.ValidateJson(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	var model []list.Model
+	err = json.Unmarshal(buf.Bytes(), &model)
+	if err != nil {
+		return err
+	}
+
+	err = list.ValidateList(model)
+	if err != nil {
+		return err
+	}
+
+	listIDs := make([]string, len(model))
+	for i, listItem := range model {
+		listIDs[i] = *listItem.ID
+	}
+
+	assetsPath := fmt.Sprintf("blockchains/%s/validators/assets", f.Chain().Handle)
+	assetFolder := s.fileService.GetAssetFile(assetsPath)
+
+	file2, err := os.Open(assetFolder.Path())
+	if err != nil {
+		return err
+	}
+	defer file2.Close()
+
+	dirAssetFolderFiles, err := file2.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirAssetFolderFiles, listIDs)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isStackingChain(c coin.Coin) bool {
+	for _, stackingChain := range config.StackingChains {
+		if c.ID == stackingChain.ID {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (s *Service) ValidateTokenListFile(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	buf := bytes.NewBuffer(nil)
+	if _, err = buf.ReadFrom(file); err != nil {
+		return err
+	}
+
+	err = validation.ValidateJson(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateInfoFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainInfoFolder.HasFiles)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateValidatorsAssetFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	compErr := validation.NewErrComposite()
+	err = validation.ValidateValidatorsAddress(f.Chain(), f.Asset())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainValidatorsAssetFolder.HasFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}

internal/service/service.go

@@ -0,0 +1,102 @@
+package service
+
+import (
+	"github.com/trustwallet/assets-go-libs/pkg/validation"
+	"github.com/trustwallet/assets/internal/file"
+	"github.com/trustwallet/assets/internal/processor"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Service struct {
+	fileService      *file.Service
+	processorService *processor.Service
+}
+
+func NewService(fs *file.Service, cs *processor.Service) *Service {
+	return &Service{
+		fileService:      fs,
+		processorService: cs,
+	}
+}
+
+func (s *Service) RunJob(paths []string, job func(*file.AssetFile)) {
+	for _, path := range paths {
+		f := s.fileService.GetAssetFile(path)
+		job(f)
+	}
+}
+
+func (s *Service) Check(f *file.AssetFile) {
+	validator := s.processorService.GetValidator(f)
+
+	if validator != nil {
+		if err := validator.Run(f); err != nil {
+			// TODO: somehow return an error from Check if there are any errors.
+			HandleError(err, f, validator.Name)
+		}
+	}
+}
+
+func (s *Service) Fix(f *file.AssetFile) {
+	fixers := s.processorService.GetFixers(f)
+
+	for _, fixer := range fixers {
+		if err := fixer.Run(f); err != nil {
+			HandleError(err, f, fixer.Name)
+		}
+	}
+}
+
+func (s *Service) RunUpdateAuto() {
+	updaters := s.processorService.GetUpdatersAuto()
+	s.runUpdaters(updaters)
+}
+
+func (s *Service) RunUpdateManual() {
+	updaters := s.processorService.GetUpdatersManual()
+	s.runUpdaters(updaters)
+}
+
+func (s *Service) runUpdaters(updaters []processor.Updater) {
+	for _, updater := range updaters {
+		err := updater.Run()
+		if err != nil {
+			log.WithError(err).Error()
+		}
+	}
+}
+
+func HandleError(err error, info *file.AssetFile, valName string) {
+	errors := UnwrapComposite(err)
+
+	for _, err := range errors {
+		logFields := log.Fields{
+			"type":       info.Type(),
+			"chain":      info.Chain().Handle,
+			"asset":      info.Asset(),
+			"path":       info.Path(),
+			"validation": valName,
+		}
+
+		if warn, ok := err.(*validation.Warning); ok {
+			log.WithFields(logFields).Warning(warn)
+		} else {
+			log.WithFields(logFields).Error(err)
+		}
+	}
+}
+
+func UnwrapComposite(err error) []error {
+	compErr, ok := err.(*validation.ErrComposite)
+	if !ok {
+		return []error{err}
+	}
+
+	var errors []error
+	for _, e := range compErr.GetErrors() {
+		errors = append(errors, UnwrapComposite(e)...)
+	}
+
+	return errors
+}