diff --git a/.github/workflows/fix-dryrun.yml b/.github/workflows/fix-dryrun.yml index 078b53fb3..43d30940e 100644 --- a/.github/workflows/fix-dryrun.yml +++ b/.github/workflows/fix-dryrun.yml @@ -1,4 +1,4 @@ -name: Fixes Dryrun +name: Fixes and Consistency Updates - Dry run on: push: branches: @@ -7,7 +7,7 @@ on: pull_request: branches: [master] jobs: - fix-consistency: + scripts: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -16,12 +16,11 @@ jobs: node-version: 12 - name: Install Dependencies run: npm ci - - name: Run fix scripts - run: | - npm run resize - npm run format:all - npm run gen:list + - name: Run fix script + run: npm run fix - name: Show git status (diff) + if: success() run: git status - name: Run test run: npm t + diff --git a/.github/workflows/fix.yml b/.github/workflows/fix.yml index 9fb034252..a3024b30c 100644 --- a/.github/workflows/fix.yml +++ b/.github/workflows/fix.yml @@ -3,23 +3,19 @@ on: push: branches: [ master ] jobs: - fix-consistency: + scripts: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - token: ${{ secrets.DANGER_GITHUB_API_TOKEN }} - uses: actions/setup-node@v1 with: node-version: 12 - name: Install Dependencies run: npm ci - - name: Run fix scripts - run: | - npm run resize - npm run format:all - npm run gen:list + - name: Run fix script + run: npm run fix - name: Show git status (diff) + if: success() run: git status - name: Run test run: npm t diff --git a/.github/workflows/periodic-update.yml b/.github/workflows/periodic-update.yml index a89ee425b..a1e1ef39d 100644 --- a/.github/workflows/periodic-update.yml +++ b/.github/workflows/periodic-update.yml @@ -4,7 +4,7 @@ on: # Run twice per day (at 7:00UTC/12amPST, 19:00UTC/12pmPST) - cron: '0 7,19 * * *' jobs: - update: + scripts: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -17,10 +17,10 @@ jobs: - name: Install Dependencies run: npm ci - name: Run scripts - run: | - npm run gen:validators:tezos - npm run update:bep2 - npm run map:cmc + run: npm run update + - name: Show git status (diff) + if: success() + run: git status - name: Run test run: npm t - name: Commit changes test pased diff --git a/blockchains/tron/whitelist.json b/blockchains/tron/whitelist.json index 5ddf25188..bce6408bd 100644 --- a/blockchains/tron/whitelist.json +++ b/blockchains/tron/whitelist.json @@ -152,8 +152,8 @@ "TLyqzVGLV1srkB7dToTAEqgDSfPtXRJZYH", "TMhjbHzJCiMWqa5oqn2DF3mGw5yh9oQNf2", "TMwFHYXLJaRUPeW6421aqXL4ZEzPRFGkGT", - "TNZXVQUKQ8Gnjq2BqLvt2kC5WbeDQc1q3j", "TNjt5fShPVJ4YpsLuU4THuBbg58g2bZoLk", + "TNZXVQUKQ8Gnjq2BqLvt2kC5WbeDQc1q3j", "TPRuU2GbXPwvvSuggE4xKMRtzsfwYfvBWq", "TQhfmGiZvRt3oe9vKv6W9WXRr4oErxc8RE", "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t", diff --git a/package.json b/package.json index e7929358c..4fd9e8272 100644 --- a/package.json +++ b/package.json @@ -5,16 +5,8 @@ "main": "index.js", "scripts": { "test": "jest", - "update:bep2": "node ./script/updateBEP2", - "checksum": "ts-node ./script/erc20_to_checksum", - "format:all": "ts-node ./script/format_files_name", - "arrange:all": "ts-node ./script/arrange_files", - "gen:list": "ts-node ./script/gen_list", - "gen:info": "ts-node ./script/gen_info", - "gen:validators:tezos": "ts-node ./script/gen_validators_tezos", - "resize": "ts-node ./script/resize_images", - "map:cmc": "ts-node ./pricing/coinmarketcap/script", - "magic": "npm run update:bep2 && npm run checksum && npm run resize && npm run gen:list && npm t" + "fix": "ts-node ./script/main/fix", + "update": "ts-node ./script/main/update" }, "repository": { "type": "git", diff --git a/pricing/coinmarketcap/script.ts b/pricing/coinmarketcap/script.ts index 4d0b9105a..cc83f4518 100644 --- a/pricing/coinmarketcap/script.ts +++ b/pricing/coinmarketcap/script.ts @@ -1,5 +1,5 @@ import { toChecksum } from "../../src/test/helpers" -const BluebirbPromise = require("bluebird") +const BluebirdPromise = require("bluebird") const axios = require("axios") const chalk = require('chalk') const fs = require("fs") @@ -20,7 +20,7 @@ import { TickerType, mapTiker, PlatformType } from "../../src/test/models"; // Steps required to run this: // 1. (Optional) CMC API key already setup, use yours if needed. Install script deps "npm i" if hasn't been run before. // 2. Pull down tokens repo https://github.com/trustwallet/assets and point COIN_IMAGE_BASE_PATH and TOKEN_IMAGE_BASE_PATH to it. -// 3. Run: `npm run gen:list` +// 3. Run: `npm run update` const CMC_PRO_API_KEY = `df781835-e5f4-4448-8b0a-fe31402ab3af` // Free Basic Plan api key is enough to run script const CMC_LATEST_BASE_URL = `https://pro-api.coinmarketcap.com/v1/global-metrics/quotes/latest?` @@ -61,14 +61,13 @@ const allContracts: mapTiker[] = [] // Temp storage for mapped assets let bnbOwnerToSymbol = {} // e.g: bnb1tawge8u97slduhhtumm03l4xl4c46dwv5m9yzk: WISH-2D5 let bnbOriginalSymbolToSymbol = {} // e.g: WISH: WISH-2D5 -run() async function run() { try { await Promise.all([initState(), setBinanceTokens()]) const [totalCrypto, coins] = await Promise.all([getTotalActiveCryptocurrencies(), getTickers()]) // setBIP44Constants() log(`Found ${totalCrypto} on CMC`, chalk.yellowBright) - await BluebirbPromise.mapSeries(coins, processCoin) + await BluebirdPromise.mapSeries(coins, processCoin) addCustom() printContracts() @@ -351,4 +350,8 @@ function log(string, cb?) { // coinName: row[2] // } // }) -// } \ No newline at end of file +// } + +export async function update() { + await run(); +} diff --git a/script/arrange_files.ts b/script-old/arrange_files.ts similarity index 100% rename from script/arrange_files.ts rename to script-old/arrange_files.ts diff --git a/script/bep2_to_checksum.sh b/script-old/bep2_to_checksum.sh similarity index 100% rename from script/bep2_to_checksum.sh rename to script-old/bep2_to_checksum.sh diff --git a/script/gen_info.ts b/script-old/gen_info.ts similarity index 100% rename from script/gen_info.ts rename to script-old/gen_info.ts diff --git a/script/opesea_contrats.ts b/script-old/opensea_contracts.ts similarity index 100% rename from script/opesea_contrats.ts rename to script-old/opensea_contracts.ts diff --git a/script/remove_empthy_folder.ts b/script-old/remove_empthy_folder.ts similarity index 100% rename from script/remove_empthy_folder.ts rename to script-old/remove_empthy_folder.ts diff --git a/script/trc20_to_checksum.js b/script-old/trc20_to_checksum.js similarity index 100% rename from script/trc20_to_checksum.js rename to script-old/trc20_to_checksum.js diff --git a/script/action/binance.ts b/script/action/binance.ts new file mode 100644 index 000000000..ca83fe9b8 --- /dev/null +++ b/script/action/binance.ts @@ -0,0 +1,87 @@ +import axios from "axios"; +import * as bluebird from "bluebird"; +import * as fs from "fs"; +import * as path from "path"; +import * as chalk from 'chalk'; +import * as config from "../common/config"; + +import { + getChainAssetLogoPath, + getChainBlacklistPath +} from "../common/repo-structure"; + +const binanceChain = "binance" +const binanceAssetsUrl = config.getConfig("binance_assets_url", "https://explorer.binance.org/api/v1/assets?page=1&rows=1000"); + +async function retrieveAssetList() { + console.log(`Retrieving assets info from: ${binanceAssetsUrl}`); + const { assetInfoList } = await axios.get(binanceAssetsUrl).then(r => r.data); + console.log(`Retrieved ${assetInfoList.length} asset infos`); + return assetInfoList +} + +function fetchImage(url) { + return axios.get(url, { responseType: "stream" }) + .then(r => r.data) + .catch(err => { + throw `Error fetchImage: ${url} ${err.message}`; + }); +} + +/// Return: array with images to fetch; {asset, assetImg} +export function findImagesToFetch(assetInfoList: any, blacklist: string[]): any[] { + let toFetch: any[] = []; + console.log(`Checking for asset images to be fetched`); + assetInfoList.forEach(({asset, assetImg}) => { + process.stdout.write(`.${asset} `); + if (assetImg) { + if (blacklist.indexOf(asset) != -1) { + console.log(); + console.log(`${asset} is blacklisted`); + } else { + const imagePath = getChainAssetLogoPath(binanceChain, asset); + if (!fs.existsSync(imagePath)) { + console.log(chalk.red(`Missing image: ${asset}`)); + toFetch.push({asset, assetImg}); + } + } + } + }); + console.log(); + console.log(`${toFetch.length} asset image(s) to be fetched`); + return toFetch; +} + + +async function fetchMissingImages(toFetch: any[]): Promise { + console.log(`Attempting to fetch ${toFetch.length} asset image(s)`); + let fetchedAssets: string[] = []; + await bluebird.each(toFetch, async ({ asset, assetImg }) => { + if (assetImg) { + const imagePath = getChainAssetLogoPath(binanceChain, asset); + fs.mkdir(path.dirname(imagePath), err => { + if (err && err.code != `EEXIST`) throw err; + }); + await fetchImage(assetImg).then(buffer => { + buffer.pipe(fs.createWriteStream(imagePath)); + fetchedAssets.push(asset) + console.log(`Fetched image ${asset} ${imagePath} from ${assetImg}`) + }); + } + }); + console.log(); + return fetchedAssets; +} + +export async function update() { + const assetInfoList = await retrieveAssetList(); + const blacklist: string[] = require(getChainBlacklistPath(binanceChain)); + + const toFetch = findImagesToFetch(assetInfoList, blacklist); + const fetchedAssets = await fetchMissingImages(toFetch); + + if (fetchedAssets.length > 0) { + console.log(`Fetched ${fetchedAssets.length} asset(s):`); + fetchedAssets.forEach(asset => console.log(` ${asset}`)); + } +} diff --git a/script/action/eth-forks.ts b/script/action/eth-forks.ts new file mode 100644 index 000000000..517730bf3 --- /dev/null +++ b/script/action/eth-forks.ts @@ -0,0 +1,66 @@ +import { ethForkChains } from "../common/blockchains"; +import { + getChainAssetsPath, + getChainAssetsList, + getChainAssetPath, + getChainAssetInfoPath, + getChainAssetFilesList, + isChainAssetInfoExistSync, + logoName, + logoExtension, + logoFullName +} from "../common/repo-structure"; +import { formatJsonFile } from "../common/json"; +import { + getFileName, + getFileExt, + gitMove, + readDirSync +} from "../common/filesystem"; +import { isChecksum, toChecksum } from "../common/eth-web3"; + +function formatInfos() { + console.log(`Formatting info files...`); + ethForkChains.forEach(chain => { + let count: number = 0; + const chainAssets = getChainAssetsList(chain); + chainAssets.forEach(address => { + if (isChainAssetInfoExistSync(chain, address)) { + const chainAssetInfoPath = getChainAssetInfoPath(chain, address); + formatJsonFile(chainAssetInfoPath, true); + ++count; + } + }) + console.log(`Formatted ${count} info files for chain ${chain} (total ${chainAssets.length})`); + }) +} + +function checkAddressChecksum(assetsFolderPath: string, address: string) { + if (!isChecksum(address)) { + const checksumAddress = toChecksum(address); + gitMove(assetsFolderPath, address, checksumAddress); + console.log(`Renamed to checksum format ${checksumAddress}`); + } +} + +function checkAddressChecksums() { + console.log(`Checking for checksum formats ...`); + ethForkChains.forEach(chain => { + const assetsPath = getChainAssetsPath(chain); + + readDirSync(assetsPath).forEach(address => { + getChainAssetFilesList(chain, address).forEach(file => { + if (getFileName(file) == logoName && getFileExt(file) !== logoExtension) { + console.log(`Renaming incorrect asset logo extension ${file} ...`); + gitMove(getChainAssetPath(chain, address), file, logoFullName); + } + }); + checkAddressChecksum(assetsPath, address); + }); + }); +} + +export async function fix() { + formatInfos(); + checkAddressChecksums(); +} diff --git a/script/action/logo-size.ts b/script/action/logo-size.ts new file mode 100644 index 000000000..035cf9212 --- /dev/null +++ b/script/action/logo-size.ts @@ -0,0 +1,62 @@ +import * as bluebird from "bluebird"; +import { + chainsPath, + getChainLogoPath, + getChainAssetsPath, + getChainAssetLogoPath, + getChainValidatorsListPath, + getChainValidatorAssetLogoPath +} from "../common/repo-structure"; +import { + readDirSync, + readFileSync, + isPathExistsSync +} from "../common/filesystem"; +import { resizeIfTooLarge } from "../common/image"; + +async function downsize(chains) { + console.log(`Checking all logos for downsizing ...`); + let totalCountChecked: number = 0; + let totalCountUpdated: number = 0; + await bluebird.map(chains, async chain => { + let countChecked: number = 0; + let countUpdated: number = 0; + + const path = getChainLogoPath(chain); + countChecked++; + countUpdated += await resizeIfTooLarge(path) ? 1 : 0; + + // Check and resize if needed chain assets + const assetsPath = getChainAssetsPath(chain); + if (isPathExistsSync(assetsPath)) { + await bluebird.mapSeries(readDirSync(assetsPath), async asset => { + const path = getChainAssetLogoPath(chain, asset); + countChecked++; + countUpdated += await resizeIfTooLarge(path) ? 1 : 0; + }) + } + + // Check and resize if needed chain validators image + const chainValidatorsList = getChainValidatorsListPath(chain); + if (isPathExistsSync(chainValidatorsList)) { + const validatorsList = JSON.parse(readFileSync(getChainValidatorsListPath(chain))); + await bluebird.mapSeries(validatorsList, async ({ id }) => { + const path = getChainValidatorAssetLogoPath(chain, id); + countChecked++; + countUpdated += await resizeIfTooLarge(path) ? 1 : 0; + }) + } + + totalCountChecked += countChecked; + totalCountUpdated += countUpdated; + if (countUpdated > 0) { + console.log(`Checking logos on chain ${chain} completed, ${countChecked} checked, ${countUpdated} logos updated`); + } + }); + console.log(`Checking logos completed, ${totalCountChecked} logos checked, ${totalCountUpdated} logos updated`); +} + +export async function fix() { + const foundChains = readDirSync(chainsPath); + await downsize(foundChains); +} diff --git a/script/gen_validators_tezos.ts b/script/action/tezos.ts similarity index 68% rename from script/gen_validators_tezos.ts rename to script/action/tezos.ts index 2c56fef40..91086d4e7 100644 --- a/script/gen_validators_tezos.ts +++ b/script/action/tezos.ts @@ -1,13 +1,23 @@ -const axios = require('axios') -import { - getChainValidatorsList, - getChainValidatorsListPath, - Tezos, - writeJSONToPath -} from "../src/test/helpers"; -import { BakingBadBaker } from "../src/test/models"; +import axios from "axios"; +import { + validatorsList, + getChainValidatorsPath, + getChainValidatorsListPath +} from "../common/repo-structure"; +import { Tezos } from "../common/blockchains"; +import { readFileSync } from "../common/filesystem"; +import { writeJsonFile } from "../common/json"; -(async function(){ +import { + BakingBadBaker, + ValidatorModel +} from "../../src/test/models"; + +function getChainValidatorsList(chain: string): ValidatorModel[] { + return JSON.parse(readFileSync(`${(getChainValidatorsPath(chain))}/${validatorsList}`)); +} + +async function gen_validators_tezos() { const bakers: BakingBadBaker[] = await axios.get(`https://api.baking-bad.org/v2/bakers`).then(res => res.data) const bakersMap: {[key: string]: BakingBadBaker} = bakers.reduce((acm, val) => { acm[val.address] = val @@ -51,5 +61,9 @@ import { BakingBadBaker } from "../src/test/models"; return acm }, []) - writeJSONToPath(getChainValidatorsListPath(Tezos), newbakers) -})() + writeJsonFile(getChainValidatorsListPath(Tezos), newbakers) +} + +export async function update() { + await gen_validators_tezos(); +} diff --git a/script/action/update-all.ts b/script/action/update-all.ts new file mode 100644 index 000000000..0ea81d836 --- /dev/null +++ b/script/action/update-all.ts @@ -0,0 +1,22 @@ +import * as eth_forks from "./eth-forks"; +import * as logo_size from "./logo-size"; +import * as validators from "./validators"; +import * as whitelists from "./whitelists"; +import * as binance from "./binance"; +import * as coinmarketcap from "../../pricing/coinmarketcap/script"; +import * as tezos from "./tezos"; + +export function fixAll() { + console.log("Running fixes..."); + eth_forks.fix(); + logo_size.fix(); + validators.fix(); + whitelists.fix(); +} + +export function updateAll() { + console.log("Running updates (using external data sources) ..."); + tezos.update(); + binance.update(); + coinmarketcap.update(); +} diff --git a/script/action/validators.ts b/script/action/validators.ts new file mode 100644 index 000000000..2b4326176 --- /dev/null +++ b/script/action/validators.ts @@ -0,0 +1,14 @@ +import { stakingChains } from "../common/blockchains"; +import { getChainValidatorsListPath } from "../common/repo-structure"; +import { formatSortJsonFile } from "../common/json"; + +function formatValidators() { + stakingChains.forEach(chain => { + const validatorsPath = getChainValidatorsListPath(chain); + formatSortJsonFile(validatorsPath); + }) +} + +export async function fix() { + formatValidators(); +} diff --git a/script/action/whitelists.ts b/script/action/whitelists.ts new file mode 100644 index 000000000..2f9913470 --- /dev/null +++ b/script/action/whitelists.ts @@ -0,0 +1,39 @@ +import { chainsWithBlacklist } from "../common/blockchains"; +import { getChainAssetsList, getChainWhitelistPath, getChainBlacklistPath } from "../common/repo-structure"; +import { readFileSync, writeFileSync } from "../common/filesystem"; +import { sortElements, makeUnique, arrayDiff } from "../common/types"; + +function formatWhiteBlackList() { + chainsWithBlacklist.forEach(async chain => { + const assets = getChainAssetsList(chain); + + const whitelistPath = getChainWhitelistPath(chain); + const blacklistPath = getChainBlacklistPath(chain); + + const currentWhitelist = JSON.parse(readFileSync(whitelistPath)); + const currentBlacklist = JSON.parse(readFileSync(blacklistPath)); + + let newBlackList = []; + // Some chains required pulling lists from other sources + // switch (chain) { + // case Ethereum: + // const nftList = await getOpenseaCollectionAddresses() + // newBlackList = currentBlacklist.concat(nftList) + // break; + // default: + // newBlackList = newBlackList.concat(currentBlacklist) + // break; + // } + + const removedAssets = arrayDiff(currentWhitelist, assets); + newBlackList = currentBlacklist.concat(removedAssets); + + writeFileSync(whitelistPath, JSON.stringify(sortElements(assets), null, 4)); + writeFileSync(blacklistPath, JSON.stringify(makeUnique(sortElements(newBlackList)), null, 4)); + console.log(`Updated white and blacklists for chain ${chain}`); + }) +} + +export async function fix() { + formatWhiteBlackList(); +} diff --git a/script/common/blockchains.ts b/script/common/blockchains.ts new file mode 100644 index 000000000..cab651aec --- /dev/null +++ b/script/common/blockchains.ts @@ -0,0 +1,25 @@ +import { CoinType } from "@trustwallet/wallet-core"; + +export const getChainName = (id: CoinType): string => CoinType.id(id); // 60 => ethereum +export const Binance = getChainName(CoinType.binance); +export const Classic = getChainName(CoinType.classic); +export const Cosmos = getChainName(CoinType.cosmos); +export const Ethereum = getChainName(CoinType.ethereum); +export const GoChain = getChainName(CoinType.gochain); +export const IoTeX = getChainName(CoinType.iotex); +export const NEO = getChainName(CoinType.neo); +export const NULS = getChainName(CoinType.nuls); +export const POA = getChainName(CoinType.poa); +export const Tezos = getChainName(CoinType.tezos); +export const ThunderCore = getChainName(CoinType.thundertoken); +export const Terra = getChainName(CoinType.terra); +export const TomoChain = getChainName(CoinType.tomochain); +export const Tron = getChainName(CoinType.tron); +export const Kava = getChainName(CoinType.kava); +export const Wanchain = getChainName(CoinType.wanchain); +export const Waves = getChainName(CoinType.waves); +export const Solana = getChainName(CoinType.solana); + +export const ethForkChains = [Ethereum, Classic, POA, TomoChain, GoChain, Wanchain, ThunderCore]; +export const stakingChains = [Tezos, Cosmos, IoTeX, Tron, Waves, Kava, Terra]; +export const chainsWithBlacklist = ethForkChains.concat(Tron, Terra, NEO, NULS); diff --git a/script/common/config.ts b/script/common/config.ts new file mode 100644 index 000000000..867619624 --- /dev/null +++ b/script/common/config.ts @@ -0,0 +1,15 @@ + +const configFileName = "../config.json"; +const configData = require(configFileName); + +export function getConfig(key: string, defaultValue: any): any { + if (!configData) { + console.log(`Missing config, config file: ${configFileName}`); + return defaultValue; + } + if (!(key in configData)) { + console.log(`Missing config entry, key ${key}, config file: ${configFileName}`); + return defaultValue; + } + return configData[key]; +} diff --git a/script/common/eth-web3.ts b/script/common/eth-web3.ts new file mode 100644 index 000000000..dec199960 --- /dev/null +++ b/script/common/eth-web3.ts @@ -0,0 +1,6 @@ +const Web3 = require('web3'); + +const web3 = new Web3('ws://localhost:8546'); + +export const isChecksum = (address: string): boolean => web3.utils.checkAddressChecksum(address); +export const toChecksum = (address: string): string => web3.utils.toChecksumAddress(address); diff --git a/script/common/filesystem.ts b/script/common/filesystem.ts new file mode 100644 index 000000000..8280c59ee --- /dev/null +++ b/script/common/filesystem.ts @@ -0,0 +1,26 @@ +import * as fs from "fs"; +import * as path from "path"; +import { execSync } from "child_process"; + +export const getFileName = (name: string): string => path.basename(name, path.extname(name)) +export const getFileExt = (name: string): string => name.slice((Math.max(0, name.lastIndexOf(".")) || Infinity) + 1) + +export const readFileSync = (path: string) => fs.readFileSync(path, 'utf8'); +export const writeFileSync = (path: string, data: any) => fs.writeFileSync(path, data); +export const readDirSync = (path: string): string[] => fs.readdirSync(path); +export const isPathExistsSync = (path: string): boolean => fs.existsSync(path); +export const getFileSizeInKilobyte = (path: string): number => fs.statSync(path).size / 1000; + +export function execRename(command: string, cwd: string) { + console.log(`Running command ${command}`); + execSync(command, {encoding: "utf-8", cwd: cwd}); +} + +function gitMoveCommand(oldName: string, newName: string): string { + return `git mv ${oldName} ${newName}-temp && git mv ${newName}-temp ${newName}`; +} + +export function gitMove(path: string, oldName: string, newName: string) { + console.log(`Renaming file or folder at path ${path}: ${oldName} => ${newName}`); + execRename(gitMoveCommand(oldName, newName), path); +} diff --git a/script/common/image.ts b/script/common/image.ts new file mode 100644 index 000000000..250fd23bb --- /dev/null +++ b/script/common/image.ts @@ -0,0 +1,73 @@ +import * as sharp from "sharp"; +import * as tinify from "tinify"; +import * as image_size from "image-size"; +import { + writeFileSync, + getFileSizeInKilobyte +} from "./filesystem"; +import * as chalk from 'chalk'; +import * as config from "../common/config"; + +//export const minLogoWidth = 64; +//export const minLogoHeight = 64; +export const maxLogoWidth = config.getConfig("image_max_logo_width", 512); +export const maxLogoHeight = config.getConfig("image_max_logo_height", 512); +export const maxLogoSizeInKilobyte = config.getConfig("image_logo_size_kb", 100); + +export function isDimensionTooLarge(width: number, height: number): boolean { + return (width > maxLogoWidth) || (height > maxLogoHeight); +} + +export function calculateTargetSize(srcWidth: number, srcHeight: number, targetWidth: number, targetHeight: number): {width: number, height: number} { + if (srcWidth == 0 || srcHeight == 0) { + return {width: targetWidth, height: targetHeight}; + } + const ratio = Math.min(targetWidth / srcWidth, targetHeight / srcHeight); + return { + width: Math.round(srcWidth * ratio), + height: Math.round(srcHeight * ratio) + }; +} + +const getImageDimensions = (path: string) => image_size.imageSize(path); + +async function compressTinyPNG(path: string) { + console.log(`Compressing image via tinypng at path ${path}`); + const source = await tinify.fromFile(path); + await source.toFile(path); +} + +// return true if image updated +export async function resizeIfTooLarge(path: string): Promise { + let updated: boolean = false; + + const { width: srcWidth, height: srcHeight } = getImageDimensions(path); + if (isDimensionTooLarge(srcWidth, srcHeight)) { + const { width, height } = calculateTargetSize(srcWidth, srcHeight, maxLogoWidth, maxLogoHeight); + console.log(`Resizing image at ${path} from ${srcWidth}x${srcHeight} => ${width}x${height}`) + await sharp(path).resize(width, height).toBuffer() + .then(data => { + writeFileSync(path, data); + updated = true; + }) + .catch(e => { + console.log(chalk.red(e.message)); + }); + } + + // If file size > max limit, compress with tinypng + const sizeKilobyte = getFileSizeInKilobyte(path); + if (sizeKilobyte > maxLogoSizeInKilobyte) { + console.log(`Resizing image at path ${path} from ${sizeKilobyte} kB`); + await compressTinyPNG(path) + .then(() => { + updated = true; + console.log(`Resized image at path ${path} from ${sizeKilobyte} kB => ${getFileSizeInKilobyte(path)} kB`); + }) + .catch(e => { + console.log(chalk.red(e.message)); + }); + } + + return updated; +} diff --git a/script/common/json.ts b/script/common/json.ts new file mode 100644 index 000000000..61724fa10 --- /dev/null +++ b/script/common/json.ts @@ -0,0 +1,23 @@ +import { + readFileSync, + writeFileSync +} from "./filesystem"; +import { sortElements } from "./types"; + +export function formatJsonFile(filename: string, silent: boolean = false) { + const jsonContent = JSON.parse(readFileSync(filename)); + writeFileSync(filename, JSON.stringify(jsonContent, null, 4)); + if (!silent) { + console.log(`Formatted json file ${filename}`); + } +} + +export function formatSortJsonFile(filename: string) { + const jsonContent = JSON.parse(readFileSync(filename)); + writeFileSync(filename, JSON.stringify(sortElements(jsonContent), null, 4)); + console.log(`Formatted json file ${filename}`); +} + +export function writeJsonFile(path: string, data: any) { + writeFileSync(path, JSON.stringify(data, null, 4)); +} diff --git a/script/common/repo-structure.ts b/script/common/repo-structure.ts new file mode 100644 index 000000000..9b20fc84e --- /dev/null +++ b/script/common/repo-structure.ts @@ -0,0 +1,36 @@ +import * as path from "path"; +import { + isPathExistsSync, + readDirSync +} from "./filesystem"; + +export const logoName = `logo`; +export const infoName = `info`; +export const listName = `list` +export const logoExtension = "png"; +export const jsonExtension = "json"; +export const logoFullName = `${logoName}.${logoExtension}`; +export const infoFullName = `${infoName}.${jsonExtension}`; +const whiteList = `whitelist.${jsonExtension}`; +const blackList = `blacklist.${jsonExtension}`; +export const validatorsList = `${listName}.${jsonExtension}` + +export const chainsPath: string = path.join(process.cwd(), '/blockchains'); +export const getChainPath = (chain: string): string => `${chainsPath}/${chain}`; +export const getChainLogoPath = (chain: string): string => `${getChainPath(chain)}/info/${logoFullName}`; +export const getChainAssetsPath = (chain: string): string => `${getChainPath(chain)}/assets`; +export const getChainAssetPath = (chain: string, asset: string) => `${getChainAssetsPath(chain)}/${asset}`; +export const getChainAssetLogoPath = (chain: string, asset: string): string => `${getChainAssetPath(chain, asset)}/${logoFullName}`; +export const getChainAssetInfoPath = (chain: string, asset: string): string => `${getChainAssetPath(chain, asset)}/${infoFullName}`; +export const getChainWhitelistPath = (chain: string): string => `${getChainPath(chain)}/${whiteList}`; +export const getChainBlacklistPath = (chain: string): string => `${getChainPath(chain)}/${blackList}`; + +export const getChainValidatorsPath = (chain: string): string => `${getChainPath(chain)}/validators`; +export const getChainValidatorsListPath = (chain: string): string => `${getChainValidatorsPath(chain)}/list.${jsonExtension}`; +export const getChainValidatorsAssetsPath = (chain: string): string => `${getChainValidatorsPath(chain)}/assets` +export const getChainValidatorAssetLogoPath = (chain: string, asset: string): string => `${getChainValidatorsAssetsPath(chain)}/${asset}/${logoFullName}` + +export const isChainAssetInfoExistSync = (chain: string, address: string) => isPathExistsSync(getChainAssetInfoPath(chain, address)); + +export const getChainAssetsList = (chain: string): string[] => readDirSync(getChainAssetsPath(chain)); +export const getChainAssetFilesList = (chain: string, address: string) => readDirSync(getChainAssetPath(chain, address)); diff --git a/script/common/types.ts b/script/common/types.ts new file mode 100644 index 000000000..568822dd6 --- /dev/null +++ b/script/common/types.ts @@ -0,0 +1,29 @@ +export const mapList = arr => { + return arr.reduce((acm, val) => { + acm[val] = ""; + return acm; + }, {}); +} + +// Sort: treat numbers as number, strings as case-insensitive +export const sortElements = (arr: any[]): any[] => { + arr.sort((a, b) => { + if (!isNaN(a) && !isNaN(b)) { + // numerical comparison + return a - b; + } + if ((typeof a === 'string' || a instanceof String) && (typeof b === 'string' || b instanceof String)) { + return a.toLowerCase() > b.toLowerCase() ? 1 : -1; + } + return 0; + }); + return arr; +} + +export const makeUnique = (arr: any[]): any[] => Array.from(new Set(arr)); + +// Remove from set a elements of set b. +export function arrayDiff(a: string[], b: string[]): string[] { + const mappedB = mapList(b); + return a.filter(e => !mappedB.hasOwnProperty(e)); +} diff --git a/script/config.json b/script/config.json new file mode 100644 index 000000000..fc910c3ba --- /dev/null +++ b/script/config.json @@ -0,0 +1,6 @@ +{ + "image_max_logo_width": 512, + "image_max_logo_height": 512, + "image_logo_size_kb": 100, + "binance_assets_url": "https://explorer.binance.org/api/v1/assets?page=1&rows=1000" +} diff --git a/script/erc20_to_checksum.ts b/script/erc20_to_checksum.ts deleted file mode 100644 index e1b9f7319..000000000 --- a/script/erc20_to_checksum.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ethSidechains, readDirSync, getChainAssetsPath } from "../src/test/helpers" -import { checksumAssetsFolder } from './format_files_name' - -ethSidechains.forEach(chain => { - const chainAssetsPath = getChainAssetsPath(chain) - - readDirSync(chainAssetsPath).forEach(addr => { - checksumAssetsFolder(chainAssetsPath, addr) - }) -}) \ No newline at end of file diff --git a/script/format_files_name.ts b/script/format_files_name.ts deleted file mode 100644 index f8ddc339a..000000000 --- a/script/format_files_name.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { - ethSidechains, - readDirSync, - getChainAssetsPath, - getChainAssetFilesList, - isChecksum, - toChecksum, - getFileName, - getFileExt, - getMoveCommandFromTo, - execRename, - logoName, - logoExtension, - logo, - getChainAssetPath -} from "../src/test/helpers" - -ethSidechains.forEach(chain => { - const assetsPath = getChainAssetsPath(chain) - - readDirSync(assetsPath).forEach(address => { - getChainAssetFilesList(chain, address).forEach(file => { - if (getFileName(file) == logoName && getFileExt(file) !== logoExtension) { - console.log(`Renaming incorrect asset logo extension ${file} ...`) - renameAndMove(getChainAssetPath(chain, address), file, logo) - } - }) - checksumAssetsFolder(assetsPath, address) - }) -}) - -export function checksumAssetsFolder(assetsFolderPath: string, addr: string) { - if (!isChecksum(addr)) { - renameAndMove(assetsFolderPath, addr, toChecksum(addr)) - } -} - -export function renameAndMove(path: string, oldName: string, newName: string) { - console.log(` Renaming file or folder at path ${path}: ${oldName} => ${newName}`) - execRename(path, getMoveCommandFromTo(oldName, newName)) -} - diff --git a/script/gen_list.ts b/script/gen_list.ts deleted file mode 100644 index 07308ff43..000000000 --- a/script/gen_list.ts +++ /dev/null @@ -1,92 +0,0 @@ -const fs = require('fs') -import { getOpenseaCollectionAddresses } from "./opesea_contrats" - -import { - Ethereum, Terra, Tron, - getChainAssetInfoPath, - getChainAssetsList, - ethSidechains, - getChainBlacklistPath, - getChainValidatorsListPath, - getChainWhitelistPath, - getUnique, - isChainAssetInfoExistSync, - isChainBlacklistExistSync, - isChainWhitelistExistSync, - mapList, - readFileSync, - sortDesc, - stakingChains, - writeFileSync, -} from '../src/test/helpers' - -formatWhiteBlackList() -formatValidators() -formatInfo() - -function formatWhiteBlackList() { - ethSidechains.concat(Tron, Terra, "neo", 'nuls').forEach(async chain => { - const assets = getChainAssetsList(chain) - - const whitelistPath = getChainWhitelistPath(chain) - const blacklistPath = getChainBlacklistPath(chain) - - //Create inital lists if they do not exists - if (!isChainWhitelistExistSync(chain)) { - writeFileSync(whitelistPath, `[]`) - } - - if (!isChainBlacklistExistSync(chain)) { - writeFileSync(blacklistPath, `[]`) - } - - const currentWhitelist = JSON.parse(readFileSync(whitelistPath)) - const currentBlacklist = JSON.parse(readFileSync(blacklistPath)) - - let newBlackList = [] - // Some chains required pulling lists from other sources - // switch (chain) { - // case Ethereum: - // const nftList = await getOpenseaCollectionAddresses() - // newBlackList = currentBlacklist.concat(nftList) - // break; - // default: - // newBlackList = newBlackList.concat(currentBlacklist) - // break; - // } - - const removedAssets = getRemovedAddressesFromAssets(assets, currentWhitelist) - newBlackList = currentBlacklist.concat(removedAssets) - - fs.writeFileSync(whitelistPath, JSON.stringify(sortDesc(assets), null, 4)) - fs.writeFileSync(blacklistPath, JSON.stringify(getUnique(sortDesc(newBlackList)), null, 4)) - }) -} - -function formatValidators() { - stakingChains.forEach(chain => { - const validatorsPath = getChainValidatorsListPath(chain) - const currentValidatorsList = JSON.parse(readFileSync(validatorsPath)) - - fs.writeFileSync(validatorsPath, JSON.stringify(currentValidatorsList, null, 4)) - }) -} - -function formatInfo() { - ethSidechains.forEach(chain => { - const chainAssets = getChainAssetsList(chain) - chainAssets.forEach(address => { - if (isChainAssetInfoExistSync(chain, address)) { - const chainAssetInfoPath = getChainAssetInfoPath(chain, address) - const currentAssetInfo = JSON.parse(readFileSync(chainAssetInfoPath)) - fs.writeFileSync(chainAssetInfoPath, JSON.stringify(currentAssetInfo, null, 4)) - } - }) - }) -} - -function getRemovedAddressesFromAssets(assets: string[], whiteList: string[]): string[] { - const mappedAssets = mapList(assets) - const removed = whiteList.filter(a => !mappedAssets.hasOwnProperty(a)) - return removed -} diff --git a/script/main/fix.ts b/script/main/fix.ts new file mode 100644 index 000000000..11acb2d00 --- /dev/null +++ b/script/main/fix.ts @@ -0,0 +1,12 @@ +import { fixAll } from "../action/update-all"; + +export function main() { + try { + fixAll(); + } catch(err) { + console.error(err); + process.exit(1); + } +} + +main(); diff --git a/script/main/update.ts b/script/main/update.ts new file mode 100644 index 000000000..7f468d1e1 --- /dev/null +++ b/script/main/update.ts @@ -0,0 +1,12 @@ +import { updateAll } from "../action/update-all"; + +export function main() { + try { + updateAll(); + } catch(err) { + console.error(err); + process.exit(1); + } +} + +main(); diff --git a/script/resize_images.ts b/script/resize_images.ts deleted file mode 100644 index db4d32081..000000000 --- a/script/resize_images.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { - maxLogoWidth, - maxLogoHeight, - readDirSync, - chainsFolderPath, - getChainLogoPath, - calculateAspectRatioFit, - getImageDimensions, - getChainAssetsPath, - getChainAssetLogoPath, - isPathExistsSync, - writeFileSync, - readFileSync, - getChainValidatorsListPath, - getChainValidatorAssetLogoPath, - maxAssetLogoSizeInKilobyte, - getFileSizeInKilobyte -} from "../src/test/helpers" -const sharp = require('sharp') -const bluebird = require("bluebird") -const foundChains = readDirSync(chainsFolderPath) -const tinify = require("tinify"); -tinify.key = "MXxhvmhjMkMM6CVccGrfyQm2RHpTf1G7"; // Key is free to get, gives 500 uploads per month - -function downsize() { - console.log(`Start resizing`) - bluebird.map(foundChains, async chain => { - console.log(`Resizing assets on chain ${chain}`) - const chainLogoPath = getChainLogoPath(chain) - const { width: srcWidth, heigth: srcHeight } = getImageDimensions(chainLogoPath) - - // Check and resize if needed chain logo - if (isDownsizing(srcWidth, srcHeight)) { - await resize(srcWidth, srcHeight, chainLogoPath) - } - - // Check and resize if needed chain assets - const assetsPath = getChainAssetsPath(chain) - if (isPathExistsSync(assetsPath)) { - bluebird.mapSeries(readDirSync(assetsPath), async asset => { - const assetPath = getChainAssetLogoPath(chain, asset) - const { width: srcWidth, height: srcHeight } = getImageDimensions(assetPath) - if (isDownsizing(srcWidth, srcHeight)) { - await resize(srcWidth, srcHeight, assetPath) - } - - // If size still > max limit, compress with tinypng - const sizeKilobyte = getFileSizeInKilobyte(assetPath) - if (sizeKilobyte > maxAssetLogoSizeInKilobyte) { - await compressTinyPNG(assetPath) - console.log(`Successfully resized iamge at path ${assetPath} from ${sizeKilobyte} => ${getFileSizeInKilobyte(assetPath)}`) - } - }) - } - - - // Check and resize if needed chain validators image - const chainValidatorsList = getChainValidatorsListPath(chain) - if (isPathExistsSync(chainValidatorsList)) { - const validatorsList = JSON.parse(readFileSync(getChainValidatorsListPath(chain))) - bluebird.mapSeries(validatorsList, async ({ id }) => { - const path = getChainValidatorAssetLogoPath(chain, id) - const { width: srcWidth, height: srcHeight } = getImageDimensions(path) - if (isDownsizing(srcWidth, srcHeight)) { - await resize(srcWidth, srcHeight, path) - } - - // If size still > max limit, compress with tinypng - const sizeKilobyte = getFileSizeInKilobyte(path) - if (sizeKilobyte > maxAssetLogoSizeInKilobyte) { - await compressTinyPNG(path) - } - }) - } - - console.log(` Resizing assets on chain ${chain} completed`) - }) -} - -downsize() - -function isDownsizing(srcWidth: number, srcHeight: number): boolean { - return (srcWidth > maxLogoWidth) || (srcHeight > maxLogoHeight) -} - -async function resize(srcWidth: number, srcHeight: number, path: string) { - const { width, height } = calculateAspectRatioFit(srcWidth, srcHeight, maxLogoWidth, maxLogoHeight) - console.log(` Resizing image at ${path} from ${srcWidth}x${srcHeight} => ${width}x${height}`) - await sharp(path).resize(width, height).toBuffer() - .then(data => writeFileSync(path, data)) - .catch(e => { - console.log(e.message) - }) -} - -export async function compressTinyPNG(path: string) { - console.log(`Compressing image via tinypng at path ${path}`) - const source = await tinify.fromFile(path); - await source.toFile(path); - -} \ No newline at end of file diff --git a/script/updateBEP2.js b/script/updateBEP2.js deleted file mode 100644 index 2303d441f..000000000 --- a/script/updateBEP2.js +++ /dev/null @@ -1,40 +0,0 @@ -const axios = require("axios") -const bluebird = require("bluebird") -const fs = require("fs") -const path = require("path") -const chalk = require('chalk') -const blacklist = require('../blockchains/binance/blacklist.json') - -;(async () => { - const { assetInfoList } = await axios.get(`https://explorer.binance.org/api/v1/assets?page=1&rows=1000`).then(r => r.data) - - await bluebird.each(assetInfoList, async ({ asset, assetImg }) => { - if (assetImg && blacklist.indexOf(asset) == -1) { - const binanceDir = path.join(__dirname, `../blockchains/binance`) - const imagePath = `${binanceDir}/assets/${asset}/logo.png` - - if (fs.existsSync(imagePath)) { - console.log(chalk.green(`${asset}`)) - } else { - console.log(chalk.red(`${asset}`)) - fs.mkdir(`${binanceDir}/assets/${asset}`, err => { - if (err && err.code != `EEXIST`) throw err - }) - - await fetchImage(assetImg).then(buffer => { - buffer.pipe(fs.createWriteStream(imagePath)) - }) - } - } - }) - - function fetchImage(url) { - return axios.get(url, { responseType: "stream" }).then(r => r.data).catch(err => { - throw `Error fetchImage: ${url} ${err.message}` - }) - } - -})().catch(err => { - console.error(err) - process.exit(1) -}) \ No newline at end of file diff --git a/src/test/helpers.ts b/src/test/helpers.ts index a566d7b54..b1a3c6041 100644 --- a/src/test/helpers.ts +++ b/src/test/helpers.ts @@ -318,6 +318,7 @@ export const rootDirAllowedFiles = [ "dapps", "media", "node_modules", + "script-old", "script", "src", ".gitignore", diff --git a/src/test/index.test.ts b/src/test/index.test.ts index 0769012a6..b1d6fab8d 100644 --- a/src/test/index.test.ts +++ b/src/test/index.test.ts @@ -22,7 +22,6 @@ import { getChainValidatorsList, findDuplicate, findCommonElementOrDuplicate, - isChecksum, isLogoDimensionOK, isLogoSizeOK, isLowerCase, @@ -32,7 +31,6 @@ import { isValidJSON, isAssetInfoOK, isValidatorHasAllKeys, - mapList, pricingFolderPath, readDirSync, readFileSync, @@ -40,7 +38,23 @@ import { stakingChains, } from "./helpers" import { ValidatorModel, mapTiker, TickerType } from "./models"; -import { getHandle } from "../../script/gen_info"; +import { getHandle } from "../../script-old/gen_info"; + +import { + isChecksum, + toChecksum +} from "../../script/common/eth-web3"; +import { + isDimensionTooLarge, + calculateTargetSize +} from "../../script/common/image"; +import { + mapList, + sortElements, + makeUnique, + arrayDiff +} from "../../script/common/types"; +import { findImagesToFetch } from "../../script/action/binance"; describe("Check repository root dir", () => { const dirActualFiles = readDirSync(".") @@ -459,3 +473,65 @@ describe("Test helper functions", () => { expect(findCommonElementOrDuplicate([], []), `Empty lists`).toBe(null) }) }); + +describe("Test eth-web3 helpers", () => { + test(`Test isChecksum`, () => { + expect(isChecksum("0x7Bb09bC8aDE747178e95B1D035ecBeEBbB18cFee"), `checksum`).toBe(true); + expect(isChecksum("0x7bb09bc8ade747178e95b1d035ecbeebbb18cfee"), `lowercase`).toBe(false); + expect(isChecksum("0x7Bb09bC8aDE747178e95B1D035ecBe"), `too short`).toBe(false); + }); + test(`Test toChecksum`, () => { + expect(toChecksum("0x7bb09bc8ade747178e95b1d035ecbeebbb18cfee"), `from lowercase`).toEqual("0x7Bb09bC8aDE747178e95B1D035ecBeEBbB18cFee"); + expect(toChecksum("0x7Bb09bC8aDE747178e95B1D035ecBeEBbB18cFee"), `from checksum`).toEqual("0x7Bb09bC8aDE747178e95B1D035ecBeEBbB18cFee"); + }); +}); + +describe("Test image helpers", () => { + test(`Test isDimensionTooLarge`, () => { + expect(isDimensionTooLarge(256, 256), `256x256`).toBe(false); + expect(isDimensionTooLarge(64, 64), `64x64`).toBe(false); + expect(isDimensionTooLarge(800, 800), `800x800`).toBe(true); + expect(isDimensionTooLarge(256, 800), `256x800`).toBe(true); + expect(isDimensionTooLarge(800, 256), `800x256`).toBe(true); + }); + test(`Test calculateReducedSize`, () => { + expect(calculateTargetSize(256, 256, 512, 512), `small 1.0`).toEqual({width: 512, height: 512}); + expect(calculateTargetSize(800, 800, 512, 512), `large 1.0`).toEqual({width: 512, height: 512}); + expect(calculateTargetSize(200, 100, 512, 512), `small 2.0`).toEqual({width: 512, height: 256}); + expect(calculateTargetSize(100, 200, 512, 512), `small 0.5`).toEqual({width: 256, height: 512}); + expect(calculateTargetSize(1200, 600, 512, 512), `small 2.0`).toEqual({width: 512, height: 256}); + expect(calculateTargetSize(600, 1200, 512, 512), `small 0.5`).toEqual({width: 256, height: 512}); + expect(calculateTargetSize(256, 0, 512, 512), `zero`).toEqual({width: 512, height: 512}); + }); +}); + +describe("Test type helpers", () => { + test(`Test mapList`, () => { + expect(mapList(["a", "b", "c"]), `3 elems`).toEqual({"a": "", "b":"", "c": ""}); + }); + test(`Test sortElements`, () => { + expect(sortElements(["c", "a", "b"]), `3 elems`).toEqual(["a", "b", "c"]); + expect(sortElements(["C", "a", "b"]), `mixed case`).toEqual(["a", "b", "C"]); + expect(sortElements(["1", "2", "11"]), `numerical`).toEqual(["1", "2", "11"]); + expect(sortElements(["C", "a", "1", "b", "2", "11"]), `complex`).toEqual(["1", "2", "11", "a", "b", "C"]); + }); + test(`Test makeUnique`, () => { + expect(makeUnique(["a", "b", "c", "b"]), `4 elems with 1 duplicate`).toEqual(["a", "b", "c"]); + }); + test(`Test arrayDiff`, () => { + expect(arrayDiff(["a", "b", "c"], ["c"]), `4 elems with 1 duplicate`).toEqual(["a", "b"]); + }); +}); + +describe("Test action binance", () => { + test(`Test findImagesToFetch`, () => { + const assetsInfoListNonexisting: any[] = [{asset: "A1", assetImg: "imgurl1"}, {asset: "A2", assetImg: "imgurl2"}]; + const assetsInfoListExisting: any[] = [{asset: "BUSD-BD1", assetImg: "imgurlBUSD"}, {asset: "ETH-1C9", assetImg: "imgurlETH"}]; + const blackListEmpty: string[] = []; + const blackListA1: string[] = ["A1"]; + expect(findImagesToFetch(assetsInfoListNonexisting, blackListEmpty), `2 nonexisting`).toEqual(assetsInfoListNonexisting); + expect(findImagesToFetch(assetsInfoListNonexisting, blackListA1), `2 nonexisting with 1 blacklisted`).toEqual([{asset: "A2", assetImg: "imgurl2"}]); + expect(findImagesToFetch(assetsInfoListExisting, blackListEmpty), `2 existing`).toEqual([]); + expect(findImagesToFetch([], []), `empty`).toEqual([]); + }); +});