Remove Coinmarketcap mapping and logic. (#3717)

Co-authored-by: Catenocrypt <catenocrypt@users.noreply.github.com>
This commit is contained in:
Adam R 2020-08-31 18:25:21 +02:00 committed by GitHub
parent 768ebbe1c5
commit 56da272f19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 3 additions and 27757 deletions

View File

@ -1,165 +0,0 @@
import { CheckStepInterface } from "../../script/action/interface";
import { readFileSync } from "../../script/common/filesystem";
import { mapTiker, TickerType } from "../../script-old/models";
import { isChecksum } from "../../script/common/eth-web3";
import { isTRC10, isTRC20 } from "../../script/action/tron";
import { retrieveAssetSymbols } from "../../script/action/binance";
export function getSanityChecks(): CheckStepInterface[] {
const cmcMap: mapTiker[] = JSON.parse(readFileSync("./pricing/coinmarketcap/mapping.json"));
return [
{
getName: () => { return "Must have items";},
check: async () => {
if (cmcMap.length == 0) {
return ["CMC map must have items", ""];
}
return ["", ""];
}
},
{
getName: () => { return `Items must be sorted by "id" in ascending order`;},
check: async () => {
var error: string = "";
cmcMap.forEach((el, i) => {
if (i > 0) {
const prevID = cmcMap[i - 1].id;
const curID = el.id;
if (curID < prevID) {
error += `Item ${curID} must be greather or equal to ${prevID}\n`;
}
}
});
return [error, ""];
}
},
{
getName: () => { return `Items must be sorted by "coin" in ascending order if have same "id"`;},
check: async () => {
var error: string = "";
cmcMap.forEach((el, i) => {
if (i > 0) {
const prevEl = cmcMap[i - 1]
const prevCoin = prevEl.coin
const prevID = cmcMap[i - 1].id
const curCoin = el.coin
const curID = el.id
if (prevID == curID) {
if (curCoin < prevCoin) {
error += `Item ${JSON.stringify(el)} must be greather or equal to ${JSON.stringify(prevEl)}\n`;
}
}
}
});
return [error, ""];
}
},
{
getName: () => { return "Properies value shoud not contain spaces";},
check: async () => {
var error: string = "";
cmcMap.forEach((el, i) => {
Object.keys(el).forEach(key => {
const val = el[key]
if (typeof val === "string") {
if (val.indexOf(" ") >= 0) {
error += ` Property value "${val}" should not contain space\n`;
}
}
})
});
return [error, ""];
}
},
{
getName: () => { return "Params should have value and correct type";},
check: async () => {
var error: string = "";
cmcMap.forEach((el) => {
const {coin, type, id, token_id} = el;
if (typeof coin !== "number") {
error += `Coin ${coin} must be type "number"\n`;
}
if (type !== "token" && type !== "coin") {
error += `Element with id ${id} has wrong type: "${type}"\n`;
}
if (type === "token") {
if (!token_id) {
error += `token_id ${token_id} with id ${id} must be type not empty\n`;
}
}
if (type === "coin") {
if ("token_in" in el) {
error += `Element with id ${id} should not have property "token_id"\n`;
}
}
});
return [error, ""];
}
},
{
getName: () => { return `"token_id" should be in correct format`;},
check: async () => {
var error: string = "";
const bep2Symbols = await retrieveAssetSymbols();
cmcMap.forEach((el) => {
const {coin, token_id, type, id} = el
switch (coin) {
case 60:
if (type === TickerType.Token) {
if (!isChecksum(token_id)) {
error += `"token_id" ${token_id} with id ${id} must be in checksum'n`;
}
}
break;
case 195:
if (type === TickerType.Token) {
if (!isTRC10(token_id) && !isTRC20(token_id)) {
error += `"token_id" ${token_id} with id ${id} must be in TRC10 or TRC20\n`;
}
}
break;
case 714:
if (type === TickerType.Token) {
if (!(bep2Symbols.indexOf(token_id) >= 0)) {
error += `"token_id" ${token_id} with id ${id} must be BEP2 symbol\n`;
}
}
break;
default:
break;
}
});
return [error, ""];
}
},
{
getName: () => { return `"token_id" shoud be unique`;},
check: async () => {
var error: string = "";
const mappedList = cmcMap.reduce((acm, val) => {
if (val.hasOwnProperty("token_id")) {
if (acm.hasOwnProperty(val.token_id)) {
acm[val.token_id] == ++acm[val.token_id]
} else {
acm[val.token_id] = 0
}
}
return acm
}, {});
cmcMap.forEach((el) => {
if (el.hasOwnProperty("token_id")) {
if (mappedList[el.token_id] > 0) {
error += `CMC map ticker with "token_id" ${el.token_id} shoud be unique'n`;
}
}
});
return [error, ""];
}
},
];
}

View File

@ -1,22 +0,0 @@
import { ActionInterface, CheckStepInterface } from "../../script/action/interface";
import { update, mergeCmcData } from "./script";
import { getSanityChecks } from "./check";
export class Coinmarketcap implements ActionInterface {
getName(): string { return "Coinmarketcap mapping"; }
getSanityChecks(): CheckStepInterface[] { return getSanityChecks(); }
getConsistencyChecks = null;
sanityFix = null;
async consistencyFix(): Promise<void> {
// do merge, for the case exceptions or script has been changed
await mergeCmcData();
}
async update(): Promise<void> {
await update();
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,378 +0,0 @@
const BluebirdPromise = require("bluebird")
const axios = require("axios")
const chalk = require('chalk')
const fs = require("fs")
const path = require('path')
const constants = require('bip44-constants')
import { readFileSync } from "../../script/common/filesystem";
import { ethForkChains } from "../../script/common/blockchains";
import {
toChecksum,
getChainAssetLogoPath,
isPathExistsSync,
makeDirSync,
getChainAssetPath,
getChainDenylist,
getChainAllowlist,
} from "../../script-old/helpers";
import { TickerType, mapTiker, PlatformType } from "../../script-old/models";
// Steps required to run this:
// 1. (Optional) CMC API key already setup, use yours if needed. Install script deps "npm i" if hasn't been run before.
// 2. Pull down tokens repo https://github.com/trustwallet/assets and point COIN_IMAGE_BASE_PATH and TOKEN_IMAGE_BASE_PATH to it.
// 3. Run: `npm run update`
const CMC_PRO_API_KEY = `df781835-e5f4-4448-8b0a-fe31402ab3af` // Free Basic Plan api key is enough to run script
const CMC_LATEST_BASE_URL = `https://pro-api.coinmarketcap.com/v1/global-metrics/quotes/latest?`
const typeToken = TickerType.Token
const typeCoin = TickerType.Coin
const mappedChainsDenylistAssets = {} // {ethereum: {<0x...>: ""},}
const mappedChainsAllowlistAssets = {} // {ethereum: {<0x...>: ""},}
const custom: mapTiker[] = [
{"coin": 60, "type": typeToken, "token_id": "0x6758B7d441a9739b98552B373703d8d3d14f9e62", "id": 2548}, // POA ERC20 on Foundation (POA20)
{"coin": 195, "type": typeToken, "token_id": "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t", "id": 825}, // Tether (TRC20)
{"coin": 1023, "type": typeCoin, "id": 3945}, // Harmony ONE mainnet
{"coin": 60, "type": typeToken, "token_id": "0x799a4202c12ca952cB311598a024C80eD371a41e", "id": 3945}, // Harmony ONE (ERC20)
{"coin": 60, "type": typeToken, "token_id": "0xB8c77482e45F1F44dE1745F52C74426C631bDD52", "id": 1839}, // BNB (ERC20)
{"coin": 304, "type": typeCoin, "id": 2777}, // IoTex coin
{"coin": 1024, "type": typeToken, "token_id": "ong", "id": 3217}, // Ontology Gas (ONG)
{"coin": 500, "type": typeToken, "token_id": "tfuel", "id": 3822}, // Theta Fuel (TFUEL)
{"coin": 818, "type": typeToken, "token_id": "0x0000000000000000000000000000456E65726779", "id": 3012}, // VeThor Token (VTHO)
{"coin": 459, "type": typeCoin, "id": 4846}, // KAVA coin
{"coin": 60, "type": typeToken, "token_id": "0xFA1a856Cfa3409CFa145Fa4e20Eb270dF3EB21ab", "id": 2405}, // IOST (ERC20)
{"coin": 60, "type": typeToken, "token_id": "0x2fe39f22EAC6d3c1C86DD9D143640EbB94609FCE", "id": 4929}, // JDC Coin (ERC20)
{"coin": 60, "type": typeToken, "token_id": "0x5Cf04716BA20127F1E2297AdDCf4B5035000c9eb", "id": 2780}, // NKN (NKN)
// {"coin": 714, "type": typeToken, "token_id": "CHZ-ECD", "id": 4066}, // Chiliz (BEP-2)
{"coin": 60, "type": typeToken, "token_id": "0xdF1D6405df92d981a2fB3ce68F6A03baC6C0E41F", "id": 3816}, // VERA (VRA)
{"coin": 60, "type": typeToken, "token_id": "0x467Bccd9d29f223BcE8043b84E8C8B282827790F", "id": 2394}, // Telcoin (TEL)
// {"coin": 714, "type": typeToken, "token_id": "BUSD-BD1", "id": 4687}, // BUSD-BD1 (BEP2)
{"coin": 60, "type": typeToken, "token_id": "0xBD87447F48ad729C5c4b8bcb503e1395F62e8B98", "id": 3408}, // Pool Usdc (plUsdc)
{"coin": 60, "type": typeToken, "token_id": "0x49d716DFe60b37379010A75329ae09428f17118d", "id": 4943}, // Pool Dai (plDai)
{"coin": 60, "type": typeToken, "token_id": "0x589891a198195061Cb8ad1a75357A3b7DbaDD7Bc", "id": 4036}, // Contentos (COS)
{"coin": 60, "type": typeToken, "token_id": "0x30f271C9E86D2B7d00a6376Cd96A1cFBD5F0b9b3", "id": 5835}, // Decentr (DEC)
// CMC returns multiple entries with 0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5 (2020-07-28), including them in the override to avoid duplicate
// 5636 5742 5743 5744 5745 5746
{"coin": 60, "type": typeToken, "token_id": "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5", "id": 5636},
{"coin": 60, "type": typeToken, "token_id": "0x41AB1b6fcbB2fA9DCEd81aCbdeC13Ea6315F2Bf2", "id": 2634},
{"coin": 60, "type": typeToken, "token_id": "0xC011A72400E58ecD99Ee497CF89E3775d4bd732F", "id": 2586},
{"coin": 354, "type": typeCoin, "id": 4129},
// {"coin": 60, "type": typeToken, "token_id": "XXX", "id": XXX}, // XXX (XXX)
]
var allContracts: mapTiker[] = [] // Temp storage for mapped assets
let bnbOwnerToSymbol = {} // e.g: bnb1tawge8u97slduhhtumm03l4xl4c46dwv5m9yzk: WISH-2D5
let bnbOriginalSymbolToSymbol = {} // e.g: WISH: WISH-2D5
async function retrieveCmcData() {
allContracts = []
await Promise.all([initState(), setBinanceTokens()])
const [totalCrypto, coins] = await Promise.all([getTotalActiveCryptocurrencies(), getTickers()])
// setBIP44Constants()
log(`Found ${totalCrypto} on CMC`, chalk.yellowBright)
await BluebirdPromise.mapSeries(coins, processCoin)
sortContracts()
fs.writeFileSync(path.join(__dirname, 'cmc-data.json'), JSON.stringify(allContracts, null, 4))
allContracts = []
}
export async function mergeCmcData() {
try {
allContracts = JSON.parse(readFileSync(path.join(__dirname, 'cmc-data.json')))
addCustom()
printContracts()
} catch (error) {
log(`Exception: ${error.message}`)
}
}
export async function update() {
try {
await retrieveCmcData()
await mergeCmcData()
} catch (error) {
log(`Exception: ${error.message}`)
}
}
function buildCoinEntry(coin: any): any {
const { id, symbol, name, platform } = coin
const platformType: PlatformType = platform == null ? "" : platform.name
log(`${symbol}:${platformType}`)
switch (platformType) {
case PlatformType.Ethereum:
// log(`Ticker ${name}(${symbol}) is a token with address ${address} and CMC id ${id}`)
if (platform.token_address) {
try {
const checksum = toChecksum(platform.token_address)
if (!isAddressInDenyList("ethereum", checksum)) {
return {
coin: 60,
type: typeToken,
token_id: checksum,
id
}
}
} catch (error) {
console.log(`Etheruem platform error`, error)
break
}
}
break
case PlatformType.Binance:
if (symbol === "BNB") {
break
}
const ownerAddress = platform.token_address.trim()
log(`Symbol ${symbol}:${ownerAddress}:${id}`)
if (ownerAddress && (ownerAddress in bnbOwnerToSymbol)) {
return {
coin: 714,
type: typeToken,
token_id: bnbOwnerToSymbol[ownerAddress],
id
}
}
if (symbol in bnbOriginalSymbolToSymbol) {
return {
coin: 714,
type: typeToken,
token_id: bnbOriginalSymbolToSymbol[symbol].trim(),
id
}
}
break
case PlatformType.TRON:
if (symbol === "TRX") {
break
}
const tokenAddr = platform.token_address.trim()
log(`tron: ${tokenAddr}`)
if (tokenAddr.length > 0) {
return {
coin: 195,
type: typeToken,
token_id: tokenAddr,
id
}
}
break
// case PlatformType.VeChain:
// if (symbol === "VET") {
// break
// }
// const addr = platform.token_address.trim()
// log(`vechain: ${tokenAddr}`)
// addToContractsList({
// coin: 0,
// type: typeCoin,
// token_id: addr,
// id
// })
// break
default:
const coinIndex = getSlip44Index(symbol, name)
if (coinIndex >= 0) {
log(`Ticker ${name}(${symbol}) is a coin with id ${coinIndex}`)
return {
coin: coinIndex,
type: typeCoin,
id
}
}
log(`Coin ${coinIndex} ${name}(${symbol}) not listed in slip44`)
break
}
log(`Could not process entry ${symbol}:${name}:${platformType}`)
return null
}
async function processCoin(coin) {
const entry = buildCoinEntry(coin)
if (!entry) {
return
}
// check if it is in custom, in that case omit it
if (entry.token_id && custom.find(elem => elem.coin == entry.coin && elem.token_id === entry.token_id)) {
log(`Entry ${entry.token_id} is in custom, omitting`)
return
}
addToContractsList(entry)
log(`Added entry ${entry.token_id}`)
}
// Iniitalize state necessary for faster data looup during script run
async function initState () {
await mapChainsAssetsLists()
}
async function mapChainsAssetsLists() {
ethForkChains.forEach(chain => {
Object.assign(mappedChainsAllowlistAssets, {[chain]: {}})
Object.assign(mappedChainsDenylistAssets, {[chain]: {}})
getChainAllowlist(chain).forEach(addr => {
Object.assign(mappedChainsAllowlistAssets[chain], {[addr]: ""})
})
getChainDenylist(chain).forEach(addr => {
Object.assign(mappedChainsDenylistAssets[chain], {[addr]: ""})
})
})
}
function addCustom() {
custom.forEach(c => {
addToContractsList(c)
})
}
function addToContractsList(ticker: mapTiker) {
allContracts.push(ticker)
}
function sortContracts() {
const sortedById = allContracts.sort((a,b) => {
if (a.id < b.id) return -1
if (a.id > b.id) return 1
if (a.hasOwnProperty("coin") && b.hasOwnProperty("coin")) {
if (a.coin < b.coin) return -1
if (a.coin > b.coin) return 1
}
if (a.token_id < b.token_id) return -1
if (a.token_id > b.token_id) return 1
})
allContracts = sortedById
}
function printContracts() {
sortContracts()
const wstream = fs.createWriteStream(path.join(__dirname, 'mapping.json'))
wstream.write(JSON.stringify(allContracts, null, 4))
}
function getSlip44Index(symbol: string, name: string): number {
const coins = constants.filter(item => item[1] === symbol)
if (coins.length == 0) return
if (coins.length == 1) {
const hex = '0x' + (coins[0][0]).toString(16)
return parseInt(hex, 16) - ((1<<31)>>>0)
}
const coin = coins.filter(c => c[2] === name || c[2].includes(name))
if (coin.length == 0) return
const hex = '0x' + (coin[0][0]).toString(16)
return parseInt(hex, 16) - ((1<<31)>>>0)
}
// id referes to cmc internal id
const getImageURL = (id: string | number): string => `https://s2.coinmarketcap.com/static/img/coins/128x128/${id}.png`
async function getImageIfMissing(chain: string, address: string, id: string) {
try {
const logoPath = getChainAssetLogoPath(chain, String(address))
if (!isPathExistsSync(logoPath) && !isAddressInDenyList(chain, address)) {
const imageStream = await fetchImage(getImageURL(id))
if (imageStream) {
const logoFolderPath = getChainAssetPath(chain, address)
if(!isPathExistsSync(logoFolderPath)) {
makeDirSync(logoFolderPath)
}
imageStream.pipe(fs.createWriteStream(logoPath))
log(`Image saved to: ${logoPath}`, chalk.green)
}
}
} catch (error) {
log(`Failed getImage to save token image ${error.message}`)
exit(2)
}
}
function isAddressInDenyList(chain: string, address: string): boolean {
return mappedChainsDenylistAssets[chain].hasOwnProperty(address)
}
async function fetchImage(url: string) {
try {
return axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
},
responseType: "stream"
}).then(res => res.data).catch(error => {
console.log(`Error getRemoteResource ${error.message}`)
})
} catch (error) {
log(`${error.message}`)
exit(3)
return false
}
}
function exit(code?: number) {
process.exit(code ?? 1)
}
function getTotalActiveCryptocurrencies() {
return axios.get(`${CMC_LATEST_BASE_URL}CMC_PRO_API_KEY=${CMC_PRO_API_KEY}`).then((res) => res.data.data.active_cryptocurrencies).catch(e => {
throw `Error getTotalActiveCryptocurrencies ${e.message}`
})
}
async function setBinanceTokens () {
return axios.get(`https://dex.binance.org/api/v1/tokens?limit=1000`).then(({ data }) => {
bnbOwnerToSymbol = data.reduce((acm, token) => {
log(`Token owner ${token.owner}:${token.symbol}`)
acm[token.owner] = token.symbol
return acm
}, {})
bnbOriginalSymbolToSymbol = data.reduce((acm, token) => {
acm[token.original_symbol] = token.symbol
return acm
}, {})
}).catch(error => {throw Error(`Error fetching Binance markets : ${error.message}`)})
}
function readBEP2() {
// Fetch https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?CMC_PRO_API_KEY=YOUR_KEYc&limit=5000 and store full response
// in file
const validatorsList = JSON.parse(readFileSync("./pricing/coinmarketcap/cryptocurrency_map.json"))
return validatorsList.data
}
async function getTickers() {
const url = `https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?&limit=3500&CMC_PRO_API_KEY=${CMC_PRO_API_KEY}`
return axios.get(url).then(res => res.data.data).catch(e => {throw `Error getTickers ${e.message}`})
}
function log(string, cb?) {
if (cb) {
console.log(cb(string))
} else {
console.log(string)
}
const saveToLogs = fs.createWriteStream(path.join(__dirname, '.syncTokensLog.txt'))
saveToLogs.write(`${string}\n`)
}
// function setBIP44Constants() {
// require('bip44-constants').forEach(row => {
// bip44Constants[row[1]] = {
// constant: row[0],
// coinName: row[2]
// }
// })
// }

View File

@ -1,4 +1,4 @@
import { chainsPath, pricingFolderPath } from "../common/repo-structure";
import { chainsPath } from "../common/repo-structure";
import { findFiles } from "../common/filesystem";
import { ActionInterface, CheckStepInterface } from "./interface";
import { isValidJSON } from "../common/json";
@ -15,7 +15,6 @@ export class JsonAction implements ActionInterface {
var error: string = "";
const files = [
...findFiles(chainsPath, 'json'),
...findFiles(pricingFolderPath, 'json')
];
await bluebird.each(files, async file => {

View File

@ -11,7 +11,6 @@ import { TronAction } from "./tron";
import { Validators } from "./validators";
import { WavesAction } from "./waves";
import { Allowlist } from "./allowlists";
import { Coinmarketcap } from "../../pricing/coinmarketcap/cmc-action";
import { ActionInterface, CheckStepInterface } from "./interface";
import * as chalk from 'chalk';
import * as bluebird from "bluebird";
@ -30,8 +29,7 @@ const actionList: ActionInterface[] = [
new TerraAction(),
new TezosAction(),
new TronAction(),
new WavesAction(),
new Coinmarketcap()
new WavesAction()
];
async function checkStepList(steps: CheckStepInterface[]): Promise<[string[], string[]]> {

View File

@ -3,7 +3,7 @@ export const imageMaxLogoHeight: number = 512;
export const imageMinLogoWidth: number = 64;
export const imageMinLogoHeight: number = 64;
export const imageMaxLogoSizeKb: number = 100;
export const foldersRootdirAllowedFiles: string[] = [".github", "blockchains", "dapps", "media", "node_modules", "script-old", "script", "test", ".gitignore", "azure-pipelines.yml", "jest.config.js", "LICENSE", "package-lock.json", "package.json", "README.md", ".git", "pricing", "dangerfile.ts", "Gemfile", "Gemfile.lock"];
export const foldersRootdirAllowedFiles: string[] = [".github", "blockchains", "dapps", "media", "node_modules", "script-old", "script", "test", ".gitignore", "azure-pipelines.yml", "jest.config.js", "LICENSE", "package-lock.json", "package.json", "README.md", ".git", "dangerfile.ts", "Gemfile", "Gemfile.lock"];
export const binanceUrlTokens2: string = "https://dex-atlantic.binance.org/api/v1/tokens?limit=1000";
export const binanceUrlTokens8: string = "https://dex-atlantic.binance.org/api/v1/mini/tokens?limit=1000";
export const binanceUrlTokenAssets: string = "https://explorer.binance.org/api/v1/assets?page=1&rows=1000";