trustwallet-assets/script/generic/update-all.ts

236 lines
9.1 KiB
TypeScript
Raw Normal View History

import { BinanceAction } from "../blockchain/binance";
[Internal] Retrieve trading pairs from PancakeSwap (#5355) * Retrieve trading pairs from PancakeSwap. * Retrieve trading pairs from Uniswap. * Add doc link * Lint fix * Include decimals * Move binance tokenlist generation to BinanceAction. * Generate smartChain tokenlist, from base and pancakeswap pairs. * Fix array types in tokenlists.ts * Common writeToFile method, sort. * Type fixes * Revert tokenlist.json to master * Include pairs with allowlisted coins only. * Move assetID functions to common asset.ts * Use common assetID functions. * Use dynamic generation time. * Keep constant generation time; Version in tokenlist. * Count additions * Update tokenlist version and timestamp if needed before save. * No counting is needed in addXxxIfNeeded() functions. * Tokenlist timestamp: take over if previous * Update ethereum tokenlist with pairs from Uniswap. * Increase query limit to 400 pairs. * iDecimals always number, add decimal fields to Eth also. * Include only pairs with primary tokens, add pair info to primary token only. * Update base tokenlist (BSC) to current full list, in order not to remove any coins. * Update base tokenlist (ETH) to current full list, in order not to remove any coins. * Move out pair checks into common code. * Add checks for volume and txCount. * Assets tokenlist: Adjust token names for added tokens. * Move parameters to central config.ts * Nicer query string, compact * Prevent change if subgraph API fails * Reduce max limit in Pancakeswap query, with 400 often times out. * Stricter error handling * Reduce code duplication. * Minor comment * Display number of original tokens * Lint fix Co-authored-by: Catenocrypt <catenocrypt@users.noreply.github.com>
2021-01-29 06:45:43 +00:00
import { SmartchainAction } from "../blockchain/smartchain";
import { EthereumAction } from "../blockchain/ethereum";
import { CosmosAction } from "../blockchain/cosmos";
import { AssetInfos } from "../generic/asset-infos";
import { EthForks } from "../generic/eth-forks";
import { FoldersFiles } from "../generic/folders-and-files";
import { JsonAction } from "../generic/json-format";
import { KavaAction } from "../blockchain/kava";
import { LogoSize } from "../generic/logo-size";
import { TerraAction } from "../blockchain/terra";
import { TezosAction } from "../blockchain/tezos";
import { TronAction } from "../blockchain/tron";
import { Validators } from "../generic/validators";
import { WavesAction } from "../blockchain/waves";
import { Allowlists } from "../generic/allowlists";
import { ActionInterface, CheckStepInterface } from "../generic/interface";
import * as chalk from 'chalk';
import * as bluebird from "bluebird";
const actionList: ActionInterface[] = [
new FoldersFiles(),
new AssetInfos(),
new EthForks(),
new LogoSize(),
new Allowlists(),
new Validators(),
new JsonAction(),
// chains:
new BinanceAction(),
[Internal] Retrieve trading pairs from PancakeSwap (#5355) * Retrieve trading pairs from PancakeSwap. * Retrieve trading pairs from Uniswap. * Add doc link * Lint fix * Include decimals * Move binance tokenlist generation to BinanceAction. * Generate smartChain tokenlist, from base and pancakeswap pairs. * Fix array types in tokenlists.ts * Common writeToFile method, sort. * Type fixes * Revert tokenlist.json to master * Include pairs with allowlisted coins only. * Move assetID functions to common asset.ts * Use common assetID functions. * Use dynamic generation time. * Keep constant generation time; Version in tokenlist. * Count additions * Update tokenlist version and timestamp if needed before save. * No counting is needed in addXxxIfNeeded() functions. * Tokenlist timestamp: take over if previous * Update ethereum tokenlist with pairs from Uniswap. * Increase query limit to 400 pairs. * iDecimals always number, add decimal fields to Eth also. * Include only pairs with primary tokens, add pair info to primary token only. * Update base tokenlist (BSC) to current full list, in order not to remove any coins. * Update base tokenlist (ETH) to current full list, in order not to remove any coins. * Move out pair checks into common code. * Add checks for volume and txCount. * Assets tokenlist: Adjust token names for added tokens. * Move parameters to central config.ts * Nicer query string, compact * Prevent change if subgraph API fails * Reduce max limit in Pancakeswap query, with 400 often times out. * Stricter error handling * Reduce code duplication. * Minor comment * Display number of original tokens * Lint fix Co-authored-by: Catenocrypt <catenocrypt@users.noreply.github.com>
2021-01-29 06:45:43 +00:00
new SmartchainAction(),
new EthereumAction(),
new CosmosAction(),
new KavaAction(),
new TerraAction(),
new TezosAction(),
new TronAction(),
new WavesAction()
];
const maxErrosFromOneCheck = 5;
const markerError = chalk.red('XXX');
const markerWarning = chalk.yellow('!!');
const markerOK = chalk.green('✓');
async function checkStepList(steps: CheckStepInterface[]): Promise<[string[], string[]]> {
const errorsAll: string[] = [];
const warningsAll: string[] = [];
await bluebird.each(steps, async (step) => {
try {
//console.log(` Running check step '${step.getName()}'...`);
const [errors, warnings] = await step.check();
if (errors && errors.length > 0) {
console.log(`- ${markerError} '${step.getName()}': ${errors.length} errors`);
let cnt = 0;
errors.forEach(err => {
if (cnt < maxErrosFromOneCheck) {
console.log(` ${markerError} '${err}'`);
errorsAll.push(err);
} else if (cnt == maxErrosFromOneCheck) {
console.log(` ${markerError} ${errors.length} errors in total, omitting rest ...`);
}
cnt++;
});
}
if (warnings && warnings.length > 0) {
console.log(`- ${markerWarning} '${step.getName()}': ${warnings.length} warnings`);
let cnt = 0;
warnings.forEach(warn => {
if (cnt < maxErrosFromOneCheck) {
console.log(` ${markerWarning} '${warn}'`);
warningsAll.push(warn);
} else if (cnt == maxErrosFromOneCheck) {
console.log(` ${markerWarning} ${warnings.length} warnings in total, omitting rest ...`);
}
cnt++;
});
}
if (errors.length == 0 && warnings.length == 0) {
console.log(`- ${markerOK} '${step.getName()}' OK`);
}
} catch (error) {
console.log(`- ${markerError} '${step.getName()}': Caught error: ${error.message}`);
errorsAll.push(`${step.getName()}: Exception: ${error.message}`);
}
});
return [errorsAll, warningsAll];
}
async function sanityCheckByActionList(actions: ActionInterface[]): Promise<[string[], string[]]> {
console.log("Running sanity checks...");
const errors: string[] = [];
const warnings: string[] = [];
await bluebird.each(actions, async (action) => {
try {
if (action.getSanityChecks) {
const steps = action.getSanityChecks();
if (steps && steps.length > 0) {
console.log(` Action '${action.getName()}' has ${steps.length} check steps`);
const [errors1, warnings1] = await checkStepList(steps);
if (errors1.length > 0) {
errors1.forEach(e => errors.push(e));
}
if (warnings1.length > 0) {
warnings1.forEach(w => warnings.push(w));
}
if (errors1.length == 0 && warnings1.length == 0) {
console.log(`- ${markerOK} Action '${action.getName()}' OK, all ${steps.length} steps`);
}
}
}
} catch (error) {
console.log(`- ${markerError} '${action.getName()}' Caught error: ${error.message}`);
errors.push(`${action.getName()}: Exception: ${error.message}`);
}
});
console.log(`All sanity checks done, found ${errors.length} errors, ${warnings.length} warnings`);
return [errors, warnings];
}
async function consistencyCheckByActionList(actions: ActionInterface[]): Promise<[string[], string[]]> {
console.log("Running consistency checks...");
const errors: string[] = [];
const warnings: string[] = [];
await bluebird.each(actions, async (action) => {
try {
if (action.getConsistencyChecks) {
const steps = action.getConsistencyChecks();
if (steps && steps.length > 0) {
console.log(` Action '${action.getName()}' has ${steps.length} check steps`);
const [errors1, warnings1] = await checkStepList(steps);
if (errors1.length > 0) {
errors1.forEach(e => errors.push(e));
}
if (warnings1.length > 0) {
warnings1.forEach(w => warnings.push(w));
}
if (errors1.length == 0 && warnings1.length == 0) {
console.log(`- ${markerOK} Action '${action.getName()}' OK, all ${steps.length} steps`);
}
}
}
} catch (error) {
console.log(`- ${markerError} '${action.getName()}' Caught error: ${error.message}`);
errors.push(`${action.getName()}: Exception: ${error.message}`);
}
});
console.log(`All consistency checks done, found ${errors.length} errors, ${warnings.length} warnings`);
return [errors, warnings];
}
async function sanityFixByList(actions: ActionInterface[]) {
console.log("Running sanity fixes...");
await bluebird.each(actions, async (action) => {
try {
if (action.sanityFix) {
console.log(`Sanity fix '${action.getName()}':`);
await action.sanityFix();
}
} catch (error) {
console.log(`Caught error: ${error.message}`);
}
});
console.log("All sanity fixes done.");
}
async function consistencyFixByList(actions: ActionInterface[]) {
console.log("Running consistency fixes...");
await bluebird.each(actions, async (action) => {
try {
if (action.consistencyFix) {
console.log(`Consistency fix '${action.getName()}':`);
await action.consistencyFix();
}
} catch (error) {
console.log(`Caught error: ${error.message}`);
}
});
console.log("All consistency fixes done.");
[Internal] Refactored fix builds (#2920) * Empty infrastructure for new-housekeeping build. * Move updateBEP action to new new-housekeeping build infra. * Remove old updateBEP. * New-housekeeping-dryrun run. * Include new top-level folder name script-new. * Remove update:bep2 from old daily-run. * Use imports instead of require. * Small refactor for testability. * Organize scripts into subfolders. * iUpdateBEP2: refactor and add tests. * Move formatting validators to new-housekeeping,add new helpers. * Move info and black/whitelist fixing to new-housekeeping. * New fix command. * New 'fix' target; Move ETH checksum fix to new-housekeeping. * Move logo size check and resize to new-housekeeping. * Improved async error handling. * Build renames. * Move (old) BEP2 and CMC update to periodic update build. * Rename (add missing). * Rename builds. * Renames ('fix'). * rename * Invoke new scripts (as well) from period-update. * Move cmc update to new-periodic. * Move tezos validator update to new-periodic. * Missing file. * Leftover. * Cleanup * Rename of unused openseacontracts. * CMC should not be run always. * Break main/fixAndUpdate function into two. * Show diff in build after changes. * Cleanup * Rename, script-old. * Cleanup, remove old fix build definitions. * Renames, remove new- prefix. * CMC mapping update. * Config infrastructure; add binance URL to config. * Add image size parameters to config. * Rename. Co-authored-by: Catenocrypt <catenocrypt@users.noreply.github.com> Co-authored-by: Andrew M <35627271+zachzwei@users.noreply.github.com>
2020-07-29 13:42:51 +00:00
}
async function updateAutoByList(actions: ActionInterface[]) {
console.log("Running auto updates (using external data sources) ...");
await bluebird.each(actions, async (action) => {
try {
if (action.updateAuto) {
console.log(`Auto update '${action.getName()}':`);
await action.updateAuto();
}
} catch (error) {
console.log(`Caught error: ${error.message}`);
}
});
console.log("All auto updates done.");
}
async function updateManualByList(actions: ActionInterface[]) {
console.log("Running manual updates (using external data sources) ...");
await bluebird.each(actions, async (action) => {
try {
if (action.updateManual) {
console.log(`Manual update '${action.getName()}':`);
await action.updateManual();
}
} catch (error) {
console.log(`Caught error: ${error.message}`);
}
});
console.log("All manual updates done.");
}
export async function sanityCheckAll(): Promise<[string[], string[]]> {
return await sanityCheckByActionList(actionList);
}
export async function consistencyCheckAll(): Promise<[string[], string[]]> {
return await consistencyCheckByActionList(actionList);
}
export async function sanityFixAll(): Promise<void> {
await sanityFixByList(actionList);
}
export async function consistencyFixAll(): Promise<void> {
await consistencyFixByList(actionList);
}
export async function updateAutoAll(): Promise<void> {
await updateAutoByList(actionList);
}
export async function updateManualAll(): Promise<void> {
await updateManualByList(actionList);
[Internal] Refactored fix builds (#2920) * Empty infrastructure for new-housekeeping build. * Move updateBEP action to new new-housekeeping build infra. * Remove old updateBEP. * New-housekeeping-dryrun run. * Include new top-level folder name script-new. * Remove update:bep2 from old daily-run. * Use imports instead of require. * Small refactor for testability. * Organize scripts into subfolders. * iUpdateBEP2: refactor and add tests. * Move formatting validators to new-housekeeping,add new helpers. * Move info and black/whitelist fixing to new-housekeeping. * New fix command. * New 'fix' target; Move ETH checksum fix to new-housekeeping. * Move logo size check and resize to new-housekeeping. * Improved async error handling. * Build renames. * Move (old) BEP2 and CMC update to periodic update build. * Rename (add missing). * Rename builds. * Renames ('fix'). * rename * Invoke new scripts (as well) from period-update. * Move cmc update to new-periodic. * Move tezos validator update to new-periodic. * Missing file. * Leftover. * Cleanup * Rename of unused openseacontracts. * CMC should not be run always. * Break main/fixAndUpdate function into two. * Show diff in build after changes. * Cleanup * Rename, script-old. * Cleanup, remove old fix build definitions. * Renames, remove new- prefix. * CMC mapping update. * Config infrastructure; add binance URL to config. * Add image size parameters to config. * Rename. Co-authored-by: Catenocrypt <catenocrypt@users.noreply.github.com> Co-authored-by: Andrew M <35627271+zachzwei@users.noreply.github.com>
2020-07-29 13:42:51 +00:00
}