From bda2c0a236abfcc5a5d7335580d8b547218c97cd Mon Sep 17 00:00:00 2001 From: Adam R <13562139+catenocrypt@users.noreply.github.com> Date: Fri, 7 Aug 2020 16:01:57 +0200 Subject: [PATCH] [internal] Make whitelist check more reliable, case-insensitive. (#3130) Co-authored-by: Catenocrypt --- blockchains/ethereum/blacklist.json | 7 ------ script/action/whitelists.ts | 19 ++++++++------- script/common/types.ts | 31 ++++++++++++++++------- test/index.test.ts | 38 +++++++++++++++++------------ 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/blockchains/ethereum/blacklist.json b/blockchains/ethereum/blacklist.json index 5400c6d9b..a896fb354 100644 --- a/blockchains/ethereum/blacklist.json +++ b/blockchains/ethereum/blacklist.json @@ -151,7 +151,6 @@ "0x08982CACf14157183055F1d6A51b8bB34BB0cBa8", "0x08995eDb5dACA2A856B82647318e30E304aa0Aa8", "0x08aC7aAc4C5Ab59aC13439E1FccEb31238590151", - "0x08ceed1e8db59acbb687a5752f0a7db815cfda5e", "0x08dAD07075efA7db37eb3057B3598F37dFbBAdcd", "0x08dD2D1278b46d423003552C7ba29324076097Ed", "0x0911d4B5Ad88e063C9302355E5015d041f7C2B20", @@ -305,7 +304,6 @@ "0x134C93f35D96E67f35B638E5aB9011DB8DCD2E77", "0x13599DB9810D89Cf25CaDd558F782895FD6E8539", "0x13793e8c72f81999fdc89f2731c9a4c895AaC2De", - "0x139838869d3e1ab94c798917840da1c079103c84", "0x139E292433A733Bd278846C0D7CCC81B58246C1B", "0x13c8a7817ab3E74C28F5ef0eb7D7a3bdB329c2A0", "0x13dccb630bA92989299bBc276486C28673CA69E8", @@ -1596,7 +1594,6 @@ "0x66c2e323590E7c00744B4880343d91cbe6F93C0F", "0x66e35862E882bFc30cde5ADdCd104191e3B44506", "0x66e48832224FF412462aD6D84f716861e4908B07", - "0x66eb65d7ab8e9567ba0fa6e37c305956c5341574", "0x66EfD47948fBBb19A9D2a6FD4124a35b26c19825", "0x66f6F9Cb5c079D084993f4762Afc0eD6E1c3cB8c", "0x67162A5075C6E1E3ec24124fD9f4e8B8EdB87ec4", @@ -2222,7 +2219,6 @@ "0x8dc3be37a1D83A174A4d417980BCB455e347BA65", "0x8Dc739defeD007966302f87820baaF0e9c04646D", "0x8DC9947B6662485252E49B04F2e445100Ab5804a", - "0x8de0f21eaa95f589a8a2fe69c5752ffdada83ead", "0x8E00439B5Ebb030d4300aC3E35b0846ACc9dF578", "0x8e08c7542E144e4082Ed2981B53Dd373539CE43F", "0x8E44eBb62Ef1d1191eDA8E5f1cB2FFC279182C5B", @@ -2310,7 +2306,6 @@ "0x9355Fb9693ffF9bB6f06721C82fe0B5F49E6c956", "0x93870f0af858206d01a3162c289F0bbdFFA4ce69", "0x938742e3d2eDd342A059d758767489279Ab20878", - "0x9388f54fa978aa9e24395a8b69033304eccea4df", "0x93a38240dd3EDa2935e9D6e429236BbD799357c1", "0x93c25b0a4D515a6040f4fc42eec066b012412df2", "0x93EFeb9578591F8418fB88F56d25B65e95619DC3", @@ -2927,7 +2922,6 @@ "0xb932f8AB034E2fC6d019E041039AeE41a02343f6", "0xB95C316b0bF5729b1cb2db8865eD9355b9966E55", "0xb961d0f24505b5FF9a794Cb781290FA55E78C61a", - "0xb96207e0a45a953953109b6e2dcb8f4b72ec7462", "0xb985B80EBb5Af0cd4f5b31e3018302ac15b697E4", "0xb9b5D9C3325b5D68eBe923362e98d4F65e6a0b43", "0xB9beae3262691225c7Aba1694D0297dA05f0DAF4", @@ -3028,7 +3022,6 @@ "0xbFaf35E5Af21528a9537eD9A78fff6eaC282A4c4", "0xBfb1dfe508f52664401032f7A0667a451c712FE7", "0xbFbD4D68BCB5D80572C332f64ce23cf72909eb5e", - "0xbfd04aacd54c0213d3ba6cbb390898ac50a2dcb2", "0xBfdE6246Df72d3ca86419628CaC46a9d2B60393C", "0xBfdf0DD4B90193Cd3c34afB788dFf02788BC03D7", "0xbfe3C9768BB1D24804d5b047F378ac6096cfdfA7", diff --git a/script/action/whitelists.ts b/script/action/whitelists.ts index f946ffe89..0061d2d24 100644 --- a/script/action/whitelists.ts +++ b/script/action/whitelists.ts @@ -3,7 +3,8 @@ import { getChainAssetsList, getChainWhitelistPath, getChainBlacklistPath } from import { readFileSync, writeFileSync } from "../common/filesystem"; import { arrayDiff, - findCommonElementOrDuplicate, + arrayDiffNocase, + findCommonElementsOrDuplicates, makeUnique } from "../common/types"; import { ActionInterface, CheckStepInterface } from "./interface"; @@ -22,9 +23,9 @@ async function checkUpdateWhiteBlackList(chain: string, checkOnly: boolean ): Pr const currentWhitelist = JSON.parse(currentWhitelistText); const currentBlacklist = JSON.parse(currentBlacklistText); - const commonElementsOrDuplicated = findCommonElementOrDuplicate(currentWhitelist, currentBlacklist); - if (commonElementsOrDuplicated && commonElementsOrDuplicated.length > 0) { - wrongMsg += `Blacklist and whitelist for chain ${chain} should have no common elements or duplicates, found ${commonElementsOrDuplicated}\n`; + const commonElementsOrDuplicates = findCommonElementsOrDuplicates(currentWhitelist, currentBlacklist); + if (commonElementsOrDuplicates && commonElementsOrDuplicates.length > 0) { + wrongMsg += `Blacklist and whitelist for chain ${chain} should have no common elements or duplicates, found ${commonElementsOrDuplicates.length} ${commonElementsOrDuplicates[0]}\n`; } const whitelistOrphan = arrayDiff(currentWhitelist, assets); if (whitelistOrphan && whitelistOrphan.length > 0) { @@ -32,24 +33,24 @@ async function checkUpdateWhiteBlackList(chain: string, checkOnly: boolean ): Pr } const newBlack = makeUnique(currentBlacklist.concat(whitelistOrphan)); - const newWhite = makeUnique(arrayDiff(assets, newBlack)); + const newWhite = makeUnique(arrayDiffNocase(assets, newBlack)); //console.log(currentWhitelist.length, "vs.", newWhite.length); //console.log(currentBlacklist.length, "vs.", newBlack.length); - const wDiff1 = arrayDiff(newWhite, currentWhitelist); + const wDiff1 = arrayDiffNocase(newWhite, currentWhitelist); if (wDiff1.length > 0) { wrongMsg += `Some elements are missing from whitelist for chain ${chain}: ${wDiff1.length} ${wDiff1[0]}\n`; } - const wDiff2 = arrayDiff(currentWhitelist, newWhite); + const wDiff2 = arrayDiffNocase(currentWhitelist, newWhite); if (wDiff2.length > 0) { wrongMsg += `Some elements should be removed from whitelist for chain ${chain}: ${wDiff2.length} ${wDiff2[0]}\n`; } - const bDiff1 = arrayDiff(newBlack, currentBlacklist); + const bDiff1 = arrayDiffNocase(newBlack, currentBlacklist); if (bDiff1.length > 0) { wrongMsg += `Some elements are missing from blacklist for chain ${chain}: ${bDiff1.length} ${bDiff1[0]}\n`; } - const bDiff2 = arrayDiff(currentBlacklist, newBlack); + const bDiff2 = arrayDiffNocase(currentBlacklist, newBlack); if (bDiff2.length > 0) { wrongMsg += `Some elements should be removed from blacklist for chain ${chain}: ${bDiff2.length} ${bDiff2[0]}\n`; } diff --git a/script/common/types.ts b/script/common/types.ts index c5f6f2247..f1a0c1704 100644 --- a/script/common/types.ts +++ b/script/common/types.ts @@ -8,6 +8,13 @@ export const mapList = arr => { }, {}); } +export function mapListTolower(arr: string[]): {} { + return arr.reduce((acm, val) => { + acm[val.toLowerCase()] = ""; + return acm; + }, {}); +} + // Sort: treat numbers as number, strings as case-insensitive export const sortElements = (arr: any[]): any[] => { arr.sort((a, b) => { @@ -25,29 +32,35 @@ export const sortElements = (arr: any[]): any[] => { export const makeUnique = (arr: any[]): any[] => Array.from(new Set(arr)); -// Remove from set a elements of set b. +// Remove from set A elements of set B. export function arrayDiff(a: string[], b: string[]): string[] { const mappedB = mapList(b); return a.filter(e => !mappedB.hasOwnProperty(e)); } -export function findDuplicate(list: string[]): string { +// Remove from set A elements of set B, case insensitive +export function arrayDiffNocase(a: string[], b: string[]): string[] { + const mappedB = mapListTolower(b); + return a.filter(e => !mappedB.hasOwnProperty(e.toLowerCase())); +} + +export function findDuplicates(list: string[]): string[] { let m = new Map(); - let duplicate: string = null; + let duplicates: string[] = []; list.forEach(val => { - if (m.has(val)) { - duplicate = val; + if (m.has(val.toLowerCase())) { + duplicates.push(val); } else { - m.set(val, 0); + m.set(val.toLowerCase(), 0); } }); - return duplicate; + return makeUnique(duplicates); } // Check that two lists have no common elements, and no duplicates in either. // Do a single check: checking for duplicates in the concatenated list. -export function findCommonElementOrDuplicate(list1: string[], list2: string[]): string { - return findDuplicate(list1.concat(list2)); +export function findCommonElementsOrDuplicates(list1: string[], list2: string[]): string[] { + return findDuplicates(list1.concat(list2)); } // Compare two arrays, order does not matter diff --git a/test/index.test.ts b/test/index.test.ts index 0b7df4160..b9d6da1a1 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -1,6 +1,6 @@ import { - findDuplicate, - findCommonElementOrDuplicate, + findDuplicates, + findCommonElementsOrDuplicates, } from "../script/common/types"; import { isChecksum, @@ -13,9 +13,11 @@ import { } from "../script/common/image"; import { mapList, + mapListTolower, sortElements, makeUnique, arrayDiff, + arrayDiffNocase, arrayEqual } from "../script/common/types"; import { findImagesToFetch } from "../script/action/binance"; @@ -64,6 +66,8 @@ describe("Test image helpers", () => { describe("Test type helpers", () => { test(`Test mapList`, () => { expect(mapList(["a", "b", "c"]), `3 elems`).toEqual({"a": "", "b":"", "c": ""}); + expect(mapList(["a", "b", "C"]), `3 elems`).toEqual({"a": "", "b":"", "C": ""}); + expect(mapListTolower(["a", "b", "C"]), `3 elems`).toEqual({"a": "", "b":"", "c": ""}); }); test(`Test sortElements`, () => { expect(sortElements(["c", "a", "b"]), `3 elems`).toEqual(["a", "b", "c"]); @@ -77,22 +81,24 @@ describe("Test type helpers", () => { test(`Test arrayDiff`, () => { expect(arrayDiff(["a", "b", "c"], ["c"]), `4 elems with 1 duplicate`).toEqual(["a", "b"]); expect(arrayDiff(["a", "b", "c"], ["d"]), `4 elems with 0 duplicate`).toEqual(["a", "b", "c"]); + expect(arrayDiff(["a", "B", "c"], ["C"]), `4 elems with 0 duplicate`).toEqual(["a", "B", "c"]); + expect(arrayDiffNocase(["a", "B", "c"], ["C"]), `4 elems with 0 duplicate`).toEqual(["a", "B"]); }); - test(`Test findDuplicate`, () => { - expect(findDuplicate(["a", "bb", "ccc"]), `No duplicates`).toBe(null) - expect(findDuplicate(["a", "bb", "ccc", "bb"]), `One double duplicate`).toBe("bb") - expect(findDuplicate([]), `Empty array`).toBe(null) - expect(findDuplicate(["a"]), `One element`).toBe(null) - expect(findDuplicate(["a", "bb", "ccc", "bb", "d", "bb"]), `One triple duplicate`).toBe("bb") - expect(findDuplicate(["a", "bb", "ccc", "bb", "a"]), `Two double duplicates`).toBe("a") + test(`Test findDuplicates`, () => { + expect(findDuplicates(["a", "bb", "ccc"]), `No duplicates`).toEqual([]); + expect(findDuplicates(["a", "bb", "ccc", "bb"]), `One double duplicate`).toEqual(["bb"]); + expect(findDuplicates([]), `Empty array`).toEqual([]); + expect(findDuplicates(["a"]), `One element`).toEqual([]); + expect(findDuplicates(["a", "bb", "ccc", "bb", "d", "bb"]), `One triple duplicate`).toEqual(["bb"]); + expect(findDuplicates(["a", "bb", "ccc", "bb", "a"]), `Two double duplicates`).toEqual(["bb", "a"]); }); - test(`Test findCommonElementOrDuplicate`, () => { - expect(findCommonElementOrDuplicate(["a", "bb", "ccc"], ["1", "22", "333"]), `No intersection or duplicates`).toBe(null) - expect(findCommonElementOrDuplicate(["a", "bb", "ccc"], ["1", "bb", "333"]), `Common element`).toBe("bb") - expect(findCommonElementOrDuplicate(["a", "bb", "ccc", "bb"], ["1", "22", "333"]), `Duplicate in first`).toBe("bb") - expect(findCommonElementOrDuplicate(["a", "bb", "ccc"], ["1", "22", "333", "22"]), `Duplicate in second`).toBe("22") - expect(findCommonElementOrDuplicate(["a", "bb", "ccc", "1", "bb"], ["1", "22", "333", "22"]), `Intersection and duplicates`).toBe("22") - expect(findCommonElementOrDuplicate([], []), `Empty lists`).toBe(null) + test(`Test findCommonElementsOrDuplicates`, () => { + expect(findCommonElementsOrDuplicates(["a", "bb", "ccc"], ["1", "22", "333"]), `No intersection or duplicates`).toEqual([]); + expect(findCommonElementsOrDuplicates(["a", "bb", "ccc"], ["1", "bb", "333"]), `Common element`).toEqual(["bb"]); + expect(findCommonElementsOrDuplicates(["a", "bb", "ccc", "bb"], ["1", "22", "333"]), `Duplicate in first`).toEqual(["bb"]); + expect(findCommonElementsOrDuplicates(["a", "bb", "ccc"], ["1", "22", "333", "22"]), `Duplicate in second`).toEqual(["22"]); + expect(findCommonElementsOrDuplicates(["a", "bb", "ccc", "1", "bb"], ["1", "22", "333", "22"]), `Intersection and duplicates`).toEqual(["bb", "1", "22"]); + expect(findCommonElementsOrDuplicates([], []), `Empty lists`).toEqual([]); }); test(`Test arrayEqual`, () => { expect(arrayEqual(["a", "b", "c"], ["a", "b", "c"]), `equal`).toBe(true);