tokenlist/image_download.py at main · foxwallet/tokenlist · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import os
import json
import requests
import re
from urllib.parse import urlparse
from os.path import basename, splitext
from PIL import Image


proxies = {
    "http": "http://127.0.0.1:7897",
    "https": "http://127.0.0.1:7897"
}


def download():
    replace_list = [
        r"^https://pbs\.twimg\.com",
        r"^https://assets\.coingecko\.com",
        r"^https://s2\.coinmarketcap\.com",
        r"^https://silkswap\.me",
        r"^https://minerva\.digital",
        r"^https://basescan\.org",
        r"^https://img\.cryptorank.io",
        r"^https://raw\.githubusercontent\.com/",
        r"^https://static\.oklink\.com",
        r"^https://swap\.zchains\.com",
        r"^https://static\.tronscan\.org",
        r"^https://archive\.cetus\.zone",
        r"^https://assets\.playdarktimes\.com",
        r"^https://.*?\.arweave\.net",
        r"^https://.*?\.amazonaws\.com",
        r"^https://coin-images\.coingecko\.com",
        r"^https://image-cdn\.solana\.fm",
    ]

    for f in os.listdir("."):
        if not f.endswith(".json"):
            continue
        need_update = False
        with open(f, "r") as reader:
            tokens = json.load(reader)
            for index, token in enumerate(tokens):
                symbol: str = token.get("display", token["symbol"])
                symbol = symbol.replace(" ", "_")
                logoURI: str = token.get("logoURI")
                if not logoURI:
                    print(f"No logoURI of {symbol}")
                    continue
                if "foxwallet/tokenlist" in logoURI:
                    continue
                need_replace = False
                for patten in replace_list:
                    if re.search(patten, logoURI):
                        need_replace = True
                        break
                if not need_replace:
                    continue
                if "assets.coingecko.com" in logoURI:
                    logoURI = logoURI.replace("/standard/", "/large/")
                _, suffix = splitext(basename(urlparse(logoURI).path))
                if suffix not in [".jpg", ".png", ".jpeg", ".webp"]:
                    continue
                filename = f"./img/{symbol}{suffix}"
                webpname = f"./img/{symbol}.webp"
                if not os.path.exists(webpname):
                    try:
                        print("donwloading", logoURI, "to", filename)
                        resp = requests.get(logoURI, proxies=proxies, timeout=10)
                        if resp.status_code != 200:
                            raise Exception(f"request failed {resp.status_code}")
                        with open(filename, "wb+") as writer:
                            writer.write(resp.content)
                        im = Image.open(filename)
                        im.save(webpname, "WEBP")
                        os.remove(filename)
                    except BaseException as e:
                        print(f"cannot download image for {token['address']} due to {e}")
                        continue
                else:
                    print(f"reuse {symbol}.webp")
                tokens[index]["logoURI"] = f"https://raw.githubusercontent.com/foxwallet/tokenlist/main/img/{symbol}.webp"
                need_update = True
        if need_update:
            with open(f, "w") as writer:
                json.dump(tokens, writer, indent=2)
            print("updated", f)


def is_image_url_valid(url):
    try:
        response = requests.head(url, proxies=proxies)
        if 200 <= response.status_code < 300:
            return True
        return False
    except requests.RequestException as e:
        return False


def check_dead_link():
    for f in os.listdir("."):
        if not f.endswith(".json"):
            continue
        print(f"working on {f}")
        with open(f, "r") as reader:
            tokens = json.load(reader)
            for token in tokens:
                logoURI: str = token.get("logoURI")
                if not logoURI:
                    continue
                if logoURI.startswith("https://tokens-data.1inch.io"):
                    continue
                if not is_image_url_valid(logoURI):
                    print(f"{logoURI} is dead")


if __name__ == "__main__":
    download()
    # check_dead_link()