chore: import upstream snapshot with attribution

internal/processor/fixers.go

@@ -0,0 +1,185 @@
+package processor
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/trustwallet/assets-go-libs/file"
+	"github.com/trustwallet/assets-go-libs/image"
+	"github.com/trustwallet/assets-go-libs/path"
+	"github.com/trustwallet/assets-go-libs/validation"
+	"github.com/trustwallet/assets-go-libs/validation/info"
+	"github.com/trustwallet/go-primitives/address"
+	"github.com/trustwallet/go-primitives/coin"
+	"github.com/trustwallet/go-primitives/types"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func (s *Service) FixJSON(f *file.AssetFile) error {
+	return file.FormatJSONFile(f.Path())
+}
+
+func (s *Service) FixETHAddressChecksum(f *file.AssetFile) error {
+	if !coin.IsEVM(f.Chain().ID) {
+		return nil
+	}
+
+	assetDir := filepath.Base(f.Path())
+
+	err := validation.ValidateETHForkAddress(f.Chain(), assetDir)
+	if err != nil {
+		checksum, e := address.EIP55Checksum(assetDir)
+		if e != nil {
+			return fmt.Errorf("failed to get checksum: %s", e)
+		}
+
+		newName := path.GetAssetPath(f.Chain().Handle, checksum)
+
+		if e = os.Rename(f.Path(), newName); e != nil {
+			return fmt.Errorf("failed to rename dir: %s", e)
+		}
+
+		s.fileService.UpdateFile(f, checksum)
+
+		log.WithField("from", assetDir).
+			WithField("to", checksum).
+			Debug("Renamed asset")
+	}
+
+	return nil
+}
+
+func (s *Service) FixLogo(f *file.AssetFile) error {
+	width, height, err := image.GetPNGImageDimensions(f.Path())
+	if err != nil {
+		return err
+	}
+
+	var isLogoTooLarge bool
+	if width > validation.MaxW || height > validation.MaxH {
+		isLogoTooLarge = true
+	}
+
+	if isLogoTooLarge {
+		log.WithField("path", f.Path()).Debug("Fixing too large image")
+
+		targetW, targetH := calculateTargetDimension(width, height)
+
+		err = image.ResizePNG(f.Path(), targetW, targetH)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = validation.ValidateLogoFileSize(f.Path())
+	if err != nil { //nolint:staticcheck
+		// TODO: Compress images.
+	}
+
+	return nil
+}
+
+func calculateTargetDimension(width, height int) (targetW, targetH int) {
+	widthFloat := float32(width)
+	heightFloat := float32(height)
+
+	maxEdge := widthFloat
+	if heightFloat > widthFloat {
+		maxEdge = heightFloat
+	}
+
+	ratio := validation.MaxW / maxEdge
+
+	targetW = int(widthFloat * ratio)
+	targetH = int(heightFloat * ratio)
+
+	return targetW, targetH
+}
+
+func (s *Service) FixChainInfoJSON(f *file.AssetFile) error {
+	var chainInfo info.CoinModel
+
+	err := file.ReadJSONFile(f.Path(), &chainInfo)
+	if err != nil {
+		return err
+	}
+
+	expectedType := string(types.Coin)
+	if chainInfo.Type == nil || *chainInfo.Type != expectedType {
+		chainInfo.Type = &expectedType
+
+		data, err := file.PrepareJSONData(&chainInfo)
+		if err != nil {
+			return err
+		}
+
+		return file.CreateJSONFile(f.Path(), data)
+	}
+
+	return nil
+}
+
+func (s *Service) FixAssetInfo(f *file.AssetFile) error {
+	var assetInfo info.AssetModel
+
+	err := file.ReadJSONFile(f.Path(), &assetInfo)
+	if err != nil {
+		return err
+	}
+
+	var isModified bool
+
+	// Fix asset type.
+	var assetType string
+	if assetInfo.Type != nil {
+		assetType = *assetInfo.Type
+	}
+
+	// We need to skip error check to fix asset type if it's incorrect or empty.
+	chain, _ := types.GetChainFromAssetType(assetType)
+
+	expectedTokenType, ok := types.GetTokenType(f.Chain().ID, f.Asset())
+	if !ok {
+		expectedTokenType = strings.ToUpper(assetType)
+	}
+
+	// https://github.com/trustwallet/backend/issues/2561
+	if chain.ID != coin.CRYPTOORG && chain.ID != coin.CRONOS {
+		if chain.ID != f.Chain().ID || !strings.EqualFold(assetType, expectedTokenType) {
+			assetInfo.Type = &expectedTokenType
+			isModified = true
+		}
+	}
+
+	// Fix asset id.
+	assetID := f.Asset()
+	if assetInfo.ID == nil || *assetInfo.ID != assetID {
+		assetInfo.ID = &assetID
+		isModified = true
+	}
+
+	expectedExplorerURL, err := coin.GetCoinExploreURL(f.Chain(), f.Asset(), assetType)
+	if err != nil {
+		return err
+	}
+
+	// Fix asset explorer url.
+	if assetInfo.Explorer == nil || !strings.EqualFold(expectedExplorerURL, *assetInfo.Explorer) {
+		assetInfo.Explorer = &expectedExplorerURL
+		isModified = true
+	}
+
+	if isModified {
+		data, err := file.PrepareJSONData(&assetInfo)
+		if err != nil {
+			return err
+		}
+
+		return file.CreateJSONFile(f.Path(), data)
+	}
+
+	return nil
+}

internal/processor/model.go

@@ -0,0 +1,51 @@
+package processor
+
+import (
+	"github.com/trustwallet/assets-go-libs/file"
+)
+
+type (
+	Validator struct {
+		Name string
+		Run  func(f *file.AssetFile) error
+	}
+
+	Fixer struct {
+		Name string
+		Run  func(f *file.AssetFile) error
+	}
+
+	Updater struct {
+		Name string
+		Run  func() error
+	}
+)
+
+type (
+	ForceListPair struct {
+		Token0 string
+		Token1 string
+	}
+
+	TradingPairs struct {
+		Data struct {
+			Pairs []TradingPair `json:"pairs"`
+		} `json:"data"`
+	}
+
+	TradingPair struct {
+		ID         string     `json:"id"`
+		ReserveUSD string     `json:"reserveUSD"`
+		VolumeUSD  string     `json:"volumeUSD"`
+		TxCount    string     `json:"txCount"`
+		Token0     *TokenInfo `json:"token0"`
+		Token1     *TokenInfo `json:"token1"`
+	}
+
+	TokenInfo struct {
+		ID       string `json:"id"`
+		Symbol   string `json:"symbol"`
+		Name     string `json:"name"`
+		Decimals string `json:"decimals"`
+	}
+)

internal/processor/service.go

@@ -0,0 +1,119 @@
+package processor
+
+import (
+	assetsmanager "github.com/trustwallet/assets-go-libs/client/assets-manager"
+	"github.com/trustwallet/assets-go-libs/file"
+	"github.com/trustwallet/assets/internal/config"
+)
+
+type Service struct {
+	fileService   *file.Service
+	assetsManager assetsmanager.Client
+}
+
+func NewService(fileProvider *file.Service) *Service {
+	return &Service{
+		fileService:   fileProvider,
+		assetsManager: assetsmanager.InitClient(config.Default.ClientURLs.AssetsManagerAPI, nil),
+	}
+}
+
+func (s *Service) GetValidator(f *file.AssetFile) []Validator {
+	jsonValidator := Validator{Name: "JSON validation", Run: s.ValidateJSON}
+
+	switch f.Type() {
+	case file.TypeAssetFolder:
+		return []Validator{
+			{Name: "Each asset folder has valid asset address and contains only allowed files", Run: s.ValidateAssetFolder},
+		}
+	case file.TypeChainFolder:
+		return []Validator{
+			{Name: "Chain folders are lowercase and contains only allowed files", Run: s.ValidateChainFolder},
+		}
+	case file.TypeChainInfoFolder:
+		return []Validator{
+			{Name: "Chain Info Folder (has files)", Run: s.ValidateInfoFolder},
+		}
+	case file.TypeDappsFolder:
+		return []Validator{
+			{Name: "Dapps folder contains only allowed png files in lowercase", Run: s.ValidateDappsFolder},
+		}
+	case file.TypeRootFolder:
+		return []Validator{
+			{Name: "Root folder contains only allowed files", Run: s.ValidateRootFolder},
+		}
+	case file.TypeValidatorsAssetFolder:
+		return []Validator{
+			{Name: "Validators asset folder has logo and valid asset address)", Run: s.ValidateValidatorsAssetFolder},
+		}
+
+	case file.TypeAssetLogoFile, file.TypeChainLogoFile, file.TypeDappsLogoFile, file.TypeValidatorsLogoFile:
+		return []Validator{
+			{Name: "Logos size and dimension are valid", Run: s.ValidateImage},
+		}
+	case file.TypeAssetInfoFile:
+		return []Validator{
+			jsonValidator,
+			{Name: "Asset info file is valid", Run: s.ValidateAssetInfoFile},
+		}
+	case file.TypeChainInfoFile:
+		return []Validator{
+			{Name: "Chain info file is valid", Run: s.ValidateChainInfoFile},
+		}
+	case file.TypeTokenListFile:
+		return []Validator{
+			jsonValidator,
+			{Name: "Tokenlist file is valid", Run: s.ValidateTokenListFile},
+		}
+	case file.TypeTokenListExtendedFile:
+		return []Validator{
+			jsonValidator,
+			{Name: "Tokenlist Extended file is valid", Run: s.ValidateTokenListExtendedFile},
+		}
+	case file.TypeValidatorsListFile:
+		return []Validator{
+			jsonValidator,
+			{Name: "Validators list file is valid", Run: s.ValidateValidatorsListFile},
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) GetFixers(f *file.AssetFile) []Fixer {
+	jsonFixer := Fixer{
+		Name: "Formatting all json files",
+		Run:  s.FixJSON,
+	}
+
+	switch f.Type() {
+	case file.TypeAssetFolder:
+		return []Fixer{
+			{Name: "Renaming EVM's asset folder to valid address checksum", Run: s.FixETHAddressChecksum},
+		}
+	case file.TypeAssetInfoFile:
+		return []Fixer{
+			jsonFixer,
+			{Name: "Fixing asset info.json files", Run: s.FixAssetInfo},
+		}
+	case file.TypeChainInfoFile:
+		return []Fixer{
+			jsonFixer,
+			{Name: "Fixing chain info.json files", Run: s.FixChainInfoJSON},
+		}
+	case file.TypeChainLogoFile, file.TypeAssetLogoFile, file.TypeValidatorsLogoFile, file.TypeDappsLogoFile:
+		return []Fixer{
+			{Name: "Resizing and compressing logo images", Run: s.FixLogo},
+		}
+	case file.TypeValidatorsListFile:
+		return []Fixer{
+			jsonFixer,
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) GetUpdatersAuto() []Updater {
+	return []Updater{}
+}

internal/processor/validators.go

@@ -0,0 +1,351 @@
+package processor
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"github.com/trustwallet/assets-go-libs/file"
+	"github.com/trustwallet/assets-go-libs/path"
+	"github.com/trustwallet/assets-go-libs/validation"
+	"github.com/trustwallet/assets-go-libs/validation/list"
+	"github.com/trustwallet/assets-go-libs/validation/tokenlist"
+	"github.com/trustwallet/assets/internal/config"
+	"github.com/trustwallet/assets/internal/info"
+	"github.com/trustwallet/go-primitives/coin"
+)
+
+func (s *Service) ValidateJSON(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	buf := bytes.NewBuffer(nil)
+	_, err = buf.ReadFrom(file)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateJSON(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateRootFolder(f *file.AssetFile) error {
+	dirFiles, err := file.ReadDir(f.Path())
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.RootFolder.AllowedFiles)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateChainFolder(f *file.AssetFile) error {
+	file, err := os.Open(f.Path())
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+
+	err = validation.ValidateLowercase(fileInfo.Name())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	dirFiles, err := file.ReadDir(0)
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.ChainFolder.AllowedFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateImage(f *file.AssetFile) error {
+	var compErr = validation.NewErrComposite()
+
+	err := validation.ValidateLogoFileSize(f.Path())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	// TODO: Replace it with validation.ValidatePngImageDimension when "assets" repo is fixed.
+	// Read comments in ValidatePngImageDimensionForCI.
+	err = validation.ValidatePngImageDimensionForCI(f.Path())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateAssetFolder(f *file.AssetFile) error {
+	dirFiles, err := file.ReadDir(f.Path())
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+
+	err = validation.ValidateAllowedFiles(dirFiles, config.Default.ValidatorsSettings.AssetFolder.AllowedFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	err = validation.ValidateAssetAddress(f.Chain(), f.Asset())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	errInfo := validation.ValidateHasFiles(dirFiles, []string{"info.json"})
+	errLogo := validation.ValidateHasFiles(dirFiles, []string{"logo.png"})
+
+	if errLogo != nil || errInfo != nil {
+		assetInfoPath := path.GetAssetInfoPath(f.Chain().Handle, f.Asset())
+
+		var infoJson info.AssetModel
+		if err = file.ReadJSONFile(assetInfoPath, &infoJson); err != nil {
+			return err
+		}
+
+		if infoJson.GetStatus() != "spam" && infoJson.GetStatus() != "abandoned" {
+			compErr.Append(fmt.Errorf("%w: logo.png for non-spam assest", validation.ErrMissingFile))
+		}
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateDappsFolder(f *file.AssetFile) error {
+	dirFiles, err := file.ReadDir(f.Path())
+	if err != nil {
+		return err
+	}
+
+	var compErr = validation.NewErrComposite()
+
+	for _, dirFile := range dirFiles {
+		err = validation.ValidateExtension(dirFile.Name(), config.Default.ValidatorsSettings.DappsFolder.Ext)
+		if err != nil {
+			compErr.Append(err)
+		}
+
+		err = validation.ValidateLowercase(dirFile.Name())
+		if err != nil {
+			compErr.Append(err)
+		}
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateChainInfoFile(f *file.AssetFile) error {
+	var coinInfo info.CoinModel
+	if err := file.ReadJSONFile(f.Path(), &coinInfo); err != nil {
+		return err
+	}
+
+	receivedTags, err := s.assetsManager.GetTagValues()
+	if err != nil {
+		return fmt.Errorf("failed to get tag values: %w", err)
+	}
+
+	tags := make([]string, len(receivedTags.Tags))
+	for i, t := range receivedTags.Tags {
+		tags[i] = t.ID
+	}
+
+	err = info.ValidateCoin(coinInfo, tags)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateAssetInfoFile(f *file.AssetFile) error {
+	var assetInfo info.AssetModel
+	if err := file.ReadJSONFile(f.Path(), &assetInfo); err != nil {
+		return err
+	}
+
+	// https://github.com/trustwallet/backend/issues/2561
+	if f.Chain() != coin.Cryptoorg() && f.Chain() != coin.Cronos() {
+		err := info.ValidateAsset(assetInfo, f.Chain(), f.Asset())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateValidatorsListFile(f *file.AssetFile) error {
+	if !isStackingChain(f.Chain()) {
+		return nil
+	}
+
+	var model []list.Model
+	if err := file.ReadJSONFile(f.Path(), &model); err != nil {
+		return err
+	}
+
+	err := list.ValidateList(model)
+	if err != nil {
+		return err
+	}
+
+	listIDs := make([]string, len(model))
+	for i, listItem := range model {
+		listIDs[i] = *listItem.ID
+	}
+
+	assetsPath := path.GetValidatorAssetsPath(f.Chain().Handle)
+	assetFolder := s.fileService.GetAssetFile(assetsPath)
+
+	dirFiles, err := file.ReadDir(assetFolder.Path())
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateAllowedFiles(dirFiles, listIDs)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isStackingChain(c coin.Coin) bool {
+	for _, stackingChain := range config.StackingChains {
+		if c.ID == stackingChain.ID {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (s *Service) ValidateTokenListFile(f *file.AssetFile) error {
+	tokenListPath := f.Path()
+	tokenListExtendedPath := path.GetTokenListPath(f.Chain().Handle, path.TokenlistExtended)
+
+	return validateTokenList(tokenListPath, tokenListExtendedPath, f.Chain())
+}
+
+func (s *Service) ValidateTokenListExtendedFile(f *file.AssetFile) error {
+	tokenListPathExtended := f.Path()
+	tokenListPath := path.GetTokenListPath(f.Chain().Handle, path.TokenlistDefault)
+
+	return validateTokenList(tokenListPathExtended, tokenListPath, f.Chain())
+}
+
+func validateTokenList(path1, path2 string, chain1 coin.Coin) error {
+	var tokenList1 tokenlist.Model
+	err := file.ReadJSONFile(path1, &tokenList1)
+	if err != nil {
+		return err
+	}
+
+	if file.Exists(path2) {
+		var tokenList2 tokenlist.Model
+		err = file.ReadJSONFile(path2, &tokenList2)
+		if err != nil {
+			return err
+		}
+
+		tokensMap := make(map[string]bool)
+		for _, token := range tokenList2.Tokens {
+			tokensMap[token.Asset] = true
+		}
+
+		for _, token := range tokenList1.Tokens {
+			if _, exists := tokensMap[token.Asset]; exists {
+				return fmt.Errorf("duplicate asset: %s from %s, already exist in %s",
+					token.Asset, path1, path2)
+			}
+		}
+	}
+
+	err = tokenlist.ValidateTokenList(tokenList1, chain1, path1)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateInfoFolder(f *file.AssetFile) error {
+	dirFiles, err := file.ReadDir(f.Path())
+	if err != nil {
+		return err
+	}
+
+	err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainInfoFolder.HasFiles)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) ValidateValidatorsAssetFolder(f *file.AssetFile) error {
+	dirFiles, err := file.ReadDir(f.Path())
+	if err != nil {
+		return err
+	}
+
+	compErr := validation.NewErrComposite()
+	err = validation.ValidateValidatorsAddress(f.Chain(), f.Asset())
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	err = validation.ValidateHasFiles(dirFiles, config.Default.ValidatorsSettings.ChainValidatorsAssetFolder.HasFiles)
+	if err != nil {
+		compErr.Append(err)
+	}
+
+	if compErr.Len() > 0 {
+		return compErr
+	}
+
+	return nil
+}